diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_drain.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_drain.rs index d46eede852..736118abaf 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_drain.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_inbox/store_drain.rs @@ -30,6 +30,16 @@ impl AgentInboxStore { recipient_member_id: &str, org_run_id: &str, task_id: &str, + ) -> Result { + Self::list_unread_task_input_for_turn(recipient_member_id, org_run_id, task_id, "", "") + } + + pub fn list_unread_task_input_for_turn( + recipient_member_id: &str, + org_run_id: &str, + task_id: &str, + session_id: &str, + turn_intent_id: &str, ) -> Result { let conn = get_connection().map_err(|err| err.to_string())?; let mut stmt = conn @@ -64,6 +74,12 @@ impl AgentInboxStore { AND json_type(inbox.payload_json,'$.task_id')='text' AND json_extract(inbox.payload_json,'$.task_id')=?3) OR + (task.status='in_progress' + AND inbox.payload_kind='task_assigned' + AND json_valid(inbox.payload_json) + AND json_type(inbox.payload_json,'$.task_id')='text' + AND json_extract(inbox.payload_json,'$.task_id')=?3) + OR (task.status='in_progress' AND task.execution_mode='plan' AND inbox.payload_kind='plan_approval_response' @@ -93,6 +109,17 @@ impl AgentInboxStore { .map_err(|err| err.to_string())? .collect::, _>>() .map_err(|err| err.to_string())?; + let task_is_pending = task_status_is_pending(&conn, org_run_id, task_id)?; + let mut filtered_rows = Vec::with_capacity(rows.len()); + for row in rows { + if row.payload_kind != "task_assigned" + || task_is_pending + || resume_continuation_owns_assignment(&conn, session_id, turn_intent_id, row.id)? + { + filtered_rows.push(row); + } + } + let rows = filtered_rows; let has_more = rows.len() > 1; Ok(AgentInboxBatch { rows: rows.into_iter().take(1).collect(), @@ -303,7 +330,7 @@ impl AgentInboxStore { /// Used by the turn-processor drain hook after rendering the /// attachment, so the next turn's drain returns an empty list. pub fn mark_many_read(ids: &[i64]) -> Result { - Self::mark_many_read_internal(ids, None) + Self::mark_many_read_internal(ids, None, None) } /// Production acknowledgement for transcript-backed delivery. Only the @@ -311,12 +338,23 @@ impl AgentInboxStore { /// it read. A stale Guard from an older/replaced Session therefore cannot /// acknowledge a row after ownership moved elsewhere. pub fn mark_many_read_for_session(ids: &[i64], session_id: &str) -> Result { - Self::mark_many_read_internal(ids, Some(session_id)) + Self::mark_many_read_internal(ids, Some(session_id), None) + } + + /// Formal Turn acknowledgement guarded by the exact current lifecycle + /// generation inside the same IMMEDIATE write transaction. + pub fn mark_many_read_for_turn( + ids: &[i64], + session_id: &str, + turn_intent_id: &str, + ) -> Result { + Self::mark_many_read_internal(ids, Some(session_id), Some(turn_intent_id)) } fn mark_many_read_internal( ids: &[i64], materialization_session_id: Option<&str>, + formal_turn_intent_id: Option<&str>, ) -> Result { if ids.is_empty() { return Ok(0); @@ -327,6 +365,46 @@ impl AgentInboxStore { let tx = conn .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) .map_err(|err| err.to_string())?; + if let (Some(session_id), Some(turn_intent_id)) = + (materialization_session_id, formal_turn_intent_id) + { + crate::coordination::agent_org_turn_contexts::validate_formal_turn_generation_with_connection( + &tx, + session_id, + turn_intent_id, + )?; + let is_resume_continuation = + turn_is_resume_continuation(&tx, session_id, turn_intent_id)?; + for id in ids { + let assignment_status: Option = tx + .query_row( + "SELECT task.status + FROM agent_org_runtime_inbox inbox + JOIN agent_org_runtime_tasks task + ON task.org_run_id=inbox.org_run_id + AND task.id=json_extract(inbox.payload_json,'$.task_id') + WHERE inbox.id=?1 AND inbox.payload_kind='task_assigned'", + [id], + |row| row.get(0), + ) + .optional() + .map_err(|err| err.to_string())?; + if is_resume_continuation && assignment_status.is_some() { + let owns_assignment = resume_continuation_owns_assignment( + &tx, + session_id, + turn_intent_id, + *id, + )?; + if assignment_status.as_deref() != Some("completed") || !owns_assignment + { + return Err(format!( + "Agent Org Inbox row {id} remains unread because its exact Resume continuation did not complete the Task successfully" + )); + } + } + } + } let now = chrono::Utc::now().to_rfc3339(); let mut updated = 0usize; let mut changed_run_ids = HashSet::new(); @@ -466,6 +544,107 @@ impl AgentInboxStore { } } +fn task_status_is_pending( + conn: &rusqlite::Connection, + org_run_id: &str, + task_id: &str, +) -> Result { + conn.query_row( + "SELECT status='pending' FROM agent_org_runtime_tasks + WHERE org_run_id=?1 AND id=?2", + params![org_run_id, task_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string()) +} + +/// Exact authority for the one exceptional TaskAssigned transition: the Task +/// was already moved to in_progress by the pre-Pause Turn, so only the +/// durable continuation created from that same handoff may consume it. +fn resume_continuation_owns_assignment( + conn: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + inbox_id: i64, +) -> Result { + conn.query_row( + "SELECT EXISTS( + SELECT 1 + FROM agent_org_runtime_pause_handoffs handoff + JOIN agent_org_runtime_pause_episodes episode + ON episode.episode_id=handoff.episode_id + JOIN agent_org_runtime_runs run ON run.id=handoff.org_run_id + JOIN agent_org_runtime_turn_contexts continuation + ON continuation.session_id=handoff.session_id + AND continuation.turn_intent_id=handoff.continuation_turn_intent_id + JOIN agent_org_runtime_turn_contexts original + ON original.session_id=handoff.session_id + AND original.turn_intent_id=handoff.original_turn_intent_id + JOIN session_turn_intents intent + ON intent.session_id=handoff.session_id + AND intent.turn_intent_id=handoff.continuation_turn_intent_id + JOIN agent_org_runtime_tasks task + ON task.org_run_id=handoff.org_run_id + AND task.id=handoff.task_id + JOIN agent_org_runtime_inbox inbox + ON inbox.id=?3 + AND inbox.org_run_id=handoff.org_run_id + AND inbox.recipient_member_id=handoff.participant_id + JOIN agent_org_runtime_inbox_materializations materialization + ON materialization.inbox_id=inbox.id + AND materialization.session_id=handoff.session_id + WHERE handoff.session_id=?1 + AND handoff.continuation_turn_intent_id=?2 + AND handoff.continuation_status='dispatched' + AND handoff.drain_status IN ('released','runtime_absent') + AND episode.status='consumed' + AND run.status='running' + AND intent.status IN ('queued','running') + AND continuation.turn_kind='task_execution' + AND continuation.source_kind='task' + AND continuation.task_id=handoff.task_id + AND continuation.owner_member_id=handoff.participant_id + AND continuation.activation_generation=run.activation_generation + AND original.activation_generation=handoff.original_activation_generation + AND original.task_id=handoff.task_id + AND original.owner_member_id=handoff.participant_id + AND task.status IN ('in_progress','completed') + AND task.owner=handoff.participant_id + AND inbox.read_at IS NULL + AND inbox.payload_kind='task_assigned' + AND json_valid(inbox.payload_json) + AND json_type(inbox.payload_json,'$.task_id')='text' + AND json_extract(inbox.payload_json,'$.task_id')=handoff.task_id + AND NOT EXISTS ( + SELECT 1 + FROM agent_org_runtime_inbox_delivery_resolutions resolution + WHERE resolution.inbox_id=inbox.id + ) + )", + params![session_id, turn_intent_id, inbox_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string()) +} + +fn turn_is_resume_continuation( + conn: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, +) -> Result { + conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM agent_org_runtime_pause_handoffs + WHERE session_id=?1 + AND continuation_turn_intent_id=?2 + AND continuation_status='dispatched' + )", + params![session_id, turn_intent_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string()) +} + // Other read-side store methods (`mark_read` for a single id, // `find_by_request_id`) will land alongside the next consumer that // actually needs them. They are intentionally not added here because diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_pause.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_pause.rs new file mode 100644 index 0000000000..6bc5382d50 --- /dev/null +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_pause.rs @@ -0,0 +1,1180 @@ +//! Durable Pause/Resume episodes for Agent Org formal work. +//! +//! The run lifecycle fence and the list of captured formal Turns are committed +//! together. Runtime teardown is deliberately post-commit and records evidence +//! back into these receipts; it never owns the Paused decision. + +use rusqlite::{params, Connection, OptionalExtension, TransactionBehavior}; +use serde::Serialize; + +use crate::coordination::agent_org_turn_contexts::{accept_with_connection, AgentOrgTurnAdmission}; + +const FORMAL_TURN_KINDS: [&str; 2] = ["coordinator", "task_execution"]; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PauseRunOutcome { + pub request_id: String, + pub run_id: String, + pub episode_id: String, + pub transitioned: bool, + pub pause_generation: i64, + pub captured_turn_count: usize, + pub draining_turn_count: usize, + pub timed_out_turn_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ResumeRunOutcome { + pub request_id: String, + pub run_id: String, + pub episode_id: String, + pub transitioned: bool, + pub resume_generation: i64, + pub continuation_count: usize, + pub skipped_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PauseHandoffSummary { + pub episode_id: String, + pub pause_generation: i64, + pub total_count: usize, + pub draining_count: usize, + pub timed_out_count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RunningPauseHandoff { + pub episode_id: String, + pub run_id: String, + pub session_id: String, + pub turn_intent_id: String, +} + +/// Internal ownership result for the process that actually committed the +/// Paused fence. Historical/idempotent callers receive the same wire outcome +/// but never start a second teardown owner. +pub(crate) struct PauseCommit { + pub outcome: PauseRunOutcome, + pub teardown_owner_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ContinuationDispatch { + pub episode_id: String, + pub run_id: String, + pub session_id: String, + pub turn_intent_id: String, + pub turn_kind: String, + pub task_id: Option, + pub member_dispatch_sequence: Option, +} + +pub(super) fn create_schema(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS agent_org_runtime_pause_episodes ( + episode_id TEXT PRIMARY KEY CHECK(length(trim(episode_id)) > 0), + org_run_id TEXT NOT NULL, + pause_request_id TEXT NOT NULL CHECK(length(trim(pause_request_id)) > 0), + pause_generation INTEGER NOT NULL CHECK(pause_generation >= 2), + status TEXT NOT NULL CHECK(status IN ('active','consumed')), + resume_request_id TEXT, + resume_generation INTEGER, + teardown_owner_id TEXT NOT NULL CHECK(length(trim(teardown_owner_id)) > 0), + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + resumed_at TEXT, + UNIQUE(org_run_id, pause_request_id), + UNIQUE(resume_request_id), + FOREIGN KEY(org_run_id) REFERENCES agent_org_runtime_runs(id) ON DELETE CASCADE, + CHECK( + (status='active' AND resume_request_id IS NULL + AND resume_generation IS NULL AND resumed_at IS NULL) + OR + (status='consumed' AND resume_request_id IS NOT NULL + AND resume_generation IS NOT NULL AND resume_generation > pause_generation + AND resumed_at IS NOT NULL) + ) + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_org_runtime_pause_one_active + ON agent_org_runtime_pause_episodes(org_run_id) + WHERE status='active'; + CREATE INDEX IF NOT EXISTS idx_agent_org_runtime_pause_request + ON agent_org_runtime_pause_episodes(org_run_id, pause_request_id); + + CREATE TABLE IF NOT EXISTS agent_org_runtime_pause_handoffs ( + handoff_id TEXT PRIMARY KEY CHECK(length(trim(handoff_id)) > 0), + episode_id TEXT NOT NULL, + org_run_id TEXT NOT NULL, + session_id TEXT NOT NULL, + original_turn_intent_id TEXT NOT NULL, + turn_kind TEXT NOT NULL CHECK(turn_kind IN ('coordinator','task_execution')), + participant_id TEXT NOT NULL, + task_id TEXT, + original_owner_member_id TEXT, + original_activation_generation INTEGER NOT NULL CHECK(original_activation_generation >= 1), + original_intent_status TEXT NOT NULL CHECK(original_intent_status IN ('queued','running')), + drain_status TEXT NOT NULL CHECK(drain_status IN ( + 'waiting','released','runtime_absent','timed_out' + )), + runtime_lease_id TEXT, + dialog_turn_generation TEXT, + yield_requested_at TEXT, + released_at TEXT, + drain_timeout_at TEXT, + drain_error TEXT, + continuation_turn_intent_id TEXT, + continuation_status TEXT CHECK(continuation_status IN ('queued','dispatched','skipped')), + skip_reason TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE(episode_id, session_id, original_turn_intent_id), + UNIQUE(continuation_turn_intent_id), + FOREIGN KEY(episode_id) REFERENCES agent_org_runtime_pause_episodes(episode_id) ON DELETE CASCADE, + FOREIGN KEY(org_run_id) REFERENCES agent_org_runtime_runs(id) ON DELETE CASCADE, + FOREIGN KEY(session_id, original_turn_intent_id) + REFERENCES agent_org_runtime_turn_contexts(session_id, turn_intent_id) + ON DELETE CASCADE, + CHECK( + (turn_kind='coordinator' AND task_id IS NULL AND original_owner_member_id IS NULL) + OR + (turn_kind='task_execution' AND task_id IS NOT NULL + AND original_owner_member_id=participant_id) + ), + CHECK( + (runtime_lease_id IS NULL AND dialog_turn_generation IS NULL) + OR + (runtime_lease_id IS NOT NULL AND dialog_turn_generation IS NOT NULL) + ), + CHECK( + (continuation_status IS NULL AND continuation_turn_intent_id IS NULL AND skip_reason IS NULL) + OR + (continuation_status IN ('queued','dispatched') + AND continuation_turn_intent_id IS NOT NULL AND skip_reason IS NULL) + OR + (continuation_status='skipped' + AND continuation_turn_intent_id IS NULL AND skip_reason IS NOT NULL) + ) + ); + CREATE INDEX IF NOT EXISTS idx_agent_org_runtime_pause_capture + ON agent_org_runtime_turn_contexts( + org_run_id, activation_generation, turn_kind, session_id, turn_intent_id + ) + WHERE turn_kind IN ('coordinator','task_execution'); + CREATE INDEX IF NOT EXISTS idx_agent_org_runtime_pause_drain + ON agent_org_runtime_pause_handoffs(episode_id, drain_status, session_id); + CREATE INDEX IF NOT EXISTS idx_agent_org_runtime_pause_dispatch + ON agent_org_runtime_pause_handoffs(continuation_status, org_run_id, session_id);", + ) +} + +pub fn pause_run(run_id: &str, request_id: &str) -> Result { + pause_run_commit(run_id, request_id).map(|commit| commit.outcome) +} + +pub(crate) fn pause_run_commit(run_id: &str, request_id: &str) -> Result { + validate_request_id(request_id)?; + database::db::with_sessions_writer(|| { + let mut conn = database::db::get_connection().map_err(|error| error.to_string())?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|error| error.to_string())?; + + if let Some(outcome) = pause_outcome_for_request(&tx, run_id, request_id, true)? { + tx.commit().map_err(|error| error.to_string())?; + return Ok(PauseCommit { + outcome, + teardown_owner_id: None, + }); + } + + let run: Option<(String, i64)> = tx + .query_row( + "SELECT status, activation_generation FROM agent_org_runtime_runs WHERE id=?1", + [run_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((status, generation)) = run else { + return Err(format!("Agent Org run {run_id} does not exist")); + }; + if status == "paused" { + let mut outcome = active_pause_outcome(&tx, run_id)? + .ok_or_else(|| format!("paused Agent Org run {run_id} has no active episode"))?; + outcome.request_id = request_id.to_string(); + outcome.transitioned = false; + tx.commit().map_err(|error| error.to_string())?; + return Ok(PauseCommit { + outcome, + teardown_owner_id: None, + }); + } + if status != "running" { + return Err(format!( + "Agent Org run {run_id} is {status}; only a Working Team can be paused" + )); + } + + let episode_id = uuid::Uuid::new_v4().to_string(); + let teardown_owner_id = uuid::Uuid::new_v4().to_string(); + let pause_generation = generation + .checked_add(1) + .ok_or_else(|| format!("Agent Org run {run_id} generation overflow"))?; + let now = chrono::Utc::now().to_rfc3339(); + + let changed = tx + .execute( + "UPDATE agent_org_runtime_runs + SET status='paused', activation_generation=?2, updated_at=?3 + WHERE id=?1 AND status='running' AND activation_generation=?4", + params![run_id, pause_generation, &now, generation], + ) + .map_err(|error| error.to_string())?; + if changed != 1 { + return Err(format!( + "Agent Org run {run_id} changed while Pause was committing" + )); + } + tx.execute( + "INSERT INTO agent_org_runtime_pause_episodes ( + episode_id,org_run_id,pause_request_id,pause_generation,status, + teardown_owner_id,created_at,updated_at + ) VALUES (?1,?2,?3,?4,'active',?5,?6,?6)", + params![ + &episode_id, + run_id, + request_id, + pause_generation, + &teardown_owner_id, + &now + ], + ) + .map_err(|error| error.to_string())?; + + let mut statement = tx + .prepare( + "SELECT context.session_id,context.turn_intent_id,context.turn_kind, + context.participant_id,context.task_id,context.owner_member_id, + context.activation_generation,intent.status + FROM agent_org_runtime_turn_contexts context + JOIN session_turn_intents intent + ON intent.session_id=context.session_id + AND intent.turn_intent_id=context.turn_intent_id + WHERE context.org_run_id=?1 + AND context.activation_generation=?2 + AND context.turn_kind IN (?3,?4) + AND intent.org_run_id=?1 + AND intent.status IN ('queued','running') + ORDER BY context.context_id ASC", + ) + .map_err(|error| error.to_string())?; + let captures = statement + .query_map( + params![ + run_id, + generation, + FORMAL_TURN_KINDS[0], + FORMAL_TURN_KINDS[1] + ], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + row.get::<_, i64>(6)?, + row.get::<_, String>(7)?, + )) + }, + ) + .map_err(|error| error.to_string())? + .collect::, _>>() + .map_err(|error| error.to_string())?; + drop(statement); + + for ( + session_id, + intent_id, + kind, + participant, + task_id, + owner, + turn_generation, + intent_status, + ) in &captures + { + let drain_status = if intent_status == "queued" { + "runtime_absent" + } else { + "waiting" + }; + let released_at = (drain_status == "runtime_absent").then_some(now.as_str()); + tx.execute( + "INSERT INTO agent_org_runtime_pause_handoffs ( + handoff_id,episode_id,org_run_id,session_id,original_turn_intent_id, + turn_kind,participant_id,task_id,original_owner_member_id, + original_activation_generation,original_intent_status,drain_status, + released_at,created_at,updated_at + ) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?14)", + params![ + uuid::Uuid::new_v4().to_string(), + &episode_id, + run_id, + session_id, + intent_id, + kind, + participant, + task_id, + owner, + turn_generation, + intent_status, + drain_status, + released_at, + &now, + ], + ) + .map_err(|error| error.to_string())?; + } + + tx.execute( + "UPDATE session_turn_intents + SET status='stale', updated_at=?2 + WHERE org_run_id=?1 AND status='queued' + AND EXISTS ( + SELECT 1 FROM agent_org_runtime_pause_handoffs handoff + WHERE handoff.episode_id=?3 + AND handoff.session_id=session_turn_intents.session_id + AND handoff.original_turn_intent_id=session_turn_intents.turn_intent_id + )", + params![run_id, &now, &episode_id], + ) + .map_err(|error| error.to_string())?; + + tx.commit().map_err(|error| error.to_string())?; + let draining = captures.iter().filter(|item| item.7 == "running").count(); + Ok(PauseCommit { + outcome: PauseRunOutcome { + request_id: request_id.to_string(), + run_id: run_id.to_string(), + episode_id, + transitioned: true, + pause_generation, + captured_turn_count: captures.len(), + draining_turn_count: draining, + timed_out_turn_count: 0, + }, + teardown_owner_id: Some(teardown_owner_id), + }) + }) +} + +pub fn resume_run(run_id: &str, request_id: &str) -> Result { + validate_request_id(request_id)?; + database::db::with_sessions_writer(|| { + let mut conn = database::db::get_connection().map_err(|error| error.to_string())?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .map_err(|error| error.to_string())?; + + if let Some(outcome) = resume_outcome_for_request(&tx, run_id, request_id, true)? { + tx.commit().map_err(|error| error.to_string())?; + return Ok(outcome); + } + + let run: Option<(String, i64, Option)> = tx + .query_row( + "SELECT status,activation_generation,root_session_id + FROM agent_org_runtime_runs WHERE id=?1", + [run_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((status, generation, root_session_id)) = run else { + return Err(format!("Agent Org run {run_id} does not exist")); + }; + if status != "paused" { + return Err(format!( + "Agent Org run {run_id} is {status}; only a Paused Team can be resumed" + )); + } + let episode: Option<(String, i64)> = tx + .query_row( + "SELECT episode_id,pause_generation + FROM agent_org_runtime_pause_episodes + WHERE org_run_id=?1 AND status='active'", + [run_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((episode_id, pause_generation)) = episode else { + return Err(format!( + "paused Agent Org run {run_id} has no active episode" + )); + }; + if generation != pause_generation { + return Err(format!( + "Pause episode generation {pause_generation} does not match run generation {generation}" + )); + } + let resume_generation = generation + .checked_add(1) + .ok_or_else(|| format!("Agent Org run {run_id} generation overflow"))?; + let now = chrono::Utc::now().to_rfc3339(); + let changed = tx + .execute( + "UPDATE agent_org_runtime_runs + SET status='running',activation_generation=?2,updated_at=?3 + WHERE id=?1 AND status='paused' AND activation_generation=?4", + params![run_id, resume_generation, &now, generation], + ) + .map_err(|error| error.to_string())?; + if changed != 1 { + return Err(format!( + "Agent Org run {run_id} changed while Resume was committing" + )); + } + + let handoffs = load_handoffs_for_resume(&tx, &episode_id)?; + let mut continuation_count = 0usize; + let mut skipped_count = 0usize; + for handoff in handoffs { + let skip_reason = + continuation_skip_reason(&tx, run_id, root_session_id.as_deref(), &handoff)?; + if let Some(reason) = skip_reason { + tx.execute( + "UPDATE agent_org_runtime_pause_handoffs + SET continuation_status='skipped',skip_reason=?2,updated_at=?3 + WHERE handoff_id=?1 AND continuation_status IS NULL", + params![&handoff.handoff_id, reason, &now], + ) + .map_err(|error| error.to_string())?; + resolve_terminal_task_assignment_after_resume_skip( + &tx, run_id, &handoff, &reason, &now, + )?; + skipped_count += 1; + continue; + } + + let continuation_turn_intent_id = format!("agent-org-cont-{}", uuid::Uuid::new_v4()); + let admission = match handoff.turn_kind.as_str() { + "coordinator" => AgentOrgTurnAdmission::coordinator( + run_id, + &handoff.session_id, + &continuation_turn_intent_id, + Some(continuation_turn_intent_id.clone()), + crate::foundation::session_bridge::TurnIntentBridgeSource::Resume, + ), + "task_execution" => AgentOrgTurnAdmission::task_continuation( + run_id, + &handoff.session_id, + &continuation_turn_intent_id, + Some(continuation_turn_intent_id.clone()), + handoff + .task_id + .as_deref() + .ok_or_else(|| "Task handoff has no task_id".to_string())?, + handoff + .owner_member_id + .as_deref() + .ok_or_else(|| "Task handoff has no owner".to_string())?, + resume_generation, + ), + other => return Err(format!("unknown formal handoff kind {other:?}")), + }; + accept_with_connection(&tx, &admission)?; + tx.execute( + "UPDATE agent_org_runtime_pause_handoffs + SET continuation_turn_intent_id=?2,continuation_status='queued',updated_at=?3 + WHERE handoff_id=?1 AND continuation_status IS NULL", + params![&handoff.handoff_id, &continuation_turn_intent_id, &now], + ) + .map_err(|error| error.to_string())?; + continuation_count += 1; + } + + tx.execute( + "UPDATE agent_org_runtime_pause_episodes + SET status='consumed',resume_request_id=?2,resume_generation=?3, + resumed_at=?4,updated_at=?4 + WHERE episode_id=?1 AND status='active'", + params![&episode_id, request_id, resume_generation, &now], + ) + .map_err(|error| error.to_string())?; + tx.commit().map_err(|error| error.to_string())?; + Ok(ResumeRunOutcome { + request_id: request_id.to_string(), + run_id: run_id.to_string(), + episode_id, + transitioned: true, + resume_generation, + continuation_count, + skipped_count, + }) + }) +} + +#[derive(Debug)] +struct ResumeHandoff { + handoff_id: String, + session_id: String, + turn_kind: String, + task_id: Option, + owner_member_id: Option, + original_intent_status: String, +} + +fn load_handoffs_for_resume( + conn: &Connection, + episode_id: &str, +) -> Result, String> { + let mut statement = conn + .prepare( + "SELECT handoff_id,session_id,turn_kind,task_id,original_owner_member_id, + original_intent_status + FROM agent_org_runtime_pause_handoffs + WHERE episode_id=?1 ORDER BY created_at ASC,handoff_id ASC", + ) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map([episode_id], |row| { + Ok(ResumeHandoff { + handoff_id: row.get(0)?, + session_id: row.get(1)?, + turn_kind: row.get(2)?, + task_id: row.get(3)?, + owner_member_id: row.get(4)?, + original_intent_status: row.get(5)?, + }) + }) + .map_err(|error| error.to_string())?; + let result = rows + .collect::, _>>() + .map_err(|error| error.to_string())?; + Ok(result) +} + +fn continuation_skip_reason( + conn: &Connection, + run_id: &str, + root_session_id: Option<&str>, + handoff: &ResumeHandoff, +) -> Result, String> { + let materialized = |member_id: &str| -> Result { + conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM agent_org_runtime_member_materializations materialization + JOIN agent_sessions session + ON session.session_id=materialization.session_id + WHERE materialization.org_run_id=?1 + AND materialization.member_id=?2 + AND materialization.session_id=?3 + AND materialization.status='succeeded' + AND session.agent_definition_id=materialization.agent_id + AND session.org_member_id=materialization.member_id + AND NOT EXISTS ( + SELECT 1 FROM agent_org_runtime_member_materializations newer + WHERE newer.org_run_id=materialization.org_run_id + AND newer.member_id=materialization.member_id + AND newer.status='succeeded' + AND newer.generation>materialization.generation + ) + )", + params![run_id, member_id, &handoff.session_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string()) + }; + + if handoff.turn_kind == "coordinator" { + if root_session_id != Some(handoff.session_id.as_str()) { + return Ok(Some("coordinator_session_changed".to_string())); + } + if !materialized(super::agent_org_runs::COORDINATOR_MEMBER_ID)? { + return Ok(Some("coordinator_materialization_changed".to_string())); + } + return Ok(None); + } + + let task_id = handoff + .task_id + .as_deref() + .ok_or_else(|| "Task handoff has no task_id".to_string())?; + let owner = handoff + .owner_member_id + .as_deref() + .ok_or_else(|| "Task handoff has no owner".to_string())?; + let task: Option<(String, Option)> = conn + .query_row( + "SELECT status,owner FROM agent_org_runtime_tasks WHERE org_run_id=?1 AND id=?2", + params![run_id, task_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((status, current_owner)) = task else { + return Ok(Some("task_missing".to_string())); + }; + if !matches!(status.as_str(), "pending" | "in_progress") { + return Ok(Some(format!("task_{status}"))); + } + if current_owner.as_deref() != Some(owner) { + return Ok(Some("task_owner_changed".to_string())); + } + if !materialized(owner)? { + return Ok(Some("member_materialization_changed".to_string())); + } + let _ = &handoff.original_intent_status; + Ok(None) +} + +/// A Task can become terminal after its assignment was materialized but before +/// the old Turn acknowledges that Inbox row. Pause correctly rejects the late +/// acknowledgement, and Resume correctly skips the terminal Task; resolve the +/// now-undeliverable assignment in the same Resume transaction so it cannot +/// keep the Run non-quiescent or wake the old owner again. +fn resolve_terminal_task_assignment_after_resume_skip( + conn: &Connection, + run_id: &str, + handoff: &ResumeHandoff, + skip_reason: &str, + now: &str, +) -> Result<(), String> { + if handoff.turn_kind != "task_execution" + || !matches!( + skip_reason, + "task_completed" | "task_failed" | "task_cancelled" + ) + { + return Ok(()); + } + let task_id = handoff + .task_id + .as_deref() + .ok_or_else(|| "terminal Task handoff has no task_id".to_string())?; + let owner = handoff + .owner_member_id + .as_deref() + .ok_or_else(|| "terminal Task handoff has no owner".to_string())?; + let resolution_reason = format!("pause_resume_{skip_reason}"); + conn.execute( + "INSERT OR IGNORE INTO agent_org_runtime_inbox_delivery_resolutions ( + inbox_id,org_run_id,resolution_kind,resolved_by_member_id,reason, + replacement_inbox_id,replacement_task_id,created_at + ) + SELECT inbox.id,inbox.org_run_id,'cancelled','coordinator',?4,NULL,NULL,?5 + FROM agent_org_runtime_inbox inbox + WHERE inbox.org_run_id=?1 + AND inbox.recipient_member_id=?2 + AND inbox.payload_kind='task_assigned' + AND json_extract(inbox.payload_json,'$.task_id')=?3 + AND inbox.read_at IS NULL", + params![run_id, owner, task_id, &resolution_reason, now], + ) + .map_err(|error| error.to_string())?; + conn.execute( + "DELETE FROM agent_org_runtime_inbox_materializations + WHERE inbox_id IN ( + SELECT resolution.inbox_id + FROM agent_org_runtime_inbox_delivery_resolutions resolution + WHERE resolution.org_run_id=?1 + AND resolution.resolution_kind='cancelled' + AND resolution.reason=?2 + )", + params![run_id, &resolution_reason], + ) + .map_err(|error| error.to_string())?; + Ok(()) +} + +pub(crate) fn list_running_handoffs( + episode_id: &str, + teardown_owner_id: &str, +) -> Result, String> { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + let mut statement = conn + .prepare( + "SELECT handoff.episode_id,handoff.org_run_id,handoff.session_id, + handoff.original_turn_intent_id + FROM agent_org_runtime_pause_handoffs handoff + JOIN agent_org_runtime_pause_episodes episode + ON episode.episode_id=handoff.episode_id + WHERE handoff.episode_id=?1 AND episode.teardown_owner_id=?2 + AND handoff.original_intent_status='running' + AND handoff.drain_status='waiting' + ORDER BY handoff.session_id ASC,handoff.created_at ASC", + ) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map(params![episode_id, teardown_owner_id], |row| { + Ok(RunningPauseHandoff { + episode_id: row.get(0)?, + run_id: row.get(1)?, + session_id: row.get(2)?, + turn_intent_id: row.get(3)?, + }) + }) + .map_err(|error| error.to_string())?; + let result = rows + .collect::, _>>() + .map_err(|error| error.to_string())?; + Ok(result) +} + +pub(crate) fn bind_runtime_and_request_yield( + episode_id: &str, + session_id: &str, + turn_intent_id: &str, + runtime_lease_id: &str, + dialog_turn_generation: &str, +) -> Result { + let now = chrono::Utc::now().to_rfc3339(); + database::db::with_sessions_writer(|| { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + conn.execute( + "UPDATE agent_org_runtime_pause_handoffs + SET runtime_lease_id=?4,dialog_turn_generation=?5,yield_requested_at=?6,updated_at=?6 + WHERE episode_id=?1 AND session_id=?2 AND original_turn_intent_id=?3 + AND drain_status='waiting' AND runtime_lease_id IS NULL", + params![ + episode_id, + session_id, + turn_intent_id, + runtime_lease_id, + dialog_turn_generation, + &now + ], + ) + .map(|changed| changed == 1) + .map_err(|error| error.to_string()) + }) +} + +pub(crate) fn mark_runtime_absent( + episode_id: &str, + session_id: &str, + turn_intent_id: &str, +) -> Result { + let now = chrono::Utc::now().to_rfc3339(); + database::db::with_sessions_writer(|| { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + conn.execute( + "UPDATE agent_org_runtime_pause_handoffs + SET drain_status='runtime_absent',released_at=?4,updated_at=?4 + WHERE episode_id=?1 AND session_id=?2 AND original_turn_intent_id=?3 + AND drain_status IN ('waiting','timed_out') AND runtime_lease_id IS NULL", + params![episode_id, session_id, turn_intent_id, &now], + ) + .map(|changed| changed == 1) + .map_err(|error| error.to_string()) + }) +} + +pub(crate) fn mark_released( + session_id: &str, + turn_intent_id: &str, + runtime_lease_id: &str, + dialog_turn_generation: &str, +) -> Result, String> { + let now = chrono::Utc::now().to_rfc3339(); + database::db::with_sessions_writer(|| { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + conn.query_row( + "UPDATE agent_org_runtime_pause_handoffs + SET drain_status='released',released_at=?5,updated_at=?5 + WHERE session_id=?1 AND original_turn_intent_id=?2 + AND runtime_lease_id=?3 AND dialog_turn_generation=?4 + AND drain_status IN ('waiting','timed_out') + RETURNING episode_id", + params![ + session_id, + turn_intent_id, + runtime_lease_id, + dialog_turn_generation, + &now + ], + |row| row.get(0), + ) + .optional() + .map_err(|error| error.to_string()) + }) +} + +pub(crate) fn bound_episode_for_runtime( + session_id: &str, + turn_intent_id: &str, + runtime_lease_id: &str, + dialog_turn_generation: &str, +) -> Result, String> { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + conn.query_row( + "SELECT episode_id FROM agent_org_runtime_pause_handoffs + WHERE session_id=?1 AND original_turn_intent_id=?2 + AND runtime_lease_id=?3 AND dialog_turn_generation=?4 + AND drain_status IN ('waiting','timed_out') + ORDER BY created_at DESC LIMIT 1", + params![ + session_id, + turn_intent_id, + runtime_lease_id, + dialog_turn_generation + ], + |row| row.get(0), + ) + .optional() + .map_err(|error| error.to_string()) +} + +pub(crate) fn mark_unresolved_timed_out(episode_id: &str) -> Result { + let now = chrono::Utc::now().to_rfc3339(); + database::db::with_sessions_writer(|| { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + conn.execute( + "UPDATE agent_org_runtime_pause_handoffs + SET drain_status='timed_out',drain_timeout_at=?2, + drain_error='runtime did not yield within 10 seconds',updated_at=?2 + WHERE episode_id=?1 AND drain_status='waiting'", + params![episode_id, &now], + ) + .map_err(|error| error.to_string()) + }) +} + +pub fn pause_summary_for_run(run_id: &str) -> Result, String> { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + pause_summary_with_connection(&conn, run_id) +} + +pub fn pause_summary_with_connection( + conn: &Connection, + run_id: &str, +) -> Result, String> { + conn.query_row( + "SELECT episode.episode_id,episode.pause_generation, + COUNT(handoff.handoff_id), + COALESCE(SUM(CASE WHEN handoff.drain_status IN ('waiting','timed_out') THEN 1 ELSE 0 END),0), + COALESCE(SUM(CASE WHEN handoff.drain_timeout_at IS NOT NULL THEN 1 ELSE 0 END),0) + FROM agent_org_runtime_pause_episodes episode + LEFT JOIN agent_org_runtime_pause_handoffs handoff ON handoff.episode_id=episode.episode_id + WHERE episode.org_run_id=?1 + GROUP BY episode.episode_id,episode.pause_generation,episode.created_at + ORDER BY episode.created_at DESC LIMIT 1", + [run_id], + |row| { + Ok(PauseHandoffSummary { + episode_id: row.get(0)?, + pause_generation: row.get(1)?, + total_count: row.get::<_, i64>(2)? as usize, + draining_count: row.get::<_, i64>(3)? as usize, + timed_out_count: row.get::<_, i64>(4)? as usize, + }) + }, + ) + .optional() + .map_err(|error| error.to_string()) +} + +pub(crate) fn list_dispatchable_continuations( + limit: usize, +) -> Result, String> { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + let mut statement = conn + .prepare( + "SELECT handoff.episode_id,handoff.org_run_id,handoff.session_id, + handoff.continuation_turn_intent_id,handoff.turn_kind,handoff.task_id, + context.member_dispatch_sequence + FROM agent_org_runtime_pause_handoffs handoff + JOIN agent_org_runtime_pause_episodes episode ON episode.episode_id=handoff.episode_id + JOIN agent_org_runtime_runs run ON run.id=handoff.org_run_id + JOIN agent_org_runtime_turn_contexts context + ON context.session_id=handoff.session_id + AND context.turn_intent_id=handoff.continuation_turn_intent_id + JOIN session_turn_intents intent + ON intent.session_id=handoff.session_id + AND intent.turn_intent_id=handoff.continuation_turn_intent_id + WHERE episode.status='consumed' AND run.status='running' + AND handoff.continuation_status='queued' + AND handoff.drain_status IN ('released','runtime_absent') + AND intent.status='queued' + ORDER BY CASE WHEN context.member_dispatch_sequence IS NULL THEN 0 ELSE 1 END, + context.dispatch_member_id,context.member_dispatch_sequence,handoff.created_at + LIMIT ?1", + ) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map([i64::try_from(limit).unwrap_or(i64::MAX)], |row| { + Ok(ContinuationDispatch { + episode_id: row.get(0)?, + run_id: row.get(1)?, + session_id: row.get(2)?, + turn_intent_id: row.get(3)?, + turn_kind: row.get(4)?, + task_id: row.get(5)?, + member_dispatch_sequence: row.get(6)?, + }) + }) + .map_err(|error| error.to_string())?; + let result = rows + .collect::, _>>() + .map_err(|error| error.to_string())?; + Ok(result) +} + +pub(crate) fn continuation_participant_ids(episode_id: &str) -> Result, String> { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + let mut statement = conn + .prepare( + "SELECT DISTINCT participant_id + FROM agent_org_runtime_pause_handoffs + WHERE episode_id=?1 AND continuation_status IN ('queued','dispatched') + ORDER BY participant_id ASC", + ) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map([episode_id], |row| row.get::<_, String>(0)) + .map_err(|error| error.to_string())?; + let result = rows + .collect::, _>>() + .map_err(|error| error.to_string())?; + Ok(result) +} + +/// Resolve the transient provider instruction for one already-claimed +/// continuation. The receipt, current run fence, and base intent must all +/// still agree; callers persist neither this text nor a synthetic Inbox row. +pub(crate) fn continuation_nudge_for_turn( + session_id: &str, + turn_intent_id: &str, +) -> Result, String> { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + let continuation: Option<(String, Option)> = conn + .query_row( + "SELECT handoff.turn_kind,handoff.task_id + FROM agent_org_runtime_pause_handoffs handoff + JOIN agent_org_runtime_pause_episodes episode + ON episode.episode_id=handoff.episode_id + JOIN agent_org_runtime_runs run ON run.id=handoff.org_run_id + JOIN session_turn_intents intent + ON intent.session_id=handoff.session_id + AND intent.turn_intent_id=handoff.continuation_turn_intent_id + WHERE handoff.session_id=?1 + AND handoff.continuation_turn_intent_id=?2 + AND handoff.continuation_status='dispatched' + AND episode.status='consumed' + AND run.status='running' + AND intent.status IN ('queued','running')", + params![session_id, turn_intent_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + match continuation { + None => Ok(None), + Some((kind, Some(task_id))) if kind == "task_execution" => Ok(Some(format!( + "Continue the paused Agent Org task {task_id} from its persisted Task, Inbox, and conversation state. Do not create a replacement Task." + ))), + Some((kind, None)) if kind == "coordinator" => Ok(Some( + "Continue coordinating the paused Agent Org run from its persisted Task, Inbox, and conversation state. Do not restart work that is already terminal." + .to_string(), + )), + Some((kind, task_id)) => Err(format!( + "invalid Agent Org continuation receipt kind={kind:?}, task_id={task_id:?}" + )), + } +} + +/// Atomically grant one dispatcher ownership of a queued continuation. +pub(crate) fn claim_continuation_dispatch( + episode_id: &str, + turn_intent_id: &str, +) -> Result { + database::db::with_sessions_writer(|| { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + conn.execute( + "UPDATE agent_org_runtime_pause_handoffs + SET continuation_status='dispatched',updated_at=?3 + WHERE episode_id=?1 AND continuation_turn_intent_id=?2 + AND continuation_status='queued'", + params![episode_id, turn_intent_id, chrono::Utc::now().to_rfc3339()], + ) + .map(|changed| changed == 1) + .map_err(|error| error.to_string()) + }) +} + +/// Return a failed in-process dispatch to the durable queue. The exact base +/// intent must still be queued; once it starts or terminates, replaying it +/// would be unsafe. +pub(crate) fn requeue_continuation_dispatch( + episode_id: &str, + turn_intent_id: &str, +) -> Result { + database::db::with_sessions_writer(|| { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + conn.execute( + "UPDATE agent_org_runtime_pause_handoffs + SET continuation_status='queued',updated_at=?3 + WHERE episode_id=?1 AND continuation_turn_intent_id=?2 + AND continuation_status='dispatched' + AND EXISTS ( + SELECT 1 FROM session_turn_intents intent + WHERE intent.session_id=agent_org_runtime_pause_handoffs.session_id + AND intent.turn_intent_id=?2 AND intent.status='queued' + )", + params![episode_id, turn_intent_id, chrono::Utc::now().to_rfc3339()], + ) + .map(|changed| changed == 1) + .map_err(|error| error.to_string()) + }) +} + +/// A process restart proves every pre-restart in-memory runtime is absent. +/// Keep timeout evidence, but unblock any durable continuation that was +/// correctly waiting for that old process-owned lease. +pub(crate) fn reconcile_runtime_absence_after_restart(conn: &Connection) -> Result { + let now = chrono::Utc::now().to_rfc3339(); + let tx = database::db::begin_immediate(conn).map_err(|error| error.to_string())?; + let runtime_rows = tx + .execute( + "UPDATE agent_org_runtime_pause_handoffs + SET drain_status='runtime_absent',released_at=COALESCE(released_at,?1),updated_at=?1 + WHERE drain_status IN ('waiting','timed_out')", + [&now], + ) + .map_err(|error| error.to_string())?; + // A claimed continuation may have crossed the scheduler boundary before + // the process died. Requeue the same durable intent (never insert a new + // continuation) so restart recovery is at-most-one by receipt while still + // making progress from persisted Task/Inbox state. + let intent_rows = tx + .execute( + "UPDATE session_turn_intents AS intent + SET status='queued',updated_at=?1 + WHERE intent.status='running' + AND EXISTS ( + SELECT 1 FROM agent_org_runtime_pause_handoffs handoff + JOIN agent_org_runtime_pause_episodes episode + ON episode.episode_id=handoff.episode_id + JOIN agent_org_runtime_runs run ON run.id=handoff.org_run_id + WHERE handoff.session_id=intent.session_id + AND handoff.continuation_turn_intent_id=intent.turn_intent_id + AND handoff.continuation_status='dispatched' + AND episode.status='consumed' + AND run.status='running' + )", + [&now], + ) + .map_err(|error| error.to_string())?; + let dispatch_rows = tx + .execute( + "UPDATE agent_org_runtime_pause_handoffs + SET continuation_status='queued',updated_at=?1 + WHERE continuation_status='dispatched' + AND EXISTS ( + SELECT 1 FROM session_turn_intents intent + WHERE intent.session_id=agent_org_runtime_pause_handoffs.session_id + AND intent.turn_intent_id=agent_org_runtime_pause_handoffs.continuation_turn_intent_id + AND intent.status='queued' + )", + [&now], + ) + .map_err(|error| error.to_string())?; + tx.commit().map_err(|error| error.to_string())?; + Ok(runtime_rows + intent_rows + dispatch_rows) +} + +fn validate_request_id(request_id: &str) -> Result<(), String> { + uuid::Uuid::parse_str(request_id) + .map(|_| ()) + .map_err(|_| "Pause/Resume request_id must be a UUID".to_string()) +} + +fn active_pause_outcome( + conn: &Connection, + run_id: &str, +) -> Result, String> { + let request_id: Option = conn + .query_row( + "SELECT pause_request_id FROM agent_org_runtime_pause_episodes + WHERE org_run_id=?1 AND status='active'", + [run_id], + |row| row.get(0), + ) + .optional() + .map_err(|error| error.to_string())?; + request_id + .map(|request| pause_outcome_for_request(conn, run_id, &request, false)) + .transpose() + .map(|value| value.flatten()) +} + +fn pause_outcome_for_request( + conn: &Connection, + run_id: &str, + request_id: &str, + transitioned: bool, +) -> Result, String> { + conn.query_row( + "SELECT episode.episode_id,episode.pause_generation, + COUNT(handoff.handoff_id), + COALESCE(SUM(CASE WHEN handoff.drain_status IN ('waiting','timed_out') THEN 1 ELSE 0 END),0), + COALESCE(SUM(CASE WHEN handoff.drain_timeout_at IS NOT NULL THEN 1 ELSE 0 END),0) + FROM agent_org_runtime_pause_episodes episode + LEFT JOIN agent_org_runtime_pause_handoffs handoff ON handoff.episode_id=episode.episode_id + WHERE episode.org_run_id=?1 AND episode.pause_request_id=?2 + GROUP BY episode.episode_id,episode.pause_generation", + params![run_id, request_id], + |row| { + Ok(PauseRunOutcome { + request_id: request_id.to_string(), + run_id: run_id.to_string(), + episode_id: row.get(0)?, + transitioned, + pause_generation: row.get(1)?, + captured_turn_count: row.get::<_, i64>(2)? as usize, + draining_turn_count: row.get::<_, i64>(3)? as usize, + timed_out_turn_count: row.get::<_, i64>(4)? as usize, + }) + }, + ) + .optional() + .map_err(|error| error.to_string()) +} + +fn resume_outcome_for_request( + conn: &Connection, + run_id: &str, + request_id: &str, + transitioned: bool, +) -> Result, String> { + conn.query_row( + "SELECT episode.episode_id,episode.resume_generation, + COALESCE(SUM(CASE WHEN handoff.continuation_status IN ('queued','dispatched') THEN 1 ELSE 0 END),0), + COALESCE(SUM(CASE WHEN handoff.continuation_status='skipped' THEN 1 ELSE 0 END),0) + FROM agent_org_runtime_pause_episodes episode + LEFT JOIN agent_org_runtime_pause_handoffs handoff ON handoff.episode_id=episode.episode_id + WHERE episode.org_run_id=?1 AND episode.resume_request_id=?2 + GROUP BY episode.episode_id,episode.resume_generation", + params![run_id, request_id], + |row| { + Ok(ResumeRunOutcome { + request_id: request_id.to_string(), + run_id: run_id.to_string(), + episode_id: row.get(0)?, + transitioned, + resume_generation: row.get(1)?, + continuation_count: row.get::<_, i64>(2)? as usize, + skipped_count: row.get::<_, i64>(3)? as usize, + }) + }, + ) + .optional() + .map_err(|error| error.to_string()) +} diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/tests.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/tests.rs index 3aa6ac0bfb..7a3a66b5d0 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/tests.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_plan_approvals/tests.rs @@ -36,6 +36,7 @@ fn setup(policy: PlanApprovalPolicy) -> (test_helpers::test_env::SandboxGuard, A crate::coordination::agent_org_runs::init_schema(&conn).expect("run schema"); crate::coordination::agent_org_turn_contexts::create_schema(&conn) .expect("Agent Org Turn context schema"); + crate::coordination::agent_org_pause::create_schema(&conn).expect("Agent Org Pause schema"); crate::coordination::agent_org_tasks::init_schema(&conn).expect("task schema"); crate::coordination::agent_inbox::init_schema(&conn).expect("inbox schema"); init_schema(&conn).expect("approval schema"); @@ -719,7 +720,11 @@ fn paused_run_rejects_plan_decisions_without_mutating_task() { let (_sandbox, context) = setup(PlanApprovalPolicy::User); create_plan_task(&context); let pending = create_pending_approval(&context); - AgentOrgRunStore::mark_paused(&context.run_id).expect("pause run"); + crate::coordination::agent_org_pause::pause_run( + &context.run_id, + &uuid::Uuid::new_v4().to_string(), + ) + .expect("pause run"); let error = AgentOrgPlanApprovalStore::approve( &pending.approval_id, @@ -743,7 +748,11 @@ fn startup_cleanup_preserves_pending_approval_for_paused_run() { let (_sandbox, context) = setup(PlanApprovalPolicy::User); create_plan_task(&context); let pending = create_pending_approval(&context); - AgentOrgRunStore::mark_paused(&context.run_id).expect("pause run"); + crate::coordination::agent_org_pause::pause_run( + &context.run_id, + &uuid::Uuid::new_v4().to_string(), + ) + .expect("pause run"); let cancelled = AgentOrgPlanApprovalStore::cancel_pending_for_terminal_or_missing_runs() .expect("run startup approval cleanup"); diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs index bead69d5de..0d1b701e7d 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_runs/store.rs @@ -594,32 +594,6 @@ impl AgentOrgRunStore { Self::list_runs_by_status(AgentOrgRunStatus::Starting, limit) } - /// Pause a running run. Only transitions `running → paused`; already - /// non-running runs are left unchanged and return `Ok(false)` (idempotent). - pub fn mark_paused(run_id: &str) -> Result { - let paused = validate_status(AgentOrgRunStatus::Paused.as_str())?; - let running = validate_status(AgentOrgRunStatus::Running.as_str())?; - let now = chrono::Utc::now().to_rfc3339(); - let changed = with_sessions_writer(|| -> Result { - let conn = get_connection().map_err(|err| err.to_string())?; - let rows_changed = conn - .execute( - "UPDATE agent_org_runtime_runs - SET status = ?1, - updated_at = ?2 - WHERE id = ?3 - AND status = ?4", - params![paused.as_str(), now, run_id, running.as_str()], - ) - .map_err(|err| err.to_string())?; - Ok(rows_changed > 0) - })?; - if changed { - crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); - } - Ok(changed) - } - /// Apply failed-Turn recovery after crash cleanup has converted a stranded /// Member session to Abandoned. Recovery proceeds only when one persisted /// running TaskExecution identifies the exact Task; a missing or ambiguous @@ -662,32 +636,6 @@ impl AgentOrgRunStore { Ok(changed) } - /// Resume a paused run. Only transitions `paused → running`; already - /// non-paused runs are left unchanged and return `Ok(false)` (idempotent). - pub fn mark_resumed(run_id: &str) -> Result { - let running = validate_status(AgentOrgRunStatus::Running.as_str())?; - let paused = validate_status(AgentOrgRunStatus::Paused.as_str())?; - let now = chrono::Utc::now().to_rfc3339(); - let changed = with_sessions_writer(|| -> Result { - let conn = get_connection().map_err(|err| err.to_string())?; - let rows_changed = conn - .execute( - "UPDATE agent_org_runtime_runs - SET status = ?1, - updated_at = ?2 - WHERE id = ?3 - AND status = ?4", - params![running.as_str(), now, run_id, paused.as_str()], - ) - .map_err(|err| err.to_string())?; - Ok(rows_changed > 0) - })?; - if changed { - crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); - } - Ok(changed) - } - pub fn fail_starting( run_id: &str, expected_generation: i64, diff --git a/src-tauri/crates/agent-core/src/core/coordination/agent_org_turn_contexts.rs b/src-tauri/crates/agent-core/src/core/coordination/agent_org_turn_contexts.rs index 4e790dc211..353845b173 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/agent_org_turn_contexts.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/agent_org_turn_contexts.rs @@ -194,6 +194,28 @@ impl AgentOrgTurnAdmission { } } + pub(crate) fn task_continuation( + org_run_id: impl Into, + session_id: impl Into, + turn_intent_id: impl Into, + client_message_id: Option, + task_id: impl Into, + owner_member_id: impl Into, + activation_generation: i64, + ) -> Self { + let mut request = Self::task_execution( + org_run_id, + session_id, + turn_intent_id, + client_message_id, + task_id, + owner_member_id, + activation_generation, + ); + request.base_source = TurnIntentBridgeSource::Resume; + request + } + #[cfg_attr(not(test), allow(dead_code))] pub(crate) fn direct_member( org_run_id: impl Into, @@ -400,6 +422,11 @@ fn accept_wake_with_connection( client_message_id: Option, member_id: &str, ) -> Result { + if has_live_pause_continuation(conn, org_run_id, member_id)? { + return Err(invariant_error(format!( + "Participant {member_id} already has a durable Pause continuation" + ))); + } if member_id == COORDINATOR_MEMBER_ID { return accept_with_connection( conn, @@ -429,6 +456,41 @@ fn accept_wake_with_connection( ) } +/// A Resume receipt is the sole owner of its participant until the persisted +/// continuation intent leaves the scheduler. This check lives inside the same +/// IMMEDIATE transaction as ordinary Wake admission, so the watchdog, unread +/// Inbox hooks, and restart recovery cannot enqueue a second formal Turn for +/// work that Resume is already continuing. +fn has_live_pause_continuation( + conn: &Connection, + org_run_id: &str, + participant_id: &str, +) -> Result { + conn.query_row( + "SELECT EXISTS( + SELECT 1 + FROM agent_org_runtime_pause_handoffs handoff + JOIN agent_org_runtime_pause_episodes episode + ON episode.episode_id=handoff.episode_id + JOIN agent_org_runtime_runs run ON run.id=handoff.org_run_id + JOIN agent_org_runtime_turn_contexts context + ON context.session_id=handoff.session_id + AND context.turn_intent_id=handoff.continuation_turn_intent_id + JOIN session_turn_intents intent + ON intent.session_id=handoff.session_id + AND intent.turn_intent_id=handoff.continuation_turn_intent_id + WHERE handoff.org_run_id=?1 AND handoff.participant_id=?2 + AND episode.status='consumed' AND run.status='running' + AND handoff.continuation_status IN ('queued','dispatched') + AND intent.status IN ('queued','running') + AND context.activation_generation=run.activation_generation + )", + params![org_run_id, participant_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string()) +} + /// Re-check the persisted authority immediately before a queued Agent Org /// Turn is promoted to Running. Admission is intentionally not a lease: Task /// cancellation/reassignment, dependency changes, member replacement, or an @@ -1407,6 +1469,64 @@ pub(crate) fn require_context_with_connection( }) } +/// Load an already-admitted typed Turn without creating or rewriting either +/// lifecycle row. Durable Pause continuations use this before enqueue; the +/// execute-time path still performs the full Running/generation revalidation. +pub(crate) fn require_existing_context( + org_run_id: &str, + session_id: &str, + turn_intent_id: &str, +) -> Result { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + let context = require_context_with_connection(&conn, session_id, turn_intent_id)?; + if context.org_run_id != org_run_id { + return Err(invariant_error(format!( + "continuation context run mismatch: expected {org_run_id}, found {}", + context.org_run_id + ))); + } + Ok(context) +} + +/// Validate only the lifecycle fence for a formal Turn. This narrower check +/// is used by the post-provider Inbox acknowledgement: the Turn may have +/// legitimately completed its Task, but it must still belong to the current +/// Working generation before it can consume formal input. +pub(crate) fn validate_formal_turn_generation_with_connection( + conn: &Connection, + session_id: &str, + turn_intent_id: &str, +) -> Result { + let context = require_context_with_connection(conn, session_id, turn_intent_id)?; + if !matches!( + context.turn_kind, + AgentOrgTurnKind::Coordinator | AgentOrgTurnKind::TaskExecution + ) { + return Err(invariant_error( + "Inbox acknowledgement requires a formal Turn".to_string(), + )); + } + let run: Option<(String, i64)> = conn + .query_row( + "SELECT status,activation_generation FROM agent_org_runtime_runs WHERE id=?1", + [&context.org_run_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let Some((status, generation)) = run else { + return Err(invariant_error("formal Turn run disappeared".to_string())); + }; + if status != AgentOrgRunStatus::Running.as_str() + || context.activation_generation != Some(generation) + { + return Err(invariant_error(format!( + "formal Turn generation fence rejected status={status}, generation={generation}" + ))); + } + Ok(context) +} + fn read_context_optional( conn: &Connection, session_id: &str, @@ -1502,7 +1622,8 @@ pub fn reconcile_in_flight_after_restart(conn: &Connection) -> Result Result(0), + ) + .expect("count rejected duplicate intent"), + 0 + ); + + conn.execute( + "UPDATE session_turn_intents SET status='completed' + WHERE session_id=?1 AND turn_intent_id=?2", + params![MEMBER_SESSION_ID, continuation_id], + ) + .expect("finish continuation"); + let transaction = conn + .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) + .expect("post-continuation Wake transaction"); + let next = accept_wake_with_connection( + &transaction, + RUN_ID, + MEMBER_SESSION_ID, + "turn-after-continuation", + None, + MEMBER_ID, + ) + .expect("ordinary Wake may proceed after continuation is terminal"); + transaction.commit().expect("commit post-continuation Wake"); + assert_eq!(next.member_dispatch_sequence, Some(3)); +} + #[test] fn failed_or_cancelled_blockers_never_unlock_a_task_wake() { let mut conn = connection(); diff --git a/src-tauri/crates/agent-core/src/core/coordination/mod.rs b/src-tauri/crates/agent-core/src/core/coordination/mod.rs index ea0b1f18cc..e1d465a005 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/mod.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/mod.rs @@ -21,6 +21,7 @@ pub mod agent_inbox; pub mod agent_member_interventions; +pub mod agent_org_pause; pub mod agent_org_payload_limits; pub mod agent_org_plan_approvals; pub mod agent_org_run_events; @@ -48,5 +49,7 @@ pub fn init_agent_org_schemas(conn: &rusqlite::Connection) -> rusqlite::Result<( pub fn reconcile_agent_org_turns_after_restart( conn: &rusqlite::Connection, ) -> Result { - agent_org_turn_contexts::reconcile_in_flight_after_restart(conn) + let runtime_absence = agent_org_pause::reconcile_runtime_absence_after_restart(conn)?; + let turn_reconciliation = agent_org_turn_contexts::reconcile_in_flight_after_restart(conn)?; + Ok(runtime_absence + turn_reconciliation) } diff --git a/src-tauri/crates/agent-core/src/core/coordination/schema.rs b/src-tauri/crates/agent-core/src/core/coordination/schema.rs index 5f8d297625..ee2c27517a 100644 --- a/src-tauri/crates/agent-core/src/core/coordination/schema.rs +++ b/src-tauri/crates/agent-core/src/core/coordination/schema.rs @@ -11,11 +11,11 @@ use std::collections::BTreeMap; use rusqlite::{ffi, Connection, Error as SqliteError, Result as SqliteResult}; use super::{ - agent_inbox, agent_member_interventions, agent_org_plan_approvals, agent_org_runs, - agent_org_tasks, agent_org_turn_contexts, agent_org_watchdog, + agent_inbox, agent_member_interventions, agent_org_pause, agent_org_plan_approvals, + agent_org_runs, agent_org_tasks, agent_org_turn_contexts, agent_org_watchdog, }; -const RUNTIME_TABLES: [&str; 15] = [ +const RUNTIME_TABLES: [&str; 17] = [ "agent_org_runtime_runs", "agent_org_runtime_run_progress", "agent_org_runtime_member_materializations", @@ -31,6 +31,8 @@ const RUNTIME_TABLES: [&str; 15] = [ "agent_org_runtime_member_interventions", "agent_org_runtime_member_dispatch_allocators", "agent_org_runtime_turn_contexts", + "agent_org_runtime_pause_episodes", + "agent_org_runtime_pause_handoffs", ]; const LEGACY_TABLES: [&str; 13] = [ @@ -128,7 +130,8 @@ fn create_runtime_schema(conn: &Connection) -> SqliteResult<()> { agent_org_plan_approvals::create_schema(conn)?; agent_member_interventions::create_schema(conn)?; agent_org_watchdog::create_schema(conn)?; - agent_org_turn_contexts::create_schema(conn) + agent_org_turn_contexts::create_schema(conn)?; + agent_org_pause::create_schema(conn) } fn expected_manifest() -> SqliteResult { @@ -610,7 +613,7 @@ mod tests { DROP TABLE agent_org_runtime_member_dispatch_allocators;", ) .expect("make partial schema"); - assert_eq!(count_known_tables(&conn, &RUNTIME_TABLES).unwrap(), 13); + assert_eq!(count_known_tables(&conn, &RUNTIME_TABLES).unwrap(), 15); } "changed" => { conn.execute_batch( @@ -637,6 +640,26 @@ mod tests { } } + #[test] + fn previous_fifteen_table_manifest_requires_an_isolated_database() { + let conn = connection(); + initialize(&conn).expect("canonical pause runtime"); + conn.execute_batch( + "DROP TABLE agent_org_runtime_pause_handoffs; + DROP TABLE agent_org_runtime_pause_episodes;", + ) + .expect("simulate the previous strict fifteen-table manifest"); + assert_eq!(count_known_tables(&conn, &RUNTIME_TABLES).unwrap(), 15); + + let error = initialize(&conn).expect_err("previous runtime must not be migrated in place"); + assert!( + error + .to_string() + .contains("found 15 of 17 canonical tables"), + "unexpected strict-schema error: {error}" + ); + } + #[test] fn create_failure_rolls_back_every_legacy_drop() { let conn = connection(); @@ -701,7 +724,7 @@ mod tests { let conn = Connection::open(path).expect("reopen shared database"); verify_manifest(&conn, &expected_manifest().expect("expected manifest")) .expect("canonical manifest after concurrent init"); - assert_eq!(count_known_tables(&conn, &RUNTIME_TABLES).unwrap(), 15); + assert_eq!(count_known_tables(&conn, &RUNTIME_TABLES).unwrap(), 17); } #[test] diff --git a/src-tauri/crates/agent-core/src/core/providers/e2e_fake.rs b/src-tauri/crates/agent-core/src/core/providers/e2e_fake.rs index 985fae3a34..b702840dab 100644 --- a/src-tauri/crates/agent-core/src/core/providers/e2e_fake.rs +++ b/src-tauri/crates/agent-core/src/core/providers/e2e_fake.rs @@ -16,9 +16,11 @@ const ADDRESS_COMMENTS_MARKER: &str = const ADDRESS_COMMENT_ID_MARKER: &str = " — id: "; const REPLY_SESSION_COMMENT_TOOL: &str = "reply_session_comment"; const AGENT_ORG_TASK_FSM_MARKER: &str = "E2E_AGENT_ORG_TASK_FSM:"; +const AGENT_ORG_PAUSE_MARKER: &str = "E2E_AGENT_ORG_PAUSE:"; const CONTROL_WAIT_MARKER: &str = "Create a stoppable window by waiting for about "; const TASK_GRAPH_CREATE_TOOL: &str = "task_graph_create"; const TASK_UPDATE_TOOL: &str = "task_update"; +const RUN_SHELL_TOOL: &str = "run_shell"; fn task_update_arguments_with_empty_placeholders(arguments: Value) -> Value { let Value::Object(mut arguments) = arguments else { @@ -350,6 +352,70 @@ impl E2eFakeProvider { } } + fn agent_org_pause_tool_calls( + messages: &[Value], + tools: Option<&[Value]>, + ) -> Vec { + let Some((latest_user_index, latest_user)) = latest_pause_user(messages) else { + return Vec::new(); + }; + let tool_result_count = messages[latest_user_index + 1..] + .iter() + .filter(|message| message.get("role").and_then(Value::as_str) == Some("tool")) + .count(); + if tool_result_count != 0 { + return Vec::new(); + } + let Some(scenario_id) = pause_scenario_id(&latest_user) else { + return Vec::new(); + }; + if is_task_assignment(&latest_user) { + if !Self::has_tool(tools, RUN_SHELL_TOOL) { + return Vec::new(); + } + let member_id = + task_assignment_value(&latest_user, "Owner member ID:", "owner_member_id") + .unwrap_or_else(|| "member".to_string()); + return vec![ToolCallRequest { + id: format!("e2e-pause-shell-{scenario_id}-{member_id}"), + name: RUN_SHELL_TOOL.to_string(), + arguments: serde_json::json!({ + "command": format!( + "trap '' TERM; sh -c 'trap \"\" TERM; while :; do sleep 120; done' & child=$!; printf 'E2E_PAUSE_PROCESS scenario={scenario_id} parent=%s child=%s\\n' \"$$\" \"$child\"; wait" + ), + "description": "Hold Pause process group", + "mode": "background" + }), + thought_signature: None, + }]; + } + if !Self::has_tool(tools, TASK_GRAPH_CREATE_TOOL) { + return Vec::new(); + } + let tasks = (1..=9) + .map(|index| { + serde_json::json!({ + "key": format!("pause-{index:02}"), + "subject": format!("E2E_PAUSE_TASK:{scenario_id}:{index:02}"), + "description": format!( + "{AGENT_ORG_PAUSE_MARKER}{scenario_id}\nCreate a stoppable window by waiting for about 30 seconds before the final answer." + ), + "owner_member_id": format!("pause-worker-{index:02}"), + "execution_mode": "build" + }) + }) + .collect::>(); + vec![ToolCallRequest { + id: format!("e2e-pause-graph-{scenario_id}"), + name: TASK_GRAPH_CREATE_TOOL.to_string(), + arguments: serde_json::json!({ + "allow_parallel_with_existing_open_tasks": true, + "tasks": tasks, + }), + thought_signature: None, + }] + } + async fn delay_task_fsm_race_stage(messages: &[Value]) { let Some((latest_user_index, latest_user)) = latest_task_fsm_user(messages) else { return; @@ -377,6 +443,74 @@ impl E2eFakeProvider { sleep(Duration::from_millis(delay_ms)).await; } } + + async fn delay_agent_org_pause_stage(messages: &[Value]) { + let Some((latest_user_index, latest_user)) = latest_pause_user(messages) else { + return; + }; + let tool_result_count = messages[latest_user_index + 1..] + .iter() + .filter(|message| message.get("role").and_then(Value::as_str) == Some("tool")) + .count(); + if !is_task_assignment(&latest_user) && tool_result_count > 0 { + sleep(Duration::from_secs(30)).await; + } + } + + fn pause_wait_required(messages: &[Value]) -> bool { + let Some((latest_user_index, _latest_user)) = latest_pause_user(messages) else { + return false; + }; + // The first provider response must be free to emit the Task graph or + // real run_shell call. Hold the Turn only after that tool completed. + messages[latest_user_index + 1..] + .iter() + .any(|message| message.get("role").and_then(Value::as_str) == Some("tool")) + } + + fn build_response(messages: &[Value], tools: Option<&[Value]>) -> LLMResponse { + let mut tool_calls = Self::address_comment_tool_calls(messages, tools); + if tool_calls.is_empty() { + tool_calls = Self::agent_org_task_fsm_tool_calls(messages, tools); + } + if tool_calls.is_empty() { + tool_calls = Self::agent_org_pause_tool_calls(messages, tools); + } + let content = if tool_calls.is_empty() { + Some(Self::response_for(messages)) + } else { + None + }; + let prompt_tokens = messages + .iter() + .map(|message| message.to_string().len() as i64 / 4) + .sum::(); + let completion_tokens = content + .as_deref() + .map_or(tool_calls.len() as i64 * 12, |text| text.len() as i64 / 4) + .max(1); + let mut usage = HashMap::new(); + usage.insert(usage_key::PROMPT_TOKENS.to_string(), prompt_tokens); + usage.insert(usage_key::COMPLETION_TOKENS.to_string(), completion_tokens); + usage.insert( + usage_key::TOTAL_TOKENS.to_string(), + prompt_tokens + completion_tokens, + ); + LLMResponse { + content, + finish_reason: if tool_calls.is_empty() { + finish_reason::STOP.to_string() + } else { + finish_reason::TOOL_CALLS.to_string() + }, + tool_calls, + usage, + reasoning_content: None, + blocks: Vec::new(), + stream_error_kind: None, + retry_after_ms: None, + } + } } fn latest_model_user(messages: &[Value]) -> Option { @@ -418,6 +552,34 @@ fn latest_task_fsm_user(messages: &[Value]) -> Option<(usize, String)> { }) } +fn latest_pause_user(messages: &[Value]) -> Option<(usize, String)> { + messages + .iter() + .enumerate() + .rev() + .filter(|(_, message)| message.get("role").and_then(Value::as_str) == Some("user")) + .filter_map(|(index, message)| { + message + .get("content") + .and_then(content_text) + .map(|content| (index, content)) + }) + .find(|(_, content)| { + content.contains(AGENT_ORG_PAUSE_MARKER) + && !content.trim_start().starts_with("") + }) +} + +fn pause_scenario_id(text: &str) -> Option { + let suffix = text.split(AGENT_ORG_PAUSE_MARKER).nth(1)?; + let id = suffix + .chars() + .take_while(|character| character.is_ascii_alphanumeric() || matches!(character, '-' | '_')) + .take(48) + .collect::(); + (!id.is_empty()).then_some(id) +} + fn task_fsm_scenario_id(text: &str) -> Option { let suffix = text.split(AGENT_ORG_TASK_FSM_MARKER).nth(1)?; let id = suffix @@ -493,45 +655,8 @@ impl LLMProvider for E2eFakeProvider { sleep(duration).await; } Self::delay_task_fsm_race_stage(messages).await; - let mut tool_calls = Self::address_comment_tool_calls(messages, tools); - if tool_calls.is_empty() { - tool_calls = Self::agent_org_task_fsm_tool_calls(messages, tools); - } - let content = if tool_calls.is_empty() { - Some(Self::response_for(messages)) - } else { - None - }; - let prompt_tokens = messages - .iter() - .map(|message| message.to_string().len() as i64 / 4) - .sum::(); - let completion_tokens = content - .as_deref() - .map_or(tool_calls.len() as i64 * 12, |text| text.len() as i64 / 4) - .max(1); - let mut usage = HashMap::new(); - usage.insert(usage_key::PROMPT_TOKENS.to_string(), prompt_tokens); - usage.insert(usage_key::COMPLETION_TOKENS.to_string(), completion_tokens); - usage.insert( - usage_key::TOTAL_TOKENS.to_string(), - prompt_tokens + completion_tokens, - ); - - Ok(LLMResponse { - content, - finish_reason: if tool_calls.is_empty() { - finish_reason::STOP.to_string() - } else { - finish_reason::TOOL_CALLS.to_string() - }, - tool_calls, - usage, - reasoning_content: None, - blocks: Vec::new(), - stream_error_kind: None, - retry_after_ms: None, - }) + Self::delay_agent_org_pause_stage(messages).await; + Ok(Self::build_response(messages, tools)) } async fn chat_streaming( @@ -548,9 +673,39 @@ impl LLMProvider for E2eFakeProvider { return Err(ProviderError::Cancelled); } - let response = self - .chat(messages, tools, model, max_tokens, temperature) - .await?; + let cancellable_wait = if Self::pause_wait_required(messages) { + // Real shell-process materialization across all nine Members can + // take longer than the old 30-second fake response window on a + // packaged build. Keep every formal Turn cancellably in flight + // until the test clicks Pause; this is still interrupted + // immediately through the normal provider cancel flag. + Some(Duration::from_secs(120)) + } else { + control_wait_duration(messages) + }; + let response = if let Some(wait_duration) = cancellable_wait { + if let Some(flag) = cancel_flag { + tokio::select! { + _ = sleep(wait_duration) => {} + _ = async { + while !flag.load(std::sync::atomic::Ordering::Relaxed) { + sleep(Duration::from_millis(25)).await; + } + } => { + // Keep the rendered Draining phase observable while + // still proving ten providers yield in parallel. + sleep(Duration::from_millis(350)).await; + return Err(ProviderError::Cancelled); + } + } + } else { + sleep(wait_duration).await; + } + Self::build_response(messages, tools) + } else { + self.chat(messages, tools, model, max_tokens, temperature) + .await? + }; if let Some(content) = response.content.clone() { on_delta(StreamDelta { content: Some(content), @@ -761,6 +916,58 @@ mod tests { ); } + #[test] + fn pause_marker_creates_nine_owned_long_running_tasks_once() { + let tools = [named_tool(TASK_GRAPH_CREATE_TOOL)]; + let messages = vec![json!({ + "role": "user", + "content": "Run E2E_AGENT_ORG_PAUSE:episode_1" + })]; + let calls = E2eFakeProvider::agent_org_pause_tool_calls(&messages, Some(&tools)); + assert_eq!(calls.len(), 1); + let tasks = calls[0].arguments["tasks"] + .as_array() + .expect("pause task array"); + assert_eq!(tasks.len(), 9); + assert_eq!(tasks[0]["owner_member_id"], "pause-worker-01"); + assert_eq!(tasks[8]["owner_member_id"], "pause-worker-09"); + + let replay = vec![ + messages[0].clone(), + json!({ "role": "tool", "content": "{}" }), + ]; + assert!(E2eFakeProvider::agent_org_pause_tool_calls(&replay, Some(&tools)).is_empty()); + } + + #[test] + fn pause_task_assignment_starts_one_real_background_process_group_once() { + let tools = [named_tool(RUN_SHELL_TOOL)]; + let assigned = json!({ + "role": "user", + "content": concat!( + "Task assigned by coordinator: E2E_PAUSE_TASK:episode_1:01\n", + "Owner member ID: pause-worker-01\n", + "E2E_AGENT_ORG_PAUSE:episode_1" + ) + }); + let calls = E2eFakeProvider::agent_org_pause_tool_calls( + std::slice::from_ref(&assigned), + Some(&tools), + ); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].name, RUN_SHELL_TOOL); + assert_eq!(calls[0].arguments["mode"], "background"); + assert!(calls[0].arguments["command"] + .as_str() + .is_some_and(|command| command.contains("sleep 120") && command.contains("child=$!"))); + + let replay = vec![assigned, json!({ "role": "tool", "content": "{}" })]; + assert!( + E2eFakeProvider::agent_org_pause_tool_calls(&replay, Some(&tools)).is_empty(), + "the continuation must not restart the background command" + ); + } + #[test] fn task_owner_lifecycle_uses_only_task_update_operations() { let tools = [named_tool(TASK_UPDATE_TOOL)]; diff --git a/src-tauri/crates/agent-core/src/core/session/compaction/manual.rs b/src-tauri/crates/agent-core/src/core/session/compaction/manual.rs index ddbd1d7a34..2cf82fa372 100644 --- a/src-tauri/crates/agent-core/src/core/session/compaction/manual.rs +++ b/src-tauri/crates/agent-core/src/core/session/compaction/manual.rs @@ -125,8 +125,7 @@ pub async fn run_manual_compact( } }; let runtime = { - let guard = session.runtime.read().await; - match guard.clone() { + match session.get_runtime().await { Some(r) => r, None => { warn!( diff --git a/src-tauri/crates/agent-core/src/core/session/gateway_pipeline.rs b/src-tauri/crates/agent-core/src/core/session/gateway_pipeline.rs index ba769e8f56..4cc69ccdf6 100644 --- a/src-tauri/crates/agent-core/src/core/session/gateway_pipeline.rs +++ b/src-tauri/crates/agent-core/src/core/session/gateway_pipeline.rs @@ -33,12 +33,9 @@ pub async fn process_gateway_message( let session_key = msg.session_key(); let runtime = session - .runtime - .read() + .get_runtime() .await - .as_ref() - .ok_or_else(|| format!("Session {} runtime not initialized", session.id))? - .clone(); + .ok_or_else(|| format!("Session {} runtime not initialized", session.id))?; let effective_model = runtime.model.clone(); diff --git a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs index 00a84bb86a..23e9cc0a53 100644 --- a/src-tauri/crates/agent-core/src/core/session/launch/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/launch/mod.rs @@ -1233,6 +1233,7 @@ pub fn spawn_agent_org_startup_recovery(state: AgentAppState) { if let Err(error) = recover_agent_org_initial_dispatches(&state).await { tracing::warn!(error = %error, "[agent-org-startup] initial dispatch recovery failed"); } + crate::state::commands::session::org_tasks::schedule_ready_continuations(state); }); } diff --git a/src-tauri/crates/agent-core/src/core/session/persistence/messages.rs b/src-tauri/crates/agent-core/src/core/session/persistence/messages.rs index 66320a8394..e0d1b46487 100644 --- a/src-tauri/crates/agent-core/src/core/session/persistence/messages.rs +++ b/src-tauri/crates/agent-core/src/core/session/persistence/messages.rs @@ -97,6 +97,37 @@ pub fn materialize_agent_org_inbox_transcript( message_id: &str, intent_id: &str, content: &str, +) -> Result<(AgentOrgInboxTranscriptMaterialization, bool), String> { + materialize_agent_org_inbox_transcript_internal( + session_id, None, inbox_ids, message_id, intent_id, content, + ) +} + +pub fn materialize_agent_org_inbox_transcript_for_turn( + session_id: &str, + turn_intent_id: &str, + inbox_ids: &[i64], + message_id: &str, + intent_id: &str, + content: &str, +) -> Result<(AgentOrgInboxTranscriptMaterialization, bool), String> { + materialize_agent_org_inbox_transcript_internal( + session_id, + Some(turn_intent_id), + inbox_ids, + message_id, + intent_id, + content, + ) +} + +fn materialize_agent_org_inbox_transcript_internal( + session_id: &str, + turn_intent_id: Option<&str>, + inbox_ids: &[i64], + message_id: &str, + intent_id: &str, + content: &str, ) -> Result<(AgentOrgInboxTranscriptMaterialization, bool), String> { if inbox_ids.is_empty() { return Err("cannot materialize an empty Agent Org Inbox batch".to_string()); @@ -107,6 +138,14 @@ pub fn materialize_agent_org_inbox_transcript( .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) .map_err(|err| err.to_string())?; + if let Some(turn_intent_id) = turn_intent_id { + crate::coordination::agent_org_turn_contexts::revalidate_context_with_connection( + &tx, + session_id, + turn_intent_id, + )?; + } + let mut existing_receipts = Vec::new(); { let mut stmt = tx diff --git a/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs b/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs index ffaf217141..bc95a33453 100644 --- a/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/persistence/mod.rs @@ -48,10 +48,11 @@ pub use messages::{ load_agent_org_inbox_transcript_materializations, load_llm_history, load_llm_history_start_sequences, load_llm_history_text_only, load_llm_history_text_only_bounded, load_messages, load_session_memory_state, - mark_turn_cancelled, materialize_agent_org_inbox_transcript, message_anchor, - message_created_at, save_assistant_msg, save_compact_summary_msg, save_session_memory_state, - save_snapshot, save_subagent_transcript, save_tool_call_msg, save_tool_result_msg, - save_user_msg, save_user_msg_with_id, seed_session_with_messages, take_turn_cancelled, + mark_turn_cancelled, materialize_agent_org_inbox_transcript, + materialize_agent_org_inbox_transcript_for_turn, message_anchor, message_created_at, + save_assistant_msg, save_compact_summary_msg, save_session_memory_state, save_snapshot, + save_subagent_transcript, save_tool_call_msg, save_tool_result_msg, save_user_msg, + save_user_msg_with_id, seed_session_with_messages, take_turn_cancelled, truncate_messages_from_sequence, update_compact_boundary_token_delta, AgentOrgInboxTranscriptMaterialization, MessageAnchor, }; diff --git a/src-tauri/crates/agent-core/src/core/session/turn/entry.rs b/src-tauri/crates/agent-core/src/core/session/turn/entry.rs index c82ed01eba..dd1ed00ffb 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/entry.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/entry.rs @@ -118,12 +118,9 @@ pub async fn process_message( app_handle: Option, ) -> Result { let runtime = session - .runtime - .read() + .get_runtime() .await - .as_ref() - .ok_or_else(|| format!("Session {} runtime not initialized", session.id))? - .clone(); + .ok_or_else(|| format!("Session {} runtime not initialized", session.id))?; let workspace_path = runtime.workspace_state.read().working_dir().to_path_buf(); diff --git a/src-tauri/crates/agent-core/src/core/session/turn/mod.rs b/src-tauri/crates/agent-core/src/core/session/turn/mod.rs index 559ee6cc37..abf51202fa 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/mod.rs @@ -75,7 +75,7 @@ pub async fn debug_prompt_cache_benchmark( use crate::core::session::turn::event_handler::EventHandlerConfig; use crate::core::session::turn::processor::{ProcessorParams, UnifiedMessageProcessor}; - let runtime = match session.runtime.read().await.clone() { + let runtime = match session.get_runtime().await { Some(runtime) => runtime, None => { return serde_json::json!({ diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs index 744d0c2d52..1cb6786eee 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/execute.rs @@ -70,9 +70,24 @@ impl UnifiedMessageProcessor { None => self.runtime.model.clone(), }; + let mut turn_process_control = self.session.turn_process_control(); + if self.runtime.agent_org_context.is_some() { + let control = turn_process_control.as_mut().ok_or_else(|| { + "Agent Org Turn execution requires an exact process owner".to_string() + })?; + if control.owner.session_id != session_id + || control.owner.turn_intent_id != turn_intent_id + { + return Err( + "Agent Org Turn process owner does not match the dispatched Turn".to_string(), + ); + } + control.require_owned_job_finality = true; + } let turn_config = TurnConfig { turn_intent_id: turn_intent_id.to_string(), projected_inbox_ids, + turn_process_control, model: turn_model, account_id: self.runtime.account_id.clone(), context_window_override: self diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/drain.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/drain.rs index 915d431fe2..980b5de248 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/drain.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/drain.rs @@ -85,6 +85,7 @@ pub(crate) fn drain_and_render_deferred_for_turn( session, Some(turn_context), ) + .bind_formal_turn(turn_context) } fn drain_and_render_deferred_impl( @@ -133,13 +134,17 @@ fn drain_and_render_deferred_impl( let unread_result = match turn_context.map(|context| context.turn_kind) { Some(crate::coordination::agent_org_turn_contexts::AgentOrgTurnKind::TaskExecution) => { - match turn_context.and_then(|context| context.task_id.as_deref()) { - Some(task_id) => AgentInboxStore::list_unread_task_input_for_member( - recipient_member_id_value, - &org_context.run_id, - task_id, - ), - None => Err("TaskExecution context has no canonical task_id".to_string()), + match turn_context { + Some(context) if context.task_id.is_some() => { + AgentInboxStore::list_unread_task_input_for_turn( + recipient_member_id_value, + &org_context.run_id, + context.task_id.as_deref().expect("guarded Task id"), + &context.session_id, + &context.turn_intent_id, + ) + } + _ => Err("TaskExecution context has no canonical task_id".to_string()), } } Some(crate::coordination::agent_org_turn_contexts::AgentOrgTurnKind::UserDirectedWork) => { diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs index feb04ce8ff..10abf51ecb 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/inbox_drain/guard.rs @@ -25,6 +25,7 @@ pub struct DrainGuard { run_id: String, recipient_member_id: String, materialization_session_id: Option, + formal_turn_intent_id: Option, pending_ids: Vec, new_materialization_ids: Vec, transcript_content: Option, @@ -37,6 +38,7 @@ impl DrainGuard { run_id: run_id.to_string(), recipient_member_id: recipient_member_id.to_string(), materialization_session_id: None, + formal_turn_intent_id: None, pending_ids: Vec::new(), new_materialization_ids: Vec::new(), transcript_content: None, @@ -57,6 +59,7 @@ impl DrainGuard { run_id: run_id.to_string(), recipient_member_id: recipient_member_id.to_string(), materialization_session_id: materialization_session_id.map(str::to_string), + formal_turn_intent_id: None, pending_ids, new_materialization_ids, transcript_content: transcript, @@ -64,6 +67,20 @@ impl DrainGuard { } } + pub(super) fn bind_formal_turn( + mut self, + turn_context: &crate::coordination::agent_org_turn_contexts::AgentOrgTurnContext, + ) -> Self { + if matches!( + turn_context.turn_kind, + crate::coordination::agent_org_turn_contexts::AgentOrgTurnKind::Coordinator + | crate::coordination::agent_org_turn_contexts::AgentOrgTurnKind::TaskExecution + ) { + self.formal_turn_intent_id = Some(turn_context.turn_intent_id.clone()); + } + self + } + pub fn transcript_content(&self) -> Option<&str> { self.transcript_content.as_deref() } @@ -144,9 +161,14 @@ impl DrainGuard { return; } let result = match self.materialization_session_id.as_deref() { - Some(session_id) => { - AgentInboxStore::mark_many_read_for_session(&self.pending_ids, session_id) - } + Some(session_id) => match self.formal_turn_intent_id.as_deref() { + Some(turn_intent_id) => AgentInboxStore::mark_many_read_for_turn( + &self.pending_ids, + session_id, + turn_intent_id, + ), + None => AgentInboxStore::mark_many_read_for_session(&self.pending_ids, session_id), + }, None => AgentInboxStore::mark_many_read(&self.pending_ids), }; match result { diff --git a/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs b/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs index 35df4ee8b0..c330e53f71 100644 --- a/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs +++ b/src-tauri/crates/agent-core/src/core/session/turn/processor/mod.rs @@ -761,7 +761,7 @@ impl UnifiedMessageProcessor { let persisted_turn_context = if self.runtime.agent_org_context.is_some() { Some(tokio::task::block_in_place(|| { let conn = database::db::get_connection().map_err(|error| error.to_string())?; - crate::coordination::agent_org_turn_contexts::require_context_with_connection( + crate::coordination::agent_org_turn_contexts::revalidate_context_with_connection( &conn, session_id, &context.turn_intent_id, @@ -790,8 +790,9 @@ impl UnifiedMessageProcessor { .transcript_identity(session_id) .expect("a non-empty drained transcript has stable source row ids"); let (materialization, inserted) = tokio::task::block_in_place(|| { - unified_persistence::materialize_agent_org_inbox_transcript( + unified_persistence::materialize_agent_org_inbox_transcript_for_turn( session_id, + &context.turn_intent_id, guard.new_materialization_ids(), &stable_message_id, &stable_intent_id, @@ -844,6 +845,30 @@ impl UnifiedMessageProcessor { } } + // A pause continuation is durable work even when the original Task + // assignment Inbox was consumed before Pause. Supply its instruction + // only in the provider request: Resume must not create a fake user + // transcript row or a second Inbox source. + if context.is_resume && content.trim().is_empty() && persisted_turn_context.is_some() { + let continuation_nudge = tokio::task::block_in_place(|| { + crate::coordination::agent_org_pause::continuation_nudge_for_turn( + session_id, + &context.turn_intent_id, + ) + })?; + if let Some(nudge) = continuation_nudge { + messages.push(serde_json::json!({ + "role": "user", + "content": nudge, + })); + info!( + session_id = %session_id, + turn_intent_id = %context.turn_intent_id, + "[unified_processor] Injected transient Agent Org Pause continuation" + ); + } + } + // An Agent Org wake is only a doorbell. If another worker consumed the // work before this turn started, do not manufacture an empty user // nudge and spend a provider call. A later unread inbox row or diff --git a/src-tauri/crates/agent-core/src/core/tools/call_context.rs b/src-tauri/crates/agent-core/src/core/tools/call_context.rs index f08cab7938..4d871ceeaa 100644 --- a/src-tauri/crates/agent-core/src/core/tools/call_context.rs +++ b/src-tauri/crates/agent-core/src/core/tools/call_context.rs @@ -40,6 +40,43 @@ //! dispatch always constructs a populated ctx in //! `turn_executor::tool_execution`. +use tokio_util::sync::CancellationToken; + +/// Exact runtime owner of subprocesses started by one dialog Turn. +/// +/// All four fields travel together so a delayed Pause callback cannot target +/// a newer runtime or a different Turn that happens to reuse the Session. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TurnProcessOwner { + pub session_id: String, + pub turn_intent_id: String, + pub runtime_lease_id: String, + pub dialog_turn_generation: String, +} + +/// Per-Turn process lifecycle control threaded to tool execution. +/// +/// The token is level-triggered and never reset. A shell that transitions to +/// background after Pause therefore observes the cancellation immediately +/// instead of missing a short pulse on the Session's ordinary cancel flag. +#[derive(Debug, Clone)] +pub struct TurnProcessControl { + pub owner: TurnProcessOwner, + pub background_cancel: CancellationToken, + /// Agent Org work consumes every owned background result inside this + /// exact Turn. Ordinary SDE controls leave this false. + pub require_owned_job_finality: bool, +} + +impl PartialEq for TurnProcessControl { + fn eq(&self, other: &Self) -> bool { + self.owner == other.owner + && self.require_owned_job_finality == other.require_owned_job_finality + } +} + +impl Eq for TurnProcessControl {} + /// Per-call framework metadata. /// /// Threaded explicitly by `turn_executor::tool_execution` to every @@ -59,6 +96,9 @@ pub struct CallContext { /// Exact Agent Org Inbox rows held by this turn's deferred drain guard. /// These rows are acknowledged only when the turn succeeds. pub projected_inbox_ids: Vec, + /// Exact owner and level-triggered cancellation for shell processes + /// started by this Turn. Direct/maintenance calls intentionally use None. + pub turn_process_control: Option, } impl CallContext { @@ -69,6 +109,7 @@ impl CallContext { session_id: session_id.into(), turn_intent_id: String::new(), projected_inbox_ids: Vec::new(), + turn_process_control: None, } } @@ -77,12 +118,29 @@ impl CallContext { session_id: impl Into, turn_intent_id: impl Into, projected_inbox_ids: Vec, + ) -> Self { + Self::for_runtime_turn( + call_id, + session_id, + turn_intent_id, + projected_inbox_ids, + None, + ) + } + + pub fn for_runtime_turn( + call_id: impl Into, + session_id: impl Into, + turn_intent_id: impl Into, + projected_inbox_ids: Vec, + turn_process_control: Option, ) -> Self { Self { call_id: call_id.into(), session_id: session_id.into(), turn_intent_id: turn_intent_id.into(), projected_inbox_ids, + turn_process_control, } } } diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/mod.rs b/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/mod.rs index 4a7996d044..dbc247fb63 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/mod.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/mod.rs @@ -591,7 +591,8 @@ impl Tool for ExecTool { .to_string(), )); } - let identity = subprocess::ExecIdentity::new(&ctx.session_id, &ctx.call_id); + let identity = subprocess::ExecIdentity::new(&ctx.session_id, &ctx.call_id) + .with_turn_process_control(ctx.turn_process_control.clone()); let replay_root = self.shell_replays_root.as_ref().ok_or_else(|| { ToolError::ExecutionFailed("Shell replay storage root is not configured.".to_string()) })?; diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/registry.rs b/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/registry.rs index 57cb5920da..1c42aedb7a 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/registry.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/registry.rs @@ -6,13 +6,16 @@ //! and query status using a unified string handle. use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, VecDeque}; +use std::collections::{HashMap, HashSet, VecDeque}; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, LazyLock, Mutex}; -use std::time::Instant; -use tokio::sync::broadcast; +use std::time::{Duration, Instant}; +use tokio::sync::{broadcast, watch}; use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; + +use crate::tools::call_context::{TurnProcessControl, TurnProcessOwner}; /// Status of a background job. #[derive(Debug, Clone)] @@ -24,6 +27,29 @@ pub enum JobStatus { Failed, } +#[derive(Debug, Clone, PartialEq, Eq)] +enum ShellCompletionState { + Running, + Terminated, + Failed(String), +} + +/// Completion half retained by the subprocess monitor. The registry keeps +/// the receiver so Pause can await OS-level process-group finality without +/// taking ownership away from the one task that owns the child handle. +pub struct ShellMonitorCompletion { + tx: watch::Sender, +} + +impl ShellMonitorCompletion { + pub fn finish(self, result: Result<(), String>) { + self.tx.send_replace(match result { + Ok(()) => ShellCompletionState::Terminated, + Err(error) => ShellCompletionState::Failed(error), + }); + } +} + /// What kind of background job this is. #[derive(Debug, Clone)] pub enum JobKind { @@ -75,6 +101,10 @@ pub struct BackgroundJob { recent_lines: VecDeque, /// Tokio JoinHandle for background subagents — `abort()` cancels the task. join_handle: Option>, + /// False only during the narrow register-to-spawn handoff. Exact-owner + /// teardown cannot report terminal until the spawned task is attached and + /// its JoinHandle has actually finished. + join_handle_attached: bool, /// Per-job cancel flag for background subagents. Owned by the job (NOT /// the parent session's flag — that one is pulsed back to `false` at the /// parent's turn boundary, which a slow worker can miss entirely). @@ -83,6 +113,22 @@ pub struct BackgroundJob { /// (LinkedSession terminal write, worktree cleanup, registry grace /// period). `None` for shell jobs. cancel_flag: Option>, + /// Exact dialog Turn that created this job. Shells always carry it when + /// launched from a durable Turn; subagents carry it only when Agent Org + /// requires same-Turn convergence. + turn_owner: Option, + /// Agent Org jobs are consumed by their owner Turn and never participate + /// in the ordinary SDE idle-wake or retention paths. + requires_in_turn_finality: bool, + /// Per-process cancellation. This is distinct from the Turn token so an + /// explicit kill_handle request terminates only the selected process. + shell_cancel: Option, + /// Reaches a terminal state only after the process group is absent and + /// replay readers/writer have drained. + shell_completion: Option>, + /// Cancellation was requested, but the monitor has not yet proved the + /// process group and replay pipeline are terminal. + shell_kill_requested: bool, /// Set to `true` once the agent has read the completed job's output via /// `AwaitTool` (monitor/wait_for). Acknowledged completed jobs are excluded /// from the per-turn system reminder to avoid the stale-reminder @@ -177,16 +223,139 @@ pub fn acknowledge_outputs(handles: &[String]) { let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); for handle in handles { if let Some(job) = reg.get_mut(handle) { + if job.requires_in_turn_finality { + continue; + } job.output_acknowledged = true; } } } +/// Snapshot only the jobs that must converge inside one exact Agent Org +/// Turn. This is an in-memory owner lookup over the already-bounded active +/// registry; it performs no database query and creates no timer. +pub fn list_jobs_for_owner(owner: &TurnProcessOwner) -> Vec { + let reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + let index = OWNER_INDEX.lock().unwrap_or_else(|e| e.into_inner()); + index + .get(owner) + .into_iter() + .flatten() + .filter_map(|handle| reg.get(handle)) + .filter(|job| job.requires_in_turn_finality) + .filter(|job| job.is_running() || !job.output_acknowledged) + .map(|job| { + let mut snapshot = job.snapshot(); + if matches!(job.kind, JobKind::Subagent { .. }) && !owned_job_execution_finished(job) { + // `finish_subagent` publishes the result before the spawned + // task returns. Keep that narrow cleanup tail logically + // Running so the parent cannot consume the result twice or + // finalize before the JoinHandle is terminal. + snapshot.status = JobStatus::Running; + snapshot.final_result = None; + snapshot.has_unread_output = false; + } + snapshot + }) + .collect() +} + +/// Active foreground handoff only: wait for an exact subagent whose result +/// was already delivered inline to finish its spawned task before removing +/// the registry row. This bounded owner-local wait creates no retained timer. +pub async fn await_subagent_execution_for_owner( + owner: &TurnProcessOwner, + handle: &str, + timeout: Duration, +) -> Result<(), String> { + let wait = async { + loop { + let finished = { + let reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + let Some(job) = reg.get(handle) else { + return Err(format!( + "owned subagent {handle} disappeared before finality" + )); + }; + if !job.requires_in_turn_finality || job.turn_owner.as_ref() != Some(owner) { + return Err(format!( + "owned subagent {handle} no longer matches its parent Turn" + )); + } + if !matches!(job.kind, JobKind::Subagent { .. }) { + return Err(format!("owned job {handle} is not a subagent")); + } + owned_job_execution_finished(job) + }; + if finished { + return Ok(()); + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }; + tokio::time::timeout(timeout, wait) + .await + .map_err(|_| format!("timed out waiting for owned subagent {handle} to finish"))? +} + +/// Acknowledge terminal results only when both the handle and exact owner +/// match, then remove them immediately. Agent Org jobs never enter the +/// ordinary 5-second acknowledgement poll or 30-minute retention tail. +pub fn acknowledge_outputs_for_owner(owner: &TurnProcessOwner, handles: &[String]) { + let removable = { + let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + handles + .iter() + .filter_map(|handle| { + let job = reg.get_mut(handle)?; + if !job.requires_in_turn_finality + || job.turn_owner.as_ref() != Some(owner) + || !owned_job_execution_finished(job) + { + return None; + } + job.output_acknowledged = true; + Some(handle.clone()) + }) + .collect::>() + }; + for handle in removable { + remove(&handle); + } +} + +/// Remove already-terminal exact-owner jobs after a cancelled Turn (Pause) +/// has proved their external work is gone. No terminal result is delivered +/// because cancellation makes the Turn unsuccessful. +pub fn remove_terminal_jobs_for_owner(owner: &TurnProcessOwner) { + let handles = { + let reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + let index = OWNER_INDEX.lock().unwrap_or_else(|e| e.into_inner()); + index + .get(owner) + .into_iter() + .flatten() + .filter_map(|handle| reg.get(handle)) + .filter(|job| job.requires_in_turn_finality && owned_job_execution_finished(job)) + .map(|job| job.handle.clone()) + .collect::>() + }; + for handle in handles { + remove(&handle); + } +} + const BROADCAST_CAPACITY: usize = 512; static REGISTRY: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); +/// Secondary index owned by the same registry module. Every access locks +/// `REGISTRY` first and this map second, so owner lookups are O(k) in that +/// Turn's jobs without a process-wide scan or lock-order inversion. +static OWNER_INDEX: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + /// How long a finished job's tombstone is retained after it leaves the live /// registry. Long enough that an `await_output` arriving just after the grace /// eviction still gets a precise "completed" answer (with the real kind), short @@ -216,7 +385,17 @@ pub fn register_shell( log_path: PathBuf, session_id: String, ) -> broadcast::Sender { - register_shell_inner(pid, command, log_path, session_id, None) + register_shell_inner(ShellRegistration { + pid, + command, + log_path, + session_id, + replay_identity: None, + turn_owner: None, + requires_in_turn_finality: false, + shell_cancel: None, + shell_completion: None, + }) } /// Register a new durable shell replay job by exact Session/call identity. @@ -228,22 +407,71 @@ pub fn register_shell_replay( call_id: String, ) -> broadcast::Sender { let replay_session_id = session_id.clone(); - register_shell_inner( + register_shell_inner(ShellRegistration { pid, command, log_path, session_id, - Some((replay_session_id, call_id)), - ) + replay_identity: Some((replay_session_id, call_id)), + turn_owner: None, + requires_in_turn_finality: false, + shell_cancel: None, + shell_completion: None, + }) +} + +/// Register a production shell with its exact Turn/runtime owner and a +/// completion barrier controlled by the background monitor. +pub fn register_owned_shell_replay( + pid: u32, + command: String, + log_path: PathBuf, + session_id: String, + call_id: String, + turn_control: &TurnProcessControl, + process_cancel: CancellationToken, +) -> ShellMonitorCompletion { + let replay_session_id = session_id.clone(); + let (completion_tx, completion_rx) = watch::channel(ShellCompletionState::Running); + register_shell_inner(ShellRegistration { + pid, + command, + log_path, + session_id, + replay_identity: Some((replay_session_id, call_id)), + turn_owner: Some(turn_control.owner.clone()), + requires_in_turn_finality: turn_control.require_owned_job_finality, + shell_cancel: Some(process_cancel), + shell_completion: Some(completion_rx), + }); + ShellMonitorCompletion { tx: completion_tx } } -fn register_shell_inner( +struct ShellRegistration { pid: u32, command: String, log_path: PathBuf, session_id: String, replay_identity: Option<(String, String)>, -) -> broadcast::Sender { + turn_owner: Option, + requires_in_turn_finality: bool, + shell_cancel: Option, + shell_completion: Option>, +} + +fn register_shell_inner(registration: ShellRegistration) -> broadcast::Sender { + let ShellRegistration { + pid, + command, + log_path, + session_id, + replay_identity, + turn_owner, + requires_in_turn_finality, + shell_cancel, + shell_completion, + } = registration; + let indexed_owner = turn_owner.clone(); let handle = pid.to_string(); let (tx, _) = broadcast::channel(BROADCAST_CAPACITY); let sender = tx.clone(); @@ -263,7 +491,13 @@ fn register_shell_inner( output_tx: tx, recent_lines: VecDeque::new(), join_handle: None, + join_handle_attached: true, cancel_flag: None, + turn_owner, + requires_in_turn_finality, + shell_cancel, + shell_completion, + shell_kill_requested: false, output_acknowledged: false, wake_dispatched: false, output_seq: 0, @@ -271,7 +505,15 @@ fn register_shell_inner( stall_delivered: false, }; let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); - reg.insert(handle, job); + reg.insert(handle.clone(), job); + if let Some(owner) = indexed_owner { + OWNER_INDEX + .lock() + .unwrap_or_else(|e| e.into_inner()) + .entry(owner) + .or_default() + .insert(handle); + } sender } @@ -286,12 +528,37 @@ pub fn register_subagent( session_id: String, ) -> (broadcast::Sender, Arc) { let cancel_flag = Arc::new(AtomicBool::new(false)); - let sender = register_subagent_with_flag( + let sender = register_subagent_inner( handle, subagent_type, agent_name, session_id, Arc::clone(&cancel_flag), + None, + false, + ); + (sender, cancel_flag) +} + +/// Register an Agent Org worker owned by one exact parent Turn. Its terminal +/// result is consumed by that Turn and therefore never starts a later generic +/// background-job wake. +pub fn register_owned_subagent( + handle: String, + subagent_type: String, + agent_name: String, + session_id: String, + owner: TurnProcessOwner, +) -> (broadcast::Sender, Arc) { + let cancel_flag = Arc::new(AtomicBool::new(false)); + let sender = register_subagent_inner( + handle, + subagent_type, + agent_name, + session_id, + Arc::clone(&cancel_flag), + Some(owner), + true, ); (sender, cancel_flag) } @@ -311,6 +578,27 @@ pub fn register_subagent_with_flag( session_id: String, cancel_flag: Arc, ) -> broadcast::Sender { + register_subagent_inner( + handle, + subagent_type, + agent_name, + session_id, + cancel_flag, + None, + false, + ) +} + +fn register_subagent_inner( + handle: String, + subagent_type: String, + agent_name: String, + session_id: String, + cancel_flag: Arc, + turn_owner: Option, + requires_in_turn_finality: bool, +) -> broadcast::Sender { + let indexed_owner = turn_owner.clone(); let (tx, _) = broadcast::channel(BROADCAST_CAPACITY); let sender = tx.clone(); let job = BackgroundJob { @@ -327,7 +615,13 @@ pub fn register_subagent_with_flag( output_tx: tx, recent_lines: VecDeque::new(), join_handle: None, + join_handle_attached: false, cancel_flag: Some(Arc::clone(&cancel_flag)), + turn_owner, + requires_in_turn_finality, + shell_cancel: None, + shell_completion: None, + shell_kill_requested: false, output_acknowledged: false, wake_dispatched: false, output_seq: 0, @@ -336,6 +630,14 @@ pub fn register_subagent_with_flag( }; let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); reg.insert(handle.clone(), job); + if let Some(owner) = indexed_owner { + OWNER_INDEX + .lock() + .unwrap_or_else(|e| e.into_inner()) + .entry(owner) + .or_default() + .insert(handle.clone()); + } drop(reg); broadcast_subagent_job_changed(&session_id, &handle, &agent_name, &subagent_type, "running"); sender @@ -380,7 +682,11 @@ pub fn mark_exited(handle: &str, status: JobStatus) { if matches!(job.status, JobStatus::Killed) { return; } - job.status = status; + job.status = if job.shell_kill_requested && matches!(job.kind, JobKind::Shell { .. }) { + JobStatus::Killed + } else { + status + }; if let JobKind::Subagent { subagent_type, agent_name, @@ -402,6 +708,17 @@ pub fn mark_exited(handle: &str, status: JobStatus) { } } +/// Latch cancellation before signalling the process monitor. The public job +/// remains Running until OS/process-output finality is confirmed. +pub fn mark_shell_cancel_requested(handle: &str) { + let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(job) = reg.get_mut(handle) { + if matches!(job.kind, JobKind::Shell { .. }) && job.is_running() { + job.shell_kill_requested = true; + } + } +} + /// Store the final result text for a completed subagent job. pub fn set_final_result(handle: &str, result: String) { let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); @@ -410,6 +727,51 @@ pub fn set_final_result(handle: &str, result: String) { } } +/// Atomically publish a subagent's terminal status and final result. Exact +/// owner finality must never observe a terminal worker before its result is +/// available for same-Turn consumption. +pub fn finish_subagent(handle: &str, status: JobStatus, result: String) { + let broadcast = { + let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + let Some(job) = reg.get_mut(handle) else { + return; + }; + let JobKind::Subagent { + subagent_type, + agent_name, + } = &job.kind + else { + return; + }; + job.final_result = Some(result); + if !matches!(job.status, JobStatus::Killed) { + job.status = status; + } + let wire_status = match &job.status { + JobStatus::Completed | JobStatus::Exited(_) => "completed", + JobStatus::Failed => "failed", + JobStatus::Killed => "killed", + JobStatus::Running => "running", + }; + Some(( + job.session_id.clone(), + job.handle.clone(), + agent_name.clone(), + subagent_type.clone(), + wire_status, + )) + }; + if let Some((session_id, handle, agent_name, subagent_type, wire_status)) = broadcast { + broadcast_subagent_job_changed( + &session_id, + &handle, + &agent_name, + &subagent_type, + wire_status, + ); + } +} + /// Remove a job from the registry (called after grace period). /// /// Leaves a short-lived [`Tombstone`] behind so a late `await_output` can @@ -420,7 +782,17 @@ pub fn set_final_result(handle: &str, result: String) { pub fn remove(handle: &str) { let removed = { let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); - reg.remove(handle) + let removed = reg.remove(handle); + if let Some(owner) = removed.as_ref().and_then(|job| job.turn_owner.as_ref()) { + let mut index = OWNER_INDEX.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(handles) = index.get_mut(owner) { + handles.remove(handle); + if handles.is_empty() { + index.remove(owner); + } + } + } + removed }; if let Some(job) = removed { let mut tombs = TOMBSTONES.lock().unwrap_or_else(|e| e.into_inner()); @@ -495,6 +867,74 @@ pub fn list_shell_for_session(session_id: &str) -> Vec<(u32, String)> { .collect() } +async fn wait_for_shell_completion( + pid: u32, + mut completion: watch::Receiver, +) -> Result<(), String> { + loop { + let state = completion.borrow().clone(); + match state { + ShellCompletionState::Running => {} + ShellCompletionState::Terminated => return Ok(()), + ShellCompletionState::Failed(error) => { + return Err(format!( + "background shell process group {pid} failed to stop: {error}" + )) + } + } + completion.changed().await.map_err(|_| { + format!("background shell process group {pid} lost its completion owner") + })?; + } +} + +/// Wait until every background shell owned by the exact Pause Turn has +/// reached OS/process-output finality. No matching jobs means foreground work +/// already completed through the synchronous tool path. +pub async fn await_shells_terminated_for_owner( + owner: &TurnProcessOwner, + timeout: Duration, +) -> Result<(), String> { + let completions = { + let reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + let index = OWNER_INDEX.lock().unwrap_or_else(|e| e.into_inner()); + index + .get(owner) + .into_iter() + .flatten() + .filter_map(|handle| reg.get(handle)) + .filter_map(|job| match (&job.kind, &job.shell_completion) { + (JobKind::Shell { pid, .. }, Some(completion)) => Some((*pid, completion.clone())), + _ => None, + }) + .collect::>() + }; + let wait_all = async move { + let results = futures::future::join_all( + completions + .into_iter() + .map(|(pid, completion)| wait_for_shell_completion(pid, completion)), + ) + .await; + let failures = results + .into_iter() + .filter_map(Result::err) + .collect::>(); + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("; ")) + } + }; + tokio::time::timeout(timeout, wait_all).await.map_err(|_| { + format!( + "timed out after {}ms waiting for background shell process groups owned by Turn {}", + timeout.as_millis(), + owner.dialog_turn_generation + ) + })? +} + /// List all jobs (shells + subagents). Pass `Some(session_id)` for session /// scope, `None` for global scope. pub fn list_jobs(session_id: Option<&str>) -> Vec { @@ -515,7 +955,9 @@ pub fn list_jobs(session_id: Option<&str>) -> Vec { pub fn acknowledge_output(handle: &str) { let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); if let Some(job) = reg.get_mut(handle) { - job.output_acknowledged = true; + if !job.requires_in_turn_finality { + job.output_acknowledged = true; + } } } @@ -539,7 +981,9 @@ pub fn list_jobs_for_reminder(session_id: &str) -> Vec { let reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); reg.values() .filter(|job| { - job.session_id == session_id && (job.is_running() || !job.output_acknowledged) + job.session_id == session_id + && !job.requires_in_turn_finality + && (job.is_running() || !job.output_acknowledged) }) .map(|job| job.snapshot()) .collect() @@ -587,6 +1031,9 @@ pub fn claim_completion_wake_for_session(session_id: &str) -> bool { if job.session_id != session_id { continue; } + if job.requires_in_turn_finality { + continue; + } if !job.is_running() && job_completion_is_wakeworthy(job) && !job.output_acknowledged @@ -618,6 +1065,9 @@ pub fn release_completion_wake_for_session(session_id: &str) { if job.session_id != session_id { continue; } + if job.requires_in_turn_finality { + continue; + } if !job.is_running() && !job.output_acknowledged { job.wake_dispatched = false; } @@ -787,7 +1237,32 @@ pub fn push_output_line(handle: &str, line: String) { pub fn set_join_handle(handle: &str, jh: JoinHandle<()>) { let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); if let Some(job) = reg.get_mut(handle) { + job.join_handle_attached = true; + if matches!(job.status, JobStatus::Killed) { + jh.abort(); + } job.join_handle = Some(jh); + } else { + // A concurrent exact-owner teardown removed the registration before + // the spawning call could attach its handle. Dropping JoinHandle would + // detach the task, so abort it explicitly. + jh.abort(); + } +} + +fn owned_job_execution_finished(job: &BackgroundJob) -> bool { + if job.is_running() { + return false; + } + match job.kind { + JobKind::Shell { .. } => true, + JobKind::Subagent { .. } => { + job.join_handle_attached + && job + .join_handle + .as_ref() + .is_some_and(JoinHandle::is_finished) + } } } @@ -806,21 +1281,33 @@ fn send_signal_to_process_tree(pid: u32, signal: libc::c_int) -> Result<(), std: } let process_error = std::io::Error::last_os_error(); - if group_error.raw_os_error() == Some(libc::ESRCH) - && process_error.raw_os_error() == Some(libc::ESRCH) - { - return Err(process_error); + if group_error.raw_os_error() == Some(libc::ESRCH) { + Err(process_error) + } else { + Err(group_error) } - - Err(process_error) } #[cfg(unix)] -fn process_tree_exists(pid: u32) -> bool { +pub(crate) fn process_tree_exists(pid: u32) -> bool { let pid = pid as libc::pid_t; unsafe { libc::kill(-pid, 0) == 0 || libc::kill(pid, 0) == 0 } } +#[cfg(unix)] +async fn wait_for_process_tree_exit(pid: u32, timeout: Duration) -> bool { + let deadline = tokio::time::Instant::now() + timeout; + loop { + if !process_tree_exists(pid) { + return true; + } + if tokio::time::Instant::now() >= deadline { + return false; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } +} + #[cfg(unix)] pub async fn terminate_shell_process_tree(pid: u32) -> Result { if pid == 0 { @@ -835,17 +1322,25 @@ pub async fn terminate_shell_process_tree(pid: u32) -> Result { Err(err) => return Err(format!("Failed to send SIGTERM to {}: {}", pid, err)), } - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - if process_tree_exists(pid) { - match send_signal_to_process_tree(pid, libc::SIGKILL) { - Ok(()) => Ok(format!("Process {} killed (SIGKILL)", pid)), - Err(err) if err.raw_os_error() == Some(libc::ESRCH) => { - Ok(format!("Process {} terminated (SIGTERM)", pid)) - } - Err(err) => Err(format!("Failed to send SIGKILL to {}: {}", pid, err)), + // Preserve the existing ordinary SDE kill contract: give cooperative + // processes the full two-second SIGTERM grace before escalating. + if wait_for_process_tree_exit(pid, Duration::from_secs(2)).await { + return Ok(format!("Process {} terminated (SIGTERM)", pid)); + } + match send_signal_to_process_tree(pid, libc::SIGKILL) { + Ok(()) => {} + Err(err) if err.raw_os_error() == Some(libc::ESRCH) => { + return Ok(format!("Process {} terminated (SIGTERM)", pid)); } + Err(err) => return Err(format!("Failed to send SIGKILL to {}: {}", pid, err)), + } + if wait_for_process_tree_exit(pid, Duration::from_secs(2)).await { + Ok(format!("Process {} killed (SIGKILL)", pid)) } else { - Ok(format!("Process {} terminated (SIGTERM)", pid)) + Err(format!( + "Process group {} still exists after SIGKILL verification window", + pid + )) } } @@ -880,7 +1375,7 @@ pub async fn terminate_shell_process_tree(pid: u32) -> Result { /// Returns `Ok(())` on success or `Err(msg)` if the handle is not found or /// not a shell job. pub async fn kill_shell(handle: &str) -> Result<(), String> { - let pid = { + let (pid, cancel, completion) = { let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); let job = reg .get_mut(handle) @@ -894,10 +1389,21 @@ pub async fn kill_shell(handle: &str) -> Result<(), String> { if !job.is_running() { return Err(format!("job '{handle}' already exited")); } - job.status = JobStatus::Killed; - pid + job.shell_kill_requested = true; + (pid, job.shell_cancel.clone(), job.shell_completion.clone()) }; + if let Some(cancel) = cancel { + cancel.cancel(); + if let Some(completion) = completion { + return tokio::time::timeout( + Duration::from_secs(10), + wait_for_shell_completion(pid, completion), + ) + .await + .map_err(|_| format!("timed out waiting for shell process group {pid} to stop"))?; + } + } terminate_shell_process_tree(pid).await.map(|_| ()) } @@ -916,7 +1422,7 @@ pub async fn kill_shell(handle: &str) -> Result<(), String> { pub fn kill_subagent(handle: &str) -> Result<(), String> { const HARD_ABORT_GRACE_SECS: u64 = 10; - let (cancel_flag, join_handle, broadcast_info) = { + let (cancel_flag, abort_handle, broadcast_info) = { let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); let job = reg .get_mut(handle) @@ -940,7 +1446,7 @@ pub fn kill_subagent(handle: &str) -> Result<(), String> { job.status = JobStatus::Killed; ( job.cancel_flag.clone(), - job.join_handle.take(), + job.join_handle.as_ref().map(JoinHandle::abort_handle), broadcast_info, ) }; @@ -950,22 +1456,22 @@ pub fn kill_subagent(handle: &str) -> Result<(), String> { if let Some(flag) = cancel_flag { flag.store(true, Ordering::SeqCst); - if let Some(jh) = join_handle { + if let Some(abort_handle) = abort_handle { // Watchdog: give the cooperative path a grace window, then abort. tokio::spawn(async move { tokio::time::sleep(std::time::Duration::from_secs(HARD_ABORT_GRACE_SECS)).await; - if !jh.is_finished() { + if !abort_handle.is_finished() { tracing::warn!( "[job-registry] background subagent did not stop within {}s of cancel; hard-aborting task", HARD_ABORT_GRACE_SECS ); - jh.abort(); + abort_handle.abort(); } }); } - } else if let Some(jh) = join_handle { + } else if let Some(abort_handle) = abort_handle { // Legacy job registered without a flag — hard abort is all we have. - jh.abort(); + abort_handle.abort(); } Ok(()) } @@ -1011,3 +1517,137 @@ pub fn cancel_subagents_for_session(session_id: &str) -> usize { } cancelled } + +/// Fan out ordinary user Stop to every running background shell in the +/// Session. OrgPause does not call this broad API; it cancels only the active +/// Turn's token and then awaits the exact owner tuple. +pub fn cancel_shells_for_session(session_id: &str) -> usize { + let cancellations = { + let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + reg.values_mut() + .filter(|job| { + job.session_id == session_id + && job.is_running() + && matches!(job.kind, JobKind::Shell { .. }) + }) + .filter_map(|job| { + job.shell_kill_requested = true; + match (&job.kind, job.shell_cancel.clone()) { + (JobKind::Shell { pid, .. }, cancel) => Some((*pid, cancel)), + _ => None, + } + }) + .collect::>() + }; + for (pid, cancel) in &cancellations { + if let Some(cancel) = cancel { + cancel.cancel(); + } else { + let pid = *pid; + tokio::spawn(async move { + if let Err(error) = terminate_shell_process_tree(pid).await { + tracing::warn!(pid, error = %error, "failed to stop legacy background shell"); + } + }); + } + } + if !cancellations.is_empty() { + tracing::info!( + session_id, + count = cancellations.len(), + "requested cancellation for background shell process groups" + ); + } + cancellations.len() +} + +/// Cancel every background job owned by one exact Agent Org Turn and wait +/// until the shell process groups and worker tasks are actually terminal. +/// This is a bounded failure-recovery path, not a steady-state poller. +pub async fn cancel_and_await_jobs_for_owner( + owner: &TurnProcessOwner, + timeout: Duration, +) -> Result<(), String> { + let (shell_cancels, subagent_handles) = { + let mut reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + let index = OWNER_INDEX.lock().unwrap_or_else(|e| e.into_inner()); + let mut shell_cancels = Vec::new(); + let mut subagent_handles = Vec::new(); + let handles = index.get(owner).cloned().unwrap_or_default(); + for handle in handles { + let Some(job) = reg.get_mut(&handle) else { + continue; + }; + if !job.requires_in_turn_finality || !job.is_running() { + continue; + } + match &job.kind { + JobKind::Shell { .. } => { + job.shell_kill_requested = true; + if let Some(cancel) = job.shell_cancel.clone() { + shell_cancels.push(cancel); + } + } + JobKind::Subagent { .. } => subagent_handles.push(( + job.handle.clone(), + job.join_handle.as_ref().map(JoinHandle::abort_handle), + )), + } + } + (shell_cancels, subagent_handles) + }; + + for cancel in shell_cancels { + cancel.cancel(); + } + for (handle, abort_handle) in subagent_handles { + if let Err(error) = kill_subagent(&handle) { + if get_status(&handle).is_some_and(|(status, _)| matches!(status, JobStatus::Running)) { + return Err(format!("failed to cancel owned subagent {handle}: {error}")); + } + } + if let Some(abort_handle) = abort_handle { + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(2)).await; + if !abort_handle.is_finished() { + abort_handle.abort(); + } + }); + } + } + + let wait = async { + await_shells_terminated_for_owner(owner, timeout).await?; + loop { + let all_subagents_terminal = { + let reg = REGISTRY.lock().unwrap_or_else(|e| e.into_inner()); + let index = OWNER_INDEX.lock().unwrap_or_else(|e| e.into_inner()); + index + .get(owner) + .into_iter() + .flatten() + .filter_map(|handle| reg.get(handle)) + .filter(|job| { + job.requires_in_turn_finality + && matches!(job.kind, JobKind::Subagent { .. }) + }) + .all(owned_job_execution_finished) + }; + if all_subagents_terminal { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + Ok::<(), String>(()) + }; + + tokio::time::timeout(timeout, wait).await.map_err(|_| { + format!( + "timed out after {}ms stopping background jobs owned by Turn {}", + timeout.as_millis(), + owner.dialog_turn_generation + ) + })??; + remove_terminal_jobs_for_owner(owner); + Ok(()) +} diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/subprocess.rs b/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/subprocess.rs index ddf7c12715..d25ddd46a1 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/subprocess.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/coding/exec/subprocess.rs @@ -9,6 +9,7 @@ use core_types::session_event::ShellReplayStatus; use tauri::AppHandle; use tokio::io::{AsyncRead, AsyncReadExt}; use tokio::sync::{mpsc, watch}; +use tokio_util::sync::CancellationToken; use tracing::warn; use crate::bus::event_pipeline_bridge; @@ -47,6 +48,8 @@ const STALL_THRESHOLD: Duration = Duration::from_secs(45); pub struct ExecIdentity { pub session_id: String, pub call_id: String, + pub turn_process_control: Option, + process_cancel: CancellationToken, } impl ExecIdentity { @@ -54,9 +57,27 @@ impl ExecIdentity { Self { session_id: session_id.into(), call_id: call_id.into(), + turn_process_control: None, + process_cancel: CancellationToken::new(), } } + pub fn with_turn_process_control( + mut self, + control: Option, + ) -> Self { + self.turn_process_control = control; + self + } + + fn cancellation_requested(&self) -> bool { + self.process_cancel.is_cancelled() + || self + .turn_process_control + .as_ref() + .is_some_and(|control| control.background_cancel.is_cancelled()) + } + fn replay_target(&self) -> ShellReplayTarget { ShellReplayTarget::new(self.session_id.clone(), self.call_id.clone()) } @@ -117,6 +138,9 @@ pub(super) fn broadcast_exec_output( sequence: u64, persisted_bytes: u64, ) { + if identity.cancellation_requested() { + return; + } crate::bus::broadcast_event( "agent:exec_output", serde_json::json!({ @@ -238,53 +262,35 @@ fn broadcast_process_exited( ); } -#[cfg(unix)] -fn signal_process_group(pid: u32, signal: libc::c_int) -> std::io::Result<()> { - let group_result = unsafe { libc::kill(-(pid as libc::pid_t), signal) }; - if group_result == 0 { - return Ok(()); - } - let group_error = std::io::Error::last_os_error(); - if unsafe { libc::kill(pid as libc::pid_t, signal) } == 0 { - return Ok(()); - } - let process_error = std::io::Error::last_os_error(); - if group_error.raw_os_error() == Some(libc::ESRCH) { - Err(process_error) - } else { - Err(group_error) +async fn terminate_child_tree(pid: u32, child: &mut tokio::process::Child) -> Result<(), String> { + if pid == 0 { + child + .kill() + .await + .map_err(|error| format!("failed to kill child without PID: {error}"))?; + return child + .wait() + .await + .map(|_| ()) + .map_err(|error| format!("failed to reap child without PID: {error}")); } -} -#[cfg(unix)] -async fn terminate_child_tree(pid: u32, child: &mut tokio::process::Child) { - if pid != 0 { - if let Err(err) = signal_process_group(pid, libc::SIGTERM) { - if err.raw_os_error() != Some(libc::ESRCH) { - warn!("[subprocess] failed to SIGTERM process group {pid}: {err}"); - } - } - tokio::time::sleep(Duration::from_millis(250)).await; - if matches!(child.try_wait(), Ok(Some(_))) { - return; - } - if let Err(err) = signal_process_group(pid, libc::SIGKILL) { - if err.raw_os_error() != Some(libc::ESRCH) { - warn!("[subprocess] failed to SIGKILL process group {pid}: {err}"); - } - } + // The process-tree helper owns signalling and verifies group absence; + // this task concurrently reaps the direct child so a zombie group leader + // cannot make that verification wait forever. + let (tree_result, child_result) = + tokio::join!(registry::terminate_shell_process_tree(pid), child.wait()); + let mut failures = Vec::new(); + if let Err(error) = tree_result { + failures.push(error); } - if let Err(err) = child.kill().await { - if err.kind() != std::io::ErrorKind::InvalidInput { - warn!("[subprocess] failed to kill child process: {err}"); - } + if let Err(error) = child_result { + failures.push(format!("failed to reap shell process {pid}: {error}")); } -} - -#[cfg(windows)] -async fn terminate_child_tree(_pid: u32, child: &mut tokio::process::Child) { - if let Err(err) = child.kill().await { - warn!("[subprocess] failed to kill child process: {err}"); + if failures.is_empty() { + Ok(()) + } else { + Err(failures.join("; ")) } } @@ -603,6 +609,11 @@ pub async fn execute_via_command( app_handle: Option, cancel_flag: Option<&AtomicBool>, ) -> Result { + if identity.cancellation_requested() { + return Err(ToolError::ExecutionFailed( + "Command was not started because its Turn is cancelled".to_string(), + )); + } let mut replay = ShellReplayWriter::create( shell_replays_root, identity.replay_target(), @@ -679,8 +690,10 @@ pub async fn execute_via_command( let wait_started_at = Instant::now(); let mut runtime = Some(runtime); loop { - if cancel_flag.is_some_and(|flag| flag.load(Ordering::Relaxed)) { - terminate_child_tree(pid, &mut child).await; + if identity.cancellation_requested() + || cancel_flag.is_some_and(|flag| flag.load(Ordering::Relaxed)) + { + let termination_error = terminate_child_tree(pid, &mut child).await.err(); let drain = match drain_output(runtime.take().expect("output runtime present")).await { Ok(drain) => drain, Err(err) => { @@ -704,6 +717,11 @@ pub async fn execute_via_command( "Command cancelled; shell replay is incomplete: {err}" ))); } + if let Some(error) = termination_error { + return Err(ToolError::ExecutionFailed(format!( + "Command cancellation did not terminate its process group: {error}" + ))); + } return Err(ToolError::ExecutionFailed( "Command cancelled by user".to_string(), )); @@ -713,7 +731,7 @@ pub async fn execute_via_command( .as_ref() .and_then(|runtime| runtime.failure_rx.borrow().clone()) { - terminate_child_tree(pid, &mut child).await; + let termination_error = terminate_child_tree(pid, &mut child).await.err(); let drain = match drain_output(runtime.take().expect("output runtime present")).await { Ok(drain) => drain, Err(writer_err) => { @@ -728,7 +746,10 @@ pub async fn execute_via_command( .finalize(ShellReplayStatus::Incomplete, Some(err.clone())); broadcast_process_exited(identity, pid, None, true, app_handle.as_ref()); return Err(ToolError::ExecutionFailed(format!( - "Command stopped because complete shell replay failed: {err}" + "Command stopped because complete shell replay failed: {err}{}", + termination_error + .map(|error| format!("; process-group termination failed: {error}")) + .unwrap_or_default() ))); } @@ -798,7 +819,7 @@ pub async fn execute_via_command( } Ok(None) => {} Err(err) => { - terminate_child_tree(pid, &mut child).await; + let termination_error = terminate_child_tree(pid, &mut child).await.err(); let drain = match drain_output(runtime.take().expect("output runtime present")) .await { @@ -806,11 +827,20 @@ pub async fn execute_via_command( Err(writer_err) => { broadcast_process_exited(identity, pid, None, true, app_handle.as_ref()); return Err(ToolError::ExecutionFailed(format!( - "Failed to wait for process; shell replay writer failed: {writer_err}" + "Failed to wait for process; shell replay writer failed: {writer_err}{}", + termination_error + .as_ref() + .map(|error| format!("; process-group termination failed: {error}")) + .unwrap_or_default() ))); } }; - let message = format!("Failed to wait for process: {err}"); + let message = format!( + "Failed to wait for process: {err}{}", + termination_error + .map(|error| format!("; process-group termination failed: {error}")) + .unwrap_or_default() + ); let _ = drain .replay .finalize(ShellReplayStatus::Incomplete, Some(message.clone())); @@ -856,15 +886,28 @@ fn handle_backgrounded( broadcast_system_output(&identity, &human_line); broadcast_process_backgrounded(&identity, pid, reason, app_handle.as_ref()); + let mut monitor_completion = None; if pid != 0 { let registry_path = log_path.clone().unwrap_or_default(); - let _ = registry::register_shell_replay( - pid, - command.to_string(), - registry_path, - identity.session_id.clone(), - identity.call_id.clone(), - ); + if let Some(control) = identity.turn_process_control.as_ref() { + monitor_completion = Some(registry::register_owned_shell_replay( + pid, + command.to_string(), + registry_path, + identity.session_id.clone(), + identity.call_id.clone(), + control, + identity.process_cancel.clone(), + )); + } else { + let _ = registry::register_shell_replay( + pid, + command.to_string(), + registry_path, + identity.session_id.clone(), + identity.call_id.clone(), + ); + } } let preview = active_state(&identity.session_id, &identity.call_id) @@ -899,38 +942,71 @@ fn handle_backgrounded( let mut runtime = Some(runtime); let started = Instant::now(); let mut stall_watchdog = StallWatchdog::new(); - let (exit_code, killed, replay_failure) = loop { + let mut parent_exit = None; + let (exit_code, mut killed, replay_failure, termination_result) = loop { + if identity.cancellation_requested() { + if pid != 0 { + registry::mark_shell_cancel_requested(&pid.to_string()); + } + let termination_result = terminate_child_tree(pid, &mut child).await; + break (None, true, None, termination_result); + } if let Some(err) = runtime .as_ref() .and_then(|runtime| runtime.failure_rx.borrow().clone()) { - terminate_child_tree(pid, &mut child).await; - break (None, true, Some(err)); + let termination_result = terminate_child_tree(pid, &mut child).await; + break (None, true, Some(err), termination_result); } - match child.try_wait() { - Ok(Some(status)) => break (status.code(), status.code().is_none(), None), - Ok(None) => {} - Err(err) => { - terminate_child_tree(pid, &mut child).await; - break ( - None, - true, - Some(format!("wait for background process: {err}")), - ); + if parent_exit.is_none() { + match child.try_wait() { + Ok(Some(status)) => parent_exit = Some(status), + Ok(None) => {} + Err(err) => { + let termination_result = terminate_child_tree(pid, &mut child).await; + break ( + None, + true, + Some(format!("wait for background process: {err}")), + termination_result, + ); + } + } + } + + if let Some(status) = parent_exit.as_ref() { + #[cfg(unix)] + let process_tree_gone = pid == 0 || !registry::process_tree_exists(pid); + #[cfg(windows)] + let process_tree_gone = true; + if process_tree_gone { + break (status.code(), status.code().is_none(), None, Ok(())); } } if started.elapsed() >= Duration::from_secs(BACKGROUND_SAFETY_TIMEOUT_SECS) { - terminate_child_tree(pid, &mut child).await; + let termination_result = if parent_exit.is_some() { + registry::terminate_shell_process_tree(pid) + .await + .map(|_| ()) + } else { + terminate_child_tree(pid, &mut child).await + }; break ( None, true, Some("background process exceeded 1h safety timeout".to_string()), + termination_result, ); } stall_watchdog.probe(&identity, pid); tokio::time::sleep(Duration::from_millis(50)).await; }; + // Natural exit and Pause can cross between the last process check and + // finalization. A late cancellation still owns the terminal verdict, + // suppressing output/wake for the paused Turn. + killed |= identity.cancellation_requested(); + let drain = match drain_output(runtime.take().expect("output runtime present")).await { Ok(drain) => drain, Err(writer_err) => { @@ -946,8 +1022,21 @@ fn handle_backgrounded( &identity, &format!("[background shell replay writer failed: {writer_err}]"), ); - broadcast_process_exited(&identity, pid, exit_code, killed, app_handle.as_ref()); - finish_background_job(pid, &identity.session_id).await; + if termination_result.is_ok() { + broadcast_process_exited( + &identity, + pid, + exit_code, + killed, + app_handle.as_ref(), + ); + } + if let Some(completion) = monitor_completion.take() { + completion.finish(Err(format!( + "shell replay output did not drain: {writer_err}" + ))); + } + finish_background_job(pid, &identity).await; return; } }; @@ -959,6 +1048,18 @@ fn handle_backgrounded( } else { drain.replay.finalize(ShellReplayStatus::Complete, None) }; + if identity + .turn_process_control + .as_ref() + .is_some_and(|control| control.require_owned_job_finality) + { + if let Ok(summary) = replay_result.as_ref() { + registry::set_final_result( + &pid.to_string(), + format_summary(summary.clone(), exit_code.unwrap_or(-1)), + ); + } + } let replay_incomplete = replay_result.is_err(); if pid != 0 { @@ -986,8 +1087,20 @@ fn handle_backgrounded( "[Session Replay is incomplete even though process termination status is known]", ); } - broadcast_process_exited(&identity, pid, exit_code, killed, app_handle.as_ref()); - finish_background_job(pid, &identity.session_id).await; + if termination_result.is_ok() { + broadcast_process_exited(&identity, pid, exit_code, killed, app_handle.as_ref()); + } else if let Err(error) = &termination_result { + tracing::warn!( + session_id = %identity.session_id, + pid, + error = %error, + "background shell monitor could not prove process-group termination" + ); + } + if let Some(completion) = monitor_completion.take() { + completion.finish(termination_result); + } + finish_background_job(pid, &identity).await; }); Ok(bounded_background_result(preview, &header, &log_info)) @@ -1001,12 +1114,19 @@ fn handle_backgrounded( /// resumed turn can still see it. The old flat 60s eviction raced exactly /// that window: a session idle for longer than a minute lost the entry /// before any turn could read it. -async fn finish_background_job(pid: u32, session_id: &str) { +async fn finish_background_job(pid: u32, identity: &ExecIdentity) { if pid == 0 { return; } + if identity + .turn_process_control + .as_ref() + .is_some_and(|control| control.require_owned_job_finality) + { + return; + } crate::tools::impls::orchestration::job_wake::current_job_completion_wake_hook() - .wake_owner(session_id); + .wake_owner(&identity.session_id); registry::retain_until_acknowledged_then_remove( &pid.to_string(), Duration::from_secs(30 * 60), @@ -1155,6 +1275,56 @@ mod tests { use super::*; use std::sync::Arc; + #[cfg(unix)] + fn test_turn_control( + session_id: &str, + generation: &str, + ) -> crate::tools::call_context::TurnProcessControl { + crate::tools::call_context::TurnProcessControl { + owner: crate::tools::call_context::TurnProcessOwner { + session_id: session_id.to_string(), + turn_intent_id: format!("intent-{generation}"), + runtime_lease_id: format!("lease-{generation}"), + dialog_turn_generation: generation.to_string(), + }, + background_cancel: CancellationToken::new(), + require_owned_job_finality: false, + } + } + + #[cfg(unix)] + async fn wait_for_pid_marker(path: &Path) -> (u32, u32) { + for _ in 0..200 { + if let Ok(contents) = std::fs::read_to_string(path) { + let mut values = contents.split_whitespace(); + let parent = values.next().unwrap().parse().unwrap(); + let child = values.next().unwrap().parse().unwrap(); + return (parent, child); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("shell did not write PID marker {}", path.display()); + } + + #[cfg(unix)] + fn assert_pid_absent(pid: u32) { + let result = unsafe { libc::kill(pid as libc::pid_t, 0) }; + assert_eq!(result, -1, "PID {pid} is still alive"); + assert_eq!( + std::io::Error::last_os_error().raw_os_error(), + Some(libc::ESRCH), + "PID {pid} still exists or could not be inspected" + ); + } + + #[cfg(unix)] + fn parent_child_command(marker: &Path) -> String { + format!( + "trap '' TERM; sh -c 'trap \"\" TERM; while :; do sleep 120; done' & child=$!; printf '%s %s\\n' \"$$\" \"$child\" > \"{}\"; wait", + marker.display() + ) + } + #[test] fn interactive_prompt_detection_matches_common_prompts() { for tail in [ @@ -1340,6 +1510,315 @@ mod tests { ); } + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial] + async fn exact_owner_background_shell_stays_in_turn_and_skips_idle_wake() { + let _sandbox = test_helpers::test_env::sandbox(); + let temp = tempfile::tempdir().unwrap(); + let mut control = test_turn_control("owned-shell-finality", "owned-shell-turn"); + control.require_owned_job_finality = true; + let owner = control.owner.clone(); + let identity = ExecIdentity::new(&owner.session_id, "owned-shell-call") + .with_turn_process_control(Some(control)); + + execute_via_command( + "printf owned-shell-output", + temp.path().to_path_buf(), + 10, + None, + ExecMode::Background, + &identity, + &temp.path().join("replays"), + None, + None, + ) + .await + .expect("launch exact-owner background shell"); + assert_eq!( + wait_for_terminal_replay(&owner.session_id, "owned-shell-call").await, + ShellReplayStatus::Complete + ); + + let terminal = loop { + let jobs = registry::list_jobs_for_owner(&owner); + if jobs + .iter() + .all(|job| !matches!(job.status, registry::JobStatus::Running)) + { + break jobs; + } + tokio::time::sleep(Duration::from_millis(10)).await; + }; + assert_eq!(terminal.len(), 1); + let handle = terminal[0].handle.clone(); + assert!(terminal[0].has_unread_output); + assert!(!registry::claim_completion_wake_for_session( + &owner.session_id + )); + assert!(registry::list_jobs_for_reminder(&owner.session_id).is_empty()); + + // `await_output` calls this exact acknowledgement after returning the + // replay. Exact-owner terminal jobs are removed immediately. + registry::acknowledge_outputs_for_owner(&owner, std::slice::from_ref(&handle)); + assert!(registry::list_jobs_for_owner(&owner).is_empty()); + assert!(registry::get_status(&handle).is_none()); + } + + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial] + async fn turn_cancel_before_spawn_never_starts_the_shell() { + let _sandbox = test_helpers::test_env::sandbox(); + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join("must-not-exist.pid"); + let control = test_turn_control("cancel-before-spawn", "turn-before"); + control.background_cancel.cancel(); + let identity = ExecIdentity::new(&control.owner.session_id, "call-before") + .with_turn_process_control(Some(control)); + + let result = execute_via_command( + &format!("printf started > \"{}\"", marker.display()), + temp.path().to_path_buf(), + 5, + None, + ExecMode::Blocking, + &identity, + &temp.path().join("replays"), + None, + None, + ) + .await; + + assert!(result.is_err()); + assert!(!marker.exists(), "cancelled Turn spawned a shell"); + } + + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial] + async fn foreground_turn_cancel_reaps_parent_child_and_process_group() { + let _sandbox = test_helpers::test_env::sandbox(); + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join("foreground.pid"); + let control = test_turn_control("cancel-foreground", "turn-foreground"); + let identity = ExecIdentity::new(&control.owner.session_id, "call-foreground") + .with_turn_process_control(Some(control.clone())); + let command = parent_child_command(&marker); + let replay_root = temp.path().join("replays"); + let cancel = async { + let pids = wait_for_pid_marker(&marker).await; + control.background_cancel.cancel(); + pids + }; + let execute = execute_via_command( + &command, + temp.path().to_path_buf(), + 120, + None, + ExecMode::Blocking, + &identity, + &replay_root, + None, + None, + ); + + let (result, (parent, child)) = tokio::join!(execute, cancel); + assert!(result.is_err()); + assert!(!registry::process_tree_exists(parent)); + assert_pid_absent(parent); + assert_pid_absent(child); + } + + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial] + async fn background_turn_cancel_escalates_and_waits_for_parent_child_exit() { + let _sandbox = test_helpers::test_env::sandbox(); + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join("background.pid"); + let control = test_turn_control("cancel-background", "turn-background"); + let owner = control.owner.clone(); + let identity = ExecIdentity::new(&owner.session_id, "call-background") + .with_turn_process_control(Some(control.clone())); + + execute_via_command( + &parent_child_command(&marker), + temp.path().to_path_buf(), + 120, + None, + ExecMode::Background, + &identity, + &temp.path().join("replays"), + None, + None, + ) + .await + .unwrap(); + let (parent, child) = wait_for_pid_marker(&marker).await; + assert!(registry::process_tree_exists(parent)); + + let started = Instant::now(); + control.background_cancel.cancel(); + registry::await_shells_terminated_for_owner(&owner, Duration::from_secs(5)) + .await + .unwrap(); + assert!( + started.elapsed() < Duration::from_secs(5), + "process drain exceeded its bounded wait" + ); + assert!(!registry::process_tree_exists(parent)); + assert_pid_absent(parent); + assert_pid_absent(child); + assert!(matches!( + registry::get_status(&parent.to_string()).map(|value| value.0), + Some(registry::JobStatus::Killed) + )); + assert_ne!( + wait_for_terminal_replay(&owner.session_id, "call-background").await, + ShellReplayStatus::Running + ); + registry::remove(&parent.to_string()); + } + + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial] + async fn timeout_background_turn_cancel_reaps_the_process_group() { + let _sandbox = test_helpers::test_env::sandbox(); + let temp = tempfile::tempdir().unwrap(); + let marker = temp.path().join("timeout-background.pid"); + let control = test_turn_control("cancel-timeout-background", "turn-timeout-background"); + let owner = control.owner.clone(); + let identity = ExecIdentity::new(&owner.session_id, "call-timeout-background") + .with_turn_process_control(Some(control.clone())); + + execute_via_command( + &parent_child_command(&marker), + temp.path().to_path_buf(), + 120, + Some(0), + ExecMode::Blocking, + &identity, + &temp.path().join("replays"), + None, + None, + ) + .await + .unwrap(); + let (parent, child) = wait_for_pid_marker(&marker).await; + control.background_cancel.cancel(); + + registry::await_shells_terminated_for_owner(&owner, Duration::from_secs(5)) + .await + .unwrap(); + assert!(!registry::process_tree_exists(parent)); + assert_pid_absent(parent); + assert_pid_absent(child); + registry::remove(&parent.to_string()); + } + + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial] + async fn natural_exit_racing_turn_cancel_has_one_terminal_barrier() { + let _sandbox = test_helpers::test_env::sandbox(); + let temp = tempfile::tempdir().unwrap(); + for index in 0..10 { + let marker = temp.path().join(format!("exit-race-{index}.pid")); + let control = test_turn_control("cancel-exit-race", &format!("turn-race-{index}")); + let owner = control.owner.clone(); + let call_id = format!("call-exit-race-{index}"); + let identity = ExecIdentity::new(&owner.session_id, &call_id) + .with_turn_process_control(Some(control.clone())); + let command = format!( + "printf '%s 0\\n' \"$$\" > \"{}\"; sleep 0.03", + marker.display() + ); + + execute_via_command( + &command, + temp.path().to_path_buf(), + 10, + None, + ExecMode::Background, + &identity, + &temp.path().join("replays"), + None, + None, + ) + .await + .unwrap(); + let (pid, _) = wait_for_pid_marker(&marker).await; + tokio::time::sleep(Duration::from_millis(25)).await; + control.background_cancel.cancel(); + registry::await_shells_terminated_for_owner(&owner, Duration::from_secs(3)) + .await + .unwrap(); + + assert!(!registry::process_tree_exists(pid)); + assert!(matches!( + registry::get_status(&pid.to_string()).map(|value| value.0), + Some(registry::JobStatus::Killed | registry::JobStatus::Exited(0)) + )); + registry::remove(&pid.to_string()); + } + } + + #[cfg(unix)] + #[tokio::test] + #[serial_test::serial] + async fn latched_cancel_between_spawn_and_background_registration_is_not_lost() { + let _sandbox = test_helpers::test_env::sandbox(); + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().join("replays"); + let control = test_turn_control("cancel-transition", "turn-transition"); + let owner = control.owner.clone(); + let identity = ExecIdentity::new(&owner.session_id, "call-transition") + .with_turn_process_control(Some(control.clone())); + let command = "trap '' TERM; sh -c 'trap \"\" TERM; while :; do sleep 120; done' & wait"; + let replay = + ShellReplayWriter::create(&root, identity.replay_target(), command, temp.path(), None) + .unwrap(); + let mut shell = tokio::process::Command::new("sh"); + shell + .arg("-c") + .arg(command) + .current_dir(temp.path()) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0); + let mut child = shell.spawn().unwrap(); + let pid = child.id().unwrap(); + let runtime = spawn_output_runtime( + identity.clone(), + child.stdout.take(), + child.stderr.take(), + replay, + ); + + control.background_cancel.cancel(); + handle_backgrounded( + command, + pid, + 0, + BackgroundReason::Timeout, + child, + runtime, + identity, + None, + ) + .unwrap(); + + registry::await_shells_terminated_for_owner(&owner, Duration::from_secs(5)) + .await + .unwrap(); + assert!(!registry::process_tree_exists(pid)); + assert_pid_absent(pid); + registry::remove(&pid.to_string()); + } + #[cfg(unix)] #[tokio::test] #[serial_test::serial] diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/background.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/background.rs index fd5b528587..a2f225a9f1 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/background.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/background.rs @@ -39,6 +39,8 @@ pub(super) struct BackgroundSpawnArgs<'a> { pub provider: Arc, pub work_item_id: Option, pub parent_cancel_flag: Option>, + /// Exact parent Turn owner for Agent Org same-Turn convergence. + pub parent_turn_owner: Option, pub handler: UnifiedSubagentHandler, /// When the subagent runs inside a worktree isolation, this is the repo /// root needed to call `remove_session_worktree` after the task exits. @@ -69,6 +71,7 @@ impl AgentTool { provider, work_item_id, parent_cancel_flag, + parent_turn_owner, handler, worktree_workspace_root, } = args; @@ -97,12 +100,21 @@ impl AgentTool { // - ForceSend (Send Now) pulses the parent flag but must NOT stop // background workers (`boundary_effect().cancel_background_workers // == false`), which a shared flag cannot express. - let (broadcast_tx, job_cancel_flag) = job_registry::register_subagent( - bg_session_id.clone(), - subagent_type_label, - bg_agent_name.clone(), - parent_session_id.clone(), - ); + let (broadcast_tx, job_cancel_flag) = match parent_turn_owner.clone() { + Some(owner) => job_registry::register_owned_subagent( + bg_session_id.clone(), + subagent_type_label, + bg_agent_name.clone(), + parent_session_id.clone(), + owner, + ), + None => job_registry::register_subagent( + bg_session_id.clone(), + subagent_type_label, + bg_agent_name.clone(), + parent_session_id.clone(), + ), + }; // Parent flag is delivered via the explicit fan-out above, not by // sharing the Arc. Drop it here so nobody reintroduces the pulse race. drop(bg_cancel_flag); @@ -239,8 +251,11 @@ impl AgentTool { ); } } - job_registry::set_final_result(&bg_session_id, resp.clone()); - job_registry::mark_exited(&bg_session_id, job_registry::JobStatus::Completed); + job_registry::finish_subagent( + &bg_session_id, + job_registry::JobStatus::Completed, + resp.clone(), + ); info!( "[agent:bg] '{}' done (model={}, cancelled={}): {} tokens", bg_agent_name, bg_model, was_cancelled, result.total_tokens @@ -285,8 +300,11 @@ impl AgentTool { )); let msg = super::helpers::prepend_worktree_note(msg, kept_worktree.as_ref()); broadcasting_handler.broadcast_error(); - job_registry::set_final_result(&bg_session_id, msg.clone()); - job_registry::mark_exited(&bg_session_id, job_registry::JobStatus::Failed); + job_registry::finish_subagent( + &bg_session_id, + job_registry::JobStatus::Failed, + msg.clone(), + ); warn!("[agent:bg] '{}' failed: {}", bg_agent_name, err); if let Some(ref wid) = bg_work_item_id { @@ -313,8 +331,10 @@ impl AgentTool { // parent is still running (the next turn's reminder covers that) // or when no app handle is installed (headless / tests). Mirrors // Claude Code's task-notification → idle-queue-processor design. - crate::tools::impls::orchestration::job_wake::current_job_completion_wake_hook() - .wake_owner(&bg_parent_session_id); + if parent_turn_owner.is_none() { + crate::tools::impls::orchestration::job_wake::current_job_completion_wake_hook() + .wake_owner(&bg_parent_session_id); + } // Remove from registry once the parent has consumed the result, // or after a hard cap if it never does. @@ -325,12 +345,14 @@ impl AgentTool { // before the Background Jobs reminder could ever surface it, so // the parent never learned the worker completed. Retaining until // `acknowledge_output` (bounded) closes that race. - job_registry::retain_until_acknowledged_then_remove( - &bg_session_id, - Duration::from_secs(30 * 60), - "agent:bg", - ) - .await; + if parent_turn_owner.is_none() { + job_registry::retain_until_acknowledged_then_remove( + &bg_session_id, + Duration::from_secs(30 * 60), + "agent:bg", + ) + .await; + } }); // Store JoinHandle so registry::kill_subagent can abort it diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/foreground.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/foreground.rs index 9486cea881..d657f8ce7a 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/foreground.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/foreground.rs @@ -51,6 +51,8 @@ pub(super) struct ForegroundRunArgs { /// needed to remove the worktree after the run — owned by the worker /// task now that it can outlive the parent's tool call. pub worktree_workspace_root: Option, + /// Exact parent Turn owner for Agent Org same-Turn convergence. + pub parent_turn_owner: Option, } impl AgentTool { @@ -76,6 +78,7 @@ impl AgentTool { model, provider, worktree_workspace_root, + parent_turn_owner, } = args; // Register in the job registry so the pin bar / kill chokepoint can @@ -86,12 +89,21 @@ impl AgentTool { // worker; after a fg→bg transition the mirroring stops and the // worker survives parent-turn boundaries (matching background // semantics, where ForceSend must NOT kill workers). - let (_job_tx, job_cancel_flag) = job_registry::register_subagent( - subagent_session_id.clone(), - subagent_type_label, - agent.name.clone(), - parent_session_id.clone(), - ); + let (_job_tx, job_cancel_flag) = match parent_turn_owner.clone() { + Some(owner) => job_registry::register_owned_subagent( + subagent_session_id.clone(), + subagent_type_label, + agent.name.clone(), + parent_session_id.clone(), + owner, + ), + None => job_registry::register_subagent( + subagent_session_id.clone(), + subagent_type_label, + agent.name.clone(), + parent_session_id.clone(), + ), + }; let parent_cancel_flag = self.config.parent_cancel_flag.clone(); let agent_name = agent.name.clone(); @@ -105,6 +117,7 @@ impl AgentTool { // is consumed, never who finalizes. let task_session_id = subagent_session_id.clone(); let task_parent_session_id = parent_session_id.clone(); + let task_parent_turn_owner = parent_turn_owner.clone(); let task_cancel_flag = Arc::clone(&job_cancel_flag); let (result_tx, mut result_rx) = tokio::sync::oneshot::channel::>(); @@ -233,16 +246,14 @@ impl AgentTool { ); } } - // completed-first invariant (gh-20236 class): terminal - // status is written BEFORE final-result storage, wake, - // and LinkedSession writes, so anything blocking on - // "is it still running?" can never deadlock against - // slow post-processing. - job_registry::mark_exited(&task_session_id, job_registry::JobStatus::Completed); - // Store the final result so a transitioned parent reads - // it from the Background Jobs reminder exactly like a - // native background worker. - job_registry::set_final_result(&task_session_id, resp.clone()); + // Publish status + result atomically before wake and + // LinkedSession writes. The owner can neither wait on a + // ghost Running row nor observe terminal-without-result. + job_registry::finish_subagent( + &task_session_id, + job_registry::JobStatus::Completed, + resp.clone(), + ); ( if was_cancelled { LinkedSessionStatus::Cancelled @@ -279,8 +290,11 @@ impl AgentTool { )); let msg = super::helpers::prepend_worktree_note(msg, kept_worktree.as_ref()); handler.broadcast_error(); - job_registry::mark_exited(&task_session_id, job_registry::JobStatus::Failed); - job_registry::set_final_result(&task_session_id, msg.clone()); + job_registry::finish_subagent( + &task_session_id, + job_registry::JobStatus::Failed, + msg.clone(), + ); ( LinkedSessionStatus::Failed, 0i64, @@ -330,17 +344,21 @@ impl AgentTool { "[agent] '{}' finished after fg→bg transition; delivering via background path", task_session_id ); - crate::tools::impls::orchestration::job_wake::current_job_completion_wake_hook( - ) - .wake_owner(&task_parent_session_id); + if task_parent_turn_owner.is_none() { + crate::tools::impls::orchestration::job_wake::current_job_completion_wake_hook( + ) + .wake_owner(&task_parent_session_id); + } // Registry retention: same ack-polling GC as the native // background path so the result survives until consumed. - job_registry::retain_until_acknowledged_then_remove( - &task_session_id, - Duration::from_secs(30 * 60), - "agent", - ) - .await; + if task_parent_turn_owner.is_none() { + job_registry::retain_until_acknowledged_then_remove( + &task_session_id, + Duration::from_secs(30 * 60), + "agent", + ) + .await; + } } } }); @@ -357,14 +375,30 @@ impl AgentTool { Ok(response) => { // Inline delivery — suppress the unread-output reminder // that background jobs rely on. - job_registry::acknowledge_output(&subagent_session_id); + if let Some(owner) = parent_turn_owner.as_ref() { + job_registry::await_subagent_execution_for_owner( + owner, + &subagent_session_id, + Duration::from_secs(2), + ) + .await + .map_err(ToolError::ExecutionFailed)?; + job_registry::acknowledge_outputs_for_owner( + owner, + std::slice::from_ref(&subagent_session_id), + ); + } else { + job_registry::acknowledge_output(&subagent_session_id); + } // Registry grace period so the verdict stays readable // briefly, then the row is GC'd. - let gc_handle = subagent_session_id.clone(); - tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(120)).await; - job_registry::remove(&gc_handle); - }); + if parent_turn_owner.is_none() { + let gc_handle = subagent_session_id.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_secs(120)).await; + job_registry::remove(&gc_handle); + }); + } return response; } Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {} diff --git a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/mod.rs b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/mod.rs index 7f8e46d300..1572c3fd5e 100644 --- a/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/mod.rs +++ b/src-tauri/crates/agent-core/src/core/tools/impls/orchestration/agent/mod.rs @@ -588,6 +588,28 @@ impl Tool for AgentTool { .and_then(|v| v.as_bool()) .unwrap_or(false); + let parent_turn_owner = if ctx + .turn_process_control + .as_ref() + .is_some_and(|control| control.require_owned_job_finality) + { + let control = ctx.turn_process_control.as_ref().ok_or_else(|| { + ToolError::ExecutionFailed( + "Agent Org worker launch requires an exact parent Turn owner.".to_string(), + ) + })?; + if control.owner.session_id != ctx.session_id + || control.owner.turn_intent_id != ctx.turn_intent_id + { + return Err(ToolError::ExecutionFailed( + "Agent Org worker owner does not match the dispatching Turn.".to_string(), + )); + } + Some(control.owner.clone()) + } else { + None + }; + let prompt = params .get("prompt") .and_then(|v| v.as_str()) @@ -922,6 +944,7 @@ impl Tool for AgentTool { let turn_config = TurnConfig { turn_intent_id: String::new(), projected_inbox_ids: Vec::new(), + turn_process_control: None, model: model.clone(), account_id: self.config.session_account_id.clone(), context_window_override: agent.context_window, @@ -1190,6 +1213,7 @@ impl Tool for AgentTool { provider: Arc::clone(&subagent_provider), work_item_id: self.config.work_item_id.clone(), parent_cancel_flag: self.config.parent_cancel_flag.clone(), + parent_turn_owner: parent_turn_owner.clone(), handler, worktree_workspace_root: isolation_workspace_root, }, @@ -1216,6 +1240,7 @@ impl Tool for AgentTool { model, provider: subagent_provider, worktree_workspace_root: isolation_workspace_root, + parent_turn_owner, }) .await } diff --git a/src-tauri/crates/agent-core/src/core/tools/tests/job_registry_tests.rs b/src-tauri/crates/agent-core/src/core/tools/tests/job_registry_tests.rs index d8a64377c5..1c6b028a61 100644 --- a/src-tauri/crates/agent-core/src/core/tools/tests/job_registry_tests.rs +++ b/src-tauri/crates/agent-core/src/core/tools/tests/job_registry_tests.rs @@ -1,7 +1,28 @@ use std::path::PathBuf; +use std::time::Duration; +use tokio_util::sync::CancellationToken; + +use crate::tools::call_context::{TurnProcessControl, TurnProcessOwner}; use crate::tools::impls::coding::exec::registry::{self, JobKind, JobStatus}; +fn shell_owner(session_id: &str, lease_id: &str) -> TurnProcessOwner { + TurnProcessOwner { + session_id: session_id.to_string(), + turn_intent_id: "intent-1".to_string(), + runtime_lease_id: lease_id.to_string(), + dialog_turn_generation: "turn-1".to_string(), + } +} + +fn shell_control(session_id: &str, lease_id: &str) -> TurnProcessControl { + TurnProcessControl { + owner: shell_owner(session_id, lease_id), + background_cancel: CancellationToken::new(), + require_owned_job_finality: false, + } +} + #[test] fn test_register_shell_and_get() { let pid = 99990; @@ -66,6 +87,110 @@ fn test_register_subagent() { assert!(registry::get_status(&handle).is_none()); } +#[tokio::test] +async fn exact_owner_result_bypasses_generic_wake_and_is_removed_after_consumption() { + let owner = shell_owner("owned-finality-session", "owned-finality-lease"); + let handle = "agent-owned-finality-result".to_string(); + let (_tx, _cancel) = registry::register_owned_subagent( + handle.clone(), + "delegate".into(), + "Owned Worker".into(), + owner.session_id.clone(), + owner.clone(), + ); + registry::set_join_handle(&handle, tokio::spawn(async {})); + tokio::task::yield_now().await; + registry::finish_subagent(&handle, JobStatus::Completed, "owned result".into()); + + assert!( + !registry::claim_completion_wake_for_session(&owner.session_id), + "Agent Org-owned results must not start an ordinary idle wake" + ); + assert!( + registry::list_jobs_for_reminder(&owner.session_id).is_empty(), + "ordinary SDE reminders must not consume an Agent Org-owned result" + ); + let owned = registry::list_jobs_for_owner(&owner); + assert_eq!(owned.len(), 1); + assert_eq!(owned[0].final_result.as_deref(), Some("owned result")); + + registry::acknowledge_outputs_for_owner(&owner, std::slice::from_ref(&handle)); + assert!(registry::get_status(&handle).is_none()); + assert!(registry::list_jobs_for_owner(&owner).is_empty()); +} + +#[tokio::test] +async fn exact_owner_teardown_does_not_cancel_a_new_runtime_owner() { + use std::sync::atomic::Ordering; + + let old_owner = shell_owner("owned-teardown-session", "lease-old"); + let new_owner = shell_owner("owned-teardown-session", "lease-new"); + let old_handle = "agent-owned-old-runtime".to_string(); + let new_handle = "agent-owned-new-runtime".to_string(); + let (_old_tx, old_cancel) = registry::register_owned_subagent( + old_handle.clone(), + "delegate".into(), + "Old Worker".into(), + old_owner.session_id.clone(), + old_owner.clone(), + ); + let (_new_tx, new_cancel) = registry::register_owned_subagent( + new_handle.clone(), + "delegate".into(), + "New Worker".into(), + new_owner.session_id.clone(), + new_owner.clone(), + ); + registry::set_join_handle(&old_handle, tokio::spawn(std::future::pending::<()>())); + registry::set_join_handle(&new_handle, tokio::spawn(std::future::pending::<()>())); + + registry::cancel_and_await_jobs_for_owner(&old_owner, Duration::from_secs(4)) + .await + .expect("old owner teardown"); + assert!(old_cancel.load(Ordering::SeqCst)); + assert!(!new_cancel.load(Ordering::SeqCst)); + assert!(registry::get_status(&old_handle).is_none()); + assert!(matches!( + registry::get_status(&new_handle), + Some((JobStatus::Running, JobKind::Subagent { .. })) + )); + + registry::cancel_and_await_jobs_for_owner(&new_owner, Duration::from_secs(4)) + .await + .expect("new owner cleanup"); +} + +#[tokio::test] +async fn exact_owner_teardown_waits_for_subagent_spawn_handoff() { + let owner = shell_owner("owned-spawn-handoff-session", "lease-spawn-handoff"); + let handle = "agent-owned-spawn-handoff".to_string(); + let (_tx, cancel) = registry::register_owned_subagent( + handle.clone(), + "delegate".into(), + "Spawn Handoff Worker".into(), + owner.session_id.clone(), + owner.clone(), + ); + + let teardown_owner = owner.clone(); + let teardown = tokio::spawn(async move { + registry::cancel_and_await_jobs_for_owner(&teardown_owner, Duration::from_secs(1)).await + }); + tokio::task::yield_now().await; + assert!(cancel.load(std::sync::atomic::Ordering::SeqCst)); + assert!( + !teardown.is_finished(), + "teardown must not pass before the spawned task handle is attached" + ); + + registry::set_join_handle(&handle, tokio::spawn(std::future::pending::<()>())); + teardown + .await + .expect("join teardown") + .expect("finish exact owner teardown"); + assert!(registry::get_status(&handle).is_none()); +} + #[test] fn test_list_shell_for_session() { let pid_a = 99992; @@ -91,6 +216,93 @@ fn test_list_shell_for_session() { registry::remove(&pid_b.to_string()); } +#[test] +fn user_stop_shell_fanout_is_session_scoped_and_level_triggered() { + let mine_pid = 99_981; + let other_pid = 99_982; + let mine_cancel = CancellationToken::new(); + let other_cancel = CancellationToken::new(); + let mine_completion = registry::register_owned_shell_replay( + mine_pid, + "mine".into(), + PathBuf::from("/tmp/owned-mine.txt"), + "owned-session-a".into(), + "owned-call-a".into(), + &shell_control("owned-session-a", "lease-a"), + mine_cancel.clone(), + ); + let other_completion = registry::register_owned_shell_replay( + other_pid, + "other".into(), + PathBuf::from("/tmp/owned-other.txt"), + "owned-session-b".into(), + "owned-call-b".into(), + &shell_control("owned-session-b", "lease-b"), + other_cancel.clone(), + ); + + assert_eq!(registry::cancel_shells_for_session("owned-session-a"), 1); + assert!(mine_cancel.is_cancelled()); + assert!(!other_cancel.is_cancelled()); + + mine_completion.finish(Ok(())); + other_completion.finish(Ok(())); + registry::remove(&mine_pid.to_string()); + registry::remove(&other_pid.to_string()); +} + +#[tokio::test] +async fn exact_owner_barrier_rejects_an_old_runtime_lease_completion() { + let old_pid = 99_983; + let new_pid = 99_984; + let old_owner = shell_owner("stale-owner-session", "lease-old"); + let new_owner = shell_owner("stale-owner-session", "lease-new"); + let old_completion = registry::register_owned_shell_replay( + old_pid, + "old".into(), + PathBuf::from("/tmp/owned-old.txt"), + old_owner.session_id.clone(), + "owned-call-old".into(), + &TurnProcessControl { + owner: old_owner.clone(), + background_cancel: CancellationToken::new(), + require_owned_job_finality: false, + }, + CancellationToken::new(), + ); + let new_completion = registry::register_owned_shell_replay( + new_pid, + "new".into(), + PathBuf::from("/tmp/owned-new.txt"), + new_owner.session_id.clone(), + "owned-call-new".into(), + &TurnProcessControl { + owner: new_owner.clone(), + background_cancel: CancellationToken::new(), + require_owned_job_finality: false, + }, + CancellationToken::new(), + ); + + old_completion.finish(Ok(())); + registry::await_shells_terminated_for_owner(&old_owner, Duration::from_millis(50)) + .await + .unwrap(); + assert!( + registry::await_shells_terminated_for_owner(&new_owner, Duration::from_millis(25)) + .await + .is_err(), + "old lease completion must not release the new lease barrier" + ); + + new_completion.finish(Ok(())); + registry::await_shells_terminated_for_owner(&new_owner, Duration::from_millis(50)) + .await + .unwrap(); + registry::remove(&old_pid.to_string()); + registry::remove(&new_pid.to_string()); +} + #[test] fn test_subagent_not_in_shell_list() { // Session id must be unique to this test: `test_list_shell_for_session` diff --git a/src-tauri/crates/agent-core/src/core/turn_executor/execute.rs b/src-tauri/crates/agent-core/src/core/turn_executor/execute.rs index dc2a3b21dd..df8b17e414 100644 --- a/src-tauri/crates/agent-core/src/core/turn_executor/execute.rs +++ b/src-tauri/crates/agent-core/src/core/turn_executor/execute.rs @@ -67,8 +67,13 @@ pub async fn execute_turn( cancel_flag: Option<&Arc>, policy_context_activator: Option<&SessionScopedContextActivator>, ) -> Result { + let require_owned_job_finality = config + .turn_process_control + .as_ref() + .is_some_and(|control| control.require_owned_job_finality); let mut iteration = 0u32; let mut final_content: Option = None; + let mut terminal_error: Option = None; // Set to true when the turn exits due to exhausted stream-error retries. // Prevents the error text from being persisted into the conversation // history (see TurnResult::is_stream_error). @@ -103,6 +108,12 @@ pub async fn execute_turn( // that always blocks cannot spin the loop forever (death-spiral guard). let mut stop_hook_blocks = 0u32; const MAX_STOP_HOOK_BLOCKS: u32 = 3; + // A model may prematurely declare success while its own Agent Org jobs + // are still running. Give it a small bounded correction budget, then + // cancel the exact owner and fail the Turn instead of creating a later + // continuation. + let mut owned_job_finality_blocks = 0u32; + const MAX_OWNED_JOB_FINALITY_BLOCKS: u32 = 3; // Auto-continue nudges already burned this turn, plus the cumulative // completion-token count observed at the last nudge (the // diminishing-returns baseline for `should_auto_continue`). @@ -142,7 +153,7 @@ pub async fn execute_turn( // — without this, shared tool instances still point at the parent id. tools.set_session_key(session_id).await; - loop { + 'turn_loop: loop { if let Some(max) = config.max_iterations { if iteration >= max { break; @@ -207,7 +218,38 @@ pub async fn execute_turn( // turn-boundary delivery. Claiming here consumes the same // exactly-once flags as the idle wake, so an event delivered // mid-turn never also wakes the session at turn end. - if iteration > 1 + if iteration > 1 && require_owned_job_finality { + let owner = config + .turn_process_control + .as_ref() + .ok_or_else(|| { + "Agent Org Turn finality requires an exact runtime owner".to_string() + })? + .owner + .clone(); + use crate::core::session::turn::background_reminder; + let jobs: Vec<_> = + crate::tools::impls::coding::exec::registry::list_jobs_for_owner(&owner) + .into_iter() + .filter(|job| job.has_unread_output || job.stalled_waiting_input) + .collect(); + if !jobs.is_empty() { + info!( + "[agent-core] exact-owner background-job note injected ({} job(s), session={})", + jobs.len(), + session_id + ); + let note = background_reminder::build_completion_notification(&jobs); + crate::tools::impls::coding::exec::registry::acknowledge_outputs_for_owner( + &owner, + &background_reminder::inlined_result_handles(&jobs), + ); + messages.push(serde_json::json!({ + "role": "user", + "content": note, + })); + } + } else if iteration > 1 && crate::tools::impls::coding::exec::registry::claim_completion_wake_for_session( session_id, ) @@ -526,11 +568,15 @@ pub async fn execute_turn( session_id, stripped, err ); if stripped == 0 { - return Err(format!("LLM error: {}", err)); + terminal_error = Some(format!("LLM error: {}", err)); + break 'turn_loop; } continue; } - Err(err) => return Err(format!("LLM error: {}", err)), + Err(err) => { + terminal_error = Some(format!("LLM error: {}", err)); + break 'turn_loop; + } }; if is_cancelled(cancel_flag) { @@ -815,14 +861,57 @@ pub async fn execute_turn( iterations_since_todo_use = 0; } + // A Task owner cannot publish a terminal Task while this same + // Agent Org Turn still owns background work. Blocking at the + // executor boundary keeps the Task mutation, Inbox commit, and + // final assistant response behind the same exact-owner fence. + let blocked_terminal_task_calls = if require_owned_job_finality { + let owner = &config + .turn_process_control + .as_ref() + .ok_or_else(|| { + "Agent Org Turn finality requires an exact runtime owner".to_string() + })? + .owner; + if crate::tools::impls::coding::exec::registry::list_jobs_for_owner(owner) + .is_empty() + { + std::collections::HashSet::new() + } else { + response + .tool_calls + .iter() + .filter(|call| { + call.name == crate::tools::names::TASK_UPDATE + && call + .arguments + .get("operation") + .and_then(Value::as_str) + .is_some_and(|operation| { + matches!(operation, "complete" | "fail") + }) + }) + .map(|call| call.id.clone()) + .collect() + } + } else { + std::collections::HashSet::new() + }; + let executable_tool_calls = response + .tool_calls + .iter() + .filter(|call| !blocked_terminal_task_calls.contains(&call.id)) + .cloned() + .collect::>(); let (_count, tool_execution_usage, outcome) = execute_tool_calls( messages, - &response.tool_calls, + &executable_tool_calls, tools, policy, session_id, &config.turn_intent_id, &config.projected_inbox_ids, + config.turn_process_control.as_ref(), handler, permission_provider, cancel_flag, @@ -834,6 +923,30 @@ pub async fn execute_turn( .await; usage_telemetry.record_tool_results(iteration as i64, tool_execution_usage); + for tool_call in response + .tool_calls + .iter() + .filter(|call| blocked_terminal_task_calls.contains(&call.id)) + { + let result = "Error: this Agent Org Task cannot become terminal while its exact Turn-owned background work is still active or unconsumed. Await or kill that work, consume its terminal result in this Turn, then retry task_update."; + handler.on_tool_call( + session_id, + &tool_call.id, + &tool_call.name, + &tool_call.name, + &tool_call.arguments, + ); + handler.on_tool_result( + session_id, + &tool_call.id, + &tool_call.name, + &tool_call.name, + result, + ); + add_tool_result(messages, &tool_call.id, &tool_call.name, result, true); + consecutive_errors = consecutive_errors.saturating_add(1); + } + // Backfill dummy results for any tool calls that don't have a // result yet after EarlyExit. let existing_ids: std::collections::HashSet = messages @@ -936,6 +1049,85 @@ pub async fn execute_turn( continue; } + // Agent Org same-Turn finality is an internal ownership gate, + // independent of user Stop hooks. A terminal result is fed back + // into this exact Turn; a running job blocks success and asks the + // model to await or kill it. Exhaustion fails closed after exact + // owner teardown — never by scheduling a post-terminal wake. + if require_owned_job_finality && !is_cancelled(cancel_flag) { + let owner = &config + .turn_process_control + .as_ref() + .ok_or_else(|| { + "Agent Org Turn finality requires an exact runtime owner".to_string() + })? + .owner; + let jobs = crate::tools::impls::coding::exec::registry::list_jobs_for_owner(owner); + if !jobs.is_empty() { + let at_iteration_limit = + config.max_iterations.is_some_and(|max| iteration >= max); + if owned_job_finality_blocks >= MAX_OWNED_JOB_FINALITY_BLOCKS + || at_iteration_limit + { + crate::tools::impls::coding::exec::registry::cancel_and_await_jobs_for_owner( + owner, + std::time::Duration::from_secs(12), + ) + .await + .map_err(|error| { + format!( + "Agent Org Turn could not stop unconverged background work: {error}" + ) + })?; + return Err( + "Agent Org Turn tried to finish before its background work converged" + .to_string(), + ); + } + owned_job_finality_blocks += 1; + + if let Some(ref text) = response.content { + if !text.trim().is_empty() { + handler.on_assistant_iteration_complete( + session_id, + Some(text.as_str()), + false, + &config.model, + ); + messages.push(serde_json::json!({ + "role": "assistant", + "content": text, + })); + } + } + + let completed: Vec<_> = jobs + .iter() + .filter(|job| job.has_unread_output) + .cloned() + .collect(); + let note = if completed.is_empty() { + format!( + "\nThis Agent Org Turn still owns {} running background job(s). The Turn cannot finish while they are active. Await their result or kill them, consume the terminal output in this same Turn, and only then provide the final answer.\n", + jobs.len() + ) + } else { + use crate::core::session::turn::background_reminder; + let note = background_reminder::build_completion_notification(&completed); + crate::tools::impls::coding::exec::registry::acknowledge_outputs_for_owner( + owner, + &background_reminder::inlined_result_handles(&completed), + ); + note + }; + messages.push(serde_json::json!({ + "role": "user", + "content": note, + })); + continue; + } + } + // Stop-hook gate first: user-defined `Stop` hooks may BLOCK this // completion (stdout `{"decision":"block","message":...}`), in // which case the feedback is persisted as the assistant text, @@ -1070,6 +1262,32 @@ pub async fn execute_turn( } } + // Every non-standard loop exit (repeat guard, iteration cap, cancellation, + // stream failure, or tool error) is also fenced. A cancelled Turn keeps + // its cancelled verdict after teardown; every other unconverged exit is + // an error so Task recovery, not a false assistant success, owns it. + if require_owned_job_finality { + let owner = &config + .turn_process_control + .as_ref() + .ok_or_else(|| "Agent Org Turn finality requires an exact runtime owner".to_string())? + .owner; + if !crate::tools::impls::coding::exec::registry::list_jobs_for_owner(owner).is_empty() { + crate::tools::impls::coding::exec::registry::cancel_and_await_jobs_for_owner( + owner, + std::time::Duration::from_secs(12), + ) + .await + .map_err(|error| format!("Agent Org Turn background-work teardown failed: {error}"))?; + if !is_cancelled(cancel_flag) && terminal_error.is_none() { + return Err("Agent Org Turn ended before its background work converged".to_string()); + } + } + } + if let Some(error) = terminal_error { + return Err(error); + } + let mut hit_max_iterations = false; if let Some(max) = config.max_iterations { if final_content.is_none() && iteration >= max { diff --git a/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/mod.rs b/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/mod.rs index 2157321dd9..aca291163e 100644 --- a/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/mod.rs +++ b/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/mod.rs @@ -249,6 +249,7 @@ pub(crate) async fn execute_tool_calls( session_id: &str, turn_intent_id: &str, projected_inbox_ids: &[i64], + turn_process_control: Option<&crate::tools::call_context::TurnProcessControl>, handler: &dyn TurnEventHandler, permission_provider: Option<&dyn PermissionProvider>, cancel_flag: Option<&Arc>, @@ -273,6 +274,7 @@ pub(crate) async fn execute_tool_calls( session_id, turn_intent_id, projected_inbox_ids, + turn_process_control, handler, permission_provider, cancel_flag, @@ -304,6 +306,7 @@ pub(crate) async fn execute_tool_calls( session_id, turn_intent_id, projected_inbox_ids, + turn_process_control, handler, permission_provider, cancel_flag, @@ -333,6 +336,7 @@ pub(crate) async fn execute_tool_calls( session_id, turn_intent_id, projected_inbox_ids, + turn_process_control, handler, permission_provider, cancel_flag, diff --git a/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/parallel.rs b/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/parallel.rs index 31cfad42f3..7bef86de87 100644 --- a/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/parallel.rs +++ b/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/parallel.rs @@ -46,6 +46,7 @@ pub(super) async fn execute_parallel_group( session_id: &str, turn_intent_id: &str, projected_inbox_ids: &[i64], + turn_process_control: Option<&crate::tools::call_context::TurnProcessControl>, handler: &dyn TurnEventHandler, permission_provider: Option<&dyn PermissionProvider>, cancel_flag: Option<&Arc>, @@ -227,11 +228,12 @@ pub(super) async fn execute_parallel_group( .map(|(idx, effective_args, _display_name)| { let tool_name = &calls[*idx].name; let call_id = &calls[*idx].id; - let ctx = crate::tools::call_context::CallContext::for_turn( + let ctx = crate::tools::call_context::CallContext::for_runtime_turn( call_id, session_id, turn_intent_id, projected_inbox_ids.to_vec(), + turn_process_control.cloned(), ); let args = effective_args.clone(); async move { diff --git a/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/single.rs b/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/single.rs index c309f716b4..b7ebeee6a7 100644 --- a/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/single.rs +++ b/src-tauri/crates/agent-core/src/core/turn_executor/tool_execution/single.rs @@ -45,6 +45,7 @@ pub(super) async fn execute_single_tool( session_id: &str, turn_intent_id: &str, projected_inbox_ids: &[i64], + turn_process_control: Option<&crate::tools::call_context::TurnProcessControl>, handler: &dyn TurnEventHandler, permission_provider: Option<&dyn PermissionProvider>, cancel_flag: Option<&Arc>, @@ -240,11 +241,12 @@ pub(super) async fn execute_single_tool( ); let exec_start = Instant::now(); - let ctx = crate::tools::call_context::CallContext::for_turn( + let ctx = crate::tools::call_context::CallContext::for_runtime_turn( &tool_call.id, session_id, turn_intent_id, projected_inbox_ids.to_vec(), + turn_process_control.cloned(), ); let raw_outcome = tools .execute_with_policy(&tool_call.name, effective_args.clone(), policy, &ctx) @@ -483,6 +485,7 @@ mod tests { "session-test", "", &[], + None, &handler, None, None, diff --git a/src-tauri/crates/agent-core/src/core/turn_executor/types.rs b/src-tauri/crates/agent-core/src/core/turn_executor/types.rs index 0b68aa81b9..c0514fbed1 100644 --- a/src-tauri/crates/agent-core/src/core/turn_executor/types.rs +++ b/src-tauri/crates/agent-core/src/core/turn_executor/types.rs @@ -98,6 +98,9 @@ pub struct TurnConfig { pub turn_intent_id: String, /// Agent Org Inbox rows that this turn will acknowledge on success. pub projected_inbox_ids: Vec, + /// Exact runtime owner and cancellation for background shell processes + /// started by this Turn. Background/offline callers intentionally use None. + pub turn_process_control: Option, /// Model identifier (provider-specific). pub model: String, /// KeyVault account id backing this turn. Threaded through so @@ -452,6 +455,7 @@ mod tests { let config = TurnConfig { turn_intent_id: String::new(), projected_inbox_ids: Vec::new(), + turn_process_control: None, model: "test".to_string(), account_id: None, context_window_override: None, @@ -476,6 +480,7 @@ mod tests { let config = TurnConfig { turn_intent_id: String::new(), projected_inbox_ids: Vec::new(), + turn_process_control: None, model: "test".to_string(), account_id: None, context_window_override: None, diff --git a/src-tauri/crates/agent-core/src/specialization/memory/workspace_memory/auto_dream.rs b/src-tauri/crates/agent-core/src/specialization/memory/workspace_memory/auto_dream.rs index 93cc84b447..9a9a2fdac0 100644 --- a/src-tauri/crates/agent-core/src/specialization/memory/workspace_memory/auto_dream.rs +++ b/src-tauri/crates/agent-core/src/specialization/memory/workspace_memory/auto_dream.rs @@ -182,6 +182,7 @@ pub async fn run_consolidation(params: super::super::MemoryAgentParams<'_>) -> R let turn_config = TurnConfig { turn_intent_id: String::new(), projected_inbox_ids: Vec::new(), + turn_process_control: None, model: params.model.to_string(), account_id: None, context_window_override: None, diff --git a/src-tauri/crates/agent-core/src/specialization/memory/workspace_memory/extract/runner.rs b/src-tauri/crates/agent-core/src/specialization/memory/workspace_memory/extract/runner.rs index 836b977868..405a03543d 100644 --- a/src-tauri/crates/agent-core/src/specialization/memory/workspace_memory/extract/runner.rs +++ b/src-tauri/crates/agent-core/src/specialization/memory/workspace_memory/extract/runner.rs @@ -96,6 +96,7 @@ pub async fn run_extraction( let turn_config = TurnConfig { turn_intent_id: String::new(), projected_inbox_ids: Vec::new(), + turn_process_control: None, model: params.model.to_string(), account_id: None, context_window_override: None, diff --git a/src-tauri/crates/agent-core/src/state/commands/session/compaction.rs b/src-tauri/crates/agent-core/src/state/commands/session/compaction.rs index 40d8236ed9..578dfba1b6 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/compaction.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/compaction.rs @@ -251,8 +251,7 @@ async fn run_manual_compact_exclusive( instructions: Option, ) -> ManualCompactCommandResult { let runtime = { - let guard = session.runtime.read().await; - match guard.clone() { + match session.get_runtime().await { Some(runtime) => runtime, None => { return ManualCompactCommandResult::status(ManualCompactStatus::NoRuntime); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/debug/general.rs b/src-tauri/crates/agent-core/src/state/commands/session/debug/general.rs index a1635f68a2..e268e4ad72 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/debug/general.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/debug/general.rs @@ -123,10 +123,8 @@ pub async fn debug_session_general_snapshot( .ok_or_else(|| format!("session not found: {}", session_id))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {}", session_id))?; let agent_id = session.definition.id.clone(); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/debug/model.rs b/src-tauri/crates/agent-core/src/state/commands/session/debug/model.rs index 42c4c9c1a2..d3e2fc8d8d 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/debug/model.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/debug/model.rs @@ -75,10 +75,8 @@ pub async fn debug_session_model_snapshot( .ok_or_else(|| format!("session not found: {}", session_id))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {}", session_id))?; let agent_id = session.definition.id.clone(); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/debug/org_runtime.rs b/src-tauri/crates/agent-core/src/state/commands/session/debug/org_runtime.rs index 57de7cae94..4e856fcf0b 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/debug/org_runtime.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/debug/org_runtime.rs @@ -7,6 +7,7 @@ use std::sync::Arc; +use rusqlite::{params, OptionalExtension}; use serde::Serialize; use serde_json::Value; @@ -113,10 +114,8 @@ pub async fn debug_session_org_runtime_snapshot( .ok_or_else(|| format!("session not found: {session_id}"))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {session_id}"))?; let agent_id = session.definition.id.clone(); @@ -176,10 +175,8 @@ pub async fn debug_session_execute_tool( .ok_or_else(|| format!("session not found: {session_id}"))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {session_id}"))?; // Enforce the same per-turn policy composition the LLM path uses @@ -251,10 +248,8 @@ pub async fn debug_session_execute_org_tool( .ok_or_else(|| format!("session not found: {session_id}"))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {session_id}"))?; if runtime.agent_org_context.is_none() { @@ -304,6 +299,25 @@ pub async fn debug_agent_org_execute_tool_as_agent( })?; let sender_agent_id = sender.agent_id.clone(); let org_context = Arc::new(org_context); + let default_call_context = crate::tools::call_context::CallContext::default(); + let task_call_context = match tool_name.as_str() { + names::TASK_CREATE | names::TASK_GRAPH_CREATE + if sender_member_id == COORDINATOR_MEMBER_ID => + { + Some(detached_task_call_context( + &run_id, + &sender_member_id, + None, + )?) + } + names::TASK_UPDATE => Some(detached_task_call_context( + &run_id, + &sender_member_id, + params.get("id").and_then(Value::as_str), + )?), + _ => None, + }; + let task_call_context = task_call_context.as_ref().unwrap_or(&default_call_context); let result = match tool_name.as_str() { names::ORG_SEND_MESSAGE => OrgSendMessageTool::with_hooks( @@ -312,14 +326,14 @@ pub async fn debug_agent_org_execute_tool_as_agent( Arc::new(NoopInboxWakeHook), Arc::new(NoopSelfAbortHook), ) - .execute(params, &crate::tools::call_context::CallContext::default()) + .execute(params, &default_call_context) .await .map_err(|err| err.to_string()), names::TASK_CREATE => { let context = task_tools_context(org_context, sender_agent_id, sender_member_id.clone()); TaskCreateTool::new(context) - .execute(params, &crate::tools::call_context::CallContext::default()) + .execute(params, task_call_context) .await .map_err(|err| err.to_string()) } @@ -327,7 +341,7 @@ pub async fn debug_agent_org_execute_tool_as_agent( let context = task_tools_context(org_context, sender_agent_id, sender_member_id.clone()); TaskGraphCreateTool::new(context) - .execute(params, &crate::tools::call_context::CallContext::default()) + .execute(params, task_call_context) .await .map_err(|err| err.to_string()) } @@ -335,7 +349,7 @@ pub async fn debug_agent_org_execute_tool_as_agent( let context = task_tools_context(org_context, sender_agent_id, sender_member_id.clone()); TaskUpdateTool::new(context) - .execute(params, &crate::tools::call_context::CallContext::default()) + .execute(params, task_call_context) .await .map_err(|err| err.to_string()) } @@ -343,7 +357,7 @@ pub async fn debug_agent_org_execute_tool_as_agent( let context = task_tools_context(org_context, sender_agent_id, sender_member_id.clone()); TaskListTool::new(context) - .execute(params, &crate::tools::call_context::CallContext::default()) + .execute(params, &default_call_context) .await .map_err(|err| err.to_string()) } @@ -351,7 +365,7 @@ pub async fn debug_agent_org_execute_tool_as_agent( let context = task_tools_context(org_context, sender_agent_id, sender_member_id.clone()); TaskGetTool::new(context) - .execute(params, &crate::tools::call_context::CallContext::default()) + .execute(params, &default_call_context) .await .map_err(|err| err.to_string()) } @@ -359,7 +373,7 @@ pub async fn debug_agent_org_execute_tool_as_agent( let context = task_tools_context(org_context, sender_agent_id, sender_member_id.clone()); OrgRunCompleteTool::new(context) - .execute(params, &crate::tools::call_context::CallContext::default()) + .execute(params, &default_call_context) .await .map_err(|err| err.to_string()) } @@ -369,6 +383,59 @@ pub async fn debug_agent_org_execute_tool_as_agent( Ok(DebugOrgToolResult::from_tool_result(result)) } +/// Detached WebDriver tool calls still cross the same typed Turn authority as +/// provider calls. Resolve an already-admitted current-generation Turn instead +/// of manufacturing a bypass identity for the test fixture. +fn detached_task_call_context( + run_id: &str, + sender_member_id: &str, + task_id: Option<&str>, +) -> Result { + let turn_kind = if sender_member_id == COORDINATOR_MEMBER_ID { + "coordinator" + } else { + "task_execution" + }; + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + let identity: Option<(String, String)> = conn + .query_row( + "SELECT context.turn_intent_id,context.session_id + FROM agent_org_runtime_turn_contexts context + JOIN agent_org_runtime_runs run ON run.id=context.org_run_id + JOIN session_turn_intents intent + ON intent.session_id=context.session_id + AND intent.turn_intent_id=context.turn_intent_id + WHERE context.org_run_id=?1 + AND context.participant_id=?2 + AND context.turn_kind=?3 + AND context.activation_generation=run.activation_generation + AND run.status='running' + AND (?4 IS NULL OR context.task_id=?4) + ORDER BY CASE intent.status + WHEN 'running' THEN 0 + WHEN 'queued' THEN 1 + ELSE 2 + END, + context.context_id DESC + LIMIT 1", + params![run_id, sender_member_id, turn_kind, task_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let (turn_intent_id, session_id) = identity.ok_or_else(|| { + format!( + "debug Agent Org task tool has no current typed {turn_kind} context for member {sender_member_id}" + ) + })?; + Ok(crate::tools::call_context::CallContext::for_turn( + format!("debug-detached-org-tool:{sender_member_id}"), + session_id, + turn_intent_id, + Vec::new(), + )) +} + fn task_tools_context( org_context: Arc, caller_agent_id: String, diff --git a/src-tauri/crates/agent-core/src/state/commands/session/debug/prompt.rs b/src-tauri/crates/agent-core/src/state/commands/session/debug/prompt.rs index 0623008861..7eb77908e5 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/debug/prompt.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/debug/prompt.rs @@ -145,10 +145,8 @@ pub async fn prompt_dump( .ok_or_else(|| format!("session not found: {}", session_id))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {}", session_id))?; // The processor caches `agent_soul` on the runtime at session-start diff --git a/src-tauri/crates/agent-core/src/state/commands/session/debug/security.rs b/src-tauri/crates/agent-core/src/state/commands/session/debug/security.rs index 5fe5f639ee..4a3dcdef0f 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/debug/security.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/debug/security.rs @@ -70,10 +70,8 @@ pub async fn debug_session_security_snapshot( .ok_or_else(|| format!("session not found: {}", session_id))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {}", session_id))?; let workspace = runtime.resolved.workspace.clone(); @@ -126,10 +124,8 @@ pub async fn debug_session_validate_command( .ok_or_else(|| format!("session not found: {}", session_id))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {}", session_id))?; // Rebuild a fresh `SecurityPolicy` for this validation call rather diff --git a/src-tauri/crates/agent-core/src/state/commands/session/debug/skills.rs b/src-tauri/crates/agent-core/src/state/commands/session/debug/skills.rs index 3145bb9236..f8e7dab4db 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/debug/skills.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/debug/skills.rs @@ -93,10 +93,8 @@ pub async fn debug_session_skills_snapshot( .ok_or_else(|| format!("session not found: {}", session_id))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {}", session_id))?; let agent_id = session.definition.id.clone(); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/debug/subagent.rs b/src-tauri/crates/agent-core/src/state/commands/session/debug/subagent.rs index 051202d935..f28d6a4cc4 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/debug/subagent.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/debug/subagent.rs @@ -92,10 +92,8 @@ pub async fn debug_session_subagent_snapshot( .ok_or_else(|| format!("session not found: {}", session_id))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {}", session_id))?; let agent_id = session.definition.id.clone(); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/debug/tools.rs b/src-tauri/crates/agent-core/src/state/commands/session/debug/tools.rs index 7b3694470e..67ce62ca3a 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/debug/tools.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/debug/tools.rs @@ -112,10 +112,8 @@ pub async fn debug_session_tools_snapshot( }; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {}", session_id))?; let agent_id = session.definition.id.clone(); diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/mod.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/mod.rs index a811c72455..73271cc0f9 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/mod.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/mod.rs @@ -34,6 +34,7 @@ mod resolve_agent_mode_tests; pub use entry_points::*; pub use exec_mode::*; +pub(crate) use org_wake::resolve_agent_org_wake_mode; // `send_message_impl` is `pub(crate)`, so the re-export matches its visibility // rather than widening the module root's public surface. pub(crate) use send::*; diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs index 41ef2605b6..3a85d27b00 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/org_wake.rs @@ -136,7 +136,7 @@ pub(super) fn promote_agent_org_direct_session_to_running( /// `TaskAssigned` is only a doorbell: the persisted Task context identifies /// the one authoritative row, and unknown/corrupt mode values fail closed /// instead of falling back to Build. Coordinator wakes have no Task mode. -pub(super) fn resolve_agent_org_wake_mode( +pub(crate) fn resolve_agent_org_wake_mode( session_id: &str, run_id: &str, turn_intent_id: &str, diff --git a/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs b/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs index 1a4c82b72a..71fe7f0087 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/message/send.rs @@ -112,6 +112,13 @@ pub(super) fn should_divert_to_mid_turn_steering( && is_turn_processing } +fn resume_requires_existing_agent_org_context( + source: TurnIntentBridgeSource, + wake_member_id: Option<&str>, +) -> bool { + wake_member_id.is_none() && matches!(source, TurnIntentBridgeSource::Resume) +} + async fn persist_direct_user_intervention( params: Option, ) -> Result<(), String> { @@ -267,6 +274,15 @@ pub(crate) async fn send_message_impl( admission_client_message_id, &member_id, ), + None + if resume_requires_existing_agent_org_context(source, None) => + { + crate::coordination::agent_org_turn_contexts::require_existing_context( + &admission_run_id, + &admission_session_id, + &admission_turn_intent_id, + ) + } None => { let admission = crate::coordination::agent_org_turn_contexts::AgentOrgTurnAdmission::coordinator( admission_run_id, @@ -309,13 +325,9 @@ pub(crate) async fn send_message_impl( let session_for_closure = Arc::clone(&session_handle); let load_workspace_resources = runtime.resolved.load_workspace_resources; - if !is_resume && !content.trim().is_empty() { - let _ = org_tasks::resume_paused_run_for_user_message(state, &session_id).await?; - } - let direct_user_intervention = if mark_direct_user_intervention && !is_resume && !content.trim().is_empty() { - let runtime_snapshot = session_handle.runtime.read().await.clone(); + let runtime_snapshot = session_handle.get_runtime().await; match runtime_snapshot.and_then(|runtime| runtime.agent_org_context.clone()) { Some(org_context) => { let session_id_for_intervention = session_id.clone(); @@ -358,6 +370,7 @@ pub(crate) async fn send_message_impl( }; let app_handle = state.app_handle.clone(); + let app_state_for_closure = state.clone(); // ── 3b. Mid-turn steering divert ───────────────────────────────────── // @@ -582,6 +595,7 @@ pub(crate) async fn send_message_impl( let direct_user_intervention = direct_user_intervention_for_closure; let org_wake_run_id = org_wake_run_id; let intent_org_run_id = intent_org_run_id_for_closure; + let app_state = app_state_for_closure; Box::pin(async move { // Clear a stale pre-turn cancel signal before the durable @@ -656,7 +670,9 @@ pub(crate) async fn send_message_impl( Err(err) => return Err(format!("running-status task failed: {err}")), } - let turn_id = session.begin_turn(content.clone()).await; + let turn_id = session + .begin_turn_with_intent(content.clone(), Some(turn_intent_id.clone())) + .await; let input = crate::session::TurnInput { content: content.clone(), @@ -695,6 +711,13 @@ pub(crate) async fn send_message_impl( duration: None, }) .unwrap_or_default(); + org_tasks::settle_pause_handoff_after_turn( + &app_state, + &session, + intent_org_run_id.as_deref(), + &turn_intent_id, + ) + .await; session.end_turn(final_turn_state, stats).await; // The turn processor can return Ok with an empty response after a @@ -862,3 +885,24 @@ pub(crate) async fn send_message_impl( model: effective_model, }) } + +#[cfg(test)] +mod admission_tests { + use super::*; + + #[test] + fn ordinary_resume_wake_creates_context_but_pause_continuation_reuses_it() { + assert!(!resume_requires_existing_agent_org_context( + TurnIntentBridgeSource::Resume, + Some("worker") + )); + assert!(resume_requires_existing_agent_org_context( + TurnIntentBridgeSource::Resume, + None + )); + assert!(!resume_requires_existing_agent_org_context( + TurnIntentBridgeSource::Queue, + None + )); + } +} diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/context.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/context.rs index 3692fd2b0c..27a34e75a9 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/context.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/context.rs @@ -24,10 +24,8 @@ pub(super) async fn session_org_read_context( ) -> Result, String> { let runtime_context = match state.get_session(session_id).await { Some(session) => session - .runtime - .read() + .get_runtime() .await - .as_ref() .and_then(|runtime| runtime.agent_org_context.clone()), None => None, }; diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs index 9ba30233c2..025cb8d513 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/group_chat.rs @@ -19,10 +19,7 @@ use crate::coordination::agent_org_turn_contexts::TURN_CONTEXT_INVARIANT_PREFIX; use crate::state::AgentAppState; use super::context::session_org_read_context; -use super::lifecycle::{ - clear_active_org_cancel_flags, resume_agent_org_context, schedule_run_progress_wakes, - wake_agent_org_member, -}; +use super::lifecycle::wake_agent_org_member; use super::run_view::{agent_org_session_run_view_impl, enrich_inbox_row, AgentOrgInboxRuntimeRow}; #[derive(Debug, Clone, Serialize)] @@ -362,33 +359,9 @@ async fn agent_org_send_group_chat_message_impl_with_display( .await .map_err(|err| format!("Agent Org group message worker failed: {err}"))??; - // The inbox row is already committed. Everything below is an acceleration - // hint; reporting a post-commit wake/resume error as "message failed" - // encourages callers to retry and duplicate the user's durable message. - match resume_agent_org_context(&view.context, false).await { - Ok(outcome) if outcome.transitioned => { - if let Err(err) = clear_active_org_cancel_flags(state, &view.context).await { - tracing::warn!( - run_id = %view.context.run_id, - error = %err, - "group message committed, but clearing stale cancel flags failed" - ); - } - schedule_run_progress_wakes(app_handle.clone(), &view.context); - } - Ok(outcome) if outcome.run_is_running => { - wake_agent_org_member(app_handle, &target.member_id, &view.context.run_id); - } - Ok(_) => {} - Err(err) => { - tracing::warn!( - run_id = %view.context.run_id, - error = %err, - "group message committed, but automatic run resume failed" - ); - wake_agent_org_member(app_handle, &target.member_id, &view.context.run_id); - } - } + // The transaction below accepts only a Working Team. Once committed, a + // wake is an acceleration hint; it must never change lifecycle status. + wake_agent_org_member(app_handle, &target.member_id, &view.context.run_id); let inbox_row = enrich_inbox_row(&view.context, row); @@ -459,8 +432,9 @@ pub(super) fn persist_group_chat_message( ) })?; match run_status { - AgentOrgRunStatus::Running | AgentOrgRunStatus::Paused => {} + AgentOrgRunStatus::Running => {} AgentOrgRunStatus::Starting + | AgentOrgRunStatus::Paused | AgentOrgRunStatus::Idle | AgentOrgRunStatus::Failed | AgentOrgRunStatus::Archived => { diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs index 4dc7485e95..110c903b5e 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/lifecycle.rs @@ -1,220 +1,438 @@ -//! Agent Org run lifecycle: pause, resume, cancel, and progress wakes. -//! -//! This module owns the pause/resume commands and the shared machinery that -//! keeps a resumed run making progress: clearing stale per-session cancel -//! flags, seeding a coordinator resume turn, and re-waking members that hold -//! unread inbox rows. The group-chat send path reuses the resume/wake helpers, -//! so they are visible to sibling modules. +//! Durable Agent Org Pause/Resume commands and post-commit runtime handoff. -use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::time::Duration; -use database::db::{get_connection, with_sessions_writer}; -use rusqlite::{params, OptionalExtension}; - -use crate::coordination::agent_inbox::{ - AgentInboxStore, AgentMessage, InsertInboxParams, SYSTEM_SENDER_ID, -}; -use crate::coordination::agent_org_runs::{ - AgentOrgRunContext, AgentOrgRunStore, COORDINATOR_MEMBER_ID, +use crate::coordination::agent_inbox::AgentInboxStore; +use crate::coordination::agent_org_pause::{ + ContinuationDispatch, PauseRunOutcome, ResumeRunOutcome, }; +use crate::coordination::agent_org_runs::{AgentOrgRunContext, COORDINATOR_MEMBER_ID}; +use crate::foundation::session_bridge::TurnIntentBridgeSource; +use crate::state::commands::session::identity::IdentityOverrides; use crate::state::control_flow::CancelReason; -use crate::state::AgentAppState; +use crate::state::{AgentAppState, AgentSession}; use super::context::session_org_read_context; -/// Pause the Agent Org run that the given session belongs to. Transitions -/// `running → paused`; already non-running runs return `Ok(false)` (idempotent). -/// The run remains available to explicit reads while paused, but PR1's -/// fallback poller deliberately observes only Starting and Running Teams. +const DRAIN_DEADLINE: Duration = Duration::from_secs(10); +const DRAIN_OBSERVATION_INTERVAL: Duration = Duration::from_millis(100); +const CONTINUATION_DISPATCH_LIMIT: usize = 256; + #[tauri::command] pub async fn agent_org_pause_run( state: tauri::State<'_, AgentAppState>, session_id: String, -) -> Result { + request_id: String, +) -> Result { crate::coordination::agent_org_runs::require_agent_org_redesign()?; - let Some(read_context) = session_org_read_context(&state, &session_id).await? else { - return Ok(false); - }; - let Some(ref context) = read_context.context else { - return Ok(false); - }; + let read_context = session_org_read_context(&state, &session_id) + .await? + .ok_or_else(|| format!("Session {session_id} is not part of an Agent Org run"))?; + let context = read_context + .context + .ok_or_else(|| format!("Session {session_id} has no Agent Org context"))?; let run_id = context.run_id.clone(); - let transitioned = tokio::task::spawn_blocking(move || AgentOrgRunStore::mark_paused(&run_id)) - .await - .map_err(|err| format!("Agent Org pause worker failed: {err}"))??; - cancel_active_org_turns(&state, context).await?; - Ok(transitioned) + let pause_run_id = run_id.clone(); + let commit = tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::pause_run_commit(&pause_run_id, &request_id) + }) + .await + .map_err(|error| format!("Agent Org Pause transaction worker failed: {error}"))??; + let outcome = commit.outcome; + + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(&run_id); + if let Some(teardown_owner_id) = commit.teardown_owner_id { + let teardown_state = state.inner().clone(); + let teardown_episode_id = outcome.episode_id.clone(); + let teardown_run_id = run_id; + tauri::async_runtime::spawn(async move { + if let Err(error) = teardown_pause_episode( + teardown_state, + teardown_run_id.clone(), + teardown_episode_id, + teardown_owner_id, + ) + .await + { + tracing::warn!( + run_id = %teardown_run_id, + error = %error, + "Agent Org Pause fence committed, but runtime drain owner failed" + ); + } + }); + } + Ok(outcome) } -/// Resume a paused Agent Org run. Transitions `paused → running`; already -/// non-paused runs return `Ok(false)` (idempotent). -/// -/// After marking the run as resumed and clearing pause cancel flags, re-wakes -/// members that have unread inbox rows. The coordinator also receives one -/// durable resume event. Owned or ownerless task state by -/// itself is not new input and must never cause an empty model turn. Without -/// this step the run's DB status becomes `running` but -/// no sessions start processing because `InboxWakeHook` only fires when new -/// rows are written, not when a run is un-paused. #[tauri::command] pub async fn agent_org_resume_run( - app_handle: tauri::AppHandle, state: tauri::State<'_, AgentAppState>, session_id: String, -) -> Result { + request_id: String, +) -> Result { crate::coordination::agent_org_runs::require_agent_org_redesign()?; - let Some(read_context) = session_org_read_context(&state, &session_id).await? else { - return Ok(false); - }; - let Some(ref context) = read_context.context else { - return Ok(false); - }; - let outcome = resume_agent_org_context(context, true).await?; - if outcome.run_is_running { - if let Err(err) = clear_active_org_cancel_flags(&state, context).await { - tracing::warn!( - run_id = %context.run_id, - error = %err, - "Agent Org resume committed, but clearing stale cancel flags failed" - ); - } - // Explicit Resume is also an idempotent repair signal. Even if a - // previous call already transitioned the Run, rescan durable unread - // inbox rows so a post-commit process crash cannot leave it Running - // with no scheduled consumer. - schedule_run_progress_wakes(app_handle, context); + let read_context = session_org_read_context(&state, &session_id) + .await? + .ok_or_else(|| format!("Session {session_id} is not part of an Agent Org run"))?; + let context = read_context + .context + .ok_or_else(|| format!("Session {session_id} has no Agent Org context"))?; + let run_id = context.run_id.clone(); + let resume_run_id = run_id.clone(); + let outcome = tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::resume_run(&resume_run_id, &request_id) + }) + .await + .map_err(|error| format!("Agent Org Resume transaction worker failed: {error}"))??; + + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(&run_id); + schedule_ready_continuations(state.inner().clone()); + if let Some(app_handle) = state.app_handle.clone() { + schedule_non_continuation_progress_wakes(app_handle, context, outcome.episode_id.clone()); } - Ok(outcome.transitioned) + Ok(outcome) } -pub(super) async fn clear_active_org_cancel_flags( - state: &AgentAppState, - context: &AgentOrgRunContext, +async fn teardown_pause_episode( + state: AgentAppState, + run_id: String, + episode_id: String, + teardown_owner_id: String, ) -> Result<(), String> { - let session_ids = org_session_ids(context).await?; - for session_id in session_ids { - if let Some(session) = state.get_session(&session_id).await { - session.cancel_flag.store(false, Ordering::SeqCst); + let deadline_at = tokio::time::Instant::now() + DRAIN_DEADLINE; + let read_episode_id = episode_id.clone(); + let handoffs = tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::list_running_handoffs( + &read_episode_id, + &teardown_owner_id, + ) + }) + .await + .map_err(|error| format!("Pause handoff reader failed: {error}"))??; + + let mut signals = tokio::task::JoinSet::new(); + for handoff in handoffs { + let child_state = state.clone(); + signals.spawn(async move { request_exact_handoff_yield(&child_state, handoff).await }); + } + let mut signal_deadline_elapsed = false; + loop { + let result = match tokio::time::timeout_at(deadline_at, signals.join_next()).await { + Ok(Some(result)) => result, + Ok(None) => break, + Err(_) => { + signal_deadline_elapsed = true; + signals.abort_all(); + break; + } + }; + match result { + Ok(Ok(())) => {} + Ok(Err(error)) => tracing::warn!( + run_id = %run_id, + error = %error, + "failed to signal one captured Agent Org runtime" + ), + Err(error) => tracing::warn!( + run_id = %run_id, + error = %error, + "captured Agent Org runtime signal task failed" + ), } } - Ok(()) -} -async fn org_session_ids(context: &AgentOrgRunContext) -> Result, String> { - let context = context.clone(); - tokio::task::spawn_blocking(move || { - let mut session_ids = Vec::new(); - if let Some(root_session_id) = context.root_session_id { - session_ids.push(root_session_id); + let observed_episode_id = episode_id.clone(); + let observer_run_id = run_id.clone(); + let drained = tokio::time::timeout_at(deadline_at, async move { + loop { + let summary_episode_id = observed_episode_id.clone(); + let summary_run_id = observer_run_id.clone(); + let summary = tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::pause_summary_for_run(&summary_run_id) + }) + .await + .map_err(|error| format!("Pause drain observer failed: {error}"))??; + match summary { + Some(summary) + if summary.episode_id == summary_episode_id && summary.draining_count == 0 => + { + return Ok::<(), String>(()); + } + Some(summary) if summary.episode_id != summary_episode_id => return Ok(()), + None => return Ok(()), + _ => tokio::time::sleep(DRAIN_OBSERVATION_INTERVAL).await, + } } - session_ids.extend( - AgentOrgRunStore::list_descendant_worker_sessions(&context.run_id)? - .into_iter() - .map(|session| session.session_id), - ); - Ok(session_ids) }) - .await - .map_err(|err| format!("Agent Org session-list worker failed: {err}"))? + .await; + + if signal_deadline_elapsed || drained.is_err() { + let timeout_episode_id = episode_id.clone(); + tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::mark_unresolved_timed_out(&timeout_episode_id) + }) + .await + .map_err(|error| format!("Pause timeout writer failed: {error}"))??; + } else if let Ok(Err(error)) = drained { + return Err(error); + } + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(&run_id); + Ok(()) } -async fn cancel_active_org_turns( +async fn request_exact_handoff_yield( state: &AgentAppState, - context: &AgentOrgRunContext, + handoff: crate::coordination::agent_org_pause::RunningPauseHandoff, ) -> Result<(), String> { - let session_ids = org_session_ids(context).await?; - - for session_id in session_ids { - state - .cancel_session(&session_id, CancelReason::OrgPause) - .await; + let Some(session) = state.get_session(&handoff.session_id).await else { + return persist_runtime_absent(handoff).await; + }; + let Some(identity) = session.runtime_turn_identity().await else { + // The durable Turn may have been promoted to Running just before + // Pause, while `begin_turn_with_intent` has not installed its + // in-memory identity yet. Preserve a pre-turn cancellation marker so + // that narrow provider-start window still yields. This is safe only + // for the identity-absent case; a mismatching active identity may be + // future UserDirectedWork and is deliberately not cancelled here. + session.cancel_active_turn(CancelReason::OrgPause).await; + return persist_runtime_absent(handoff).await; + }; + if identity.turn_intent_id.as_deref() != Some(handoff.turn_intent_id.as_str()) { + return persist_runtime_absent(handoff).await; } + let bind = handoff.clone(); + let runtime_lease_id = identity.runtime_lease_id.clone(); + let dialog_turn_generation = identity.dialog_turn_generation.clone(); + let bound = tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::bind_runtime_and_request_yield( + &bind.episode_id, + &bind.session_id, + &bind.turn_intent_id, + &runtime_lease_id, + &dialog_turn_generation, + ) + }) + .await + .map_err(|error| format!("Pause runtime binding worker failed: {error}"))??; + if bound { + session.cancel_active_turn(CancelReason::OrgPause).await; + } Ok(()) } -pub(crate) async fn resume_paused_run_for_user_message( +async fn persist_runtime_absent( + handoff: crate::coordination::agent_org_pause::RunningPauseHandoff, +) -> Result<(), String> { + tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::mark_runtime_absent( + &handoff.episode_id, + &handoff.session_id, + &handoff.turn_intent_id, + ) + .map(|_| ()) + }) + .await + .map_err(|error| format!("Pause runtime-absent worker failed: {error}"))? +} + +/// Completion callback for every real Turn. Ordinary SDE turns do one indexed +/// read and return; only a runtime identity already bound to a Pause receipt +/// can clear the runtime slot. +pub(crate) async fn settle_pause_handoff_after_turn( state: &AgentAppState, - session_id: &str, -) -> Result { - let Some(app_handle) = state.app_handle.clone() else { - return Ok(false); + session: &Arc, + run_id: Option<&str>, + turn_intent_id: &str, +) { + let Some(identity) = session.runtime_turn_identity().await else { + return; }; - let Some(read_context) = session_org_read_context(state, session_id).await? else { - return Ok(false); + if identity.turn_intent_id.as_deref() != Some(turn_intent_id) { + return; + } + let lookup_session_id = session.id.clone(); + let lookup_intent_id = turn_intent_id.to_string(); + let lookup_lease = identity.runtime_lease_id.clone(); + let lookup_generation = identity.dialog_turn_generation.clone(); + let episode = match tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::bound_episode_for_runtime( + &lookup_session_id, + &lookup_intent_id, + &lookup_lease, + &lookup_generation, + ) + }) + .await + { + Ok(Ok(episode)) => episode, + Ok(Err(error)) => { + tracing::warn!(session_id = %session.id, error = %error, "failed to read Pause handoff receipt"); + return; + } + Err(error) => { + tracing::warn!(session_id = %session.id, error = %error, "Pause handoff receipt worker failed"); + return; + } }; - let Some(ref context) = read_context.context else { - return Ok(false); + if episode.is_none() { + return; + } + let process_owner = crate::tools::call_context::TurnProcessOwner { + session_id: session.id.clone(), + turn_intent_id: turn_intent_id.to_string(), + runtime_lease_id: identity.runtime_lease_id.clone(), + dialog_turn_generation: identity.dialog_turn_generation.clone(), }; - let outcome = resume_agent_org_context(context, false).await?; - if outcome.run_is_running { - if let Err(err) = clear_active_org_cancel_flags(state, context).await { - tracing::warn!( - run_id = %context.run_id, - error = %err, - "user-message resume committed, but clearing stale cancel flags failed" - ); + if let Err(error) = + crate::tools::impls::coding::exec::registry::cancel_and_await_jobs_for_owner( + &process_owner, + DRAIN_DEADLINE, + ) + .await + { + tracing::warn!( + session_id = %session.id, + runtime_lease_id = %identity.runtime_lease_id, + error = %error, + "Pause handoff remains draining because exact-owner background work is not terminal" + ); + return; + } + let released_current_slot = session + .release_runtime_if_current(&identity.runtime_lease_id, &identity.dialog_turn_generation) + .await; + if !released_current_slot { + tracing::debug!( + session_id = %session.id, + runtime_lease_id = %identity.runtime_lease_id, + "Pause completion observed a replaced runtime lease; preserving the current slot" + ); + return; + } + let release_session_id = session.id.clone(); + let release_intent_id = turn_intent_id.to_string(); + let release_lease = identity.runtime_lease_id; + let release_generation = identity.dialog_turn_generation; + match tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::mark_released( + &release_session_id, + &release_intent_id, + &release_lease, + &release_generation, + ) + }) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + tracing::warn!(session_id = %session.id, error = %error, "failed to persist Pause release receipt"); + return; } - schedule_run_progress_wakes(app_handle, context); + Err(error) => { + tracing::warn!(session_id = %session.id, error = %error, "Pause release receipt worker failed"); + return; + } + } + if let Some(run_id) = run_id { + crate::coordination::agent_org_run_events::notify_agent_org_run_changed(run_id); } - Ok(outcome.transitioned) + schedule_ready_continuations(state.clone()); } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) struct AgentOrgResumeOutcome { - pub(super) transitioned: bool, - pub(super) run_is_running: bool, +pub(crate) fn schedule_ready_continuations(state: AgentAppState) { + tauri::async_runtime::spawn(async move { + if let Err(error) = dispatch_ready_continuations(&state).await { + tracing::warn!(error = %error, "Agent Org continuation dispatcher failed"); + } + }); } -pub(super) async fn resume_agent_org_context( - context: &AgentOrgRunContext, - seed_coordinator_resume_turn: bool, -) -> Result { - let context = context.clone(); - tokio::task::spawn_blocking(move || { - resume_agent_org_context_sync(&context, seed_coordinator_resume_turn) +pub(crate) async fn dispatch_ready_continuations(state: &AgentAppState) -> Result { + let dispatches = tokio::task::spawn_blocking(|| { + crate::coordination::agent_org_pause::list_dispatchable_continuations( + CONTINUATION_DISPATCH_LIMIT, + ) }) .await - .map_err(|err| format!("Agent Org resume worker failed: {err}"))? -} - -pub(super) fn resume_agent_org_context_sync( - context: &AgentOrgRunContext, - seed_coordinator_resume_turn: bool, -) -> Result { - with_sessions_writer(|| -> Result { - let mut conn = get_connection().map_err(|err| err.to_string())?; - let tx = conn - .transaction_with_behavior(rusqlite::TransactionBehavior::Immediate) - .map_err(|err| err.to_string())?; - let status: Option = tx - .query_row( - "SELECT status FROM agent_org_runtime_runs WHERE id=?1", - params![&context.run_id], - |row| row.get(0), + .map_err(|error| format!("continuation reader failed: {error}"))??; + let mut dispatched = 0usize; + for dispatch in dispatches { + let episode_id = dispatch.episode_id.clone(); + let turn_intent_id = dispatch.turn_intent_id.clone(); + let claimed = tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::claim_continuation_dispatch( + &episode_id, + &turn_intent_id, ) - .optional() - .map_err(|err| err.to_string())?; - let transitioned = status.as_deref() == Some("paused"); - let run_is_running = transitioned || status.as_deref() == Some("running"); - if transitioned { - tx.execute( - "UPDATE agent_org_runtime_runs - SET status='running', updated_at=?2 - WHERE id=?1 AND status='paused'", - params![&context.run_id, chrono::Utc::now().to_rfc3339()], - ) - .map_err(|err| err.to_string())?; + }) + .await + .map_err(|error| format!("continuation receipt worker failed: {error}"))??; + if !claimed { + continue; } - if run_is_running && seed_coordinator_resume_turn { - seed_coordinator_resume_inbox_in_tx(&tx, context)?; + if let Err(error) = dispatch_one_continuation(state, &dispatch).await { + let episode_id = dispatch.episode_id.clone(); + let turn_intent_id = dispatch.turn_intent_id.clone(); + if let Err(requeue_error) = tokio::task::spawn_blocking(move || { + crate::coordination::agent_org_pause::requeue_continuation_dispatch( + &episode_id, + &turn_intent_id, + ) + }) + .await + .map_err(|join_error| join_error.to_string()) + .and_then(|result| result.map_err(|persist_error| persist_error.to_string())) + { + tracing::warn!(error = %requeue_error, "failed to requeue continuation after dispatch error"); + } + return Err(error); } - tx.commit().map_err(|err| err.to_string())?; - Ok(AgentOrgResumeOutcome { - transitioned, - run_is_running, - }) - }) + dispatched += 1; + } + Ok(dispatched) +} + +async fn dispatch_one_continuation( + state: &AgentAppState, + dispatch: &ContinuationDispatch, +) -> Result<(), String> { + let mode = if dispatch.turn_kind == "task_execution" { + super::super::message::resolve_agent_org_wake_mode( + &dispatch.session_id, + &dispatch.run_id, + &dispatch.turn_intent_id, + )? + .map(|value| value.as_str().to_string()) + } else { + None + }; + // Resume continues the captured formal Turn; it is not a new user + // submission. Empty content prevents a synthetic transcript row. The + // processor derives a transient provider nudge from the durable receipt. + let content = String::new(); + super::super::message::send_message_impl( + state, + dispatch.session_id.clone(), + content, + None, + IdentityOverrides::default(), + mode, + None, + None, + true, + false, + Some(dispatch.turn_intent_id.clone()), + Some(dispatch.turn_intent_id.clone()), + None, + None, + Some(dispatch.run_id.clone()), + TurnIntentBridgeSource::Resume, + ) + .await?; + Ok(()) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -229,10 +447,7 @@ pub(super) struct AgentOrgWakeTarget { } pub(super) fn should_wake_member_for_progress(has_unread: bool) -> Option { - if has_unread { - return Some(AgentOrgWakeReason::UnreadInbox); - } - None + has_unread.then_some(AgentOrgWakeReason::UnreadInbox) } pub(super) fn collect_run_progress_wake_targets( @@ -241,8 +456,9 @@ pub(super) fn collect_run_progress_wake_targets( ) -> Result, String> { let mut targets = Vec::new(); for member_id in member_ids { - let has_unread = AgentInboxStore::has_unread_for_member(member_id, run_id)?; - if let Some(reason) = should_wake_member_for_progress(has_unread) { + if let Some(reason) = should_wake_member_for_progress( + AgentInboxStore::has_unread_for_member(member_id, run_id)?, + ) { targets.push(AgentOrgWakeTarget { member_id: member_id.clone(), reason, @@ -269,88 +485,39 @@ pub(super) fn wake_agent_org_member(app_handle: tauri::AppHandle, member_id: &st AppHandleInboxWakeHook::new(app_handle).wake_member(member_id, run_id); } -pub(super) fn schedule_run_progress_wakes( +fn schedule_non_continuation_progress_wakes( app_handle: tauri::AppHandle, - context: &AgentOrgRunContext, + context: AgentOrgRunContext, + episode_id: String, ) { - let run_id = context.run_id.clone(); - let member_ids = org_progress_member_ids(context); - - tokio::spawn(async move { - let target_run_id = run_id.clone(); - let targets = match tokio::task::spawn_blocking(move || { - collect_run_progress_wake_targets(&target_run_id, &member_ids) + tauri::async_runtime::spawn(async move { + let run_id = context.run_id.clone(); + let member_ids = org_progress_member_ids(&context); + let query_run_id = run_id.clone(); + let result = tokio::task::spawn_blocking(move || { + let continued = + crate::coordination::agent_org_pause::continuation_participant_ids(&episode_id)? + .into_iter() + .collect::>(); + let candidates = member_ids + .into_iter() + .filter(|member_id| !continued.contains(member_id)) + .collect::>(); + collect_run_progress_wake_targets(&query_run_id, &candidates) }) - .await - { - Ok(Ok(targets)) => targets, - Ok(Err(err)) => { - tracing::warn!( - run_id = %run_id, - error = %err, - "[agent_org_progress] failed to collect wake targets after run progress transition" - ); - return; + .await; + match result { + Ok(Ok(targets)) => { + for target in targets { + wake_agent_org_member(app_handle.clone(), &target.member_id, &run_id); + } } - Err(err) => { - tracing::warn!( - run_id = %run_id, - error = %err, - "[agent_org_progress] wake-target worker failed" - ); - return; + Ok(Err(error)) => { + tracing::warn!(run_id = %run_id, error = %error, "failed to collect Resume wake targets") + } + Err(error) => { + tracing::warn!(run_id = %run_id, error = %error, "Resume wake-target worker failed") } - }; - for target in targets { - tracing::info!( - run_id = %run_id, - member_id = %target.member_id, - reason = ?target.reason, - "[agent_org_progress] waking member for runnable Agent Org work" - ); - wake_agent_org_member(app_handle.clone(), &target.member_id, &run_id); } }); } - -fn seed_coordinator_resume_inbox_in_tx( - tx: &rusqlite::Transaction<'_>, - context: &AgentOrgRunContext, -) -> Result<(), String> { - let coordinator_member_id = COORDINATOR_MEMBER_ID; - let has_unread: bool = tx - .query_row( - "SELECT EXISTS( - SELECT 1 FROM agent_org_runtime_inbox - WHERE recipient_member_id=?1 - AND org_run_id=?2 - AND read_at IS NULL - AND NOT EXISTS ( - SELECT 1 FROM agent_org_runtime_inbox_delivery_resolutions resolution - WHERE resolution.inbox_id=agent_org_runtime_inbox.id - ) - )", - params![coordinator_member_id, &context.run_id], - |row| row.get(0), - ) - .map_err(|err| err.to_string())?; - if has_unread { - return Ok(()); - } - - AgentInboxStore::insert_in_tx( - tx, - InsertInboxParams { - recipient_agent_id: context.coordinator_agent_id.clone(), - recipient_member_id: Some(coordinator_member_id.to_string()), - sender_agent_id: SYSTEM_SENDER_ID.to_string(), - sender_member_id: None, - org_run_id: Some(context.run_id.clone()), - message: AgentMessage::Plain { - summary: "Agent Org run resumed".to_string(), - text: "The Agent Org run was resumed by the user. Continue coordinating the current work from the persisted task and member state. If all assigned work is already complete, summarize the current status instead of waiting idly.".to_string(), - }, - }, - )?; - Ok(()) -} diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs index 33941eb2e1..96f15ae402 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/run_view.rs @@ -118,6 +118,8 @@ pub struct AgentOrgRunView { pub context: AgentOrgRunContext, pub run_status: String, pub run_phase: AgentOrgRunPhase, + #[serde(skip_serializing_if = "Option::is_none")] + pub pause_handoff: Option, pub current_member_id: Option, pub members: Vec, pub tasks: Vec, @@ -153,6 +155,7 @@ pub enum AgentOrgRunPhase { Waiting, AwaitingPlanApproval, Finalizing, + Draining, Paused, Idle, Failed, @@ -322,13 +325,28 @@ pub(super) fn build_agent_org_run_view( &context.run_id, )?; - let run_phase = project_run_phase( - run_status_value, - &members, - &task_overview, - quiescence.facts.unread_inbox_count, - &pending_plan_approvals, - ); + // Running Teams are polled while active; keep pause-receipt aggregation + // entirely off that hot path. Paused Teams receive push updates only. + let pause_handoff = if run_status_value == AgentOrgRunStatus::Paused { + crate::coordination::agent_org_pause::pause_summary_with_connection(&tx, &context.run_id)? + } else { + None + }; + let run_phase = if run_status_value == AgentOrgRunStatus::Paused + && pause_handoff + .as_ref() + .is_some_and(|summary| summary.draining_count > 0) + { + AgentOrgRunPhase::Draining + } else { + project_run_phase( + run_status_value, + &members, + &task_overview, + quiescence.facts.unread_inbox_count, + &pending_plan_approvals, + ) + }; tx.commit().map_err(|err| err.to_string())?; @@ -337,6 +355,7 @@ pub(super) fn build_agent_org_run_view( context: context.clone(), run_status, run_phase, + pause_handoff, members, tasks, task_overview, diff --git a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs index f5df201700..12db147934 100644 --- a/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs +++ b/src-tauri/crates/agent-core/src/state/commands/session/org_tasks/tests.rs @@ -19,9 +19,12 @@ use crate::coordination::agent_member_interventions::{ }; use crate::coordination::agent_org_plan_approvals::AgentOrgPlanApprovalSummary; use crate::coordination::agent_org_runs::{ - AgentOrgContextMember, AgentOrgRunContext, AgentOrgRunStatus, COORDINATOR_MEMBER_ID, + AgentOrgContextMember, AgentOrgRunContext, AgentOrgRunStatus, AgentOrgRunStore, + COORDINATOR_MEMBER_ID, }; use crate::coordination::agent_org_tasks::{Task, TaskExecutionMode, TaskStatus, TaskSummary}; +use crate::definitions::orgs::{AgentOrgLaunchSnapshot, FlatOrgMember, PlanApprovalPolicy}; +use crate::foundation::session_bridge::{TurnIntentBridgeSource, TurnIntentBridgeStatus}; fn context_with_shared_member_agent_id() -> AgentOrgRunContext { AgentOrgRunContext { @@ -55,10 +58,22 @@ fn context_with_shared_member_agent_id() -> AgentOrgRunContext { fn prepare_command_run(status: &str) -> AgentOrgRunContext { let context = context_with_shared_member_agent_id(); let conn = get_connection().expect("db connection"); - crate::coordination::agent_org_runs::init_schema(&conn).expect("run schema"); - crate::coordination::agent_inbox::init_schema(&conn).expect("inbox schema"); - crate::coordination::agent_member_interventions::init_schema(&conn) - .expect("intervention schema"); + crate::foundation::persistence::test_schema::ensure_agent_sessions_schema(&conn); + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS session_turn_intents ( + session_id TEXT NOT NULL, + turn_intent_id TEXT NOT NULL, + client_message_id TEXT, + org_run_id TEXT, + source TEXT NOT NULL, + status TEXT NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (session_id, turn_intent_id) + );", + ) + .expect("base Turn lifecycle schema"); + crate::coordination::init_agent_org_schemas(&conn).expect("complete Agent Org schemas"); let now = chrono::Utc::now().to_rfc3339(); conn.execute( "INSERT INTO agent_org_runtime_runs ( @@ -94,6 +109,222 @@ fn inbox_count_for_member(context: &AgentOrgRunContext, member_id: &str) -> usiz usize::try_from(count).expect("non-negative inbox count") } +struct PauseTurnSeed<'a> { + session_id: &'a str, + turn_intent_id: &'a str, + turn_kind: &'a str, + intent_status: &'a str, + task_id: Option<&'a str>, + activation_generation: Option, + member_sequence: Option, +} + +fn seed_pause_turn_context( + conn: &rusqlite::Connection, + context: &AgentOrgRunContext, + seed: PauseTurnSeed<'_>, +) { + let PauseTurnSeed { + session_id, + turn_intent_id, + turn_kind, + intent_status, + task_id, + activation_generation, + member_sequence, + } = seed; + let now = chrono::Utc::now().to_rfc3339(); + conn.execute( + "INSERT INTO session_turn_intents ( + session_id,turn_intent_id,client_message_id,org_run_id,source,status, + created_at,updated_at + ) VALUES (?1,?2,?2,?3,'agent_org',?4,?5,?5)", + params![ + session_id, + turn_intent_id, + &context.run_id, + intent_status, + &now + ], + ) + .expect("insert Pause base Turn"); + let ( + participant_id, + task_id, + owner, + dispatch_member, + source_kind, + source_id, + root_turn, + actor_version, + ) = match turn_kind { + "coordinator" => ( + COORDINATOR_MEMBER_ID, + None, + None, + None, + "root_turn", + turn_intent_id, + None, + None, + ), + "task_execution" => { + let task_id = task_id.unwrap_or("pause-task"); + ( + "member-planner", + Some(task_id), + Some("member-planner"), + Some("member-planner"), + "task", + task_id, + None, + None, + ) + } + "user_directed_work" => ( + "member-planner", + None, + None, + Some("member-planner"), + "direct_member", + turn_intent_id, + Some(turn_intent_id), + Some(1_i64), + ), + other => panic!("unknown test Turn kind {other}"), + }; + conn.execute( + "INSERT INTO agent_org_runtime_turn_contexts ( + session_id,turn_intent_id,org_run_id,participant_id,turn_kind,task_id, + owner_member_id,dispatch_member_id,member_dispatch_sequence,source_kind, + source_id,root_authority_turn_id,actor_version,activation_generation,created_at + ) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15)", + params![ + session_id, + turn_intent_id, + &context.run_id, + participant_id, + turn_kind, + task_id, + owner, + dispatch_member, + member_sequence, + source_kind, + source_id, + root_turn, + actor_version, + activation_generation, + &now, + ], + ) + .expect("insert Pause companion context"); +} + +fn test_upsert_pause_turn_intent( + conn: &rusqlite::Connection, + session_id: &str, + turn_intent_id: &str, + client_message_id: Option<&str>, + org_run_id: Option<&str>, + source: TurnIntentBridgeSource, + status: TurnIntentBridgeStatus, +) -> Result<(), String> { + let now = chrono::Utc::now().to_rfc3339(); + conn.execute( + "INSERT OR IGNORE INTO session_turn_intents ( + session_id,turn_intent_id,client_message_id,org_run_id,source,status, + created_at,updated_at + ) VALUES (?1,?2,?3,?4,?5,?6,?7,?7)", + params![ + session_id, + turn_intent_id, + client_message_id, + org_run_id, + source.as_str(), + status.as_str(), + &now, + ], + ) + .map(|_| ()) + .map_err(|error| error.to_string()) +} + +fn configure_pause_resume_authority(conn: &rusqlite::Connection, context: &AgentOrgRunContext) { + crate::foundation::session_bridge::register_upsert_turn_intent_with_connection( + test_upsert_pause_turn_intent, + ); + let snapshot = AgentOrgLaunchSnapshot { + schema_version: 1, + org_id: context.org_id.clone(), + org_name: context.org_name.clone(), + coordinator_role: context.coordinator_role.clone(), + coordinator_agent_id: context.coordinator_agent_id.clone(), + plan_approval_policy: PlanApprovalPolicy::Coordinator, + members: context + .members + .iter() + .map(|member| FlatOrgMember { + member_id: member.member_id.clone(), + name: member.name.clone(), + role: member.role.clone(), + agent_id: member.agent_id.clone(), + runtime_config: None, + }) + .collect(), + additional_task_graph_writer_member_ids: Vec::new(), + member_communication_links: Vec::new(), + }; + conn.execute( + "UPDATE agent_org_runtime_runs SET org_snapshot_json=?2 WHERE id=?1", + params![ + &context.run_id, + serde_json::to_string(&snapshot).expect("serialize test snapshot") + ], + ) + .expect("install immutable launch snapshot"); + let now = chrono::Utc::now().to_rfc3339(); + for (session_id, agent_id, member_id) in [ + ( + context.root_session_id.as_deref().expect("root session"), + context.coordinator_agent_id.as_str(), + COORDINATOR_MEMBER_ID, + ), + ("planner-session", "builtin:sde", "member-planner"), + ] { + conn.execute( + "INSERT INTO agent_sessions ( + session_id,name,status,created_at,updated_at,session_type, + agent_definition_id,org_member_id + ) VALUES (?1,?1,'idle',?4,?4,'agent',?2,?3)", + params![session_id, agent_id, member_id, &now], + ) + .expect("seed canonical materialized Agent session"); + } + conn.execute( + "INSERT INTO agent_org_runtime_member_materializations ( + org_run_id,member_id,agent_id,generation,session_id,authority_class, + status,created_at,updated_at + ) VALUES (?1,?2,?3,1,?4,'formal','succeeded',?5,?5)", + params![ + &context.run_id, + COORDINATOR_MEMBER_ID, + &context.coordinator_agent_id, + context.root_session_id.as_deref().expect("root session"), + &now, + ], + ) + .expect("materialize coordinator"); + conn.execute( + "INSERT INTO agent_org_runtime_member_materializations ( + org_run_id,member_id,agent_id,generation,session_id,authority_class, + status,created_at,updated_at + ) VALUES (?1,'member-planner','builtin:sde',1,'planner-session','formal', + 'succeeded',?2,?2)", + params![&context.run_id, &now], + ) + .expect("materialize planner"); +} + #[test] fn pr3_group_chat_rejects_legacy_member_before_inbox_write() { let _sandbox = test_helpers::test_env::sandbox(); @@ -479,6 +710,26 @@ fn archived_group_message_writes_neither_inbox_nor_intervention_clear() { ); } +#[test] +fn paused_group_message_is_rejected_without_inbox_write_or_auto_resume() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("paused"); + let error = persist_group_chat_message( + &context, + &context.coordinator_agent_id, + COORDINATOR_MEMBER_ID, + "This must wait for explicit Resume", + None, + ) + .expect_err("Paused run rejects Group Chat submission"); + assert!(error.contains("this status does not accept")); + assert_eq!(inbox_count_for_member(&context, COORDINATOR_MEMBER_ID), 0); + assert_eq!( + AgentOrgRunStore::get_run_status(&context.run_id).expect("run status"), + Some(AgentOrgRunStatus::Paused) + ); +} + #[test] fn group_message_and_intervention_clear_commit_atomically() { let _sandbox = test_helpers::test_env::sandbox(); @@ -596,85 +847,1373 @@ fn group_chat_history_pages_all_rows_and_preserves_long_display_text_after_reloa } #[test] -fn paused_resume_and_coordinator_seed_commit_or_rollback_together() { +fn pause_episode_and_resume_request_are_durable_and_idempotent() { let _sandbox = test_helpers::test_env::sandbox(); - let context = prepare_command_run("paused"); + let context = prepare_command_run("running"); + let pause_request = "00000000-0000-4000-8000-000000000001"; + let resume_request = "00000000-0000-4000-8000-000000000002"; + + let first_commit = + crate::coordination::agent_org_pause::pause_run_commit(&context.run_id, pause_request) + .expect("pause transaction"); + let first = first_commit.outcome; + assert!(first_commit.teardown_owner_id.is_some()); + let duplicate_commit = + crate::coordination::agent_org_pause::pause_run_commit(&context.run_id, pause_request) + .expect("duplicate pause request"); + let duplicate = duplicate_commit.outcome; + assert!(duplicate_commit.teardown_owner_id.is_none()); + assert!(first.transitioned); + assert_eq!(duplicate, first); + assert_eq!(first.captured_turn_count, 0); + + let resumed = crate::coordination::agent_org_pause::resume_run(&context.run_id, resume_request) + .expect("resume transaction"); + let duplicate_resume = + crate::coordination::agent_org_pause::resume_run(&context.run_id, resume_request) + .expect("duplicate resume request"); + assert!(resumed.transitioned); + assert_eq!(duplicate_resume, resumed); + assert_eq!(resumed.resume_generation, first.pause_generation + 1); + assert_eq!(inbox_count_for_member(&context, COORDINATOR_MEMBER_ID), 0); +} + +#[test] +fn concurrent_pause_and_resume_requests_advance_each_episode_once() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let run_id = context.run_id.clone(); + let pause_request = "00000000-0000-4000-8000-000000000201".to_string(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); + let pause_threads = (0..8) + .map(|_| { + let run_id = run_id.clone(); + let request_id = pause_request.clone(); + let barrier = std::sync::Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + crate::coordination::agent_org_pause::pause_run(&run_id, &request_id) + }) + }) + .collect::>(); + let pause_results = pause_threads + .into_iter() + .map(|thread| thread.join().expect("Pause worker did not panic")) + .collect::, _>>() + .expect("same-request concurrent Pause"); + assert!(pause_results.iter().all(|outcome| outcome.transitioned)); + assert!(pause_results.windows(2).all(|pair| pair[0] == pair[1])); + + let resume_request = "00000000-0000-4000-8000-000000000202".to_string(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); + let resume_threads = (0..8) + .map(|_| { + let run_id = run_id.clone(); + let request_id = resume_request.clone(); + let barrier = std::sync::Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + crate::coordination::agent_org_pause::resume_run(&run_id, &request_id) + }) + }) + .collect::>(); + let resume_results = resume_threads + .into_iter() + .map(|thread| thread.join().expect("Resume worker did not panic")) + .collect::, _>>() + .expect("same-request concurrent Resume"); + assert!(resume_results.iter().all(|outcome| outcome.transitioned)); + assert!(resume_results.windows(2).all(|pair| pair[0] == pair[1])); + + crate::coordination::agent_org_pause::pause_run( + &run_id, + "00000000-0000-4000-8000-000000000203", + ) + .expect("second Pause episode"); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); + let different_resume_threads = (0..8) + .map(|index| { + let run_id = run_id.clone(); + let barrier = std::sync::Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + crate::coordination::agent_org_pause::resume_run( + &run_id, + &format!("00000000-0000-4000-8000-{:012}", 300 + index), + ) + }) + }) + .collect::>(); + let different_resume_results = different_resume_threads + .into_iter() + .map(|thread| { + thread + .join() + .expect("different Resume worker did not panic") + }) + .collect::>(); + assert_eq!( + different_resume_results + .iter() + .filter(|result| result.as_ref().is_ok_and(|outcome| outcome.transitioned)) + .count(), + 1 + ); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(8)); + let different_pause_threads = (0..8) + .map(|index| { + let run_id = run_id.clone(); + let barrier = std::sync::Arc::clone(&barrier); + std::thread::spawn(move || { + barrier.wait(); + crate::coordination::agent_org_pause::pause_run( + &run_id, + &format!("00000000-0000-4000-8000-{:012}", 400 + index), + ) + }) + }) + .collect::>(); + let different_pause_results = different_pause_threads + .into_iter() + .map(|thread| thread.join().expect("different Pause worker did not panic")) + .collect::, _>>() + .expect("different-request concurrent Pause"); + assert_eq!( + different_pause_results + .iter() + .filter(|outcome| outcome.transitioned) + .count(), + 1 + ); + let conn = get_connection().expect("db connection"); + let final_state: (String, i64, i64) = conn + .query_row( + "SELECT status,activation_generation, + (SELECT COUNT(*) FROM agent_org_runtime_pause_episodes WHERE org_run_id=?1) + FROM agent_org_runtime_runs WHERE id=?1", + [&run_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .expect("read concurrent lifecycle state"); + assert_eq!(final_state, ("paused".to_string(), 6, 3)); +} + +#[test] +fn pause_run_update_failure_writes_no_episode_or_handoff() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); let conn = get_connection().expect("db connection"); conn.execute_batch( - "CREATE TRIGGER reject_resume_seed - BEFORE INSERT ON agent_org_runtime_inbox - BEGIN - SELECT RAISE(ABORT, 'injected resume seed failure'); - END;", + "CREATE TRIGGER reject_pause_run_update + BEFORE UPDATE ON agent_org_runtime_runs + WHEN OLD.status='running' AND NEW.status='paused' + BEGIN SELECT RAISE(ABORT, 'injected pause run update failure'); END;", + ) + .expect("install Pause run update trigger"); + drop(conn); + + let error = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000021", + ) + .expect_err("run update failure aborts Pause"); + assert!(error.contains("injected pause run update failure")); + let conn = get_connection().expect("db connection"); + let state: (String, i64, i64, i64) = conn + .query_row( + "SELECT status,activation_generation, + (SELECT COUNT(*) FROM agent_org_runtime_pause_episodes WHERE org_run_id=?1), + (SELECT COUNT(*) FROM agent_org_runtime_pause_handoffs WHERE org_run_id=?1) + FROM agent_org_runtime_runs WHERE id=?1", + [&context.run_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .expect("read failed Pause run update state"); + assert_eq!(state, ("running".to_string(), 1, 0, 0)); +} + +#[test] +fn pause_episode_insert_failure_rolls_back_status_generation_and_episode() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + let original_generation: i64 = conn + .query_row( + "SELECT activation_generation FROM agent_org_runtime_runs WHERE id=?1", + params![&context.run_id], + |row| row.get(0), + ) + .expect("original generation"); + conn.execute_batch( + "CREATE TRIGGER reject_pause_episode + BEFORE INSERT ON agent_org_runtime_pause_episodes + BEGIN SELECT RAISE(ABORT, 'injected pause episode failure'); END;", ) .expect("install failure trigger"); drop(conn); - let error = resume_agent_org_context_sync(&context, true) - .expect_err("seed failure rolls back resume transition"); - assert!(error.contains("injected resume seed failure")); + let error = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000003", + ) + .expect_err("episode failure rolls back Pause fence"); + assert!(error.contains("injected pause episode failure")); let conn = get_connection().expect("db connection"); - let status: String = conn + let state: (String, i64, i64) = conn .query_row( - "SELECT status FROM agent_org_runtime_runs WHERE id=?1", + "SELECT status,activation_generation, + (SELECT COUNT(*) FROM agent_org_runtime_pause_episodes WHERE org_run_id=?1) + FROM agent_org_runtime_runs WHERE id=?1", params![&context.run_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .expect("rolled-back Pause state"); + assert_eq!(state, ("running".to_string(), original_generation, 0)); +} + +#[test] +fn pause_captures_only_current_generation_formal_in_flight_turns() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "root-shared-agent", + turn_intent_id: "queued-coordinator", + turn_kind: "coordinator", + intent_status: "queued", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "planner-session-running", + turn_intent_id: "running-task", + turn_kind: "task_execution", + intent_status: "running", + task_id: Some("pause-task"), + activation_generation: Some(1), + member_sequence: Some(1), + }, + ); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "planner-session-direct", + turn_intent_id: "running-user-directed", + turn_kind: "user_directed_work", + intent_status: "running", + task_id: None, + activation_generation: None, + member_sequence: Some(2), + }, + ); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "root-old-generation", + turn_intent_id: "running-old-generation", + turn_kind: "coordinator", + intent_status: "running", + task_id: None, + activation_generation: Some(2), + member_sequence: None, + }, + ); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "root-terminal", + turn_intent_id: "completed-coordinator", + turn_kind: "coordinator", + intent_status: "completed", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + drop(conn); + + let outcome = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000004", + ) + .expect("Pause selector transaction"); + assert_eq!(outcome.captured_turn_count, 2); + assert_eq!(outcome.draining_turn_count, 1); + + let conn = get_connection().expect("db connection"); + let captured: Vec<(String, String, String)> = { + let mut statement = conn + .prepare( + "SELECT original_turn_intent_id,turn_kind,drain_status + FROM agent_org_runtime_pause_handoffs + WHERE episode_id=?1 ORDER BY original_turn_intent_id", + ) + .expect("prepare receipt query"); + statement + .query_map([&outcome.episode_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) + .expect("query receipts") + .collect::, _>>() + .expect("collect receipts") + }; + assert_eq!( + captured, + vec![ + ( + "queued-coordinator".to_string(), + "coordinator".to_string(), + "runtime_absent".to_string(), + ), + ( + "running-task".to_string(), + "task_execution".to_string(), + "waiting".to_string(), + ), + ] + ); + let statuses: Vec<(String, String)> = { + let mut statement = conn + .prepare( + "SELECT turn_intent_id,status FROM session_turn_intents + ORDER BY turn_intent_id", + ) + .expect("prepare status query"); + statement + .query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("query statuses") + .collect::, _>>() + .expect("collect statuses") + }; + assert!(statuses.contains(&("queued-coordinator".to_string(), "stale".to_string()))); + assert!(statuses.contains(&("running-task".to_string(), "running".to_string()))); + assert!(statuses.contains(&("running-user-directed".to_string(), "running".to_string()))); +} + +#[test] +fn pause_release_receipt_requires_the_exact_runtime_lease_and_turn_generation() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + configure_pause_resume_authority(&conn, &context); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "release-owner-session", + turn_intent_id: "release-owner-intent", + turn_kind: "coordinator", + intent_status: "running", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + drop(conn); + let paused = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000041", + ) + .expect("Pause before owner binding"); + assert!( + crate::coordination::agent_org_pause::bind_runtime_and_request_yield( + &paused.episode_id, + "release-owner-session", + "release-owner-intent", + "lease-current", + "turn-current", + ) + .expect("bind exact runtime owner") + ); + + assert_eq!( + crate::coordination::agent_org_pause::mark_released( + "release-owner-session", + "release-owner-intent", + "lease-old", + "turn-current", + ) + .expect("stale release callback"), + None + ); + assert_eq!( + crate::coordination::agent_org_pause::mark_released( + "release-owner-session", + "release-owner-intent", + "lease-current", + "turn-old", + ) + .expect("stale generation callback"), + None + ); + let conn = get_connection().expect("db connection"); + let still_waiting: String = conn + .query_row( + "SELECT drain_status FROM agent_org_runtime_pause_handoffs + WHERE episode_id=?1 AND session_id='release-owner-session'", + [&paused.episode_id], |row| row.get(0), ) - .expect("load rolled-back run status"); - assert_eq!(status, "paused"); - assert_eq!(inbox_count_for_member(&context, COORDINATOR_MEMBER_ID), 0); - conn.execute_batch("DROP TRIGGER reject_resume_seed;") - .expect("drop failure trigger"); + .expect("read handoff after stale callbacks"); + assert_eq!(still_waiting, "waiting"); drop(conn); - let outcome = resume_agent_org_context_sync(&context, true).expect("resume run"); assert_eq!( - outcome, - AgentOrgResumeOutcome { - transitioned: true, - run_is_running: true, - } + crate::coordination::agent_org_pause::mark_released( + "release-owner-session", + "release-owner-intent", + "lease-current", + "turn-current", + ) + .expect("release exact owner"), + Some(paused.episode_id.clone()) ); let conn = get_connection().expect("db connection"); - let status: String = conn + let released: String = conn .query_row( - "SELECT status FROM agent_org_runtime_runs WHERE id=?1", - params![&context.run_id], + "SELECT drain_status FROM agent_org_runtime_pause_handoffs + WHERE episode_id=?1 AND session_id='release-owner-session'", + [&paused.episode_id], |row| row.get(0), ) - .expect("load resumed run status"); - assert_eq!(status, "running"); - assert_eq!(inbox_count_for_member(&context, COORDINATOR_MEMBER_ID), 1); + .expect("read released handoff"); + assert_eq!(released, "released"); } #[test] -fn explicit_resume_of_running_run_repairs_unread_without_duplicate_seed() { +fn pause_nth_child_failure_rolls_back_fence_and_all_receipts() { let _sandbox = test_helpers::test_env::sandbox(); let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + for (index, intent) in ["child-one", "child-two"].into_iter().enumerate() { + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: if index == 0 { + "child-session-one" + } else { + "child-session-two" + }, + turn_intent_id: intent, + turn_kind: "coordinator", + intent_status: "queued", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + } + conn.execute_batch( + "CREATE TRIGGER reject_second_pause_child + BEFORE INSERT ON agent_org_runtime_pause_handoffs + WHEN NEW.original_turn_intent_id='child-two' + BEGIN SELECT RAISE(ABORT, 'injected second child failure'); END;", + ) + .expect("install child failure trigger"); + drop(conn); + + let error = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000005", + ) + .expect_err("N-th child failure rolls back entire Pause"); + assert!(error.contains("injected second child failure"), "{error}"); + let conn = get_connection().expect("db connection"); + let state: (String, i64, i64, i64) = conn + .query_row( + "SELECT status,activation_generation, + (SELECT COUNT(*) FROM agent_org_runtime_pause_episodes WHERE org_run_id=?1), + (SELECT COUNT(*) FROM agent_org_runtime_pause_handoffs WHERE org_run_id=?1) + FROM agent_org_runtime_runs WHERE id=?1", + [&context.run_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .expect("read rolled-back child state"); + assert_eq!(state, ("running".to_string(), 1, 0, 0)); +} + +#[test] +fn stale_formal_turn_cannot_materialize_or_ack_inbox_after_pause_fence() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "root-shared-agent", + turn_intent_id: "coordinator-inbox-turn", + turn_kind: "coordinator", + intent_status: "running", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + drop(conn); + let materialize_row = AgentInboxStore::insert(InsertInboxParams { + recipient_agent_id: context.coordinator_agent_id.clone(), + recipient_member_id: Some(COORDINATOR_MEMBER_ID.to_string()), + sender_agent_id: USER_SENDER_ID.to_string(), + sender_member_id: Some(USER_SENDER_ID.to_string()), + org_run_id: Some(context.run_id.clone()), + message: AgentMessage::Plain { + summary: "materialize race".to_string(), + text: "materialize race".to_string(), + }, + }) + .expect("insert materialize race row"); + let ack_row = AgentInboxStore::insert(InsertInboxParams { + recipient_agent_id: context.coordinator_agent_id.clone(), + recipient_member_id: Some(COORDINATOR_MEMBER_ID.to_string()), + sender_agent_id: USER_SENDER_ID.to_string(), + sender_member_id: Some(USER_SENDER_ID.to_string()), + org_run_id: Some(context.run_id.clone()), + message: AgentMessage::Plain { + summary: "ack race".to_string(), + text: "ack race".to_string(), + }, + }) + .expect("insert ack race row"); + let conn = get_connection().expect("db connection"); + conn.execute( + "INSERT INTO agent_org_runtime_inbox_materializations ( + inbox_id,session_id,transcript_message_id,transcript_intent_id,materialized_at + ) VALUES (?1,'root-shared-agent','message-ack','intent-ack',?2)", + params![ack_row.id, chrono::Utc::now().to_rfc3339()], + ) + .expect("seed transcript receipt owned by old Turn session"); + drop(conn); + + crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000006", + ) + .expect("commit Pause fence before Inbox writes"); + + let materialize_error = + crate::session::persistence::materialize_agent_org_inbox_transcript_for_turn( + "root-shared-agent", + "coordinator-inbox-turn", + &[materialize_row.id], + "message-materialize", + "intent-materialize", + "materialize race", + ) + .expect_err("old Turn cannot materialize after Pause"); + assert!( + materialize_error.contains("requires a running Team"), + "{materialize_error}" + ); + let ack_error = AgentInboxStore::mark_many_read_for_turn( + &[ack_row.id], + "root-shared-agent", + "coordinator-inbox-turn", + ) + .expect_err("old Turn cannot acknowledge after Pause"); + assert!( + ack_error.contains("generation fence rejected"), + "{ack_error}" + ); + + let conn = get_connection().expect("db connection"); + let unread_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_org_runtime_inbox + WHERE id IN (?1,?2) AND read_at IS NULL", + params![materialize_row.id, ack_row.id], + |row| row.get(0), + ) + .expect("read post-race Inbox state"); + let materialize_receipt_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_org_runtime_inbox_materializations + WHERE inbox_id=?1", + [materialize_row.id], + |row| row.get(0), + ) + .expect("count rejected materialization receipts"); + assert_eq!(unread_count, 2); + assert_eq!(materialize_receipt_count, 0); +} - for _ in 0..2 { - let outcome = - resume_agent_org_context_sync(&context, true).expect("idempotent explicit resume"); - assert_eq!( - outcome, - AgentOrgResumeOutcome { - transitioned: false, - run_is_running: true, - } +#[test] +fn resume_continues_only_legal_work_and_preserves_member_fifo_without_mutating_tasks() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + configure_pause_resume_authority(&conn, &context); + let now = chrono::Utc::now().to_rfc3339(); + conn.execute_batch( + "INSERT INTO agent_org_runtime_member_dispatch_allocators + (org_run_id,member_id,next_sequence) + VALUES ('run-shared-agent','member-planner',6);", + ) + .expect("seed FIFO allocator"); + for (task_id, status, output_json) in [ + ("open-a", "pending", None), + ("open-b", "in_progress", None), + ( + "completed-c", + "completed", + Some(r#"{"summary":"finished before Pause"}"#), + ), + ("reassigned-d", "pending", None), + ("old-materialization-e", "pending", None), + ] { + conn.execute( + "INSERT INTO agent_org_runtime_tasks ( + id,org_run_id,subject,description,owner,status,execution_mode, + blocked_by_json,output_json,cancel_reason_json,created_by_participant_id, + source_turn_intent_id,created_at,updated_at + ) VALUES (?1,?2,?1,'resume legality','member-planner',?3,'build','[]', + ?4,NULL,'coordinator','seed-task',?5,?5)", + params![task_id, &context.run_id, status, output_json, &now], + ) + .expect("seed Resume legality Task"); + } + conn.execute( + "UPDATE agent_org_runtime_tasks SET owner='member-builder' + WHERE org_run_id=?1 AND id='reassigned-d'", + [&context.run_id], + ) + .expect("seed reassigned Task owner"); + conn.execute( + "INSERT INTO agent_org_runtime_inbox ( + recipient_agent_id,recipient_member_id,sender_agent_id,sender_member_id, + org_run_id,payload_kind,payload_json,created_at + ) VALUES ('planner-agent','member-planner','coordinator-agent','coordinator', + ?1,'task_assigned',?2,?3)", + params![ + &context.run_id, + r#"{"kind":"task_assigned","task_id":"completed-c"}"#, + &now + ], + ) + .expect("seed terminal Task assignment Inbox row"); + let terminal_assignment_inbox_id = conn.last_insert_rowid(); + conn.execute( + "INSERT INTO agent_org_runtime_inbox_materializations ( + inbox_id,session_id,transcript_message_id,transcript_intent_id,materialized_at + ) VALUES (?1,'planner-session','terminal-assignment-message', + 'terminal-assignment-intent',?2)", + params![terminal_assignment_inbox_id, &now], + ) + .expect("seed terminal Task assignment materialization"); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "root-shared-agent", + turn_intent_id: "resume-root", + turn_kind: "coordinator", + intent_status: "running", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + for (sequence, task_id, intent_status) in [ + (1, "open-a", "running"), + (2, "open-b", "queued"), + (3, "completed-c", "running"), + (4, "reassigned-d", "running"), + (5, "old-materialization-e", "running"), + ] { + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: if task_id == "old-materialization-e" { + "planner-old-session" + } else { + "planner-session" + }, + turn_intent_id: &format!("resume-{task_id}"), + turn_kind: "task_execution", + intent_status, + task_id: Some(task_id), + activation_generation: Some(1), + member_sequence: Some(sequence), + }, ); } + let tasks_before: Vec<(String, String, Option)> = { + let mut statement = conn + .prepare( + "SELECT id,status,owner FROM agent_org_runtime_tasks + WHERE org_run_id=?1 ORDER BY id", + ) + .expect("prepare Task snapshot"); + statement + .query_map([&context.run_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) + .expect("query Task snapshot") + .collect::, _>>() + .expect("collect Task snapshot") + }; + drop(conn); + + let paused = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000007", + ) + .expect("Pause formal work"); + assert_eq!(paused.captured_turn_count, 6); + let resumed = crate::coordination::agent_org_pause::resume_run( + &context.run_id, + "00000000-0000-4000-8000-000000000008", + ) + .expect("Resume legal formal work"); + assert_eq!(resumed.continuation_count, 3); + assert_eq!(resumed.skipped_count, 3); - assert_eq!(inbox_count_for_member(&context, COORDINATOR_MEMBER_ID), 1); - let targets = - collect_run_progress_wake_targets(&context.run_id, &org_progress_member_ids(&context)) - .expect("rescan unread inbox rows"); + let conn = get_connection().expect("db connection"); + let tasks_after: Vec<(String, String, Option)> = { + let mut statement = conn + .prepare( + "SELECT id,status,owner FROM agent_org_runtime_tasks + WHERE org_run_id=?1 ORDER BY id", + ) + .expect("prepare post-Resume Task snapshot"); + statement + .query_map([&context.run_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) + .expect("query post-Resume Task snapshot") + .collect::, _>>() + .expect("collect post-Resume Task snapshot") + }; + assert_eq!( + tasks_after, tasks_before, + "Pause/Resume must not rewrite Tasks" + ); + let planner_sequences: Vec = { + let mut statement = conn + .prepare( + "SELECT context.member_dispatch_sequence + FROM agent_org_runtime_pause_handoffs handoff + JOIN agent_org_runtime_turn_contexts context + ON context.session_id=handoff.session_id + AND context.turn_intent_id=handoff.continuation_turn_intent_id + WHERE handoff.episode_id=?1 + AND handoff.participant_id='member-planner' + AND handoff.continuation_status='queued' + ORDER BY context.member_dispatch_sequence", + ) + .expect("prepare continuation FIFO query"); + statement + .query_map([&resumed.episode_id], |row| row.get(0)) + .expect("query continuation FIFO") + .collect::, _>>() + .expect("collect continuation FIFO") + }; + assert_eq!(planner_sequences, vec![6, 7]); + let skipped_reason: String = conn + .query_row( + "SELECT skip_reason FROM agent_org_runtime_pause_handoffs + WHERE episode_id=?1 AND task_id='completed-c'", + [&resumed.episode_id], + |row| row.get(0), + ) + .expect("read terminal Task skip reason"); + assert_eq!(skipped_reason, "task_completed"); + let terminal_assignment_resolution: (String, String, Option, i64) = conn + .query_row( + "SELECT resolution.resolution_kind,resolution.reason,inbox.read_at, + (SELECT COUNT(*) + FROM agent_org_runtime_inbox_materializations materialization + WHERE materialization.inbox_id=inbox.id) + FROM agent_org_runtime_inbox inbox + JOIN agent_org_runtime_inbox_delivery_resolutions resolution + ON resolution.inbox_id=inbox.id + WHERE inbox.id=?1", + [terminal_assignment_inbox_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .expect("read terminal Task assignment resolution"); assert_eq!( - targets, - vec![AgentOrgWakeTarget { - member_id: COORDINATOR_MEMBER_ID.to_string(), - reason: AgentOrgWakeReason::UnreadInbox, - }] + terminal_assignment_resolution, + ( + "cancelled".to_string(), + "pause_resume_task_completed".to_string(), + None, + 0, + ), + "Resume must resolve the stale assignment without falsifying its read receipt" ); + let other_skip_reasons: Vec<(String, String)> = { + let mut statement = conn + .prepare( + "SELECT task_id,skip_reason FROM agent_org_runtime_pause_handoffs + WHERE episode_id=?1 AND task_id IN ('reassigned-d','old-materialization-e') + ORDER BY task_id", + ) + .expect("prepare legality skip query"); + statement + .query_map([&resumed.episode_id], |row| Ok((row.get(0)?, row.get(1)?))) + .expect("query legality skips") + .collect::, _>>() + .expect("collect legality skips") + }; + assert_eq!( + other_skip_reasons, + vec![ + ( + "old-materialization-e".to_string(), + "member_materialization_changed".to_string(), + ), + ("reassigned-d".to_string(), "task_owner_changed".to_string(),), + ] + ); +} + +#[test] +fn exact_resume_continuation_consumes_old_assignment_only_after_task_success() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + configure_pause_resume_authority(&conn, &context); + let now = chrono::Utc::now().to_rfc3339(); + conn.execute( + "INSERT INTO agent_org_runtime_member_dispatch_allocators + (org_run_id,member_id,next_sequence) + VALUES (?1,'member-planner',2)", + [&context.run_id], + ) + .expect("seed continuation FIFO"); + conn.execute( + "INSERT INTO agent_org_runtime_tasks ( + id,org_run_id,subject,description,owner,status,execution_mode, + blocked_by_json,output_json,cancel_reason_json,created_by_participant_id, + source_turn_intent_id,created_at,updated_at + ) VALUES ('resume-owned-task',?1,'Resume owned task','', + 'member-planner','in_progress','build','[]',NULL,NULL, + 'coordinator','seed-task',?2,?2)", + params![&context.run_id, &now], + ) + .expect("seed in-progress Task"); + let assignment = AgentInboxStore::insert(InsertInboxParams { + recipient_agent_id: "builtin:sde".to_string(), + recipient_member_id: Some("member-planner".to_string()), + sender_agent_id: context.coordinator_agent_id.clone(), + sender_member_id: Some(COORDINATOR_MEMBER_ID.to_string()), + org_run_id: Some(context.run_id.clone()), + message: AgentMessage::TaskAssigned { + task_id: "resume-owned-task".to_string(), + subject: "Resume owned task".to_string(), + description: String::new(), + assigned_by: "Coordinator".to_string(), + execution_mode: TaskExecutionMode::Build, + dependency_outputs: Vec::new(), + }, + }) + .expect("seed old assignment"); + let unrelated_assignment = AgentInboxStore::insert(InsertInboxParams { + recipient_agent_id: "builtin:sde".to_string(), + recipient_member_id: Some("member-planner".to_string()), + sender_agent_id: context.coordinator_agent_id.clone(), + sender_member_id: Some(COORDINATOR_MEMBER_ID.to_string()), + org_run_id: Some(context.run_id.clone()), + message: AgentMessage::TaskAssigned { + task_id: "resume-owned-task".to_string(), + subject: "Unmaterialized duplicate must stay unrelated".to_string(), + description: String::new(), + assigned_by: "Coordinator".to_string(), + execution_mode: TaskExecutionMode::Build, + dependency_outputs: Vec::new(), + }, + }) + .expect("seed unrelated unmaterialized assignment"); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "planner-session", + turn_intent_id: "resume-owned-original", + turn_kind: "task_execution", + intent_status: "running", + task_id: Some("resume-owned-task"), + activation_generation: Some(1), + member_sequence: Some(1), + }, + ); + conn.execute( + "INSERT INTO agent_org_runtime_inbox_materializations ( + inbox_id,session_id,transcript_message_id,transcript_intent_id,materialized_at + ) VALUES (?1,'planner-session','resume-owned-message', + 'resume-owned-transcript-intent',?2)", + params![assignment.id, &now], + ) + .expect("materialize assignment before Pause"); + drop(conn); + + let paused = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000021", + ) + .expect("Pause in-progress Task"); + assert!(crate::coordination::agent_org_pause::mark_runtime_absent( + &paused.episode_id, + "planner-session", + "resume-owned-original", + ) + .expect("mark old runtime absent")); + let resumed = crate::coordination::agent_org_pause::resume_run( + &context.run_id, + "00000000-0000-4000-8000-000000000022", + ) + .expect("Resume in-progress Task"); + assert_eq!(resumed.continuation_count, 1); + + let conn = get_connection().expect("db connection"); + let continuation_turn_intent_id: String = conn + .query_row( + "SELECT continuation_turn_intent_id + FROM agent_org_runtime_pause_handoffs + WHERE episode_id=?1 AND task_id='resume-owned-task'", + [&resumed.episode_id], + |row| row.get(0), + ) + .expect("read Task continuation"); + drop(conn); + assert!( + crate::coordination::agent_org_pause::claim_continuation_dispatch( + &resumed.episode_id, + &continuation_turn_intent_id, + ) + .expect("claim Task continuation") + ); + let conn = get_connection().expect("db connection"); + conn.execute( + "UPDATE session_turn_intents SET status='running' + WHERE session_id='planner-session' AND turn_intent_id=?1", + [&continuation_turn_intent_id], + ) + .expect("start Task continuation"); + + let old_turn_batch = AgentInboxStore::list_unread_task_input_for_turn( + "member-planner", + &context.run_id, + "resume-owned-task", + "planner-session", + "resume-owned-original", + ) + .expect("probe stale original Turn"); + assert!(old_turn_batch.rows.is_empty()); + let continuation_batch = AgentInboxStore::list_unread_task_input_for_turn( + "member-planner", + &context.run_id, + "resume-owned-task", + "planner-session", + &continuation_turn_intent_id, + ) + .expect("exact continuation claims old assignment"); + assert_eq!( + continuation_batch + .rows + .iter() + .map(|row| row.id) + .collect::>(), + vec![assignment.id] + ); + drop(conn); + + let early_ack = AgentInboxStore::mark_many_read_for_turn( + &[assignment.id], + "planner-session", + &continuation_turn_intent_id, + ) + .expect_err("in-progress Task must leave assignment unread"); + assert!(early_ack.contains("did not complete the Task successfully")); + + let conn = get_connection().expect("db connection"); + conn.execute( + "UPDATE agent_org_runtime_tasks + SET status='completed',output_json='{}',updated_at=?3 + WHERE org_run_id=?1 AND id=?2", + params![ + &context.run_id, + "resume-owned-task", + chrono::Utc::now().to_rfc3339() + ], + ) + .expect("complete resumed Task"); + drop(conn); + assert_eq!( + AgentInboxStore::mark_many_read_for_turn( + &[assignment.id], + "planner-session", + &continuation_turn_intent_id, + ) + .expect("successful continuation acknowledges assignment"), + 1 + ); + let conn = get_connection().expect("db connection"); + let read_at: Option = conn + .query_row( + "SELECT read_at FROM agent_org_runtime_inbox WHERE id=?1", + [assignment.id], + |row| row.get(0), + ) + .expect("read assignment receipt"); + assert!(read_at.is_some()); + let unrelated_read_at: Option = conn + .query_row( + "SELECT read_at FROM agent_org_runtime_inbox WHERE id=?1", + [unrelated_assignment.id], + |row| row.get(0), + ) + .expect("read unrelated assignment"); + assert!( + unrelated_read_at.is_none(), + "Resume must not acknowledge an unmaterialized duplicate assignment" + ); +} + +#[test] +fn resume_run_update_failure_keeps_active_episode_without_continuations() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + configure_pause_resume_authority(&conn, &context); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "root-shared-agent", + turn_intent_id: "resume-run-update-root", + turn_kind: "coordinator", + intent_status: "queued", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + drop(conn); + let paused = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000022", + ) + .expect("Pause before Resume run update fault"); + let conn = get_connection().expect("db connection"); + conn.execute_batch( + "CREATE TRIGGER reject_resume_run_update + BEFORE UPDATE ON agent_org_runtime_runs + WHEN OLD.status='paused' AND NEW.status='running' + BEGIN SELECT RAISE(ABORT, 'injected Resume run update failure'); END;", + ) + .expect("install Resume run update trigger"); + drop(conn); + + let error = crate::coordination::agent_org_pause::resume_run( + &context.run_id, + "00000000-0000-4000-8000-000000000023", + ) + .expect_err("run update failure aborts Resume"); + assert!(error.contains("injected Resume run update failure")); + let conn = get_connection().expect("db connection"); + let state: (String, i64, String, i64, i64) = conn + .query_row( + "SELECT run.status,run.activation_generation,episode.status, + (SELECT COUNT(*) FROM agent_org_runtime_pause_handoffs handoff + WHERE handoff.episode_id=episode.episode_id + AND handoff.continuation_status IS NOT NULL), + (SELECT COUNT(*) FROM session_turn_intents intent + WHERE intent.org_run_id=run.id AND intent.source='resume') + FROM agent_org_runtime_runs run + JOIN agent_org_runtime_pause_episodes episode ON episode.org_run_id=run.id + WHERE run.id=?1", + [&context.run_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + ) + .expect("read failed Resume run update state"); + assert_eq!( + state, + ( + "paused".to_string(), + paused.pause_generation, + "active".to_string(), + 0, + 0 + ) + ); +} + +#[test] +fn resume_continuation_insert_failure_rolls_back_run_and_receipts() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + configure_pause_resume_authority(&conn, &context); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "root-shared-agent", + turn_intent_id: "resume-continuation-fault-root", + turn_kind: "coordinator", + intent_status: "queued", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + drop(conn); + let paused = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000024", + ) + .expect("Pause before Resume continuation fault"); + let conn = get_connection().expect("db connection"); + conn.execute_batch( + "CREATE TRIGGER reject_resume_continuation_insert + BEFORE INSERT ON session_turn_intents + WHEN NEW.source='resume' + BEGIN SELECT RAISE(ABORT, 'injected Resume continuation failure'); END;", + ) + .expect("install Resume continuation trigger"); + drop(conn); + + let error = crate::coordination::agent_org_pause::resume_run( + &context.run_id, + "00000000-0000-4000-8000-000000000025", + ) + .expect_err("continuation failure aborts Resume"); + assert!(error.contains("injected Resume continuation failure")); + let conn = get_connection().expect("db connection"); + let state: (String, i64, String, i64, i64) = conn + .query_row( + "SELECT run.status,run.activation_generation,episode.status, + (SELECT COUNT(*) FROM agent_org_runtime_pause_handoffs handoff + WHERE handoff.episode_id=episode.episode_id + AND handoff.continuation_status IS NOT NULL), + (SELECT COUNT(*) FROM session_turn_intents intent + WHERE intent.org_run_id=run.id AND intent.source='resume') + FROM agent_org_runtime_runs run + JOIN agent_org_runtime_pause_episodes episode ON episode.org_run_id=run.id + WHERE run.id=?1", + [&context.run_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + ) + .expect("read failed Resume continuation state"); + assert_eq!( + state, + ( + "paused".to_string(), + paused.pause_generation, + "active".to_string(), + 0, + 0 + ) + ); +} + +#[test] +fn resume_precommit_failure_rolls_back_generation_episode_and_continuations() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + configure_pause_resume_authority(&conn, &context); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "root-shared-agent", + turn_intent_id: "resume-fault-root", + turn_kind: "coordinator", + intent_status: "queued", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + drop(conn); + let paused = crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000009", + ) + .expect("Pause before Resume fault"); + let conn = get_connection().expect("db connection"); + conn.execute_batch( + "CREATE TRIGGER reject_resume_precommit + BEFORE UPDATE ON agent_org_runtime_pause_episodes + WHEN NEW.status='consumed' + BEGIN SELECT RAISE(ABORT, 'injected Resume precommit failure'); END;", + ) + .expect("install Resume failure trigger"); + drop(conn); + + let error = crate::coordination::agent_org_pause::resume_run( + &context.run_id, + "00000000-0000-4000-8000-000000000010", + ) + .expect_err("Resume precommit fault rolls back all writes"); + assert!( + error.contains("injected Resume precommit failure"), + "{error}" + ); + let conn = get_connection().expect("db connection"); + let state: (String, i64, String, i64, i64) = conn + .query_row( + "SELECT run.status,run.activation_generation,episode.status, + (SELECT COUNT(*) FROM agent_org_runtime_pause_handoffs handoff + WHERE handoff.episode_id=episode.episode_id + AND handoff.continuation_status IS NOT NULL), + (SELECT COUNT(*) FROM session_turn_intents intent + WHERE intent.org_run_id=run.id AND intent.source='resume') + FROM agent_org_runtime_runs run + JOIN agent_org_runtime_pause_episodes episode ON episode.org_run_id=run.id + WHERE run.id=?1", + [&context.run_id], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + )) + }, + ) + .expect("read rolled-back Resume state"); + assert_eq!( + state, + ( + "paused".to_string(), + paused.pause_generation, + "active".to_string(), + 0, + 0 + ) + ); +} + +#[test] +fn restart_recovers_one_durable_continuation_without_replaying_it_twice() { + let _sandbox = test_helpers::test_env::sandbox(); + let context = prepare_command_run("running"); + let conn = get_connection().expect("db connection"); + configure_pause_resume_authority(&conn, &context); + seed_pause_turn_context( + &conn, + &context, + PauseTurnSeed { + session_id: "root-shared-agent", + turn_intent_id: "restart-root", + turn_kind: "coordinator", + intent_status: "running", + task_id: None, + activation_generation: Some(1), + member_sequence: None, + }, + ); + drop(conn); + crate::coordination::agent_org_pause::pause_run( + &context.run_id, + "00000000-0000-4000-8000-000000000011", + ) + .expect("Pause before restart"); + let resumed = crate::coordination::agent_org_pause::resume_run( + &context.run_id, + "00000000-0000-4000-8000-000000000012", + ) + .expect("Resume before restart"); + let conn = get_connection().expect("db connection"); + let continuation_turn_intent_id: String = conn + .query_row( + "SELECT continuation_turn_intent_id + FROM agent_org_runtime_pause_handoffs WHERE episode_id=?1", + [&resumed.episode_id], + |row| row.get(0), + ) + .expect("read continuation id"); + drop(conn); + assert!( + crate::coordination::agent_org_pause::claim_continuation_dispatch( + &resumed.episode_id, + &continuation_turn_intent_id, + ) + .expect("simulate pre-crash dispatch claim") + ); + let nudge = crate::coordination::agent_org_pause::continuation_nudge_for_turn( + "root-shared-agent", + &continuation_turn_intent_id, + ) + .expect("resolve durable continuation nudge") + .expect("claimed continuation has transient provider work"); + assert!(nudge.contains("Continue coordinating the paused Agent Org run")); + + let conn = get_connection().expect("db connection"); + conn.execute( + "UPDATE session_turn_intents SET status='running' + WHERE session_id='root-shared-agent' AND turn_intent_id=?1", + [&continuation_turn_intent_id], + ) + .expect("simulate a continuation that crossed the scheduler boundary before crash"); + let first = crate::coordination::reconcile_agent_org_turns_after_restart(&conn) + .expect("first restart reconciliation"); + assert!( + first >= 3, + "runtime absence, running intent, and dispatch claim should reconcile" + ); + let recovered: (String, String, String) = conn + .query_row( + "SELECT handoff.drain_status,handoff.continuation_status,intent.status + FROM agent_org_runtime_pause_handoffs handoff + JOIN session_turn_intents intent + ON intent.session_id=handoff.session_id + AND intent.turn_intent_id=handoff.continuation_turn_intent_id + WHERE handoff.episode_id=?1", + [&resumed.episode_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .expect("read recovered continuation"); + assert_eq!( + recovered, + ( + "runtime_absent".to_string(), + "queued".to_string(), + "queued".to_string(), + ) + ); + let second = crate::coordination::reconcile_agent_org_turns_after_restart(&conn) + .expect("idempotent restart reconciliation"); + assert_eq!(second, 0); + drop(conn); + let dispatches = crate::coordination::agent_org_pause::list_dispatchable_continuations(10) + .expect("list recovered continuations"); + assert_eq!(dispatches.len(), 1); + assert_eq!(dispatches[0].turn_intent_id, continuation_turn_intent_id); } #[test] diff --git a/src-tauri/crates/agent-core/src/state/commands/tools.rs b/src-tauri/crates/agent-core/src/state/commands/tools.rs index 2b6257deae..f56b7a5881 100644 --- a/src-tauri/crates/agent-core/src/state/commands/tools.rs +++ b/src-tauri/crates/agent-core/src/state/commands/tools.rs @@ -212,10 +212,8 @@ pub async fn list_effective_tools_for_session( .await .ok_or_else(|| format!("session not found: {}", request.session_id))?; let runtime = session - .runtime - .read() + .get_runtime() .await - .clone() .ok_or_else(|| format!("session runtime not initialized: {}", request.session_id))?; let session_record = diff --git a/src-tauri/crates/agent-core/src/state/session_runtime.rs b/src-tauri/crates/agent-core/src/state/session_runtime.rs index ff033470cb..dd8ce12ee3 100644 --- a/src-tauri/crates/agent-core/src/state/session_runtime.rs +++ b/src-tauri/crates/agent-core/src/state/session_runtime.rs @@ -29,8 +29,10 @@ use crate::session::workspace::SessionWorkspace; use crate::session::{DialogScheduler, DialogTurn, DialogTurnState, TurnStats}; use crate::specialization::policies::activation::SessionScopedContextActivator; use crate::state::control_flow::CancelReason; +use crate::tools::call_context::{TurnProcessControl, TurnProcessOwner}; use crate::tools::policy::ResolvedToolPolicy; use crate::tools::registry::ToolRegistry; +use tokio_util::sync::CancellationToken; /// Runtime resources for a single agent session. /// @@ -96,6 +98,33 @@ pub struct SessionRuntime { pub agent_definition_id: Option, } +/// One installed runtime generation for a Session. +/// +/// The lease changes whenever initialization replaces the runtime. Lifecycle +/// cleanup must present the lease it originally observed, so delayed Pause +/// teardown cannot clear a runtime installed by a later Resume. +#[derive(Clone)] +struct RuntimeSlot { + lease_id: String, + runtime: Arc, +} + +/// Exact in-memory identity of the Turn currently using a runtime lease. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct RuntimeTurnIdentity { + pub runtime_lease_id: String, + pub dialog_turn_generation: String, + pub turn_intent_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ActiveTurnIdentity { + runtime_lease_id: Option, + dialog_turn_generation: String, + turn_intent_id: Option, + process_control: Option, +} + /// An active agent session — the single source of truth for all per-session state. /// /// All sub-resources (runtime, managers, locks) live here. `AgentAppState` @@ -113,7 +142,7 @@ pub struct AgentSession { /// /// `None` briefly while the session is being registered before /// `ensure_session_initialized` completes. - pub runtime: tokio::sync::RwLock>>, + runtime: tokio::sync::RwLock>, // ── Execution Control ───────────────────────────────────────────────── /// Cancellation flag — set to `true` to abort the active turn. @@ -183,6 +212,8 @@ pub struct AgentSession { /// Synchronous mirror of `active_turn.turn_id` for non-async event-store /// write guards on hot paths. pub active_turn_generation: Arc>>, + /// Persisted Turn intent paired with the dialog generation above. + active_turn_identity: parking_lot::RwLock>, /// Per-session FIFO message queue. /// /// All incoming messages are enqueued here and processed one at a time @@ -315,6 +346,7 @@ impl AgentSession { last_active_at: tokio::sync::Mutex::new(Instant::now()), active_turn: tokio::sync::Mutex::new(None), active_turn_generation: Arc::new(parking_lot::RwLock::new(None)), + active_turn_identity: parking_lot::RwLock::new(None), scheduler: DialogScheduler::new(session_id_for_scheduler, 32), steering_queue: Arc::new(tokio::sync::Mutex::new(Vec::new())), sm_state: Arc::new(tokio::sync::Mutex::new(SessionMemoryState::default())), @@ -336,13 +368,83 @@ impl AgentSession { } /// Attach (or replace) the runtime after initialization completes. - pub async fn set_runtime(&self, runtime: Arc) { - *self.runtime.write().await = Some(runtime); + pub async fn set_runtime(&self, runtime: Arc) -> String { + let lease_id = uuid::Uuid::new_v4().to_string(); + *self.runtime.write().await = Some(RuntimeSlot { + lease_id: lease_id.clone(), + runtime, + }); + lease_id } /// Return the current runtime, if initialized. pub async fn get_runtime(&self) -> Option> { - self.runtime.read().await.clone() + self.runtime + .read() + .await + .as_ref() + .map(|slot| Arc::clone(&slot.runtime)) + } + + /// Clear whichever runtime is current. This remains the ordinary SDE + /// invalidation path; Pause uses the conditional lease method below. + pub(crate) async fn invalidate_runtime(&self) { + *self.runtime.write().await = None; + } + + /// Return the exact runtime/Turn pair currently active in this Session. + pub(crate) async fn runtime_turn_identity(&self) -> Option { + let turn = self.active_turn_identity.read().clone()?; + Some(RuntimeTurnIdentity { + runtime_lease_id: turn.runtime_lease_id?, + dialog_turn_generation: turn.dialog_turn_generation, + turn_intent_id: turn.turn_intent_id, + }) + } + + /// Return the exact, level-triggered process control for the active Turn. + /// Direct/maintenance turns without a durable intent or runtime lease do + /// not own detachable shell work through the Agent Org Pause protocol. + pub(crate) fn turn_process_control(&self) -> Option { + self.active_turn_identity + .read() + .as_ref() + .and_then(|turn| turn.process_control.clone()) + } + + /// Release only the runtime generation and dialog Turn captured by Pause. + /// A stale completion is deliberately a no-op. + pub(crate) async fn release_runtime_if_current( + &self, + runtime_lease_id: &str, + dialog_turn_generation: &str, + ) -> bool { + let mut slot = self.runtime.write().await; + let current_lease = slot.as_ref().map(|current| current.lease_id.as_str()); + let turn = self.active_turn_identity.read(); + let current_turn_lease = turn + .as_ref() + .and_then(|turn| turn.runtime_lease_id.as_deref()); + let current_generation = turn + .as_ref() + .map(|turn| turn.dialog_turn_generation.as_str()); + if runtime_release_identity_matches( + current_lease, + current_turn_lease, + current_generation, + runtime_lease_id, + dialog_turn_generation, + ) { + *slot = None; + return true; + } + false + } + + pub(crate) async fn runtime_agent_definition_id(&self) -> Option { + self.get_runtime() + .await + .and_then(|runtime| runtime.agent_definition_id.clone()) } pub async fn invalidate_prompt_cache(&self, reason: PromptCacheInvalidationReason) { @@ -404,8 +506,41 @@ impl AgentSession { /// Returns the stable `turn_id` so the caller can embed it in events /// without holding the `active_turn` lock for the duration of processing. pub async fn begin_turn(&self, user_input: String) -> String { + self.begin_turn_with_intent(user_input, None).await + } + + /// Start a dialog Turn and bind it to its durable intent when one exists. + pub async fn begin_turn_with_intent( + &self, + user_input: String, + turn_intent_id: Option, + ) -> String { + let runtime_lease_id = self + .runtime + .read() + .await + .as_ref() + .map(|slot| slot.lease_id.clone()); let turn = DialogTurn::new(user_input, Arc::clone(&self.cancel_flag)); let turn_id = turn.turn_id.clone(); + let process_control = runtime_lease_id.as_ref().zip(turn_intent_id.as_ref()).map( + |(runtime_lease_id, turn_intent_id)| TurnProcessControl { + owner: TurnProcessOwner { + session_id: self.id.clone(), + turn_intent_id: turn_intent_id.clone(), + runtime_lease_id: runtime_lease_id.clone(), + dialog_turn_generation: turn_id.clone(), + }, + background_cancel: CancellationToken::new(), + require_owned_job_finality: false, + }, + ); + *self.active_turn_identity.write() = Some(ActiveTurnIdentity { + runtime_lease_id, + dialog_turn_generation: turn_id.clone(), + turn_intent_id, + process_control, + }); *self.active_turn_generation.write() = Some(turn_id.clone()); *self.active_turn.lock().await = Some(turn); turn_id @@ -421,6 +556,7 @@ impl AgentSession { turn.finalize(turn_state, stats); } *guard = None; + *self.active_turn_identity.write() = None; *self.active_turn_generation.write() = None; } @@ -465,6 +601,25 @@ impl AgentSession { crate::tools::impls::coding::exec::registry::cancel_subagents_for_session(&self.id); } + match shell_cancellation_scope(reason) { + ShellCancellationScope::ActiveTurn => { + if let Some(control) = self.turn_process_control() { + control.background_cancel.cancel(); + } + } + ShellCancellationScope::Session => { + // Cancel the active Turn token to close the foreground→background + // race, then fan out to background jobs from earlier Turns in + // the same ordinary SDE Session. ForceSend deliberately does + // neither: it preserves intentional background processes. + if let Some(control) = self.turn_process_control() { + control.background_cancel.cancel(); + } + crate::tools::impls::coding::exec::registry::cancel_shells_for_session(&self.id); + } + ShellCancellationScope::None => {} + } + let guard: tokio::sync::MutexGuard<'_, Option> = self.active_turn.lock().await; if let Some(ref turn) = *guard { turn.cancel(); @@ -480,3 +635,101 @@ impl AgentSession { } } } + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ShellCancellationScope { + None, + ActiveTurn, + Session, +} + +const fn shell_cancellation_scope(reason: CancelReason) -> ShellCancellationScope { + match reason { + CancelReason::UserStop => ShellCancellationScope::Session, + CancelReason::OrgPause => ShellCancellationScope::ActiveTurn, + CancelReason::ForceSend + | CancelReason::AgentOrgDelete + | CancelReason::ProgrammaticShutdown + | CancelReason::SessionEviction + | CancelReason::ModeSwitchAbort => ShellCancellationScope::None, + } +} + +fn runtime_release_identity_matches( + current_lease_id: Option<&str>, + active_turn_lease_id: Option<&str>, + current_turn_generation: Option<&str>, + expected_lease_id: &str, + expected_turn_generation: &str, +) -> bool { + current_lease_id == Some(expected_lease_id) + && active_turn_lease_id == Some(expected_lease_id) + && current_turn_generation == Some(expected_turn_generation) +} + +#[cfg(test)] +mod runtime_lease_tests { + use super::{ + runtime_release_identity_matches, shell_cancellation_scope, ShellCancellationScope, + }; + use crate::state::control_flow::CancelReason; + + #[test] + fn shell_cancellation_preserves_stop_force_send_and_pause_boundaries() { + assert_eq!( + shell_cancellation_scope(CancelReason::UserStop), + ShellCancellationScope::Session + ); + assert_eq!( + shell_cancellation_scope(CancelReason::OrgPause), + ShellCancellationScope::ActiveTurn + ); + assert_eq!( + shell_cancellation_scope(CancelReason::ForceSend), + ShellCancellationScope::None + ); + assert_eq!( + shell_cancellation_scope(CancelReason::AgentOrgDelete), + ShellCancellationScope::None + ); + } + + #[test] + fn release_requires_the_same_runtime_lease_and_dialog_generation() { + assert!(runtime_release_identity_matches( + Some("lease-a"), + Some("lease-a"), + Some("turn-1"), + "lease-a", + "turn-1" + )); + assert!(!runtime_release_identity_matches( + Some("lease-b"), + Some("lease-a"), + Some("turn-1"), + "lease-a", + "turn-1" + )); + assert!(!runtime_release_identity_matches( + Some("lease-a"), + Some("lease-b"), + Some("turn-1"), + "lease-a", + "turn-1" + )); + assert!(!runtime_release_identity_matches( + Some("lease-a"), + Some("lease-a"), + Some("turn-2"), + "lease-a", + "turn-1" + )); + assert!(!runtime_release_identity_matches( + None, + Some("lease-a"), + Some("turn-1"), + "lease-a", + "turn-1" + )); + } +} diff --git a/src-tauri/crates/agent-core/src/state/unified.rs b/src-tauri/crates/agent-core/src/state/unified.rs index 906d2f14c3..d1217a21eb 100644 --- a/src-tauri/crates/agent-core/src/state/unified.rs +++ b/src-tauri/crates/agent-core/src/state/unified.rs @@ -343,7 +343,7 @@ impl AgentAppState { /// on the next request. pub async fn invalidate_session(&self, session_id: &str) { if let Some(session) = self.get_session(session_id).await { - *session.runtime.write().await = None; + session.invalidate_runtime().await; info!("[agent-state] Invalidated session runtime: {}", session_id); } } @@ -379,13 +379,8 @@ impl AgentAppState { let mut count = 0usize; for session in &sessions { let applies_to_session_definition = session.definition.id == definition_id; - let applies_to_runtime_definition = session - .runtime - .read() - .await - .as_ref() - .and_then(|runtime| runtime.agent_definition_id.as_deref()) - == Some(definition_id); + let applies_to_runtime_definition = + session.runtime_agent_definition_id().await.as_deref() == Some(definition_id); if applies_to_session_definition || applies_to_runtime_definition { session.invalidate_prompt_cache(reason).await; count += 1; @@ -408,7 +403,7 @@ impl AgentAppState { let mut count = 0usize; for (id, session) in sessions.iter() { if prefixes.iter().any(|p| id.starts_with(p)) { - *session.runtime.write().await = None; + session.invalidate_runtime().await; count += 1; } } diff --git a/src-tauri/crates/agent-core/src/tests/turn_executor_retry_tests.rs b/src-tauri/crates/agent-core/src/tests/turn_executor_retry_tests.rs index 02a3db79cc..e2565dfc58 100644 --- a/src-tauri/crates/agent-core/src/tests/turn_executor_retry_tests.rs +++ b/src-tauri/crates/agent-core/src/tests/turn_executor_retry_tests.rs @@ -9,18 +9,23 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicU32, Ordering}; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; use serde_json::Value; use crate::providers::finish_reason; -use crate::providers::traits::{LLMProvider, LLMResponse, ProviderError, StreamErrorKind}; +use crate::providers::traits::{ + LLMProvider, LLMResponse, ProviderError, StreamErrorKind, ToolCallRequest, +}; +use crate::tools::call_context::{TurnProcessControl, TurnProcessOwner}; use crate::tools::policy::ResolvedToolPolicy; use crate::tools::registry::ToolRegistry; +use crate::tools::traits::{CallContext, Tool, ToolError}; use crate::turn_executor::{ execute_turn, set_test_backoff_override_ms, TurnConfig, TurnEventHandler, }; +use tokio_util::sync::CancellationToken; // ============================================ // Mock Provider @@ -104,6 +109,262 @@ impl LLMProvider for MockRetryProvider { } } +struct PrematureFinalProvider { + calls: AtomicU32, + owner: TurnProcessOwner, + handle: String, +} + +struct NeverConvergesProvider { + calls: AtomicU32, + owner: TurnProcessOwner, + handle: String, +} + +struct ProviderErrorAfterSpawn { + owner: TurnProcessOwner, + handle: String, +} + +struct TerminalTaskBeforeJobProvider { + calls: AtomicU32, + owner: TurnProcessOwner, + handle: String, +} + +#[async_trait] +impl LLMProvider for TerminalTaskBeforeJobProvider { + async fn chat( + &self, + _messages: &[Value], + _tools: Option<&[Value]>, + _model: &str, + _max_tokens: u32, + _temperature: f32, + ) -> Result { + let call = self.calls.fetch_add(1, Ordering::SeqCst); + if call == 0 { + let _ = crate::tools::impls::coding::exec::registry::register_owned_subagent( + self.handle.clone(), + "delegate".to_string(), + "Task Finality Worker".to_string(), + self.owner.session_id.clone(), + self.owner.clone(), + ); + crate::tools::impls::coding::exec::registry::set_join_handle( + &self.handle, + tokio::spawn(async {}), + ); + tokio::task::yield_now().await; + } else if call == 1 { + crate::tools::impls::coding::exec::registry::finish_subagent( + &self.handle, + crate::tools::impls::coding::exec::registry::JobStatus::Completed, + "terminal work result".to_string(), + ); + } + let tool_calls = (call < 3) + .then(|| ToolCallRequest { + id: format!("task-complete-{call}"), + name: crate::tools::names::TASK_UPDATE.to_string(), + arguments: serde_json::json!({ + "operation": "complete", + "id": "owned-task", + "output": { "summary": "done" } + }), + thought_signature: None, + }) + .into_iter() + .collect(); + Ok(LLMResponse { + content: Some( + if call < 3 { + "trying to complete the Task" + } else { + "final after Task completion" + } + .to_string(), + ), + tool_calls, + finish_reason: finish_reason::STOP.to_string(), + usage: HashMap::new(), + reasoning_content: None, + blocks: Vec::new(), + stream_error_kind: None, + retry_after_ms: None, + }) + } + + fn default_model(&self) -> &str { + "terminal-task-before-job" + } + + fn provider_name(&self) -> &str { + "terminal-task-before-job" + } +} + +struct RecordingTaskUpdateTool { + executions: Arc, +} + +#[async_trait] +impl Tool for RecordingTaskUpdateTool { + fn name(&self) -> &str { + crate::tools::names::TASK_UPDATE + } + + fn description(&self) -> &str { + "test Task terminal mutation" + } + + fn parameters(&self) -> Value { + serde_json::json!({ "type": "object" }) + } + + async fn execute_text(&self, _params: Value, _ctx: &CallContext) -> Result { + self.executions.fetch_add(1, Ordering::SeqCst); + Ok("Task completed".to_string()) + } +} + +#[async_trait] +impl LLMProvider for ProviderErrorAfterSpawn { + async fn chat( + &self, + _messages: &[Value], + _tools: Option<&[Value]>, + _model: &str, + _max_tokens: u32, + _temperature: f32, + ) -> Result { + let _ = crate::tools::impls::coding::exec::registry::register_owned_subagent( + self.handle.clone(), + "delegate".to_string(), + "Provider Error Worker".to_string(), + self.owner.session_id.clone(), + self.owner.clone(), + ); + crate::tools::impls::coding::exec::registry::set_join_handle( + &self.handle, + tokio::spawn(std::future::pending::<()>()), + ); + Err(ProviderError::Other( + "provider failed after spawn".to_string(), + )) + } + + fn default_model(&self) -> &str { + "provider-error-after-spawn" + } + + fn provider_name(&self) -> &str { + "provider-error-after-spawn" + } +} + +#[async_trait] +impl LLMProvider for NeverConvergesProvider { + async fn chat( + &self, + _messages: &[Value], + _tools: Option<&[Value]>, + _model: &str, + _max_tokens: u32, + _temperature: f32, + ) -> Result { + if self.calls.fetch_add(1, Ordering::SeqCst) == 0 { + let _ = crate::tools::impls::coding::exec::registry::register_owned_subagent( + self.handle.clone(), + "delegate".to_string(), + "Never Converges".to_string(), + self.owner.session_id.clone(), + self.owner.clone(), + ); + crate::tools::impls::coding::exec::registry::set_join_handle( + &self.handle, + tokio::spawn(std::future::pending::<()>()), + ); + } + Ok(LLMResponse { + content: Some("done despite running work".to_string()), + tool_calls: Vec::new(), + finish_reason: finish_reason::STOP.to_string(), + usage: HashMap::new(), + reasoning_content: None, + blocks: Vec::new(), + stream_error_kind: None, + retry_after_ms: None, + }) + } + + fn default_model(&self) -> &str { + "never-converges" + } + + fn provider_name(&self) -> &str { + "never-converges" + } +} + +#[async_trait] +impl LLMProvider for PrematureFinalProvider { + async fn chat( + &self, + _messages: &[Value], + _tools: Option<&[Value]>, + _model: &str, + _max_tokens: u32, + _temperature: f32, + ) -> Result { + let call = self.calls.fetch_add(1, Ordering::SeqCst); + let content = match call { + 0 => { + let _ = crate::tools::impls::coding::exec::registry::register_owned_subagent( + self.handle.clone(), + "delegate".to_string(), + "Convergence Worker".to_string(), + self.owner.session_id.clone(), + self.owner.clone(), + ); + crate::tools::impls::coding::exec::registry::set_join_handle( + &self.handle, + tokio::spawn(async {}), + ); + tokio::task::yield_now().await; + "premature final while the worker is running" + } + 1 => { + crate::tools::impls::coding::exec::registry::finish_subagent( + &self.handle, + crate::tools::impls::coding::exec::registry::JobStatus::Completed, + "worker terminal result".to_string(), + ); + "premature final before consuming the worker result" + } + _ => "final after the owned result was consumed", + }; + Ok(LLMResponse { + content: Some(content.to_string()), + tool_calls: Vec::new(), + finish_reason: finish_reason::STOP.to_string(), + usage: HashMap::new(), + reasoning_content: None, + blocks: Vec::new(), + stream_error_kind: None, + retry_after_ms: None, + }) + } + + fn default_model(&self) -> &str { + "premature-final" + } + + fn provider_name(&self) -> &str { + "premature-final" + } +} + // ============================================ // Mock Event Handler // ============================================ @@ -210,6 +471,7 @@ fn test_config() -> TurnConfig { TurnConfig { turn_intent_id: String::new(), projected_inbox_ids: Vec::new(), + turn_process_control: None, model: "mock-model".to_string(), account_id: None, context_window_override: None, @@ -225,6 +487,237 @@ fn test_config() -> TurnConfig { } } +#[tokio::test] +async fn owned_background_result_converges_inside_the_same_turn() { + let owner = TurnProcessOwner { + session_id: "owned-finality-turn-session".to_string(), + turn_intent_id: "owned-finality-turn-intent".to_string(), + runtime_lease_id: "owned-finality-runtime".to_string(), + dialog_turn_generation: "owned-finality-generation".to_string(), + }; + let handle = "agent-owned-finality-turn-worker".to_string(); + let provider = PrematureFinalProvider { + calls: AtomicU32::new(0), + owner: owner.clone(), + handle: handle.clone(), + }; + let handler = MockRetryHandler::new(); + let tools = ToolRegistry::new(); + let policy = empty_policy(); + let mut config = test_config(); + config.turn_intent_id = owner.turn_intent_id.clone(); + config.turn_process_control = Some(TurnProcessControl { + owner: owner.clone(), + background_cancel: CancellationToken::new(), + require_owned_job_finality: true, + }); + let mut messages = vec![serde_json::json!({ + "role": "user", + "content": "finish only after your worker" + })]; + + let result = execute_turn( + &mut messages, + &provider, + &tools, + &policy, + &config, + &owner.session_id, + &handler, + None, + None, + None, + ) + .await + .expect("same-Turn convergence should succeed"); + + assert_eq!( + result.content.as_deref(), + Some("final after the owned result was consumed") + ); + assert_eq!(provider.calls.load(Ordering::SeqCst), 3); + assert!(messages.iter().any(|message| { + message + .get("content") + .and_then(Value::as_str) + .is_some_and(|text| text.contains("worker terminal result")) + })); + assert!(crate::tools::impls::coding::exec::registry::list_jobs_for_owner(&owner).is_empty()); + assert!(crate::tools::impls::coding::exec::registry::get_status(&handle).is_none()); + assert!( + !crate::tools::impls::coding::exec::registry::claim_completion_wake_for_session( + &owner.session_id + ), + "consumed Agent Org result must not schedule a later generic wake" + ); +} + +#[tokio::test] +async fn task_terminal_mutation_waits_for_owned_job_result_consumption() { + let owner = TurnProcessOwner { + session_id: "owned-task-finality-session".to_string(), + turn_intent_id: "owned-task-finality-intent".to_string(), + runtime_lease_id: "owned-task-finality-runtime".to_string(), + dialog_turn_generation: "owned-task-finality-generation".to_string(), + }; + let handle = "agent-owned-task-finality-worker".to_string(); + let provider = TerminalTaskBeforeJobProvider { + calls: AtomicU32::new(0), + owner: owner.clone(), + handle, + }; + let executions = Arc::new(AtomicU32::new(0)); + let mut tools = ToolRegistry::new(); + tools.register(Box::new(RecordingTaskUpdateTool { + executions: Arc::clone(&executions), + })); + let mut config = test_config(); + config.turn_intent_id = owner.turn_intent_id.clone(); + config.turn_process_control = Some(TurnProcessControl { + owner: owner.clone(), + background_cancel: CancellationToken::new(), + require_owned_job_finality: true, + }); + let mut messages = vec![serde_json::json!({ + "role": "user", + "content": "complete only after owned work converges" + })]; + + let result = execute_turn( + &mut messages, + &provider, + &tools, + &empty_policy(), + &config, + &owner.session_id, + &MockRetryHandler::new(), + None, + None, + None, + ) + .await + .expect("Task and Turn should converge in order"); + + assert_eq!( + result.content.as_deref(), + Some("final after Task completion") + ); + assert_eq!(provider.calls.load(Ordering::SeqCst), 4); + assert_eq!( + executions.load(Ordering::SeqCst), + 1, + "task_update must execute only after exact-owned result consumption" + ); + assert!(messages.iter().any(|message| { + message + .get("content") + .and_then(Value::as_str) + .is_some_and(|text| text.contains("cannot become terminal")) + })); + assert!(crate::tools::impls::coding::exec::registry::list_jobs_for_owner(&owner).is_empty()); +} + +#[tokio::test] +async fn unconverged_owned_job_is_cancelled_and_the_turn_fails_closed() { + let owner = TurnProcessOwner { + session_id: "owned-finality-failure-session".to_string(), + turn_intent_id: "owned-finality-failure-intent".to_string(), + runtime_lease_id: "owned-finality-failure-runtime".to_string(), + dialog_turn_generation: "owned-finality-failure-generation".to_string(), + }; + let handle = "agent-owned-finality-never-converges".to_string(); + let provider = NeverConvergesProvider { + calls: AtomicU32::new(0), + owner: owner.clone(), + handle: handle.clone(), + }; + let handler = MockRetryHandler::new(); + let tools = ToolRegistry::new(); + let policy = empty_policy(); + let mut config = test_config(); + config.max_iterations = Some(2); + config.turn_intent_id = owner.turn_intent_id.clone(); + config.turn_process_control = Some(TurnProcessControl { + owner: owner.clone(), + background_cancel: CancellationToken::new(), + require_owned_job_finality: true, + }); + let mut messages = vec![serde_json::json!({ + "role": "user", + "content": "do not finish early" + })]; + + let error = execute_turn( + &mut messages, + &provider, + &tools, + &policy, + &config, + &owner.session_id, + &handler, + None, + None, + None, + ) + .await + .err() + .expect("unconverged background work must fail the Turn"); + + assert!( + error.contains("before its background work converged"), + "{error}" + ); + assert_eq!(provider.calls.load(Ordering::SeqCst), 2); + assert!(crate::tools::impls::coding::exec::registry::list_jobs_for_owner(&owner).is_empty()); + assert!(crate::tools::impls::coding::exec::registry::get_status(&handle).is_none()); +} + +#[tokio::test] +async fn provider_error_still_tears_down_the_exact_owned_job() { + let owner = TurnProcessOwner { + session_id: "owned-provider-error-session".to_string(), + turn_intent_id: "owned-provider-error-intent".to_string(), + runtime_lease_id: "owned-provider-error-runtime".to_string(), + dialog_turn_generation: "owned-provider-error-generation".to_string(), + }; + let handle = "agent-owned-provider-error-worker".to_string(); + let provider = ProviderErrorAfterSpawn { + owner: owner.clone(), + handle: handle.clone(), + }; + let mut config = test_config(); + config.turn_intent_id = owner.turn_intent_id.clone(); + config.turn_process_control = Some(TurnProcessControl { + owner: owner.clone(), + background_cancel: CancellationToken::new(), + require_owned_job_finality: true, + }); + let mut messages = vec![serde_json::json!({ + "role": "user", + "content": "fail only after tearing down owned work" + })]; + + let error = execute_turn( + &mut messages, + &provider, + &ToolRegistry::new(), + &empty_policy(), + &config, + &owner.session_id, + &MockRetryHandler::new(), + None, + None, + None, + ) + .await + .err() + .expect("Provider failure must remain a failed Turn"); + + assert!(error.contains("provider failed after spawn"), "{error}"); + assert!(crate::tools::impls::coding::exec::registry::list_jobs_for_owner(&owner).is_empty()); + assert!(crate::tools::impls::coding::exec::registry::get_status(&handle).is_none()); +} + // ============================================ // Tests // ============================================ diff --git a/src-tauri/src/api/agent/mod.rs b/src-tauri/src/api/agent/mod.rs index da14556d58..fd4bc141dd 100644 --- a/src-tauri/src/api/agent/mod.rs +++ b/src-tauri/src/api/agent/mod.rs @@ -751,6 +751,10 @@ pub fn create_routes() -> Router { "/test/agent-org/run/resume", post(test::agent_org::test_agent_org_resume_run), ) + .route( + "/test/agent-org/pause/evidence", + post(test::agent_org::test_agent_org_pause_evidence), + ) .route( "/test/agent-org/simulate-app-restart", post(test::agent_org::test_agent_org_simulate_app_restart), diff --git a/src-tauri/src/api/agent/test/agent_org.rs b/src-tauri/src/api/agent/test/agent_org.rs index 70bbc604b5..0406a1dbd7 100644 --- a/src-tauri/src/api/agent/test/agent_org.rs +++ b/src-tauri/src/api/agent/test/agent_org.rs @@ -410,7 +410,7 @@ pub async fn test_agent_org_launch_coordinator( "error": "Session not found after sync init", })); }; - if let Some(runtime) = session_arc.runtime.read().await.clone() { + if let Some(runtime) = session_arc.get_runtime().await { runtime_tool_names = runtime.tool_registry.tool_names(); runtime_tool_names.sort(); } @@ -2898,8 +2898,9 @@ pub async fn test_agent_org_seed_cli_member_run( /// `POST /test/agent-org/run/pause` /// -/// Transitions the named run `running → paused`. Seed-only path for E2E -/// tests that verify pause/resume semantics without a live coordinator. +/// Commits the same durable Pause fence used by the product command. This +/// seed-only path does not drive runtime teardown; rendered E2E must use the +/// real product button for Pause itself. /// Body: `{ "org_run_id": "" }` pub async fn test_agent_org_pause_run( Json(body): Json, @@ -2917,8 +2918,11 @@ pub async fn test_agent_org_pause_run( } }; - let result = - tokio::task::spawn_blocking(move || AgentOrgRunStore::mark_paused(&org_run_id)).await; + let request_id = uuid::Uuid::new_v4().to_string(); + let result = tokio::task::spawn_blocking(move || { + agent_core::coordination::agent_org_pause::pause_run(&org_run_id, &request_id) + }) + .await; match result { Err(join_err) => Json(serde_json::json!({ @@ -2926,9 +2930,7 @@ pub async fn test_agent_org_pause_run( "error": format!("spawn_blocking join error: {join_err}"), })), Ok(Err(err)) => Json(serde_json::json!({ "ok": false, "error": err })), - Ok(Ok(transitioned)) => { - Json(serde_json::json!({ "ok": true, "transitioned": transitioned })) - } + Ok(Ok(outcome)) => Json(serde_json::json!({ "ok": true, "outcome": outcome })), } } @@ -3013,8 +3015,8 @@ pub async fn test_agent_org_simulate_app_restart() -> Json { /// `POST /test/agent-org/run/resume` /// -/// Transitions the named run `paused → running`. Seed-only path for E2E -/// tests that verify pause/resume semantics without a live coordinator. +/// Commits the same durable Resume transaction used by the product command. +/// Rendered E2E must use the real product button for Resume itself. /// Body: `{ "org_run_id": "" }` pub async fn test_agent_org_resume_run( Json(body): Json, @@ -3032,8 +3034,11 @@ pub async fn test_agent_org_resume_run( } }; - let result = - tokio::task::spawn_blocking(move || AgentOrgRunStore::mark_resumed(&org_run_id)).await; + let request_id = uuid::Uuid::new_v4().to_string(); + let result = tokio::task::spawn_blocking(move || { + agent_core::coordination::agent_org_pause::resume_run(&org_run_id, &request_id) + }) + .await; match result { Err(join_err) => Json(serde_json::json!({ @@ -3041,8 +3046,204 @@ pub async fn test_agent_org_resume_run( "error": format!("spawn_blocking join error: {join_err}"), })), Ok(Err(err)) => Json(serde_json::json!({ "ok": false, "error": err })), - Ok(Ok(transitioned)) => { - Json(serde_json::json!({ "ok": true, "transitioned": transitioned })) + Ok(Ok(outcome)) => Json(serde_json::json!({ "ok": true, "outcome": outcome })), + } +} + +/// `POST /test/agent-org/pause/evidence` +/// +/// Read-only evidence for the rendered Pause/Resume scenario. The endpoint +/// never performs the lifecycle transition; E2E must drive the real Tauri +/// command through the product buttons. +pub async fn test_agent_org_pause_evidence( + Json(body): Json, +) -> Json { + use rusqlite::OptionalExtension; + use tauri::Manager; + + let Some(org_run_id) = body + .get("org_run_id") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.trim().is_empty()) + .map(str::to_string) + else { + return Json(serde_json::json!({ + "ok": false, + "error": "org_run_id is required (non-empty string)" + })); + }; + let query_run_id = org_run_id.clone(); + let durable = tokio::task::spawn_blocking(move || -> Result { + let conn = database::db::get_connection().map_err(|error| error.to_string())?; + let (run_status, activation_generation): (String, i64) = conn + .query_row( + "SELECT status,activation_generation FROM agent_org_runtime_runs WHERE id=?1", + [&query_run_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|error| error.to_string())?; + let episode: Option<(String, String, i64, Option)> = conn + .query_row( + "SELECT episode_id,status,pause_generation,resume_generation + FROM agent_org_runtime_pause_episodes + WHERE org_run_id=?1 ORDER BY created_at DESC LIMIT 1", + [&query_run_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?)), + ) + .optional() + .map_err(|error| error.to_string())?; + let mut handoffs = Vec::new(); + let mut session_ids = Vec::new(); + if let Some((episode_id, _, _, _)) = episode.as_ref() { + let mut statement = conn + .prepare( + "SELECT handoff.session_id,handoff.original_turn_intent_id, + handoff.turn_kind,handoff.task_id,handoff.original_intent_status, + handoff.drain_status,handoff.runtime_lease_id, + handoff.dialog_turn_generation,handoff.drain_timeout_at, + handoff.continuation_turn_intent_id,handoff.continuation_status, + handoff.skip_reason,context.member_dispatch_sequence + FROM agent_org_runtime_pause_handoffs handoff + LEFT JOIN agent_org_runtime_turn_contexts context + ON context.session_id=handoff.session_id + AND context.turn_intent_id=handoff.continuation_turn_intent_id + WHERE handoff.episode_id=?1 + ORDER BY handoff.created_at,handoff.handoff_id", + ) + .map_err(|error| error.to_string())?; + let rows = statement + .query_map([episode_id], |row| { + Ok(serde_json::json!({ + "session_id": row.get::<_, String>(0)?, + "original_turn_intent_id": row.get::<_, String>(1)?, + "turn_kind": row.get::<_, String>(2)?, + "task_id": row.get::<_, Option>(3)?, + "original_intent_status": row.get::<_, String>(4)?, + "drain_status": row.get::<_, String>(5)?, + "runtime_lease_id": row.get::<_, Option>(6)?, + "dialog_turn_generation": row.get::<_, Option>(7)?, + "drain_timeout_at": row.get::<_, Option>(8)?, + "continuation_turn_intent_id": row.get::<_, Option>(9)?, + "continuation_status": row.get::<_, Option>(10)?, + "skip_reason": row.get::<_, Option>(11)?, + "member_dispatch_sequence": row.get::<_, Option>(12)?, + })) + }) + .map_err(|error| error.to_string())?; + for row in rows { + let value = row.map_err(|error| error.to_string())?; + if let Some(session_id) = value.get("session_id").and_then(|v| v.as_str()) { + session_ids.push(session_id.to_string()); + } + handoffs.push(value); + } + } + if session_ids.is_empty() { + let mut statement = conn + .prepare( + "SELECT session_id FROM agent_org_runtime_member_materializations + WHERE org_run_id=?1 AND status='succeeded' ORDER BY member_id", + ) + .map_err(|error| error.to_string())?; + session_ids = statement + .query_map([&query_run_id], |row| row.get::<_, String>(0)) + .map_err(|error| error.to_string())? + .collect::, _>>() + .map_err(|error| error.to_string())?; + } + session_ids.sort(); + session_ids.dedup(); + let mut task_statement = conn + .prepare( + "SELECT id,status,owner,updated_at FROM agent_org_runtime_tasks + WHERE org_run_id=?1 ORDER BY id", + ) + .map_err(|error| error.to_string())?; + let tasks = task_statement + .query_map([&query_run_id], |row| { + Ok(serde_json::json!({ + "id": row.get::<_, String>(0)?, + "status": row.get::<_, String>(1)?, + "owner": row.get::<_, Option>(2)?, + "updated_at": row.get::<_, String>(3)?, + })) + }) + .map_err(|error| error.to_string())? + .collect::, _>>() + .map_err(|error| error.to_string())?; + let inbox_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_org_runtime_inbox WHERE org_run_id=?1", + [&query_run_id], + |row| row.get(0), + ) + .map_err(|error| error.to_string())?; + Ok(serde_json::json!({ + "run_status": run_status, + "activation_generation": activation_generation, + "episode": episode.map(|(episode_id,status,pause_generation,resume_generation)| serde_json::json!({ + "episode_id": episode_id, + "status": status, + "pause_generation": pause_generation, + "resume_generation": resume_generation, + })), + "handoffs": handoffs, + "tasks": tasks, + "inbox_count": inbox_count, + "session_ids": session_ids, + })) + }) + .await; + let durable = match durable { + Err(error) => { + return Json(serde_json::json!({ + "ok": false, + "error": format!("spawn_blocking join error: {error}") + })) + } + Ok(Err(error)) => return Json(serde_json::json!({ "ok": false, "error": error })), + Ok(Ok(value)) => value, + }; + let Some(handle) = crate::api::get_app_handle() else { + return Json(serde_json::json!({ "ok": false, "error": "AppHandle not initialized" })); + }; + let state = handle.state::(); + let session_ids = durable + .get("session_ids") + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap_or_default(); + let mut active_runtime_count = 0usize; + let mut active_turns = Vec::new(); + let mut background_shells = Vec::new(); + for session_id in session_ids.iter().filter_map(serde_json::Value::as_str) { + for (pid, command) in + agent_core::tools::impls::coding::exec::registry::list_shell_for_session(session_id) + { + background_shells.push(serde_json::json!({ + "session_id": session_id, + "pid": pid, + "command": command, + })); + } + let Some(session) = state.get_session(session_id).await else { + continue; + }; + if session.get_runtime().await.is_some() { + active_runtime_count += 1; + if let Some(dialog_turn_generation) = session.active_turn_id().await { + active_turns.push(serde_json::json!({ + "session_id": session_id, + "dialog_turn_generation": dialog_turn_generation, + })); + } } } + Json(serde_json::json!({ + "ok": true, + "durable": durable, + "active_runtime_count": active_runtime_count, + "active_turns": active_turns, + "background_shells": background_shells, + })) } diff --git a/src-tauri/src/api/agent/test/sde.rs b/src-tauri/src/api/agent/test/sde.rs index 90c68e512b..d96c7b770b 100644 --- a/src-tauri/src/api/agent/test/sde.rs +++ b/src-tauri/src/api/agent/test/sde.rs @@ -782,7 +782,7 @@ pub async fn test_sde_mode_switch_seed( "error": format!("No session found: {}", session_id), })); }; - if session.runtime.read().await.is_none() { + if session.get_runtime().await.is_none() { let workspace_path = request .workspace_path .as_deref() diff --git a/src-tauri/src/api/agent/test/workspace.rs b/src-tauri/src/api/agent/test/workspace.rs index 7b3152d52a..780a419f58 100644 --- a/src-tauri/src/api/agent/test/workspace.rs +++ b/src-tauri/src/api/agent/test/workspace.rs @@ -474,7 +474,7 @@ pub async fn test_session_prompt_environment_block( "error": "session not found", })); }; - let runtime = match session.runtime.read().await.clone() { + let runtime = match session.get_runtime().await { Some(r) => r, None => { return Json(serde_json::json!({ diff --git a/src/api/realtime/codeEditorWebSocket.ts b/src/api/realtime/codeEditorWebSocket.ts index 49eff4f572..38bdc86cd7 100644 --- a/src/api/realtime/codeEditorWebSocket.ts +++ b/src/api/realtime/codeEditorWebSocket.ts @@ -190,7 +190,10 @@ export function getCodeEditorWebSocket(): CodeEditorWebSocketClient | null { // Initialize on module load in the app. Unit tests import broad UI graphs in // jsdom; opening a real socket there leaks asynchronous undici events across // test files and can fail after the owning test has already completed. -if (typeof window !== "undefined" && process.env.NODE_ENV !== "test") { +if ( + typeof window !== "undefined" && + (process.env.NODE_ENV !== "test" || process.env.ORGII_E2E === "1") +) { // Auto-connect when app loads wsClientInstance = new CodeEditorWebSocketClient(); wsClientInstance.connect().catch((err) => { diff --git a/src/api/tauri/agent/orgTasks.test.ts b/src/api/tauri/agent/orgTasks.test.ts index a8a7433fde..b576cf8e4c 100644 --- a/src/api/tauri/agent/orgTasks.test.ts +++ b/src/api/tauri/agent/orgTasks.test.ts @@ -1,12 +1,25 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { invokeTauri } from "@src/util/platform/tauri/init"; import { AGENT_ORG_TASK_STATUS, agentOrgTaskStatusSatisfiesDependency, isAgentOrgTaskOpenStatus, isAgentOrgTaskTerminalStatus, + pauseAgentOrgRun, + resumeAgentOrgRun, + subscribeAgentOrgStateChanges, } from "./orgTasks"; +vi.mock("@src/util/platform/tauri/init", () => ({ invokeTauri: vi.fn() })); + +const invokeMock = vi.mocked(invokeTauri); + +afterEach(() => { + invokeMock.mockReset(); +}); + describe("Agent Org Task status semantics", () => { it.each([ [AGENT_ORG_TASK_STATUS.PENDING, true, false, false], @@ -25,3 +38,54 @@ describe("Agent Org Task status semantics", () => { } ); }); + +describe("Agent Org durable Pause/Resume wire", () => { + it("sends the caller-stable Pause request id and returns the structured receipt", async () => { + const outcome = { + requestId: "00000000-0000-4000-8000-000000000101", + runId: "run-a", + episodeId: "episode-a", + transitioned: true, + pauseGeneration: 2, + capturedTurnCount: 10, + drainingTurnCount: 10, + timedOutTurnCount: 0, + }; + invokeMock.mockResolvedValueOnce(outcome); + const changes: string[] = []; + const unsubscribe = subscribeAgentOrgStateChanges((sessionId) => + changes.push(sessionId) + ); + + await expect( + pauseAgentOrgRun("root-session", outcome.requestId) + ).resolves.toEqual(outcome); + expect(invokeMock).toHaveBeenCalledWith("agent_org_pause_run", { + sessionId: "root-session", + requestId: outcome.requestId, + }); + expect(changes).toEqual(["root-session"]); + unsubscribe(); + }); + + it("sends the caller-stable Resume request id and exposes continuation counts", async () => { + const outcome = { + requestId: "00000000-0000-4000-8000-000000000102", + runId: "run-a", + episodeId: "episode-a", + transitioned: true, + resumeGeneration: 3, + continuationCount: 7, + skippedCount: 3, + }; + invokeMock.mockResolvedValueOnce(outcome); + + await expect( + resumeAgentOrgRun("root-session", outcome.requestId) + ).resolves.toEqual(outcome); + expect(invokeMock).toHaveBeenCalledWith("agent_org_resume_run", { + sessionId: "root-session", + requestId: outcome.requestId, + }); + }); +}); diff --git a/src/api/tauri/agent/orgTasks.ts b/src/api/tauri/agent/orgTasks.ts index 149733e11b..bdcc03b82d 100644 --- a/src/api/tauri/agent/orgTasks.ts +++ b/src/api/tauri/agent/orgTasks.ts @@ -120,6 +120,7 @@ export const AGENT_ORG_RUN_PHASE = { WAITING: "waiting", AWAITING_PLAN_APPROVAL: "awaiting_plan_approval", FINALIZING: "finalizing", + DRAINING: "draining", PAUSED: "paused", IDLE: "idle", FAILED: "failed", @@ -133,6 +134,7 @@ export interface AgentOrgRunView { context: AgentOrgRunContext; runStatus: AgentOrgRunStatus; runPhase: AgentOrgRunPhase; + pauseHandoff?: AgentOrgPauseHandoffSummary | null; currentMemberId?: string | null; members: AgentOrgRunMemberView[]; tasks: AgentOrgTask[]; @@ -142,6 +144,35 @@ export interface AgentOrgRunView { pendingPlanApprovals: AgentOrgPlanApprovalSummary[]; } +export interface AgentOrgPauseHandoffSummary { + episodeId: string; + pauseGeneration: number; + totalCount: number; + drainingCount: number; + timedOutCount: number; +} + +export interface PauseRunOutcome { + requestId: string; + runId: string; + episodeId: string; + transitioned: boolean; + pauseGeneration: number; + capturedTurnCount: number; + drainingTurnCount: number; + timedOutTurnCount: number; +} + +export interface ResumeRunOutcome { + requestId: string; + runId: string; + episodeId: string; + transitioned: boolean; + resumeGeneration: number; + continuationCount: number; + skippedCount: number; +} + export interface AgentOrgRunTaskOverview { total: number; pending: number; @@ -517,18 +548,26 @@ export async function sendAgentOrgUserMessageToMember( return response; } -export async function pauseAgentOrgRun(sessionId: string): Promise { - const changed = await invokeTauri("agent_org_pause_run", { +export async function pauseAgentOrgRun( + sessionId: string, + requestId: string = crypto.randomUUID() +): Promise { + const outcome = await invokeTauri("agent_org_pause_run", { sessionId, + requestId, }); - if (changed) publishAgentOrgStateChange(sessionId); - return changed; + publishAgentOrgStateChange(sessionId); + return outcome; } -export async function resumeAgentOrgRun(sessionId: string): Promise { - const changed = await invokeTauri("agent_org_resume_run", { +export async function resumeAgentOrgRun( + sessionId: string, + requestId: string = crypto.randomUUID() +): Promise { + const outcome = await invokeTauri("agent_org_resume_run", { sessionId, + requestId, }); - if (changed) publishAgentOrgStateChange(sessionId); - return changed; + publishAgentOrgStateChange(sessionId); + return outcome; } diff --git a/src/app/root/e2e/helpers/agentOrgs.ts b/src/app/root/e2e/helpers/agentOrgs.ts index 1f5023d470..94a8c1046c 100644 --- a/src/app/root/e2e/helpers/agentOrgs.ts +++ b/src/app/root/e2e/helpers/agentOrgs.ts @@ -461,8 +461,9 @@ export function createAgentOrgHelpers(): AgentOrgE2EHelpers { }; const agentOrgPauseRun = async ( - sessionId: string - ): Promise> => { + sessionId: string, + requestId: string = crypto.randomUUID() + ): Promise> => { try { if (!sessionId) { return { @@ -470,18 +471,20 @@ export function createAgentOrgHelpers(): AgentOrgE2EHelpers { error: "agentOrgPauseRun: `sessionId` is required", }; } - const transitioned = (await invoke("agent_org_pause_run", { + const outcome = (await invoke("agent_org_pause_run", { sessionId, - })) as boolean; - return { ok: true, transitioned }; + requestId, + })) as Json; + return { ok: true, outcome }; } catch (err) { return asError(err); } }; const agentOrgResumeRun = async ( - sessionId: string - ): Promise> => { + sessionId: string, + requestId: string = crypto.randomUUID() + ): Promise> => { try { if (!sessionId) { return { @@ -489,10 +492,11 @@ export function createAgentOrgHelpers(): AgentOrgE2EHelpers { error: "agentOrgResumeRun: `sessionId` is required", }; } - const transitioned = (await invoke("agent_org_resume_run", { + const outcome = (await invoke("agent_org_resume_run", { sessionId, - })) as boolean; - return { ok: true, transitioned }; + requestId, + })) as Json; + return { ok: true, outcome }; } catch (err) { return asError(err); } diff --git a/src/app/root/e2e/types.ts b/src/app/root/e2e/types.ts index c25031fc21..ad17a7e540 100644 --- a/src/app/root/e2e/types.ts +++ b/src/app/root/e2e/types.ts @@ -331,11 +331,13 @@ export interface E2EHelpers { content: string ) => Promise>; agentOrgPauseRun: ( - sessionId: string - ) => Promise>; + sessionId: string, + requestId?: string + ) => Promise>; agentOrgResumeRun: ( - sessionId: string - ) => Promise>; + sessionId: string, + requestId?: string + ) => Promise>; agentOrgSimulateAppRestart: () => Promise< Result<{ intentsReconciled: number; diff --git a/src/config/ideServer.ts b/src/config/ideServer.ts index f319d9215c..f42054f5db 100644 --- a/src/config/ideServer.ts +++ b/src/config/ideServer.ts @@ -14,10 +14,20 @@ export let IDE_SERVER_HTTP_URL = `http://localhost:${IDE_SERVER_PORT}`; export let IDE_SERVER_WS_URL = `ws://localhost:${IDE_SERVER_PORT}/ws`; +function exposeIdeServerUrlForE2E(): void { + if (typeof window === "undefined" || process.env.ORGII_E2E !== "1") return; + ( + window as unknown as { __ORGII_E2E_IDE_SERVER_WS_URL__: string } + ).__ORGII_E2E_IDE_SERVER_WS_URL__ = IDE_SERVER_WS_URL; +} + +exposeIdeServerUrlForE2E(); + export function configureIdeServerForIdentifier(identifier: string): number { const { ideServerPort } = runtimeInstanceProfileForIdentifier(identifier); IDE_SERVER_PORT = String(ideServerPort); IDE_SERVER_HTTP_URL = `http://localhost:${IDE_SERVER_PORT}`; IDE_SERVER_WS_URL = `ws://localhost:${IDE_SERVER_PORT}/ws`; + exposeIdeServerUrlForE2E(); return ideServerPort; } diff --git a/src/engines/ChatPanel/ChatFloatingComposer.tsx b/src/engines/ChatPanel/ChatFloatingComposer.tsx index c4da0d731e..25374c0bf9 100644 --- a/src/engines/ChatPanel/ChatFloatingComposer.tsx +++ b/src/engines/ChatPanel/ChatFloatingComposer.tsx @@ -108,6 +108,7 @@ interface ChatFloatingComposerProps { customMentionOptions: ReadonlyArray; queueEditProps: QueueEditInputAreaProps; disableStopWhenEmpty?: boolean; + submitDisabled?: boolean; } const ChatFloatingComposer: React.FC = memo( @@ -158,6 +159,7 @@ const ChatFloatingComposer: React.FC = memo( customMentionOptions, queueEditProps, disableStopWhenEmpty = false, + submitDisabled = false, }) => { const { t } = useTranslation("sessions"); const [fileChangeStats, setFileChangeStatsState] = @@ -318,6 +320,7 @@ const ChatFloatingComposer: React.FC = memo( sessionId={inputAreaSessionId} onSubmitOverride={onSubmitOverride} customMentionOptions={customMentionOptions} + submitDisabled={submitDisabled} topRowPills={ showTopRowPills ? ( = memo( customMentionOptions={groupChatMentionOptions} queueEditProps={queueEditProps} disableStopWhenEmpty={groupChatViewActive} + submitDisabled={ + groupChatViewActive && agentOrgRunView?.runStatus === "paused" + } /> )} diff --git a/src/engines/ChatPanel/ChatViewHistorySurface.tsx b/src/engines/ChatPanel/ChatViewHistorySurface.tsx index 4929095ac0..67e78a783f 100644 --- a/src/engines/ChatPanel/ChatViewHistorySurface.tsx +++ b/src/engines/ChatPanel/ChatViewHistorySurface.tsx @@ -131,7 +131,11 @@ export function ChatViewHistorySurface({ ) : null diff --git a/src/engines/ChatPanel/InputArea/components/AgentOrgOverviewPanel.tsx b/src/engines/ChatPanel/InputArea/components/AgentOrgOverviewPanel.tsx index 5ca5450172..4351f7501c 100644 --- a/src/engines/ChatPanel/InputArea/components/AgentOrgOverviewPanel.tsx +++ b/src/engines/ChatPanel/InputArea/components/AgentOrgOverviewPanel.tsx @@ -38,6 +38,11 @@ import ComposerStackHeader, { const logger = createLogger("AgentOrgOverviewPanel"); +// Keep the opposite control disabled briefly after Pause/Resume settles. The +// two controls occupy the same toolbar position, so the second click of a +// double-click can otherwise land on the newly rendered inverse action. +const PAUSE_TOGGLE_GESTURE_COOLDOWN_MS = 500; + const AGENT_SESSION_STATUS = { RUNNING: "running", WAITING_FOR_USER: "waiting_for_user", @@ -65,6 +70,11 @@ const AgentOrgOverviewPanel: React.FC = memo( ); const [historyLoading, setHistoryLoading] = useState(false); const [historyError, setHistoryError] = useState(false); + const pauseToggleLockedRef = useRef(false); + const pauseToggleCooldownRef = useRef | null>( + null + ); + const mountedRef = useRef(true); const historyRequestIdRef = useRef(0); const currentSessionIdRef = useRef(currentSessionId); const currentRunId = view?.context.runId ?? null; @@ -72,6 +82,17 @@ const AgentOrgOverviewPanel: React.FC = memo( currentSessionIdRef.current = currentSessionId; currentRunIdRef.current = currentRunId; + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + if (pauseToggleCooldownRef.current !== null) { + clearTimeout(pauseToggleCooldownRef.current); + pauseToggleCooldownRef.current = null; + } + }; + }, []); + useEffect(() => { historyRequestIdRef.current += 1; setHistoryExpanded(false); @@ -181,31 +202,48 @@ const AgentOrgOverviewPanel: React.FC = memo( } }, [rootSessionId, setActiveSessionId]); - const handlePauseRun = useCallback(async () => { - if (!currentSessionId || isTogglingPause) return; + const beginPauseToggle = useCallback(() => { + if (pauseToggleLockedRef.current) return false; + pauseToggleLockedRef.current = true; setIsTogglingPause(true); + return true; + }, []); + + const finishPauseToggle = useCallback(() => { + if (!mountedRef.current) { + pauseToggleLockedRef.current = false; + return; + } + pauseToggleCooldownRef.current = setTimeout(() => { + pauseToggleCooldownRef.current = null; + pauseToggleLockedRef.current = false; + setIsTogglingPause(false); + }, PAUSE_TOGGLE_GESTURE_COOLDOWN_MS); + }, []); + + const handlePauseRun = useCallback(async () => { + if (!currentSessionId || !beginPauseToggle()) return; try { await pauseAgentOrgRun(currentSessionId); await onRefresh(); } catch (err: unknown) { logger.error("Failed to pause Agent Team run:", err); } finally { - setIsTogglingPause(false); + finishPauseToggle(); } - }, [currentSessionId, isTogglingPause, onRefresh]); + }, [beginPauseToggle, currentSessionId, finishPauseToggle, onRefresh]); const handleResumeRun = useCallback(async () => { - if (!currentSessionId || isTogglingPause) return; - setIsTogglingPause(true); + if (!currentSessionId || !beginPauseToggle()) return; try { await resumeAgentOrgRun(currentSessionId); await onRefresh(); } catch (err: unknown) { logger.error("Failed to resume Agent Team run:", err); } finally { - setIsTogglingPause(false); + finishPauseToggle(); } - }, [currentSessionId, isTogglingPause, onRefresh]); + }, [beginPauseToggle, currentSessionId, finishPauseToggle, onRefresh]); if (!view && !error) return null; @@ -240,7 +278,8 @@ const AgentOrgOverviewPanel: React.FC = memo( data-testid="agent-org-overview-run-phase" data-run-phase={view?.runPhase ?? ""} > - {view?.runPhase === AGENT_ORG_RUN_PHASE.FINALIZING && ( + {(view?.runPhase === AGENT_ORG_RUN_PHASE.FINALIZING || + view?.runPhase === AGENT_ORG_RUN_PHASE.DRAINING) && ( ({ AGENT_ORG_RUN_PHASE: { COORDINATING: "coordinating", FINALIZING: "finalizing", + DRAINING: "draining", }, AGENT_ORG_TASK_STATUS: { PENDING: "pending", @@ -190,6 +191,8 @@ describe("Agent Org Task panel", () => { mocks.getPage.mockReset(); mocks.getDetail.mockReset(); mocks.getAnnotations.mockReset(); + mocks.pause.mockReset(); + mocks.resume.mockReset(); }); afterEach(() => { @@ -276,6 +279,117 @@ describe("Agent Org Task panel", () => { }); }); + it("shows Paused draining immediately and keeps Resume enabled", async () => { + mocks.resume.mockResolvedValue({ + requestId: "resume-request", + runId: "run-task-panel", + episodeId: "episode-a", + transitioned: true, + resumeGeneration: 3, + continuationCount: 2, + skippedCount: 0, + }); + const onRefresh = vi.fn().mockResolvedValue(undefined); + const view: AgentOrgRunView = { + ...runView(), + runStatus: "paused", + runPhase: "draining", + pauseHandoff: { + episodeId: "episode-a", + pauseGeneration: 2, + totalCount: 2, + drainingCount: 2, + timedOutCount: 0, + }, + }; + await act(async () => { + root.render( + createElement(AgentOrgOverviewPanel, { + view, + error: null, + currentSessionId: "root-session", + onRefresh, + }) + ); + }); + const phase = container.querySelector( + '[data-testid="agent-org-overview-run-phase"]' + ); + expect(phase?.dataset.runPhase).toBe("draining"); + expect(phase?.textContent).toContain( + "planner.agentOrgOverview.phase.draining" + ); + const resume = container.querySelector( + '[data-testid="agent-org-overview-resume-button"]' + ); + expect(resume?.disabled).toBe(false); + await act(async () => { + resume?.click(); + }); + expect(mocks.resume).toHaveBeenCalledWith("root-session"); + expect(onRefresh).toHaveBeenCalledTimes(1); + }); + + it("keeps the inverse Pause control locked through a Resume double-click", async () => { + vi.useFakeTimers(); + try { + mocks.resume.mockResolvedValue({ + requestId: "resume-request", + runId: "run-task-panel", + episodeId: "episode-a", + transitioned: true, + resumeGeneration: 3, + continuationCount: 2, + skippedCount: 0, + }); + mocks.pause.mockResolvedValue({ + requestId: "pause-request", + runId: "run-task-panel", + episodeId: "episode-b", + transitioned: true, + pauseGeneration: 4, + capturedCount: 0, + }); + const onRefresh = vi.fn().mockResolvedValue(undefined); + const render = (runStatus: AgentOrgRunView["runStatus"]) => + root.render( + createElement(AgentOrgOverviewPanel, { + view: { ...runView(), runStatus }, + error: null, + currentSessionId: "root-session", + onRefresh, + }) + ); + + await act(async () => render("paused")); + await act(async () => { + container + .querySelector( + '[data-testid="agent-org-overview-resume-button"]' + ) + ?.click(); + await Promise.resolve(); + await Promise.resolve(); + }); + expect(mocks.resume).toHaveBeenCalledTimes(1); + + await act(async () => render("running")); + const pause = container.querySelector( + '[data-testid="agent-org-overview-pause-button"]' + ); + expect(pause?.disabled).toBe(true); + pause?.click(); + expect(mocks.pause).not.toHaveBeenCalled(); + + await act(async () => { + await vi.advanceTimersByTimeAsync(500); + }); + expect(pause?.disabled).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + it("discards a late History response after switching teams", async () => { const oldPage = deferred<{ bucket: "history"; diff --git a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts index 7b54de26d9..59d4fc2674 100644 --- a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts +++ b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.test.ts @@ -49,6 +49,7 @@ function runView( rootSessionId: "session-root", }, runStatus, + runPhase: runStatus === "running" ? "coordinating" : runStatus, currentMemberId: "coordinator", members: [ { @@ -225,6 +226,25 @@ describe("Agent Org run-view store", () => { unsubscribe(); }); + it("performs one bounded follow-up when bootstrap first observes Starting", async () => { + mocks.subscribeAgentOrgStateChanges.mockReturnValue( + mocks.unsubscribeStateChanges + ); + mocks.getAgentOrgSessionRunView + .mockResolvedValueOnce(runView("starting")) + .mockResolvedValueOnce(runView("running")); + + const unsubscribe = subscribeAgentOrgRunView("session-root", vi.fn()); + await flushPromises(); + await flushPromises(); + + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(2); + expect(getAgentOrgRunViewSnapshot("session-root").view?.runStatus).toBe( + "running" + ); + unsubscribe(); + }); + it.each(["paused", "idle", "failed", "archived"] as const)( "does not retain a fallback interval when the initial Team is %s", async (status) => { @@ -244,6 +264,81 @@ describe("Agent Org run-view store", () => { } ); + it("does not poll a Working Team whose derived phase is Idle", async () => { + vi.useFakeTimers(); + mocks.subscribeAgentOrgStateChanges.mockReturnValue( + mocks.unsubscribeStateChanges + ); + const idleWorkingView = runView("running"); + idleWorkingView.runPhase = "idle"; + mocks.getAgentOrgSessionRunView.mockResolvedValue(idleWorkingView); + + const unsubscribe = subscribeAgentOrgRunView("session-root", vi.fn()); + await flushPromises(); + + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(false); + await vi.advanceTimersByTimeAsync(AGENT_ORG_RUN_VIEW_FALLBACK_MS * 5); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(1); + unsubscribe(); + }); + + it("reconciles a Paused Run once on WebSocket reconnect without starting polling", async () => { + vi.useFakeTimers(); + let connectedHandler: (() => void) | undefined; + mocks.subscribeAgentOrgStateChanges.mockReturnValue( + mocks.unsubscribeStateChanges + ); + mocks.websocketOn.mockImplementation( + (event: string, handler: () => void) => { + if (event === "connected") connectedHandler = handler; + return mocks.unsubscribeBackendChanges; + } + ); + const draining = runView("paused") as ReturnType & { + pauseHandoff?: { + episodeId: string; + pauseGeneration: number; + totalCount: number; + drainingCount: number; + timedOutCount: number; + }; + }; + draining.pauseHandoff = { + episodeId: "episode-1", + pauseGeneration: 2, + totalCount: 2, + drainingCount: 2, + timedOutCount: 0, + }; + const released = structuredClone(draining); + released.pauseHandoff!.drainingCount = 0; + mocks.getAgentOrgSessionRunView + .mockResolvedValueOnce(draining) + .mockResolvedValueOnce(released); + + const unsubscribe = subscribeAgentOrgRunView("session-root", vi.fn()); + await flushPromises(); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(false); + expect( + getAgentOrgRunViewSnapshot("session-root").view?.pauseHandoff + ?.drainingCount + ).toBe(2); + + connectedHandler?.(); + await flushPromises(); + + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(2); + expect( + getAgentOrgRunViewSnapshot("session-root").view?.pauseHandoff + ?.drainingCount + ).toBe(0); + expect(agentOrgRunViewStoreTestApi.hasPollingTimer()).toBe(false); + + await vi.advanceTimersByTimeAsync(AGENT_ORG_RUN_VIEW_FALLBACK_MS * 5); + expect(mocks.getAgentOrgSessionRunView).toHaveBeenCalledTimes(2); + unsubscribe(); + }); + it("destroys the shared interval when the last pollable Team becomes Idle", async () => { vi.useFakeTimers(); mocks.subscribeAgentOrgStateChanges.mockReturnValue( diff --git a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts index b5a58f6f10..9616643d22 100644 --- a/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts +++ b/src/engines/ChatPanel/InputArea/components/agentOrgRunViewStore.ts @@ -93,7 +93,11 @@ function viewForSession( } function isPollable(view: AgentOrgRunView | null): boolean { - return view !== null && POLLABLE_RUN_STATUSES.has(view.runStatus); + return ( + view !== null && + POLLABLE_RUN_STATUSES.has(view.runStatus) && + view.runPhase !== "idle" + ); } function findEntryCoveringSession(sessionId: string): RunViewEntry | undefined { @@ -372,6 +376,14 @@ function refreshAgentOrgRunViewInternal( if (requestId < latestRequestId) return; latestRequestIdByRun.set(runId, requestId); publishRunView(view, requestId); + // The lifecycle may advance while the first discovery read is in + // flight. A run-scoped push cannot target this entry until that read + // teaches the store its run id, so perform exactly one follow-up when + // bootstrap lands on Starting. Later Starting reads rely on pushes and + // the shared slow fallback instead of creating a retry loop. + if (!knownRunId && view.runStatus === "starting") { + refreshAfterInFlight.add(`run:${runId}`); + } return; } missingRunReplacement = publishMissingRun(entry, requestId); @@ -433,14 +445,16 @@ function scheduleRunRefresh(runId: string): void { pushDebounceTimers.set(runId, timer); } -function pollActiveRuns(): void { +function refreshSubscribedRuns(pollableOnly: boolean): void { if (!isDocumentVisible()) return; const representatives = new Map(); for (const entry of entriesBySessionId.values()) { if ( entry.subscribers.size === 0 || - (entry.snapshot.view !== null && !isPollable(entry.snapshot.view)) + (pollableOnly && + entry.snapshot.view !== null && + !isPollable(entry.snapshot.view)) ) continue; if ( @@ -456,12 +470,27 @@ function pollActiveRuns(): void { } } +function pollActiveRuns(): void { + refreshSubscribedRuns(true); +} + +/** + * Push notifications are transient. Reconcile every currently observed Run + * once after a transport gap so a release/timeout event missed while the + * socket was disconnected cannot leave a durable Paused view stuck in + * Draining. This is deliberately not the fallback poll: Paused/Idle Runs get + * no interval, and representatives keep the reconnect read to one per Run. + */ +function reconcileSubscribedRunsAfterTransportGap(): void { + refreshSubscribedRuns(false); +} + function handleVisibilityChange(): void { if (!isDocumentVisible()) { reconcilePollingTimer(); return; } - pollActiveRuns(); + reconcileSubscribedRunsAfterTransportGap(); reconcilePollingTimer(); } @@ -532,7 +561,7 @@ function startScheduler(): void { if (!unsubscribeWebsocketConnected) { unsubscribeWebsocketConnected = getCodeEditorWebSocket()?.on( "connected", - pollActiveRuns + reconcileSubscribedRunsAfterTransportGap ); } if (typeof document !== "undefined" && !visibilityListenerInstalled) { diff --git a/src/engines/ChatPanel/InputArea/components/useAgentOrgMemberSessionJump.ts b/src/engines/ChatPanel/InputArea/components/useAgentOrgMemberSessionJump.ts index b6d8c7dba1..cea7d438ea 100644 --- a/src/engines/ChatPanel/InputArea/components/useAgentOrgMemberSessionJump.ts +++ b/src/engines/ChatPanel/InputArea/components/useAgentOrgMemberSessionJump.ts @@ -4,10 +4,12 @@ import { useCallback } from "react"; import type { AgentOrgRunMemberView } from "@src/api/tauri/agent"; import { CliAgentTypeSchema } from "@src/api/tauri/rpc/schemas/validation"; import { DISPATCH_CATEGORY, KEY_SOURCE } from "@src/api/tauri/session"; -import { clearSessionAtom } from "@src/engines/SessionCore/core/atoms/actions"; -import { loadStatusAtom } from "@src/engines/SessionCore/core/atoms/metadata"; import { CLI_AGENT_PREFIX } from "@src/modules/MainApp/AgentOrgs/types"; -import { activeSessionIdAtom, sessionMapAtom } from "@src/store/session"; +import { + activeSessionIdAtom, + claimPipelineSessionAtom, + sessionMapAtom, +} from "@src/store/session"; import { loadSidebarSessions, upsertSession, @@ -90,9 +92,7 @@ export function useAgentOrgMemberSessionJump(_currentSessionId: string) { : (runtime.agentDefinitionId ?? undefined), }); } - set(clearSessionAtom); - set(loadStatusAtom, "loading"); - set(activeSessionIdAtom, runtime.sessionId); + set(claimPipelineSessionAtom, runtime.sessionId); markSessionVisited(runtime.sessionId); void loadSidebarSessions({ forceRefresh: true }); }, []) diff --git a/src/engines/ChatPanel/components/ChatStatusBanners.tsx b/src/engines/ChatPanel/components/ChatStatusBanners.tsx index f0f6c06eba..39e3b8025e 100644 --- a/src/engines/ChatPanel/components/ChatStatusBanners.tsx +++ b/src/engines/ChatPanel/components/ChatStatusBanners.tsx @@ -179,8 +179,7 @@ export function GroupChatPausedBanner({ defaultValue: "New work is paused", })} description={t("groupChat.pausedBanner.body", { - defaultValue: - "Pause stops active replies, send a message or press Resume to continue", + defaultValue: "Resume this Agent Team before sending a message", })} /> ), diff --git a/src/engines/ChatPanel/hooks/useAgentOrgGroupChatController.ts b/src/engines/ChatPanel/hooks/useAgentOrgGroupChatController.ts index 82e989e74f..d250094ced 100644 --- a/src/engines/ChatPanel/hooks/useAgentOrgGroupChatController.ts +++ b/src/engines/ChatPanel/hooks/useAgentOrgGroupChatController.ts @@ -281,6 +281,9 @@ export function useAgentOrgGroupChatController({ const handleGroupChatSubmitOverride = useCallback( async (input: SubmitOverrideInput): Promise => { if (!agentOrgRunView) return false; + if (agentOrgRunView.runStatus === AGENT_ORG_RUN_STATUS.PAUSED) { + throw new Error("Resume this Agent Team before sending a message"); + } // Route on the DISPLAY copy: the `@member` header is what the user // typed and what the transcript renders. The agent copy may have been // rewritten by an interceptor (canvas contract) and must only feed the diff --git a/src/i18n/locales/de/sessions.json b/src/i18n/locales/de/sessions.json index 892ba9e701..72f3018c84 100644 --- a/src/i18n/locales/de/sessions.json +++ b/src/i18n/locales/de/sessions.json @@ -2450,7 +2450,21 @@ "memberSessions": "Mitgliedersitzungen", "pauseRun": "Ausführung pausieren", "resumeRun": "Ausführung fortsetzen", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Coordinator-Chatverlauf anzeigen", + "phase": { + "starting": "Wird gestartet", + "coordinating": "Koordination", + "dispatching": "Wird verteilt", + "members_working": "Mitglieder arbeiten", + "waiting": "Wartet auf Arbeit", + "awaiting_plan_approval": "Wartet auf Planfreigabe", + "finalizing": "Wird abgeschlossen", + "draining": "Pausiert · wird beendet", + "paused": "Pausiert", + "idle": "Inaktiv", + "failed": "Fehlgeschlagen", + "archived": "Archiviert" + } }, "agentOrgInbox": { "title": "Agent-Nachrichten", @@ -2679,7 +2693,7 @@ "userMessagePending": "{{member}} nimmt deine Nachricht auf", "pausedBanner": { "title": "Neue Arbeit ist pausiert", - "body": "Pausieren stoppt aktive Antworten, sende eine Nachricht oder klicke auf Fortsetzen, um weiterzumachen", + "body": "Setze dieses Agent-Team fort, bevor du eine Nachricht sendest", "resume": "Fortsetzen" }, "toolUseSummary": { diff --git a/src/i18n/locales/en/sessions.json b/src/i18n/locales/en/sessions.json index 63ba1b8a54..42f59ae4c4 100644 --- a/src/i18n/locales/en/sessions.json +++ b/src/i18n/locales/en/sessions.json @@ -2561,6 +2561,7 @@ "waiting": "Waiting for work", "awaiting_plan_approval": "Awaiting plan approval", "finalizing": "Finalizing", + "draining": "Paused · draining", "paused": "Paused", "idle": "Idle", "failed": "Failed", @@ -2786,7 +2787,7 @@ "userMessagePending": "{{member}} is picking up your message", "pausedBanner": { "title": "New work is paused", - "body": "Pause stops active replies, send a message or press Resume to continue", + "body": "Resume this Agent Team before sending a message", "resume": "Resume" }, "toolUseSummary": { diff --git a/src/i18n/locales/es/sessions.json b/src/i18n/locales/es/sessions.json index 888c8e27d9..b286e3d73b 100644 --- a/src/i18n/locales/es/sessions.json +++ b/src/i18n/locales/es/sessions.json @@ -2452,7 +2452,21 @@ "memberSessions": "Sesiones de miembros", "pauseRun": "Pausar ejecución", "resumeRun": "Reanudar ejecución", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Ver historial del chat del coordinador", + "phase": { + "starting": "Iniciando", + "coordinating": "Coordinando", + "dispatching": "Distribuyendo", + "members_working": "Miembros trabajando", + "waiting": "Esperando trabajo", + "awaiting_plan_approval": "Esperando aprobación del plan", + "finalizing": "Finalizando", + "draining": "En pausa · finalizando", + "paused": "En pausa", + "idle": "Inactivo", + "failed": "Fallido", + "archived": "Archivado" + } }, "agentOrgInbox": { "title": "Mensajes Agent", @@ -2681,7 +2695,7 @@ "userMessagePending": "{{member}} está atendiendo tu mensaje", "pausedBanner": { "title": "El trabajo nuevo está en pausa", - "body": "Pausar detiene las respuestas activas, envía un mensaje o pulsa Reanudar para continuar", + "body": "Reanuda este Equipo Agent antes de enviar un mensaje", "resume": "Reanudar" }, "toolUseSummary": { diff --git a/src/i18n/locales/fr/sessions.json b/src/i18n/locales/fr/sessions.json index 236000105f..fb8a1d5e65 100644 --- a/src/i18n/locales/fr/sessions.json +++ b/src/i18n/locales/fr/sessions.json @@ -2452,7 +2452,21 @@ "memberSessions": "Sessions des membres", "pauseRun": "Mettre en pause", "resumeRun": "Reprendre l'exécution", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Voir l’historique du coordinateur", + "phase": { + "starting": "Démarrage", + "coordinating": "Coordination", + "dispatching": "Répartition", + "members_working": "Membres au travail", + "waiting": "En attente de travail", + "awaiting_plan_approval": "En attente d’approbation du plan", + "finalizing": "Finalisation", + "draining": "En pause · arrêt en cours", + "paused": "En pause", + "idle": "Inactif", + "failed": "Échec", + "archived": "Archivé" + } }, "agentOrgInbox": { "title": "Messages Agent", @@ -2681,7 +2695,7 @@ "userMessagePending": "{{member}} prend en charge votre message", "pausedBanner": { "title": "Le nouveau travail est en pause", - "body": "La pause arrête les réponses actives, envoyez un message ou cliquez sur Reprendre pour continuer", + "body": "Reprenez cette équipe d’agents avant d’envoyer un message", "resume": "Reprendre" }, "toolUseSummary": { diff --git a/src/i18n/locales/ja/sessions.json b/src/i18n/locales/ja/sessions.json index 507e97e1b8..bf689010ee 100644 --- a/src/i18n/locales/ja/sessions.json +++ b/src/i18n/locales/ja/sessions.json @@ -2451,7 +2451,21 @@ "memberSessions": "メンバーセッション", "pauseRun": "実行を一時停止", "resumeRun": "実行を再開", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Coordinator のチャット履歴を表示", + "phase": { + "starting": "開始中", + "coordinating": "調整中", + "dispatching": "割り当て中", + "members_working": "メンバーが作業中", + "waiting": "作業待ち", + "awaiting_plan_approval": "プラン承認待ち", + "finalizing": "終了処理中", + "draining": "一時停止中・終了処理中", + "paused": "一時停止中", + "idle": "待機中", + "failed": "失敗", + "archived": "アーカイブ済み" + } }, "agentOrgInbox": { "title": "Agent メッセージ", @@ -2680,7 +2694,7 @@ "userMessagePending": "{{member}} があなたのメッセージを処理しています", "pausedBanner": { "title": "新しい作業は一時停止中です", - "body": "一時停止すると進行中の返信が停止します、メッセージを送信するか、再開を押すと続行します", + "body": "メッセージを送信する前に、この Agent Team を再開してください", "resume": "再開" }, "toolUseSummary": { diff --git a/src/i18n/locales/ko/sessions.json b/src/i18n/locales/ko/sessions.json index 62934ff6fc..7f81a4f116 100644 --- a/src/i18n/locales/ko/sessions.json +++ b/src/i18n/locales/ko/sessions.json @@ -2452,7 +2452,21 @@ "memberSessions": "Member 세션", "pauseRun": "실행 일시 중지", "resumeRun": "실행 재개", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Coordinator 채팅 기록 보기", + "phase": { + "starting": "시작 중", + "coordinating": "조정 중", + "dispatching": "배정 중", + "members_working": "멤버 작업 중", + "waiting": "작업 대기 중", + "awaiting_plan_approval": "계획 승인 대기 중", + "finalizing": "마무리 중", + "draining": "일시 중지됨 · 종료 중", + "paused": "일시 중지됨", + "idle": "대기 중", + "failed": "실패", + "archived": "보관됨" + } }, "agentOrgInbox": { "title": "Agent 메시지", @@ -2681,7 +2695,7 @@ "userMessagePending": "{{member}} is picking up your message", "pausedBanner": { "title": "New work is paused", - "body": "Pause stops active replies, send a message or press Resume to continue", + "body": "메시지를 보내기 전에 이 Agent Team을 재개하세요", "resume": "Resume" }, "toolUseSummary": { diff --git a/src/i18n/locales/pl/sessions.json b/src/i18n/locales/pl/sessions.json index 3fa0f8fcad..fd15b50659 100644 --- a/src/i18n/locales/pl/sessions.json +++ b/src/i18n/locales/pl/sessions.json @@ -2505,7 +2505,21 @@ "memberSessions": "Sesje członków", "pauseRun": "Wstrzymaj wykonanie", "resumeRun": "Wznów wykonanie", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Pokaż historię czatu koordynatora", + "phase": { + "starting": "Uruchamianie", + "coordinating": "Koordynowanie", + "dispatching": "Przydzielanie", + "members_working": "Członkowie pracują", + "waiting": "Oczekiwanie na pracę", + "awaiting_plan_approval": "Oczekiwanie na zatwierdzenie planu", + "finalizing": "Finalizowanie", + "draining": "Wstrzymano · kończenie", + "paused": "Wstrzymano", + "idle": "Bezczynny", + "failed": "Niepowodzenie", + "archived": "Zarchiwizowano" + } }, "agentOrgInbox": { "title": "Wiadomości Agent", @@ -2718,7 +2732,7 @@ "userMessagePending": "{{member}} przejmuje Twoją wiadomość", "pausedBanner": { "title": "Nowa praca jest wstrzymana", - "body": "Pauza zatrzymuje aktywne odpowiedzi, wyślij wiadomość albo kliknij Wznów, aby kontynuować", + "body": "Wznów ten zespół agentów przed wysłaniem wiadomości", "resume": "Wznów" }, "toolUseSummary": { diff --git a/src/i18n/locales/pt/sessions.json b/src/i18n/locales/pt/sessions.json index afb55d210a..ad5c6768a3 100644 --- a/src/i18n/locales/pt/sessions.json +++ b/src/i18n/locales/pt/sessions.json @@ -2467,7 +2467,21 @@ "memberSessions": "Sessões de membros", "pauseRun": "Pausar execução", "resumeRun": "Retomar execução", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Ver histórico do chat do coordenador", + "phase": { + "starting": "Iniciando", + "coordinating": "Coordenando", + "dispatching": "Distribuindo", + "members_working": "Membros trabalhando", + "waiting": "Aguardando trabalho", + "awaiting_plan_approval": "Aguardando aprovação do plano", + "finalizing": "Finalizando", + "draining": "Pausado · finalizando", + "paused": "Pausado", + "idle": "Inativo", + "failed": "Falhou", + "archived": "Arquivado" + } }, "agentOrgInbox": { "title": "Mensagens Agent", @@ -2680,7 +2694,7 @@ "userMessagePending": "{{member}} está assumindo sua mensagem", "pausedBanner": { "title": "Novo trabalho está pausado", - "body": "Pausar interrompe as respostas ativas, envie uma mensagem ou clique em Retomar para continuar", + "body": "Retome esta Equipe de Agents antes de enviar uma mensagem", "resume": "Retomar" }, "toolUseSummary": { diff --git a/src/i18n/locales/ru/sessions.json b/src/i18n/locales/ru/sessions.json index 49dea7d09e..71282d6d55 100644 --- a/src/i18n/locales/ru/sessions.json +++ b/src/i18n/locales/ru/sessions.json @@ -2496,7 +2496,21 @@ "memberSessions": "Сессии участников", "pauseRun": "Приостановить выполнение", "resumeRun": "Возобновить выполнение", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Показать историю чата координатора", + "phase": { + "starting": "Запуск", + "coordinating": "Координация", + "dispatching": "Распределение", + "members_working": "Участники работают", + "waiting": "Ожидание работы", + "awaiting_plan_approval": "Ожидание утверждения плана", + "finalizing": "Завершение", + "draining": "Приостановлено · завершение", + "paused": "Приостановлено", + "idle": "Ожидание", + "failed": "Ошибка", + "archived": "В архиве" + } }, "agentOrgInbox": { "title": "Сообщения Agent", @@ -2725,7 +2739,7 @@ "userMessagePending": "{{member}} принимает ваше сообщение", "pausedBanner": { "title": "Новая работа приостановлена", - "body": "Пауза останавливает активные ответы, отправьте сообщение или нажмите Возобновить, чтобы продолжить", + "body": "Возобновите эту команду агентов перед отправкой сообщения", "resume": "Возобновить" }, "toolUseSummary": { diff --git a/src/i18n/locales/tr/sessions.json b/src/i18n/locales/tr/sessions.json index 44933ebfb1..f171e0202b 100644 --- a/src/i18n/locales/tr/sessions.json +++ b/src/i18n/locales/tr/sessions.json @@ -2453,7 +2453,21 @@ "memberSessions": "Üye oturumları", "pauseRun": "Çalıştırmayı duraklat", "resumeRun": "Çalıştırmayı devam ettir", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Koordinatör sohbet geçmişini görüntüle", + "phase": { + "starting": "Başlatılıyor", + "coordinating": "Koordine ediliyor", + "dispatching": "Dağıtılıyor", + "members_working": "Üyeler çalışıyor", + "waiting": "İş bekleniyor", + "awaiting_plan_approval": "Plan onayı bekleniyor", + "finalizing": "Tamamlanıyor", + "draining": "Duraklatıldı · sonlandırılıyor", + "paused": "Duraklatıldı", + "idle": "Boşta", + "failed": "Başarısız", + "archived": "Arşivlendi" + } }, "agentOrgInbox": { "title": "Agent mesajları", @@ -2682,7 +2696,7 @@ "userMessagePending": "{{member}} mesajınızı üstleniyor", "pausedBanner": { "title": "Yeni çalışma duraklatıldı", - "body": "Duraklatma aktif yanıtları durdurur, devam etmek için mesaj gönderin veya Devam et'e basın", + "body": "Mesaj göndermeden önce bu Agent Ekibini devam ettirin", "resume": "Devam et" }, "toolUseSummary": { diff --git a/src/i18n/locales/vi/sessions.json b/src/i18n/locales/vi/sessions.json index 1be829050b..67579e53a7 100644 --- a/src/i18n/locales/vi/sessions.json +++ b/src/i18n/locales/vi/sessions.json @@ -2449,7 +2449,21 @@ "memberSessions": "Phiên thành viên", "pauseRun": "Tạm dừng thực thi", "resumeRun": "Tiếp tục thực thi", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "Xem lịch sử trò chuyện của điều phối viên", + "phase": { + "starting": "Đang bắt đầu", + "coordinating": "Đang điều phối", + "dispatching": "Đang phân công", + "members_working": "Thành viên đang làm việc", + "waiting": "Đang chờ công việc", + "awaiting_plan_approval": "Đang chờ phê duyệt kế hoạch", + "finalizing": "Đang hoàn tất", + "draining": "Đã tạm dừng · đang kết thúc", + "paused": "Đã tạm dừng", + "idle": "Không hoạt động", + "failed": "Thất bại", + "archived": "Đã lưu trữ" + } }, "agentOrgInbox": { "title": "Tin nhắn Agent", @@ -2678,7 +2692,7 @@ "userMessagePending": "{{member}} đang tiếp nhận tin nhắn của bạn", "pausedBanner": { "title": "Công việc mới đang tạm dừng", - "body": "Tạm dừng sẽ dừng các phản hồi đang chạy, gửi tin nhắn hoặc nhấn Tiếp tục để tiếp tục", + "body": "Tiếp tục Nhóm Agent này trước khi gửi tin nhắn", "resume": "Tiếp tục" }, "toolUseSummary": { diff --git a/src/i18n/locales/zh-Hant/sessions.json b/src/i18n/locales/zh-Hant/sessions.json index b36068fe9a..2cf4275f81 100644 --- a/src/i18n/locales/zh-Hant/sessions.json +++ b/src/i18n/locales/zh-Hant/sessions.json @@ -2467,7 +2467,21 @@ "memberSessions": "成員會話", "pauseRun": "暫停執行", "resumeRun": "恢復執行", - "viewCoordinatorHistory": "View coordinator chat history" + "viewCoordinatorHistory": "查看 Coordinator 對話記錄", + "phase": { + "starting": "正在啟動", + "coordinating": "正在協調", + "dispatching": "正在分派", + "members_working": "成員工作中", + "waiting": "等待下一步", + "awaiting_plan_approval": "等待計畫審批", + "finalizing": "正在收尾", + "draining": "已暫停 · 正在收尾", + "paused": "已暫停", + "idle": "待命", + "failed": "失敗", + "archived": "已封存" + } }, "agentOrgInbox": { "title": "Agent 訊息", @@ -2696,7 +2710,7 @@ "userMessagePending": "{{member}} 正在接收你的訊息", "pausedBanner": { "title": "新的工作已暫停", - "body": "暫停會停止目前回覆，傳送訊息或點擊恢復即可繼續", + "body": "請先恢復這個 Agent Team，再傳送訊息", "resume": "恢復" }, "toolUseSummary": { diff --git a/src/i18n/locales/zh/sessions.json b/src/i18n/locales/zh/sessions.json index 5a7995d1a6..f6c1940844 100644 --- a/src/i18n/locales/zh/sessions.json +++ b/src/i18n/locales/zh/sessions.json @@ -2535,6 +2535,7 @@ "waiting": "等待下一步", "awaiting_plan_approval": "等待计划审批", "finalizing": "正在收尾", + "draining": "已暂停 · 正在收尾", "paused": "已暂停", "idle": "待命", "failed": "失败", @@ -2776,7 +2777,7 @@ "userMessagePending": "{{member}} 正在接收你的消息", "pausedBanner": { "title": "新的工作已暂停", - "body": "暂停会停止当前回复，发送消息或点击恢复即可继续", + "body": "请先恢复这个 Agent Team，再发送消息", "resume": "恢复" }, "toolUseSummary": { diff --git a/src/modules/__tests__/useWorkStationPipelineBridge.test.ts b/src/modules/__tests__/useWorkStationPipelineBridge.test.ts index 749e63d5a4..ba60b126c1 100644 --- a/src/modules/__tests__/useWorkStationPipelineBridge.test.ts +++ b/src/modules/__tests__/useWorkStationPipelineBridge.test.ts @@ -49,6 +49,8 @@ async function loadModule() { const bridge = await import("../useWorkStationPipelineBridge"); return { activeSessionIdAtom: atoms.activeSessionIdAtom, + claimPipelineSessionAtom: atoms.claimPipelineSessionAtom, + pipelineSessionClaimAtom: atoms.pipelineSessionClaimAtom, workstationActiveSessionIdAtom: atoms.workstationActiveSessionIdAtom, sessionViewAtom: atoms.sessionViewAtom, applyWorkStationPipelineBridge: bridge.applyWorkStationPipelineBridge, @@ -166,6 +168,53 @@ describe("applyWorkStationPipelineBridge", () => { dispose(); }); + it("preserves a claimed Agent Org member pipeline while WorkStation stays anchored", async () => { + const { + activeSessionIdAtom, + claimPipelineSessionAtom, + installWorkStationPipelineBridge, + pipelineSessionClaimAtom, + workstationActiveSessionIdAtom, + } = await loadModule(); + const store = createStore(); + const dispose = installWorkStationPipelineBridge(true, store); + store.set(workstationActiveSessionIdAtom, "coordinator-session"); + store.set(activeSessionIdAtom, "coordinator-session"); + + store.set(claimPipelineSessionAtom, "member-session"); + + expect(store.get(activeSessionIdAtom)).toBe("member-session"); + expect(store.get(workstationActiveSessionIdAtom)).toBe( + "coordinator-session" + ); + expect(store.get(pipelineSessionClaimAtom)).toEqual({ + sessionId: "member-session", + workstationSessionId: "coordinator-session", + }); + dispose(); + }); + + it("invalidates a member claim when WorkStation navigates elsewhere", async () => { + const { + activeSessionIdAtom, + claimPipelineSessionAtom, + installWorkStationPipelineBridge, + pipelineSessionClaimAtom, + workstationActiveSessionIdAtom, + } = await loadModule(); + const store = createStore(); + const dispose = installWorkStationPipelineBridge(true, store); + store.set(workstationActiveSessionIdAtom, "coordinator-session"); + store.set(activeSessionIdAtom, "coordinator-session"); + store.set(claimPipelineSessionAtom, "member-session"); + + store.set(workstationActiveSessionIdAtom, "other-session"); + + expect(store.get(activeSessionIdAtom)).toBe("other-session"); + expect(store.get(pipelineSessionClaimAtom)).toBeNull(); + dispose(); + }); + it("stops reconciling after the visible-session lifecycle ends", async () => { const { activeSessionIdAtom, diff --git a/src/modules/useWorkStationPipelineBridge.ts b/src/modules/useWorkStationPipelineBridge.ts index 6130d1bc26..53ec823629 100644 --- a/src/modules/useWorkStationPipelineBridge.ts +++ b/src/modules/useWorkStationPipelineBridge.ts @@ -32,6 +32,7 @@ import { useEffect } from "react"; import { activeSessionIdAtom, + pipelineSessionClaimAtom, workstationActiveSessionIdAtom, } from "@src/store/session"; @@ -56,7 +57,15 @@ export function applyWorkStationPipelineBridge( ): boolean { if (!isWorkStationViewActive) return false; const pipeline = store.get(activeSessionIdAtom); + const claim = store.get(pipelineSessionClaimAtom); + if ( + claim?.sessionId === pipeline && + claim.workstationSessionId === remembered + ) { + return false; + } if (remembered === pipeline) return false; + if (claim) store.set(pipelineSessionClaimAtom, null); store.set(activeSessionIdAtom, remembered); return true; } diff --git a/src/store/session/__tests__/viewAtom.test.ts b/src/store/session/__tests__/viewAtom.test.ts index bfa5c136d8..04d6142882 100644 --- a/src/store/session/__tests__/viewAtom.test.ts +++ b/src/store/session/__tests__/viewAtom.test.ts @@ -60,6 +60,7 @@ async function loadAtoms() { activeSessionIdAtom: mod.activeSessionIdAtom, workstationActiveSessionIdAtom: mod.workstationActiveSessionIdAtom, claimPipelineSessionAtom: mod.claimPipelineSessionAtom, + pipelineSessionClaimAtom: mod.pipelineSessionClaimAtom, jumpToSessionAtom: mod.jumpToSessionAtom, openSessionAtom: mod.openSessionAtom, closeSessionAtom: mod.closeSessionAtom, @@ -354,6 +355,7 @@ describe("claimPipelineSessionAtom", () => { activeSessionIdAtom, claimPipelineSessionAtom, loadStatusAtom, + pipelineSessionClaimAtom, sessionViewAtom, workstationActiveSessionIdAtom, } = await loadAtoms(); @@ -377,6 +379,10 @@ describe("claimPipelineSessionAtom", () => { expect(store.get(workstationActiveSessionIdAtom)).toBe( "osagent-workstation" ); + expect(store.get(pipelineSessionClaimAtom)).toEqual({ + sessionId: "claudecodeapp-48238728-ab4f-4697-850d-459b12e03e72", + workstationSessionId: "osagent-workstation", + }); }); it("bumps reload epoch when reclaiming the current pipeline session", async () => { diff --git a/src/store/session/viewAtom.ts b/src/store/session/viewAtom.ts index 5323b7853e..477c1afd3f 100644 --- a/src/store/session/viewAtom.ts +++ b/src/store/session/viewAtom.ts @@ -142,6 +142,20 @@ workstationActiveSessionIdAtom.debugLabel = "workstationActiveSessionIdAtom"; export const activeSessionIdAtom = atom(null); activeSessionIdAtom.debugLabel = "activeSessionIdAtom"; +export interface PipelineSessionClaim { + sessionId: string; + workstationSessionId: string | null; +} + +/** + * Identifies an intentional, temporary pipeline divergence while the + * WorkStation remains anchored to its parent session. The bridge uses this + * receipt to distinguish an Agent Org member inspection from an accidental + * or stale write to `activeSessionIdAtom`. + */ +export const pipelineSessionClaimAtom = atom(null); +pipelineSessionClaimAtom.debugLabel = "pipelineSessionClaimAtom"; + // Runtime-status writes are gated to the visible session. Both "visible" // atoms qualify: the pipeline id (what the chat surface renders) and the // SessionCore sessionIdAtom (what the event store is subscribed to) — they @@ -183,6 +197,7 @@ export const openSessionAtom = atom( set, payload: { sessionId: string; sessionName?: string; repoPath?: string } ) => { + set(pipelineSessionClaimAtom, null); set(sessionViewAtom, { activeSessionId: payload.sessionId, sessionName: payload.sessionName, @@ -197,6 +212,7 @@ openSessionAtom.debugLabel = "openSessionAtom"; * Close current session — clears both memory and pipeline. */ export const closeSessionAtom = atom(null, (_get, set) => { + set(pipelineSessionClaimAtom, null); set(sessionViewAtom, { activeSessionId: null, sessionName: undefined, @@ -237,6 +253,10 @@ export const claimPipelineSessionAtom = atom( set(clearSessionAtom); set(loadStatusAtom, "loading"); + set(pipelineSessionClaimAtom, { + sessionId, + workstationSessionId: get(workstationActiveSessionIdAtom), + }); set(activeSessionIdAtom, sessionId); if (previousPipelineSessionId === sessionId) { set(triggerSessionReloadAtom, sessionId); @@ -252,8 +272,15 @@ claimPipelineSessionAtom.debugLabel = "claimPipelineSessionAtom"; * do not keep treating the hidden session as rendered. */ export const releasePipelineSessionAtom = atom(null, (get, set) => { - if (!get(activeSessionIdAtom) && !get(sessionIdAtom)) return; + if ( + !get(activeSessionIdAtom) && + !get(sessionIdAtom) && + !get(pipelineSessionClaimAtom) + ) { + return; + } set(clearSessionAtom); + set(pipelineSessionClaimAtom, null); set(activeSessionIdAtom, null); }); releasePipelineSessionAtom.debugLabel = "releasePipelineSessionAtom"; @@ -290,6 +317,7 @@ export const jumpToSessionAtom = atom( set(clearSessionAtom); set(loadStatusAtom, sessionId ? "loading" : "idle"); + set(pipelineSessionClaimAtom, null); // WorkStation owns the navigation, so update its memory atom AND // the pipeline atom in a single underlying-storage write. When // the caller passes the rich form with name/repoPath, fold those diff --git a/tests/e2e/specs/core/agent-org-group-chat-ui.spec.mjs b/tests/e2e/specs/core/agent-org-group-chat-ui.spec.mjs index 7fa74ccfff..a12ed2626f 100644 --- a/tests/e2e/specs/core/agent-org-group-chat-ui.spec.mjs +++ b/tests/e2e/specs/core/agent-org-group-chat-ui.spec.mjs @@ -833,43 +833,32 @@ describe("Agent Org group chat and plan rendered UI", () => { await invokeE2E("agentOrgPauseRun", sessionId), "agentOrgPauseRun (group chat paused banner)" ); - if (pauseResult.transitioned !== false) { + if (pauseResult.outcome?.transitioned !== false) { await waitForAgentOrgRunView( sessionId, (view) => view?.runStatus === "paused", "group chat run paused for inline Resume" ); await refreshRenderedAgentOrgOverview("group chat paused banner refresh"); - await waitForGroupChatPausedBanner("group chat paused send resume"); - - const pausedMessage = `E2E group chat paused send resumes ${RUN_ID}`; - await sendRenderedChatPrompt(pausedMessage); - const pausedInboxRow = await waitForInboxRow( - sessionId, - (row) => { - const payload = parseInboxPayload(row, "paused group chat send"); - return ( - row.senderAgentId === "_user" && - row.recipientMemberId === AGENT_ORG_COORDINATOR_MEMBER_ID && - payload.text === pausedMessage - ); - }, - "paused group chat inbox row persisted" - ); - await waitForRenderedGroupChatUserTurn({ - text: pausedMessage, - label: "paused group chat send resumes", - }); + await waitForGroupChatPausedBanner("group chat paused requires Resume"); + const pausedSendState = await execJS(` + const visible = Array.from(document.querySelectorAll('[data-testid="chat-send-button"]')) + .find((element) => { + const rect = element.getBoundingClientRect(); + return rect.width > 0 && rect.height > 0; + }); + return visible ? { disabled: Boolean(visible.disabled) } : null; + `); + if (pausedSendState?.disabled !== true) { + throw new Error( + `Paused Group Chat send must be disabled: ${JSON.stringify(pausedSendState)}` + ); + } + await clickGroupChatResumeButton("group chat explicit Resume"); await waitForAgentOrgRunView( sessionId, (view) => view?.runStatus !== "paused", - "group chat send while paused resumes run" - ); - await waitForInboxRowRead( - sessionId, - pausedInboxRow.id, - "paused group chat inbox row drained after resume", - REPLY_TIMEOUT_MS + "explicit Group Chat Resume leaves paused state" ); await browser.waitUntil( async () => @@ -880,7 +869,7 @@ describe("Agent Org group chat and plan rendered UI", () => { timeout: RENDER_TIMEOUT_MS, interval: 250, timeoutMsg: - "group chat paused banner did not disappear after sending a message", + "group chat paused banner did not disappear after explicit Resume", } ); } @@ -930,7 +919,7 @@ describe("Agent Org group chat and plan rendered UI", () => { await invokeE2E("agentOrgPauseRun", sessionId), "agentOrgPauseRun(durable Group Chat history seed)" ); - if (pauseResult.transitioned !== false) { + if (pauseResult.outcome?.transitioned !== false) { await waitForAgentOrgRunView( sessionId, (view) => view?.runStatus === "paused", @@ -1138,7 +1127,7 @@ describe("Agent Org group chat and plan rendered UI", () => { runId, DEFAULT_AGENT_ORG_MEMBER_IDS.PLANNER, "task_update", - { id: planTaskId, status: AGENT_ORG_TASK_STATUS.IN_PROGRESS } + { operation: "start", id: planTaskId } ), "debugAgentOrgExecuteToolAsAgent(start Plan task)" ).result; @@ -1533,7 +1522,7 @@ describe("Agent Org group chat and plan rendered UI", () => { runId, plannerMemberId, "task_update", - { id: planTaskId, status: AGENT_ORG_TASK_STATUS.IN_PROGRESS } + { operation: "start", id: planTaskId } ), "debugAgentOrgExecuteToolAsAgent(start user-approved Plan task)" ).result; diff --git a/tests/e2e/specs/core/agent-org-pause-resume-live.spec.mjs b/tests/e2e/specs/core/agent-org-pause-resume-live.spec.mjs new file mode 100644 index 0000000000..bf7cb2b62f --- /dev/null +++ b/tests/e2e/specs/core/agent-org-pause-resume-live.spec.mjs @@ -0,0 +1,777 @@ +/* global browser, before, describe, it, process */ +import { execFileSync } from "node:child_process"; +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; + +import { + RENDER_TIMEOUT_MS, + REPLY_TIMEOUT_MS, + configureCreatorForAgentOrg, + execJS, + getApiAccount, + invokeE2E, + openAgentOrgOverviewPanel, + removeAgentOrgsByName, + selectPreferredModel, + selectRenderedAgentOrg, + sendFromRenderedCreator, + unwrap, + waitForAgentOrgRunView, + waitForAgentOrgRunViewByOrg, + waitForApp, +} from "../../support/core/agentOrgUiDriver.mjs"; + +const BASE_URL = `http://127.0.0.1:${process.env.E2E_IDE_SERVER_PORT ?? "13847"}`; +const PHASE = process.env.E2E_AGENT_ORG_LIVE_PHASE ?? ""; +const ROUND = (process.env.E2E_AGENT_ORG_LIVE_ROUND_ID ?? "") + .replace(/[^a-zA-Z0-9_-]/g, "-") + .slice(0, 40); +const PROVIDER_MODE = process.env.E2E_PROVIDER_MODE ?? "mock"; +const ORG_ID = `e2e-pause-resume-live-${ROUND}`; +const ORG_NAME = `E2E Pause Resume Live ${ROUND}`; +const MEMBER_ID = "pause-worker"; +// Keep the durable/process marker compatible with sendRenderedChatPrompt's +// compact marker probe. A hyphenated round id makes that helper fall back to +// comparing the entire shell-heavy prompt inside JSON-escaped state. +const MARKER_ROUND = ROUND.replaceAll("-", "_"); +const PROCESS_MARKER = `ORGII_PAUSE_LIVE_${MARKER_ROUND}`; +const FINALITY_MARKER = `ORGII_RESUME_FINALITY_${MARKER_ROUND}`; +const ORGII_HOME = process.env.E2E_ORGII_HOME ?? ""; + +async function postJson(pathname, body = {}, timeoutMs = 15_000) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(`${BASE_URL}${pathname}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + signal: controller.signal, + }); + const json = await response.json(); + if (!response.ok || json?.ok !== true) { + throw new Error(`${pathname} failed: ${JSON.stringify(json)}`); + } + return json; + } finally { + clearTimeout(timer); + } +} + +async function visibleProductButton(selector, marker) { + let state = null; + await browser.waitUntil( + async () => { + state = await execJS(` + const visible = (element) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden"; + }; + const elements = Array.from(document.querySelectorAll(${JSON.stringify(selector)})); + for (const element of elements) element.removeAttribute(${JSON.stringify(marker)}); + const element = elements.find(visible) ?? null; + if (element) element.setAttribute(${JSON.stringify(marker)}, "true"); + return { count: elements.length, marked: Boolean(element), disabled: element?.disabled ?? null }; + `); + return state?.marked && state.disabled === false; + }, + { + timeout: RENDER_TIMEOUT_MS, + interval: 50, + timeoutMsg: `No enabled visible product control for ${selector}: ${JSON.stringify(state)}`, + } + ); + return browser.$(`[${marker}="true"]`); +} + +function processGroupSnapshot(processGroupId) { + return execFileSync("ps", ["-ax", "-o", "pid=,ppid=,pgid=,command="], { + encoding: "utf8", + }) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/); + return match + ? { + pid: Number(match[1]), + parentPid: Number(match[2]), + processGroupId: Number(match[3]), + command: match[4], + } + : null; + }) + .filter((row) => row?.processGroupId === processGroupId); +} + +function pidExists(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error?.code === "ESRCH") return false; + throw error; + } +} + +function filesContainingMarker(root, marker) { + const matches = []; + const visit = (path) => { + let entries; + try { + entries = readdirSync(path, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + const child = join(path, entry.name); + if (entry.isDirectory()) { + visit(child); + continue; + } + try { + if (readFileSync(child, "utf8").includes(marker)) matches.push(child); + } catch { + // Ignore non-text runtime files; only shell replay logs can match. + } + } + }; + visit(root); + return matches; +} + +function durableRunIdForRootSession(sessionId) { + try { + return execFileSync( + "sqlite3", + [ + join(ORGII_HOME, "sessions.db"), + `SELECT id FROM agent_org_runtime_runs WHERE root_session_id='${sessionId.replaceAll("'", "''")}' ORDER BY created_at DESC LIMIT 1;`, + ], + { encoding: "utf8" } + ).trim(); + } catch { + return ""; + } +} + +function sqlLiteral(value) { + return `'${String(value).replaceAll("'", "''")}'`; +} + +function sqliteRow(sql) { + const output = execFileSync( + "sqlite3", + ["-json", "-cmd", ".timeout 5000", join(ORGII_HOME, "sessions.db"), sql], + { encoding: "utf8" } + ).trim(); + const rows = output ? JSON.parse(output) : []; + return rows[0] ?? null; +} + +function convergenceSnapshot({ + runId, + taskId, + memberSessionId, + continuationId, +}) { + const run = sqlLiteral(runId); + const task = sqlLiteral(taskId); + const session = sqlLiteral(memberSessionId); + const continuation = sqlLiteral(continuationId); + const finalityMarker = sqlLiteral(`%${FINALITY_MARKER}%`); + return sqliteRow(` + SELECT + (SELECT status FROM agent_org_runtime_runs WHERE id=${run}) AS run_status, + (SELECT status FROM agent_org_runtime_tasks + WHERE org_run_id=${run} AND id=${task}) AS task_status, + (SELECT status FROM session_turn_intents + WHERE session_id=${session} AND turn_intent_id=${continuation}) AS continuation_status, + (SELECT COUNT(*) FROM session_turn_intents + WHERE org_run_id=${run} AND status IN ('queued','running')) AS active_intent_count, + (SELECT COUNT(*) FROM session_turn_intents + WHERE org_run_id=${run}) AS intent_count, + (SELECT COUNT(*) FROM agent_org_runtime_turn_contexts + WHERE org_run_id=${run}) AS context_count, + (SELECT COUNT(*) FROM agent_org_runtime_inbox inbox + WHERE inbox.org_run_id=${run} + AND inbox.payload_kind='task_assigned' + AND json_extract(inbox.payload_json,'$.task_id')=${task} + AND inbox.read_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM agent_org_runtime_inbox_delivery_resolutions resolution + WHERE resolution.inbox_id=inbox.id + )) AS unresolved_assignment_count, + (SELECT COUNT(*) + FROM agent_org_runtime_inbox_materializations materialization + JOIN agent_org_runtime_inbox inbox ON inbox.id=materialization.inbox_id + WHERE inbox.org_run_id=${run} + AND inbox.payload_kind='task_assigned' + AND json_extract(inbox.payload_json,'$.task_id')=${task}) AS assignment_materialization_count, + (SELECT COUNT(*) FROM agent_messages + WHERE session_id=${session} AND role='assistant') AS assistant_count, + (SELECT COUNT(*) FROM agent_messages + WHERE session_id=${session} AND role='assistant' + AND content LIKE ${finalityMarker}) AS finality_assistant_count + `); +} + +async function assertFeatureGatePreflight() { + const frontendGate = await execJS(` + return { + helperPresent: Boolean(window.__e2e), + enablePresent: typeof window.__e2e?.debugAgentOrgEnableRedesign === "function", + compiledIdeUrl: window.__ORGII_E2E_IDE_SERVER_WS_URL__ ?? null, + }; + `); + if (!frontendGate.helperPresent || !frontendGate.enablePresent) { + throw new Error( + `frontend WebDriver compile gate is disabled: ${JSON.stringify(frontendGate)}` + ); + } + const enabled = unwrap( + await invokeE2E("debugAgentOrgEnableRedesign"), + "enable Agent Org redesign through webdriver-only Rust gate" + ); + if (enabled.enabled !== true) { + throw new Error( + `Rust Agent Org runtime gate did not enable: ${JSON.stringify(enabled)}` + ); + } + console.info( + `[agent-org-live-feature-gates] ${JSON.stringify({ frontendGate, cargoWebdriverAndRuntimeGate: true })}` + ); +} + +function assertLiveInputs() { + if (!ROUND) throw new Error("E2E_AGENT_ORG_LIVE_ROUND_ID is required"); + if (!ORGII_HOME) throw new Error("E2E_ORGII_HOME is required"); + if (!new Set(["pause", "resume", "task-smoke"]).has(PHASE)) { + throw new Error(`unsupported E2E_AGENT_ORG_LIVE_PHASE=${PHASE}`); + } + if (PROVIDER_MODE === "mock") { + throw new Error("live acceptance refuses E2E_PROVIDER_MODE=mock"); + } +} + +async function seedOneMemberOrg() { + await removeAgentOrgsByName(ORG_NAME); + await postJson("/agent/test/agent-org/seed", { + id: ORG_ID, + name: ORG_NAME, + coordinator_agent_id: "builtin:sde", + members: [ + { + id: MEMBER_ID, + name: "Pause Worker", + role: "Own the exact shell lifecycle acceptance task", + agent_id: "builtin:sde", + }, + ], + }); +} + +async function runPausePhase() { + const account = await getApiAccount(); + const model = selectPreferredModel(account); + await seedOneMemberOrg(); + await configureCreatorForAgentOrg({ account, model, agentOrgId: ORG_ID }); + await selectRenderedAgentOrg(ORG_ID); + + const backgroundCommand = [ + "trap '' TERM;", + `sh -c 'trap \"\" TERM; while :; do sleep 120; done' & child=$!;`, + `printf '${PROCESS_MARKER} parent=%s child=%s\\n' \"$$\" \"$child\";`, + "wait", + ].join(" "); + const convergenceCommand = `sleep 20; printf '${FINALITY_MARKER} terminal\\n'`; + const prompt = [ + `This is live acceptance round ${ROUND}.`, + "Use task_graph_create exactly once to create exactly one Task assigned to member pause-worker.", + `The Task subject must contain ${PROCESS_MARKER}.`, + "Its description must instruct the Member to do exactly this:", + `before Pause, call run_shell exactly once with mode=background and command: ${backgroundCommand}`, + "If that command is interrupted by Pause, the Resume continuation must not repeat it.", + `On Resume, call run_shell exactly once with mode=background and command: ${convergenceCommand}`, + "Immediately after launching that Resume-only background command, try task_update operation=complete and try to give the final answer without calling await_output first.", + `If the system blocks early completion, consume the terminal background result in this same Turn, retry task_update operation=complete once, and include ${FINALITY_MARKER} in the final answer.`, + "Do not run either command as coordinator and do not create another Task.", + ].join(" "); + const sessionId = await sendFromRenderedCreator(prompt); + if (!sessionId) throw new Error("live Pause launch produced no root Session"); + + const view = await waitForAgentOrgRunView( + sessionId, + (candidate) => + candidate?.runStatus === "running" && + candidate?.tasks?.length === 1 && + candidate.tasks[0]?.owner === MEMBER_ID, + "real Provider created one assigned Pause Task", + REPLY_TIMEOUT_MS * 2 + ); + const runId = view.context.runId; + let runningEvidence = null; + let targetShell = null; + await browser.waitUntil( + async () => { + runningEvidence = await postJson("/agent/test/agent-org/pause/evidence", { + org_run_id: runId, + }); + targetShell = runningEvidence.background_shells?.find( + (shell) => + shell.session_id !== sessionId && + String(shell.command).includes(PROCESS_MARKER) + ); + return ( + Boolean(targetShell) && + runningEvidence.active_turns?.some( + (turn) => turn.session_id === targetShell.session_id + ) && + processGroupSnapshot(targetShell.pid).length >= 3 + ); + }, + { + timeout: REPLY_TIMEOUT_MS * 2, + interval: 250, + timeoutMsg: `real Provider Member never started the owned parent/child process group: ${JSON.stringify(runningEvidence)}`, + } + ); + const processRows = processGroupSnapshot(targetShell.pid); + const knownPids = processRows.map((row) => row.pid); + const taskIdsBeforePause = runningEvidence.durable.tasks.map( + (task) => task.id + ); + const replayFilesBeforePause = filesContainingMarker( + join(ORGII_HOME, "shell-replays"), + PROCESS_MARKER + ); + if (replayFilesBeforePause.length !== 1) { + throw new Error( + `live background command did not start exactly once: ${JSON.stringify(replayFilesBeforePause)}` + ); + } + + await openAgentOrgOverviewPanel("real Provider Pause control"); + const pauseButton = await visibleProductButton( + '[data-testid="agent-org-overview-pause-button"]', + "data-e2e-live-pause" + ); + const pauseStartedAt = Date.now(); + await pauseButton.click(); + + let drained = null; + await browser.waitUntil( + async () => { + drained = await postJson("/agent/test/agent-org/pause/evidence", { + org_run_id: runId, + }); + return ( + drained.durable.run_status === "paused" && + drained.active_runtime_count === 0 && + drained.active_turns.length === 0 && + drained.background_shells.length === 0 && + drained.durable.handoffs.length > 0 && + drained.durable.handoffs.every((handoff) => + ["released", "runtime_absent"].includes(handoff.drain_status) + ) + ); + }, + { + timeout: 10_000, + interval: 50, + timeoutMsg: `real Provider Pause did not drain every owned runtime/process: ${JSON.stringify(drained)}`, + } + ); + const drainMs = Date.now() - pauseStartedAt; + const memberHandoff = drained.durable.handoffs.find( + (handoff) => handoff.session_id === targetShell.session_id + ); + if ( + !memberHandoff?.runtime_lease_id || + !memberHandoff?.dialog_turn_generation || + memberHandoff.drain_status !== "released" || + memberHandoff.drain_timeout_at + ) { + throw new Error( + `live Member handoff lacked exact released owner identity: ${JSON.stringify(memberHandoff)}` + ); + } + const survivors = processGroupSnapshot(targetShell.pid); + const liveKnownPids = knownPids.filter(pidExists); + if (survivors.length > 0 || liveKnownPids.length > 0) { + throw new Error( + `live Pause left parent/child processes alive: ${JSON.stringify({ survivors, liveKnownPids, processRows })}` + ); + } + if ( + drained.durable.tasks.length !== 1 || + drained.durable.tasks[0].id !== taskIdsBeforePause[0] + ) { + throw new Error( + `live Pause duplicated or replaced its Task: ${JSON.stringify(drained)}` + ); + } + + const renderedBeforeQuietWindow = await execJS(` + return { + assistants: document.querySelectorAll('[data-testid="chat-message-assistant"]').length, + groupMessages: document.querySelectorAll('[data-testid="agent-org-group-chat-message"]').length, + }; + `); + await browser.pause(2_500); + const quietEvidence = await postJson("/agent/test/agent-org/pause/evidence", { + org_run_id: runId, + }); + const renderedAfterQuietWindow = await execJS(` + return { + assistants: document.querySelectorAll('[data-testid="chat-message-assistant"]').length, + groupMessages: document.querySelectorAll('[data-testid="agent-org-group-chat-message"]').length, + }; + `); + if ( + quietEvidence.active_runtime_count !== 0 || + quietEvidence.active_turns.length !== 0 || + quietEvidence.background_shells.length !== 0 || + JSON.stringify(renderedBeforeQuietWindow) !== + JSON.stringify(renderedAfterQuietWindow) + ) { + throw new Error( + `late Provider/shell activity appeared after Pause: ${JSON.stringify({ quietEvidence, renderedBeforeQuietWindow, renderedAfterQuietWindow })}` + ); + } + + console.info( + `[agent-org-live-pause-evidence] ${JSON.stringify({ + round: ROUND, + provider: { accountId: account.id, accountName: account.name, model }, + runId, + taskIds: taskIdsBeforePause, + owner: { + sessionId: memberHandoff.session_id, + turnIntentId: memberHandoff.original_turn_intent_id, + runtimeLeaseId: memberHandoff.runtime_lease_id, + dialogTurnGeneration: memberHandoff.dialog_turn_generation, + }, + processGroupId: targetShell.pid, + processRows, + drainMs, + replayFiles: replayFilesBeforePause, + activeRuntimeCountAfter: quietEvidence.active_runtime_count, + activeTurnCountAfter: quietEvidence.active_turns.length, + })}` + ); +} + +async function runResumePhase() { + const state = await waitForAgentOrgRunViewByOrg( + ORG_ID, + (view) => view?.runStatus === "paused", + `persisted Paused run after real app restart ${ROUND}` + ); + const sessionId = state.run.rootSessionId; + const runId = state.view.context.runId; + // Opening the persisted Session is restart pre-state only. Resume itself is + // still driven through the rendered product control below. This avoids + // coupling the lifecycle acceptance to unrelated sidebar pagination. + unwrap( + await invokeE2E("openSession", sessionId), + "open persisted Paused root Session after real app restart" + ); + await openAgentOrgOverviewPanel("real restart Resume control"); + + const before = await postJson("/agent/test/agent-org/pause/evidence", { + org_run_id: runId, + }); + const taskIds = before.durable.tasks.map((task) => task.id); + const replayFilesBefore = filesContainingMarker( + join(ORGII_HOME, "shell-replays"), + PROCESS_MARKER + ); + if ( + before.durable.run_status !== "paused" || + before.active_runtime_count !== 0 || + before.active_turns.length !== 0 || + before.background_shells.length !== 0 || + before.durable.tasks.length !== 1 || + replayFilesBefore.length !== 1 + ) { + throw new Error( + `restart did not restore exact Paused state: ${JSON.stringify(before)}` + ); + } + + const resumeButton = await visibleProductButton( + '[data-testid="agent-org-overview-resume-button"]', + "data-e2e-live-resume" + ); + await resumeButton.click(); + let resumed = null; + await browser.waitUntil( + async () => { + resumed = await postJson("/agent/test/agent-org/pause/evidence", { + org_run_id: runId, + }); + return ( + resumed.durable.run_status === "running" && + resumed.durable.episode?.status === "consumed" && + resumed.durable.handoffs.every( + (handoff) => handoff.continuation_status === "dispatched" + ) + ); + }, + { + timeout: REPLY_TIMEOUT_MS, + interval: 100, + timeoutMsg: `restart Resume did not dispatch one durable continuation: ${JSON.stringify(resumed)}`, + } + ); + const continuationIds = resumed.durable.handoffs.map( + (handoff) => handoff.continuation_turn_intent_id + ); + if ( + continuationIds.some((id) => !id) || + new Set(continuationIds).size !== continuationIds.length || + resumed.durable.tasks.length !== 1 || + resumed.durable.tasks[0].id !== taskIds[0] + ) { + throw new Error( + `restart Resume duplicated a continuation or Task: ${JSON.stringify(resumed)}` + ); + } + const memberHandoff = resumed.durable.handoffs.find( + (handoff) => handoff.task_id === taskIds[0] + ); + const memberContinuationId = memberHandoff?.continuation_turn_intent_id; + if (!memberHandoff?.session_id || !memberContinuationId) { + throw new Error( + `Resume lacked exact Member continuation: ${JSON.stringify(resumed)}` + ); + } + const initialConvergence = convergenceSnapshot({ + runId, + taskId: taskIds[0], + memberSessionId: memberHandoff.session_id, + continuationId: memberContinuationId, + }); + + let heldEvidence = null; + let heldConvergence = null; + let convergenceShell = null; + await browser.waitUntil( + async () => { + heldEvidence = await postJson("/agent/test/agent-org/pause/evidence", { + org_run_id: runId, + }); + if ( + heldEvidence.background_shells.some((shell) => + String(shell.command).includes(PROCESS_MARKER) + ) + ) { + throw new Error( + `Resume restarted the Pause command: ${JSON.stringify(heldEvidence)}` + ); + } + convergenceShell = heldEvidence.background_shells.find((shell) => + String(shell.command).includes(FINALITY_MARKER) + ); + heldConvergence = convergenceSnapshot({ + runId, + taskId: taskIds[0], + memberSessionId: memberHandoff.session_id, + continuationId: memberContinuationId, + }); + return ( + Boolean(convergenceShell) && + heldConvergence?.continuation_status === "running" && + heldConvergence?.task_status === "in_progress" && + heldEvidence.active_turns.some( + (turn) => turn.session_id === memberHandoff.session_id + ) + ); + }, + { + timeout: REPLY_TIMEOUT_MS, + interval: 100, + timeoutMsg: `same-Turn finality gate never held the resumed Member while its job was running: ${JSON.stringify({ heldEvidence, heldConvergence })}`, + } + ); + + let convergedEvidence = null; + let converged = null; + await browser.waitUntil( + async () => { + convergedEvidence = await postJson( + "/agent/test/agent-org/pause/evidence", + { + org_run_id: runId, + } + ); + converged = convergenceSnapshot({ + runId, + taskId: taskIds[0], + memberSessionId: memberHandoff.session_id, + continuationId: memberContinuationId, + }); + return ( + convergedEvidence.durable.run_status === "idle" && + convergedEvidence.active_turns.length === 0 && + convergedEvidence.background_shells.length === 0 && + converged?.run_status === "idle" && + converged?.task_status === "completed" && + converged?.continuation_status === "completed" && + converged?.active_intent_count === 0 && + converged?.unresolved_assignment_count === 0 && + converged?.assignment_materialization_count === 0 && + converged?.finality_assistant_count >= 1 + ); + }, + { + timeout: REPLY_TIMEOUT_MS * 2, + interval: 250, + timeoutMsg: `Resume did not converge Task, Turn, Inbox, assistant, job, and Run to terminal/Idle: ${JSON.stringify({ convergedEvidence, converged })}`, + } + ); + const replayFilesAfter = filesContainingMarker( + join(ORGII_HOME, "shell-replays"), + PROCESS_MARKER + ); + if ( + replayFilesAfter.length !== 1 || + replayFilesAfter[0] !== replayFilesBefore[0] + ) { + throw new Error( + `Resume created another command replay: ${JSON.stringify({ replayFilesBefore, replayFilesAfter })}` + ); + } + const finalityReplayFiles = filesContainingMarker( + join(ORGII_HOME, "shell-replays"), + FINALITY_MARKER + ); + if (finalityReplayFiles.length !== 1) { + throw new Error( + `Resume finality command did not execute exactly once: ${JSON.stringify(finalityReplayFiles)}` + ); + } + + const quietBefore = { + convergence: converged, + taskUpdatedAt: convergedEvidence.durable.tasks[0]?.updated_at, + replayFilesAfter, + finalityReplayFiles, + }; + await browser.pause(5_000); + const quietEvidence = await postJson("/agent/test/agent-org/pause/evidence", { + org_run_id: runId, + }); + const quietConvergence = convergenceSnapshot({ + runId, + taskId: taskIds[0], + memberSessionId: memberHandoff.session_id, + continuationId: memberContinuationId, + }); + if ( + quietEvidence.durable.run_status !== "idle" || + quietEvidence.active_turns.length !== 0 || + quietEvidence.background_shells.length !== 0 || + quietConvergence.intent_count !== converged.intent_count || + quietConvergence.context_count !== converged.context_count || + quietConvergence.assistant_count !== converged.assistant_count || + quietEvidence.durable.tasks[0]?.updated_at !== quietBefore.taskUpdatedAt || + filesContainingMarker(join(ORGII_HOME, "shell-replays"), PROCESS_MARKER) + .length !== 1 || + filesContainingMarker(join(ORGII_HOME, "shell-replays"), FINALITY_MARKER) + .length !== 1 + ) { + throw new Error( + `Resume quiet window detected a duplicate Wake, Provider Turn, Task mutation, message, or process: ${JSON.stringify({ quietBefore, quietEvidence, quietConvergence })}` + ); + } + console.info( + `[agent-org-live-resume-evidence] ${JSON.stringify({ + round: ROUND, + runId, + taskIds, + continuationIds, + memberContinuationId, + initialConvergence, + heldConvergence, + convergenceShell, + converged, + quietConvergence, + replayFilesBefore, + replayFilesAfter, + finalityReplayFiles, + })}` + ); +} + +async function runTaskSmokePhase() { + const account = await getApiAccount(); + const model = selectPreferredModel(account); + await seedOneMemberOrg(); + await configureCreatorForAgentOrg({ account, model, agentOrgId: ORG_ID }); + await selectRenderedAgentOrg(ORG_ID); + const marker = `TASK_LIFECYCLE_SMOKE_${ROUND}`; + const sessionId = await sendFromRenderedCreator( + [ + `Use task_graph_create exactly once to create one Task assigned to ${MEMBER_ID}.`, + `The subject must be ${marker}.`, + "The Member must inspect README.md, mark the Task in progress, then mark it completed and reply briefly.", + "Do not create any other Task.", + ].join(" ") + ); + let runId = ""; + await browser.waitUntil( + async () => { + runId = durableRunIdForRootSession(sessionId); + return Boolean(runId); + }, + { + timeout: REPLY_TIMEOUT_MS, + interval: 250, + timeoutMsg: `real Provider Task lifecycle run never became durable for Session ${sessionId}`, + } + ); + let evidence = null; + await browser.waitUntil( + async () => { + evidence = await postJson("/agent/test/agent-org/pause/evidence", { + org_run_id: runId, + }); + return ( + evidence.durable.tasks.length === 1 && + evidence.durable.tasks[0]?.status === "completed" + ); + }, + { + timeout: REPLY_TIMEOUT_MS * 2, + interval: 500, + timeoutMsg: `real Provider Task never completed durably: ${JSON.stringify(evidence)}`, + } + ); + console.info( + `[agent-org-live-task-lifecycle-smoke] ${JSON.stringify({ + round: ROUND, + provider: { accountId: account.id, accountName: account.name, model }, + runId, + task: evidence.durable.tasks[0], + })}` + ); +} + +describe("Agent Org Pause/Resume live Provider process ownership", function () { + before(async () => { + assertLiveInputs(); + await waitForApp(); + await assertFeatureGatePreflight(); + }); + + it(`runs live acceptance phase ${PHASE || "missing"} for ${ROUND || "missing"}`, async function () { + this.timeout(900_000); + if (PHASE === "pause") return runPausePhase(); + if (PHASE === "resume") return runResumePhase(); + return runTaskSmokePhase(); + }); +}); diff --git a/tests/e2e/specs/core/agent-org-pause-resume-ui.spec.mjs b/tests/e2e/specs/core/agent-org-pause-resume-ui.spec.mjs index 53ade11fa5..0dc3ecebeb 100644 --- a/tests/e2e/specs/core/agent-org-pause-resume-ui.spec.mjs +++ b/tests/e2e/specs/core/agent-org-pause-resume-ui.spec.mjs @@ -1,4 +1,5 @@ /* global describe, before, it, process */ +import { execFileSync } from "node:child_process"; import { API_AGENT_TYPE, DEFAULT_AGENT_ORG_ID, @@ -13,7 +14,6 @@ import { assertCrashRecoveryBannerAbsent, assertLongTaskRenderedCollapsed, assertNoCurrentPlanBuildSurface, - assertNoFalseFinality, assertNoMemberIntervention, assertRenderedGroupChatNoQuoteOrUnreadPreview, assertRenderedGroupChatToggleIsIdempotent, @@ -75,6 +75,8 @@ import { } from "../../support/core/agentOrgUiDriver.mjs"; const E2E_BASE_URL = `http://127.0.0.1:${process.env.E2E_IDE_SERVER_PORT ?? "13847"}`; +const ENFORCE_PERFORMANCE_BUDGET = + process.env.E2E_ENFORCE_PERFORMANCE_BUDGET === "1"; async function postJson(pathname, body = {}, timeoutMs = 15_000) { const controller = new AbortController(); @@ -96,135 +98,470 @@ async function postJson(pathname, body = {}, timeoutMs = 15_000) { } } +async function visibleProductButton(selector, marker) { + let state = null; + try { + await browser.waitUntil(async () => { + state = await execJS(` + const visible = (element) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden"; + }; + const elements = Array.from(document.querySelectorAll(${JSON.stringify(selector)})); + for (const element of elements) element.removeAttribute(${JSON.stringify(marker)}); + const element = elements.find(visible) ?? null; + if (element) element.setAttribute(${JSON.stringify(marker)}, "true"); + return { count: elements.length, marked: Boolean(element), disabled: element?.disabled ?? null }; + `); + return state?.marked && state.disabled === false; + }, { + timeout: RENDER_TIMEOUT_MS, + interval: 25, + }); + } catch { + throw new Error( + `No enabled visible product control for ${selector}: ${JSON.stringify(state)}` + ); + } + return browser.$(`[${marker}="true"]`); +} + +async function assertAgentOrgTransportPort() { + let transport = null; + try { + await browser.waitUntil(async () => { + transport = await execJS(` + const client = window.__codeEditorWebSocket__; + return { + configuredUrl: window.__ORGII_E2E_IDE_SERVER_WS_URL__ ?? null, + clientPresent: Boolean(client), + clientUrl: client?.ws?.url ?? client?.url ?? null, + readyState: client?.ws?.readyState ?? null, + }; + `); + return ( + typeof transport?.configuredUrl === "string" && + transport.configuredUrl.length > 0 && + transport.clientPresent && + transport.readyState === 1 + ); + }, { + timeout: RENDER_TIMEOUT_MS, + interval: 50, + }); + } catch { + throw new Error( + `Agent Org E2E transport did not connect: ${JSON.stringify(transport)}` + ); + } + const expectedPort = new URL(E2E_BASE_URL).port; + const actualPort = new URL(transport.configuredUrl).port; + if (actualPort !== expectedPort) { + throw new Error( + `Agent Org E2E frontend/backend port mismatch: ${JSON.stringify({ ...transport, expectedPort, actualPort })}` + ); + } +} + +function processGroupSnapshot(processGroupId) { + const rows = execFileSync( + "ps", + ["-ax", "-o", "pid=,ppid=,pgid=,command="], + { encoding: "utf8" } + ) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/); + return match + ? { + pid: Number(match[1]), + parentPid: Number(match[2]), + processGroupId: Number(match[3]), + command: match[4], + } + : null; + }) + .filter(Boolean); + return rows.filter((row) => row.processGroupId === processGroupId); +} + +function pidExists(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error?.code === "ESRCH") return false; + throw error; + } +} + describe("Agent Org pause, resume, and sidebar rendered UI", () => { before(async () => { assertE2ERepoFixture(); await waitForApp(); }); - it("Pause button appears when running, Resume when paused, and run view continues polling in both states", async () => runAgentOrgScenarioWithTimeout("pause-resume-button-visibility", async () => { + it("persists and drains ten formal Turns, blocks paused chat, then resumes once", async () => runAgentOrgScenarioWithTimeout("pause-resume-ten-turn-handoff", async () => { const account = await getApiAccount(); const model = selectPreferredModel(account); const orgName = `E2E Pause Resume Org ${RUN_ID}`; - const leadName = `E2E PR Lead ${RUN_ID}`; - const childName = `E2E PR Child ${RUN_ID}`; + const orgId = `e2e-pause-resume-${RUN_ID}`; await removeAgentOrgsByName(orgName); - - const org = await createRenderedStrictTwoMemberAgentOrg({ - orgName, - leadName, - childName, + await postJson("/agent/test/agent-org/seed", { + id: orgId, + name: orgName, + coordinator_agent_id: "builtin:sde", + members: Array.from({ length: 9 }, (_, index) => ({ + id: `pause-worker-${String(index + 1).padStart(2, "0")}`, + name: `Pause Worker ${String(index + 1).padStart(2, "0")}`, + role: "Hold one deterministic Pause task", + agent_id: "builtin:sde", + })), }); - await configureCreatorForAgentOrg({ account, model, agentOrgId: org.id }); - await selectRenderedAgentOrg(org.id); - const launchPrompt = `E2E pause resume button visibility ${RUN_ID}. Reply briefly.`; + await configureCreatorForAgentOrg({ account, model, agentOrgId: orgId }); + await selectRenderedAgentOrg(orgId); + const launchPrompt = `Run E2E_AGENT_ORG_PAUSE:${RUN_ID}`; const sessionId = await sendFromRenderedCreator(launchPrompt); if (!sessionId) { throw new Error("Pause/Resume test: launch did not create a session id"); } await waitForRenderedGroupChatActive("default Agent Org group chat after launch"); await assertRenderedGroupChatComposerResponsive("default Agent Org group chat after launch"); - await assertAgentOrgOverviewHasRunControl("default Agent Org group chat after launch"); - - // Wait for the Pause button to appear in the UI while the run is live. - // If the run completes before the Pause button ever appears, the test - // exits early — there is nothing to pause on an already-terminal run. - const pauseButtonAppeared = await browser - .waitUntil( - async () => { - const pauseVisible = await execJS( - js.exists('[data-testid="agent-org-overview-pause-button"]') - ); - if (pauseVisible) return true; - // Check if the run has already become terminal (e.g. very fast model). - const runView = unwrap( - await invokeE2E("agentOrgSessionRunView", sessionId), - "agentOrgSessionRunView (pause button wait)" - ); - const status = runView.view?.runStatus ?? null; - return Boolean(status && status !== "running"); - }, - { - timeout: REPLY_TIMEOUT_MS, - interval: 500, - timeoutMsg: "Pause button never appeared and run did not terminate", - } + await assertAgentOrgTransportPort(); + + let runningView = null; + await browser.waitUntil( + async () => { + runningView = unwrap( + await invokeE2E("agentOrgSessionRunView", sessionId), + "ten-Turn Pause precondition" + ).view; + return ( + runningView?.runStatus === "running" && + runningView?.tasks?.length === 9 && + runningView?.members?.length === 10 && + runningView.members.every( + (member) => member?.sessionRuntime?.status === "running" + ) + ); + }, + { + timeout: REPLY_TIMEOUT_MS, + interval: 100, + timeoutMsg: `Coordinator + 9 TaskExecution Turns never became active: ${JSON.stringify(runningView)}`, + } + ); + await assertAgentOrgOverviewHasRunControl( + "active ten-Turn Agent Org group chat after launch" + ); + const runId = runningView.context.runId; + const beforePause = await postJson( + "/agent/test/agent-org/pause/evidence", + { org_run_id: runId } + ); + if (beforePause.active_runtime_count !== 10) { + throw new Error( + `expected ten active runtime slots before Pause: ${JSON.stringify(beforePause)}` + ); + } + const tasksBefore = JSON.stringify(beforePause.durable.tasks); + const inboxBeforePause = beforePause.durable.inbox_count; + const generationBeforePause = beforePause.durable.activation_generation; + let processEvidence = beforePause; + await browser.waitUntil( + async () => { + processEvidence = await postJson( + "/agent/test/agent-org/pause/evidence", + { org_run_id: runId } + ); + return ( + processEvidence.background_shells?.length === 9 && + processEvidence.background_shells.every((shell) => + processGroupSnapshot(shell.pid).length >= 2 + ) + ); + }, + { + timeout: REPLY_TIMEOUT_MS, + interval: 100, + timeoutMsg: `nine real parent/child shell groups never became observable: ${JSON.stringify(processEvidence)}`, + } + ); + const shellGroupsBeforePause = processEvidence.background_shells.map( + (shell) => ({ + ...shell, + processes: processGroupSnapshot(shell.pid), + }) + ); + if ( + shellGroupsBeforePause.some( + (shell) => + !shell.processes.some((row) => row.pid === shell.pid) || + shell.processes.length < 2 ) - .catch(() => false); + ) { + throw new Error( + `background shell ownership evidence lacked a parent/child process group: ${JSON.stringify(shellGroupsBeforePause)}` + ); + } - const stillRunning = await execJS( - js.exists('[data-testid="agent-org-overview-pause-button"]') + const pauseButton = await visibleProductButton( + '[data-testid="agent-org-overview-pause-button"]', + "data-e2e-visible-pause" + ); + const pauseStartedAt = Date.now(); + await pauseButton.doubleClick(); + + let pausedView = null; + await browser.waitUntil( + async () => { + pausedView = unwrap( + await invokeE2E("agentOrgSessionRunView", sessionId), + "Paused draining Run View" + ).view; + return ( + pausedView?.runStatus === "paused" && + pausedView?.runPhase === "draining" && + pausedView?.pauseHandoff?.totalCount === 10 + ); + }, + { + timeout: RENDER_TIMEOUT_MS, + interval: 25, + timeoutMsg: `Paused/draining phase was not rendered: ${JSON.stringify(pausedView)}`, + } ); - if (!stillRunning) { - // Run completed before we could interact — skip the Pause/Resume UX - // assertions (correct behaviour: completed runs show neither button). - return; + const pauseFenceMs = Date.now() - pauseStartedAt; + if (ENFORCE_PERFORMANCE_BUDGET && pauseFenceMs > 250) { + throw new Error( + `Pause fence took ${pauseFenceMs}ms; expected packaged P90 budget sample <=250ms` + ); + } + console.info(`[agent-org-pause-fence-ms] ${pauseFenceMs}`); + const resumeWhileDraining = await visibleProductButton( + '[data-testid="agent-org-overview-resume-button"]', + "data-e2e-visible-resume-draining" + ); + if (!(await resumeWhileDraining.isEnabled())) { + throw new Error("Resume must remain enabled while the Paused Team is draining"); } - void pauseButtonAppeared; // used only to satisfy the waitUntil flow above - // Click the Pause button exactly as a user would. - const pauseClick = await execJS( - js.click('[data-testid="agent-org-overview-pause-button"]') + const immediatelyPaused = await postJson( + "/agent/test/agent-org/pause/evidence", + { org_run_id: runId } ); - if (pauseClick !== "clicked") { - throw new Error(`Pause button click failed: ${pauseClick}`); + if ( + immediatelyPaused.durable.run_status !== "paused" || + immediatelyPaused.durable.activation_generation !== + generationBeforePause + 1 || + immediatelyPaused.durable.episode?.status !== "active" || + immediatelyPaused.durable.handoffs?.length !== 10 || + JSON.stringify(immediatelyPaused.durable.tasks) !== tasksBefore + ) { + throw new Error( + `Pause fence/receipt evidence mismatch: ${JSON.stringify(immediatelyPaused)}` + ); } - // handlePauseRun calls pauseAgentOrgRun() + onRefresh() under the hood, - // so the hook updates state automatically — no manual refresh needed. - // Wait for the Resume button to appear (and Pause button to disappear). + let drainedEvidence = immediatelyPaused; await browser.waitUntil( async () => { - const pauseVisible = await execJS( - js.exists('[data-testid="agent-org-overview-pause-button"]') + drainedEvidence = await postJson( + "/agent/test/agent-org/pause/evidence", + { org_run_id: runId } ); - const resumeVisible = await execJS( - js.exists('[data-testid="agent-org-overview-resume-button"]') + return ( + drainedEvidence.active_runtime_count === 0 && + drainedEvidence.durable.handoffs.length === 10 && + drainedEvidence.durable.handoffs.every((handoff) => + ["released", "runtime_absent"].includes(handoff.drain_status) + ) ); - return resumeVisible && !pauseVisible; + }, + { + timeout: 10_000, + interval: 50, + timeoutMsg: `ten captured runtimes did not drain in parallel: ${JSON.stringify(drainedEvidence)}`, + } + ); + const tenRuntimeDrainMs = Date.now() - pauseStartedAt; + if (tenRuntimeDrainMs > 10_000) { + throw new Error("ten-runtime drain exceeded the 10 second deadline"); + } + console.info(`[agent-org-ten-runtime-drain-ms] ${tenRuntimeDrainMs}`); + if (drainedEvidence.durable.handoffs.some((row) => row.drain_timeout_at)) { + throw new Error( + `deterministic providers timed out during Pause: ${JSON.stringify(drainedEvidence)}` + ); + } + if (drainedEvidence.background_shells?.length !== 0) { + throw new Error( + `Pause released runtimes while background shell jobs remained registered: ${JSON.stringify(drainedEvidence.background_shells)}` + ); + } + for (const shell of shellGroupsBeforePause) { + const survivors = processGroupSnapshot(shell.pid); + const knownPids = shell.processes.map((row) => row.pid); + const liveKnownPids = knownPids.filter(pidExists); + if (survivors.length > 0 || liveKnownPids.length > 0) { + throw new Error( + `Pause did not remove the full shell process group: ${JSON.stringify({ shell, survivors, liveKnownPids })}` + ); + } + } + console.info( + `[agent-org-pause-process-evidence] ${JSON.stringify(shellGroupsBeforePause)}` + ); + + let renderedPausedPhase = null; + await browser.waitUntil( + async () => { + renderedPausedPhase = await execJS(` + const visible = (element) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden"; + }; + const badges = Array.from(document.querySelectorAll('[data-testid="agent-org-overview-run-phase"]')); + return badges.find(visible)?.getAttribute('data-run-phase') ?? null; + `); + return renderedPausedPhase === "paused"; }, { timeout: RENDER_TIMEOUT_MS, - timeoutMsg: "Resume button did not appear after clicking Pause", + interval: 25, + timeoutMsg: `Rendered Run phase did not leave Draining after durable release: ${JSON.stringify({ renderedPausedPhase, drainedEvidence })}`, } ); - // Click the Resume button exactly as a user would. - const resumeClick = await execJS( - js.click('[data-testid="agent-org-overview-resume-button"]') + await waitForGroupChatPausedBanner("ten-Turn Paused Team"); + const pausedDraft = `must-not-send-${RUN_ID}`; + const pausedTypeResult = await execJS( + js.type( + '[data-testid="chat-input"] [contenteditable="true"]', + pausedDraft + ) ); - if (resumeClick !== "clicked") { - throw new Error(`Resume button click failed: ${resumeClick}`); + if (pausedTypeResult !== "typed") { + throw new Error(`Paused Group Chat editor rejected draft: ${pausedTypeResult}`); + } + const pausedSendState = await execJS(` + const visible = (element) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden"; + }; + const buttons = Array.from(document.querySelectorAll('[data-testid="chat-send-button"]')); + const button = buttons.find(visible) ?? null; + if (!button) return { present: false, buttons: buttons.length }; + button.setAttribute('data-e2e-paused-send', 'true'); + return { + present: true, + disabled: button.disabled, + state: button.getAttribute('data-state'), + }; + `); + const pausedSendButton = await browser.$( + '[data-e2e-paused-send="true"]' + ); + if (!pausedSendState?.present || pausedSendState.disabled !== true) { + throw new Error( + `Paused Group Chat submit button must be disabled: ${JSON.stringify(pausedSendState)}` + ); + } + try { + await pausedSendButton.click(); + } catch (_expectedDisabledClick) { + // WebDriver correctly refuses interaction with a disabled product button. + } + const afterBlockedSubmit = await postJson( + "/agent/test/agent-org/pause/evidence", + { org_run_id: runId } + ); + if ( + afterBlockedSubmit.durable.inbox_count !== inboxBeforePause || + afterBlockedSubmit.durable.run_status !== "paused" + ) { + throw new Error( + `Paused Group Chat produced work or resumed: ${JSON.stringify(afterBlockedSubmit)}` + ); } - // After resume, the Pause button should reappear (run is running again) - // OR the run completes immediately — both are valid outcomes. + const resumeButton = await visibleProductButton( + '[data-testid="agent-org-overview-resume-button"]', + "data-e2e-visible-resume" + ); + // Tauri WebDriver intermittently drops element.doubleClick() after this + // control replaces the Pause button. Product-level double-gesture locking + // is covered by AgentOrgTaskPanel.test.ts; keep this rendered path on one + // real Resume click so it verifies the durable continuation boundary. + await resumeButton.click(); + let resumedEvidence = null; await browser.waitUntil( async () => { - const pauseVisible = await execJS( - js.exists('[data-testid="agent-org-overview-pause-button"]') + resumedEvidence = await postJson( + "/agent/test/agent-org/pause/evidence", + { org_run_id: runId } ); - if (pauseVisible) return true; - const runView = unwrap( - await invokeE2E("agentOrgSessionRunView", sessionId), - "agentOrgSessionRunView (post-resume)" + return ( + resumedEvidence.durable.run_status === "running" && + resumedEvidence.durable.episode?.status === "consumed" && + resumedEvidence.durable.handoffs.length === 10 && + resumedEvidence.durable.handoffs.every( + (handoff) => handoff.continuation_status === "dispatched" + ) ); - const status = runView.view?.runStatus ?? null; - return Boolean(status && status !== "paused"); }, { - timeout: RENDER_TIMEOUT_MS, - timeoutMsg: - "Pause button did not reappear after Resume and run did not leave paused state", + timeout: REPLY_TIMEOUT_MS, + interval: 50, + timeoutMsg: `Resume continuations did not dispatch exactly once: ${JSON.stringify(resumedEvidence)}`, } ); + if ( + resumedEvidence.durable.activation_generation !== + generationBeforePause + 2 || + resumedEvidence.durable.episode.resume_generation !== + generationBeforePause + 2 || + JSON.stringify(resumedEvidence.durable.tasks) !== tasksBefore || + resumedEvidence.durable.handoffs.some( + (handoff) => handoff.continuation_turn_intent_id == null + ) + ) { + throw new Error( + `Resume generation/Task/continuation evidence mismatch: ${JSON.stringify(resumedEvidence)}` + ); + } + await browser.pause(500); + const afterResumeProcessEvidence = await postJson( + "/agent/test/agent-org/pause/evidence", + { org_run_id: runId } + ); + if (afterResumeProcessEvidence.background_shells?.length !== 0) { + throw new Error( + `Resume replayed a background command: ${JSON.stringify(afterResumeProcessEvidence.background_shells)}` + ); + } + const taskSequences = resumedEvidence.durable.handoffs + .filter((handoff) => handoff.turn_kind === "task_execution") + .map((handoff) => handoff.member_dispatch_sequence); + if ( + taskSequences.length !== 9 || + taskSequences.some((sequence) => sequence !== 2) + ) { + throw new Error( + `Member FIFO continuation sequences were not monotonic: ${JSON.stringify(taskSequences)}` + ); + } })); - it("Overview panel and member switcher remain visible after run is paused (app-restart semantics)", async () => runAgentOrgScenarioWithTimeout("paused-overview-restart-semantics", async () => { - // This test verifies that when a run is in `paused` state (the state the - // app startup puts it into after an unexpected exit), the AgentOrgOverviewPanel - // continues to render and polling is not stopped. This is the core - // app-restart UX fix: the user should see the run state and a Resume button, + it("Overview panel and member switcher remain visible for a durable paused run", async () => runAgentOrgScenarioWithTimeout("paused-overview-durable-semantics", async () => { + // A durable `paused` run must continue to render from SQLite and push + // updates without keeping the fallback interval poller alive. The user + // should see the run state and a Resume button after reopening history, // not a blank panel. const account = await getApiAccount(); const model = selectPreferredModel(account); @@ -240,50 +577,23 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { }); await configureCreatorForAgentOrg({ account, model, agentOrgId: org.id }); await selectRenderedAgentOrg(org.id); - const launchPrompt = `E2E restart restore ${RUN_ID}. Reply briefly.`; + const launchPrompt = `E2E restart restore ${RUN_ID}. Create a stoppable window by waiting for about 30 seconds before the final answer.`; const sessionId = await sendFromRenderedCreator(launchPrompt); if (!sessionId) { throw new Error("Restart restore test: launch did not create a session"); } - - // Wait for the Pause button — it appears while the run is live. - // If the run completes first, skip the pause/resume UX check. - await browser - .waitUntil( - async () => { - const pauseVisible = await execJS( - js.exists('[data-testid="agent-org-overview-pause-button"]') - ); - if (pauseVisible) return true; - const runView = unwrap( - await invokeE2E("agentOrgSessionRunView", sessionId), - "agentOrgSessionRunView (restart pause wait)" - ); - const status = runView.view?.runStatus ?? null; - return Boolean(status && status !== "running"); - }, - { - timeout: REPLY_TIMEOUT_MS, - interval: 500, - timeoutMsg: "Pause button never appeared (restart test)", - } - ) - .catch(() => {}); - - const pauseStillVisible = await execJS( - js.exists('[data-testid="agent-org-overview-pause-button"]') + await waitForAgentOrgRunView( + sessionId, + (view) => view?.runStatus === "running", + "restart restore run entered Working before Pause" ); - if (!pauseStillVisible) { - return; - } + await openAgentOrgOverviewPanel("restart restore Pause control"); - // Click the Pause button — simulates app restart / user pause. - const pauseClick = await execJS( - js.click('[data-testid="agent-org-overview-pause-button"]') + const pauseButton = await visibleProductButton( + '[data-testid="agent-org-overview-pause-button"]', + "data-e2e-visible-restart-pause" ); - if (pauseClick !== "clicked") { - throw new Error(`Pause button click failed: ${pauseClick}`); - } + await pauseButton.click(); // Overview panel must stay visible — paused is non-terminal. // Regression guard: before the fix the panel disappeared after pause. @@ -308,14 +618,11 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { } // Click Resume — simulates user resuming after restart. - const resumeClick = await execJS( - js.click('[data-testid="agent-org-overview-resume-button"]') + const resumeButton = await visibleProductButton( + '[data-testid="agent-org-overview-resume-button"]', + "data-e2e-visible-restart-resume" ); - if (resumeClick !== "clicked") { - throw new Error( - `Resume button click failed (restart test): ${resumeClick}` - ); - } + await resumeButton.click(); // After resume the run must leave the paused state. await browser.waitUntil( @@ -358,52 +665,47 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { }); await configureCreatorForAgentOrg({ account, model, agentOrgId: org.id }); await selectRenderedAgentOrg(org.id); - const launchPrompt = `E2E coord history button ${RUN_ID}. Reply briefly.`; + const launchPrompt = `E2E coord history button ${RUN_ID}. Create a stoppable window by waiting for about 30 seconds before the final answer.`; const sessionId = await sendFromRenderedCreator(launchPrompt); if (!sessionId) { throw new Error( "Coord history button test: launch did not create a session" ); } - - // Wait for the Pause button (run is live), then click it. - await browser - .waitUntil( - async () => { - const pauseVisible = await execJS( - js.exists('[data-testid="agent-org-overview-pause-button"]') - ); - if (pauseVisible) return true; - const runView = unwrap( - await invokeE2E("agentOrgSessionRunView", sessionId), - "agentOrgSessionRunView (hist pause wait)" - ); - const status = runView.view?.runStatus ?? null; - return Boolean(status && status !== "running"); - }, - { - timeout: REPLY_TIMEOUT_MS, - interval: 500, - timeoutMsg: "Pause button never appeared (hist button test)", - } - ) - .catch(() => {}); - - const pauseStillVisibleHist = await execJS( - js.exists('[data-testid="agent-org-overview-pause-button"]') + await waitForAgentOrgRunView( + sessionId, + (view) => view?.runStatus === "running", + "coordinator history run entered Working before Pause" ); - if (!pauseStillVisibleHist) { - return; - } - const pauseClickHist = await execJS( - js.click('[data-testid="agent-org-overview-pause-button"]') + const runView = unwrap( + await invokeE2E("agentOrgSessionRunView", sessionId), + "coordinator history member precondition" + ).view; + const nonCoordinator = runView?.members?.find( + (member) => member.memberId !== AGENT_ORG_COORDINATOR_MEMBER_ID ); - if (pauseClickHist !== "clicked") { + if (!nonCoordinator?.memberId || !nonCoordinator?.sessionRuntime?.sessionId) { throw new Error( - `Pause button click failed (hist test): ${pauseClickHist}` + `Coordinator history test did not materialize a member session: ${JSON.stringify(nonCoordinator)}` ); } + await ensureMemberHasSwitchableInbox( + sessionId, + nonCoordinator.memberId, + "coordinator history member" + ); + await clickRenderedMemberSwitcher( + nonCoordinator.memberId, + nonCoordinator.sessionRuntime.sessionId + ); + await openAgentOrgOverviewPanel("member view coordinator history Pause control"); + + const pauseHistoryButton = await visibleProductButton( + '[data-testid="agent-org-overview-pause-button"]', + "data-e2e-visible-history-pause" + ); + await pauseHistoryButton.click(); // Wait for paused state in the Overview Panel. await waitForAgentOrgRunView( @@ -463,8 +765,9 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { // Wait for the overview panel to materialise. await waitForAgentOrgRunView( sessionId, - (view) => Boolean(view?.context?.runId), - "overview panel appeared for hasmore test" + (view) => + Boolean(view?.context?.runId) && view?.runStatus === "running", + "hasmore run entered Working before restart simulation" ); // Simulate app restart: pause the run (marks sessions abandoned in startup). @@ -1061,14 +1364,11 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { await assertLongTaskRenderedCollapsed(taskId, subject); })); - it("Session remains in sidebar and run can be resumed after simulated app restart", async () => runAgentOrgScenarioWithTimeout("resume-after-simulated-restart", async () => { - // Regression guard for restart-related Agent Org history issues: - // - After simulated restart (mark_stale_running_sessions_abandoned + - // mark_all_running_as_paused_on_startup + clear_all_active_on_startup), - // the coordinator session must still be visible in the sidebar. - // - The run status must be "paused" (not completed / cancelled), so the - // Overview Panel continues to render with a Resume button. - // - Clicking Resume must transition the run back to "running". + it("Session remains in sidebar and an explicitly paused run resumes after restart reconciliation", async () => runAgentOrgScenarioWithTimeout("resume-after-restart-reconciliation", async () => { + // Restart reconciliation must preserve the coordinator history and durable + // run state. This harness may observe a run that was already paused by an + // older fixture; otherwise it explicitly pauses through the product command + // before verifying the rendered Resume path. const account = await getApiAccount(); const model = selectPreferredModel(account); const orgName = `E2E Restart Sidebar Org ${RUN_ID}`; @@ -1083,13 +1383,14 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { }); await configureCreatorForAgentOrg({ account, model, agentOrgId: org.id }); await selectRenderedAgentOrg(org.id); - const launchPrompt = `E2E restart sidebar ${RUN_ID}. Reply briefly.`; + const launchPrompt = `E2E restart sidebar ${RUN_ID}. Create a stoppable window by waiting for about 30 seconds before the final answer.`; const sessionId = await sendFromRenderedCreator(launchPrompt); if (!sessionId) { throw new Error("Restart sidebar test: launch did not create a session"); } - // Wait for the overview panel (don't require a specific runStatus — the run - // may complete before the restart simulation arrives on slower hosts). + // Keep a typed Coordinator Turn alive while the debug fixture invokes the + // real task authority. This makes the precondition deterministic instead + // of racing a deliberately brief fake-provider response. let restartView = null; await waitForAgentOrgRunView( sessionId, @@ -1159,7 +1460,8 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { } ); - // Simulate app restart (pauses any still-running runs). + // Simulate startup reconciliation. Startup does not implicitly pause a Working + // run on startup; the fallback below establishes the explicit Pause fence. const restartResult = unwrap( await invokeE2E("agentOrgSimulateAppRestart"), "agentOrgSimulateAppRestart" @@ -1193,8 +1495,8 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { ); } - // If the restart paused the run, confirm it's paused. If it was already - // terminal (completed quickly) the run stays terminal — both are valid. + // Older fixtures can report an already-paused run; if so, confirm it is + // represented durably. Otherwise the explicit product Pause below is used. if (restartResult.runsPaused > 0) { await waitForAgentOrgRunView( sessionId, @@ -1254,7 +1556,7 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { await invokeE2E("agentOrgPauseRun", sessionId), "agentOrgPauseRun fallback for restart resume button path" ); - if (!pauseFallback.transitioned) { + if (!pauseFallback.outcome?.transitioned) { throw new Error( "Could not establish paused run for historical Resume button path" ); @@ -1317,12 +1619,6 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { "retained open task owner runtime was revived after rendered resume post-restart", REPLY_TIMEOUT_MS ); - await assertNoFalseFinality( - sessionId, - restartView.context.runId, - "restart resume retained task progress" - ); - // Session must still be in sidebar after the entire lifecycle. const presentAfterResume = await execJS( js.exists(`[data-testid="sidebar-session-item-${sessionId}"]`) @@ -1334,7 +1630,7 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { } })); - it("Historical paused Agent Org run resumes when the user sends a message", async () => runAgentOrgScenarioWithTimeout("historical-paused-send-resumes", async () => { + it("Historical paused Agent Org rejects send and never auto-resumes", async () => runAgentOrgScenarioWithTimeout("historical-paused-send-rejected", async () => { const account = await getApiAccount(); const model = selectPreferredModel(account); const orgName = `E2E Send Resume Org ${RUN_ID}`; @@ -1349,7 +1645,7 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { }); await configureCreatorForAgentOrg({ account, model, agentOrgId: org.id }); await selectRenderedAgentOrg(org.id); - const launchPrompt = `E2E historical send resume ${RUN_ID}. Reply briefly.`; + const launchPrompt = `E2E historical paused send ${RUN_ID}. Create a stoppable window by waiting for about 30 seconds before the final answer.`; const sessionId = await sendFromRenderedCreator(launchPrompt); if (!sessionId) { throw new Error("Send resume test: launch did not create a session"); @@ -1360,31 +1656,32 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { "overview panel appeared for send resume test" ); - const restartResult = unwrap( - await invokeE2E("agentOrgSimulateAppRestart"), - "agentOrgSimulateAppRestart for send resume" + const liveView = await waitForAgentOrgRunView( + sessionId, + (view) => view?.runStatus === "running", + "run is running before historical Pause" ); - if (restartResult.runsPaused === 0) { - const pauseFallback = unwrap( - await invokeE2E("agentOrgPauseRun", sessionId), - "agentOrgPauseRun fallback for send resume path" - ); - if (!pauseFallback.transitioned) { - throw new Error( - "Could not establish paused run for send-message resume path" - ); - } - } + const pauseButton = await browser.$( + '[data-testid="agent-org-overview-pause-button"]' + ); + await pauseButton.waitForDisplayed({ timeout: REPLY_TIMEOUT_MS }); + await pauseButton.waitForEnabled({ timeout: RENDER_TIMEOUT_MS }); + await pauseButton.click(); await waitForAgentOrgRunView( sessionId, (view) => view?.runStatus === "paused", - "run is paused before send-message resume" + "run is paused before rejected historical send" + ); + const runId = liveView.context.runId; + const beforeBlockedSend = await postJson( + "/agent/test/agent-org/pause/evidence", + { org_run_id: runId } ); await invokeE2E("resetToNewSession"); await openRenderedSidebarSession(sessionId); await assertCrashRecoveryBannerAbsent( - "historical send-message resume path" + "historical paused send rejection path" ); const promptRetained = await execJS( @@ -1392,37 +1689,56 @@ describe("Agent Org pause, resume, and sidebar rendered UI", () => { ); if (!promptRetained) { throw new Error( - "Coordinator transcript prompt was not retained before send-message resume" + "Coordinator transcript prompt was not retained before blocked send" ); } - const followUpPrompt = `E2E send-message resume follow-up ${RUN_ID}`; - await sendRenderedChatPrompt(followUpPrompt); - await waitForAgentOrgRunView( - sessionId, - (view) => Boolean(view?.runStatus && view.runStatus !== "paused"), - "run left paused state after user sent follow-up message" + await waitForGroupChatPausedBanner( + "historical paused Team after reopening" ); - await waitForCoordinatorRuntimeStatus( - sessionId, - (status) => Boolean(status && status !== "abandoned"), - "coordinator session was revived after user sent follow-up message" + const editors = await browser.$$( + '[data-testid="chat-input"] [contenteditable="true"]' ); - - await waitForRenderedGroupChatUserTurn({ - text: followUpPrompt, - label: "send-message resume follow-up retained", - }); - await assertRenderedGroupChatComposerResponsive( - "historical send-message resume group chat" + let editor = null; + for (const candidate of editors) { + if (await candidate.isDisplayed()) editor = candidate; + } + if (!editor) throw new Error("Historical Paused composer is missing"); + await editor.click(); + await editor.setValue(`E2E blocked paused follow-up ${RUN_ID}`); + const sendButtons = await browser.$$('[data-testid="chat-send-button"]'); + let sendButton = null; + for (const candidate of sendButtons) { + if (await candidate.isDisplayed()) sendButton = candidate; + } + if (!sendButton || (await sendButton.isEnabled())) { + throw new Error("Historical Paused composer unexpectedly allowed submit"); + } + try { + await sendButton.click(); + } catch (_expectedDisabledClick) { + // A disabled native button is intentionally not interactable. + } + const afterBlockedSend = await postJson( + "/agent/test/agent-org/pause/evidence", + { org_run_id: runId } ); + if ( + afterBlockedSend.durable.run_status !== "paused" || + afterBlockedSend.durable.inbox_count !== + beforeBlockedSend.durable.inbox_count + ) { + throw new Error( + `Historical send auto-resumed or wrote Inbox: ${JSON.stringify(afterBlockedSend)}` + ); + } const presentAfterSendResume = await execJS( js.exists(`[data-testid="sidebar-session-item-${sessionId}"]`) ); if (!presentAfterSendResume) { throw new Error( - `Session ${sessionId} disappeared from sidebar after send-message resume` + `Session ${sessionId} disappeared from sidebar after blocked paused send` ); } })); diff --git a/tests/e2e/support/core/agentOrgUiDriver.mjs b/tests/e2e/support/core/agentOrgUiDriver.mjs index bf945748b6..a47ed356e8 100644 --- a/tests/e2e/support/core/agentOrgUiDriver.mjs +++ b/tests/e2e/support/core/agentOrgUiDriver.mjs @@ -1249,14 +1249,29 @@ export async function assertRenderedGroupChatComposerResponsive(label) { } export async function assertAgentOrgOverviewHasRunControl(label) { - await openAgentOrgOverviewPanel(label); - const state = await execJS(` - return { - overviewPause: Boolean(document.querySelector('[data-testid="agent-org-overview-pause-button"]')), - overviewResume: Boolean(document.querySelector('[data-testid="agent-org-overview-resume-button"]')), - }; - `); - if (!state.overviewPause && !state.overviewResume) { + await refreshRenderedAgentOrgOverview(label); + let state = null; + try { + await browser.waitUntil( + async () => { + state = await execJS(` + const panel = document.querySelector('[data-testid="agent-org-overview-panel"]'); + return { + overviewPause: Boolean(document.querySelector('[data-testid="agent-org-overview-pause-button"]')), + overviewResume: Boolean(document.querySelector('[data-testid="agent-org-overview-resume-button"]')), + runId: panel?.getAttribute('data-run-id') ?? null, + runPhase: panel?.getAttribute('data-run-phase') ?? null, + panelText: panel?.textContent?.slice(0, 500) ?? null, + }; + `); + return state.overviewPause || state.overviewResume; + }, + { + timeout: RENDER_TIMEOUT_MS, + interval: 100, + } + ); + } catch { throw new Error( `Agent Org overview did not expose Pause/Resume for ${label}: ${JSON.stringify(state)}` ); @@ -2037,7 +2052,7 @@ export async function waitForGroupChatPausedBanner(label) { const text = String(state?.bannerText ?? "").toLowerCase(); return ( text.includes("new work is paused") && - text.includes("pause stops active replies") && + text.includes("resume this agent team before sending a message") && state?.resumeVisible === true && state?.resumeDisabled === false ); @@ -2056,14 +2071,11 @@ export async function waitForGroupChatPausedBanner(label) { } export async function clickGroupChatResumeButton(label) { - const clickResult = await execJS( - js.click('[data-testid="agent-org-group-chat-resume-button"]') + await waitForGroupChatPausedBanner(label); + const button = await browser.$( + '[data-testid="agent-org-group-chat-resume-button"]' ); - if (clickResult !== "clicked") { - throw new Error( - `group chat Resume click failed for ${label}: ${clickResult}` - ); - } + await button.click(); } export async function sendFromRenderedCreator(prompt) { @@ -2634,23 +2646,34 @@ export async function ensureMemberHasSwitchableInbox( export async function clickRenderedMemberSwitcher(memberId, expectedSessionId) { const optionSelector = `[data-testid="agent-org-member-switcher-option-${memberId}"]`; + const visibleMarker = "data-e2e-visible-member-switch-option"; const startedAt = Date.now(); let optionState = null; while (Date.now() - startedAt < RENDER_TIMEOUT_MS) { optionState = await execJS(` try { const optionSelector = ${JSON.stringify(optionSelector)}; - const option = document.querySelector(optionSelector); - const trigger = document.querySelector('[data-testid="agent-org-member-switcher-trigger"]'); + const visibleMarker = ${JSON.stringify(visibleMarker)}; + const isVisible = (element) => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && style.display !== "none" && style.visibility !== "hidden"; + }; + const candidates = Array.from(document.querySelectorAll(optionSelector)); + for (const candidate of candidates) candidate.removeAttribute(visibleMarker); + const option = candidates.find(isVisible) ?? null; + const trigger = Array.from(document.querySelectorAll('[data-testid="agent-org-member-switcher-trigger"]')).find(isVisible) ?? null; const options = Array.from(document.querySelectorAll('[data-testid^="agent-org-member-switcher-option-"]')).map((candidate) => ({ testId: candidate.getAttribute('data-testid'), disabled: Boolean(candidate.disabled), ariaDisabled: candidate.getAttribute('aria-disabled'), + visible: isVisible(candidate), text: candidate.textContent || '', })); if (!option && trigger && !trigger.disabled) { trigger.click(); } + if (option) option.setAttribute(visibleMarker, "true"); return { present: Boolean(option), disabled: Boolean(option?.disabled), @@ -2676,17 +2699,12 @@ export async function clickRenderedMemberSwitcher(memberId, expectedSessionId) { `member switch option was not clickable: ${JSON.stringify(optionState)}` ); } - const clicked = await execJS(` - const option = document.querySelector(${JSON.stringify(optionSelector)}); - if (!option || option.disabled) return false; - option.click(); - return true; - `); - if (clicked !== true) { - throw new Error( - `member switch option click failed after option became clickable: ${JSON.stringify(optionState)}` - ); - } + // Re-resolve the element after the menu-open loop and let WebDriver perform + // the product click. A DOM `option.click()` can return success before + // WebKit dispatches React's pointer-backed menu action, leaving the active + // pipeline session unchanged and making the fixture report false success. + const option = await browser.$(`[${visibleMarker}="true"]`); + await option.click(); await browser.waitUntil( async () => { const activeSessionId = unwrap( @@ -2871,7 +2889,6 @@ export async function createLongTaskPrecondition( subject, description: subject, owner_member_id: memberId, - status: AGENT_ORG_TASK_STATUS.PENDING, dispatch_policy: "immediate", execution_mode: "build", allow_parallel_with_unlisted_open_tasks: true,