Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,8 @@ function createInteractionCoordinator(
preflightSessionSnapshot: () => true,
refreshCanonicalContinuity: async () => undefined,
onPoison: () => undefined,
onSandboxBoundarySettled: async () => undefined,
resolveSandboxBoundaryRootSession: async () => undefined,
onSandboxBoundaryGraphWake: async () => undefined,
};
return new HostInteractionCoordinator(options);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -617,7 +617,8 @@ async function createFixture(options: { recoverAdmissions?: boolean } = {}): Pro
onPoison: () => {
requestedDrain = true;
},
onSandboxBoundarySettled: async () => {},
resolveSandboxBoundaryRootSession: async () => undefined,
onSandboxBoundaryGraphWake: async () => {},
});
const backends = new BackendRegistry();
backends.register('ai-sdk', (context) => new FakeBackend(context));
Expand Down
206 changes: 201 additions & 5 deletions packages/runtime-host/src/__tests__/interaction-coordinator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

import { deferred } from '@maka/core/test-only/async-primitives';
import { deferred, withTimeout } from '@maka/core/test-only/async-primitives';
import assert from 'node:assert/strict';
import { mkdir, mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
Expand Down Expand Up @@ -294,7 +294,8 @@ describe('HostInteractionCoordinator', () => {
return true;
},
refreshCanonicalContinuity: async () => {},
onSandboxBoundarySettled: async (sessionId) => {
resolveSandboxBoundaryRootSession: async (sessionId) => sessionId,
onSandboxBoundaryGraphWake: async (sessionId) => {
assert.equal(sessionId, session.id);
graphWakes += 1;
},
Expand Down Expand Up @@ -388,6 +389,198 @@ describe('HostInteractionCoordinator', () => {
});
});

test('does not hold Session admission while graph wake reconciliation waits', async () => {
await withStore(async ({ owner, store, stores }) => {
const workspace = join(owner.capability.canonicalPath, 'wake-workspace');
await mkdir(workspace);
const session = await stores.sessionStore.create({
cwd: workspace,
llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',
llmConnectionSlug: 'fake',
model: 'fake-model',
permissionMode: 'ask',
});
const identity = { ...RUN, sessionId: session.id };
const wakeStarted = deferred();
const releaseWake = deferred();
const wakeFinished = deferred();
const resolverStarted = deferred();
const releaseResolver = deferred();
let resolvedRootSessionId: string | undefined;
let wakeNotificationStarted = false;
const coordinator = new HostInteractionCoordinator({
store,
sandboxBoundaries: stores.sessionStore,
sessionAdmission: new SessionAdmissionGate(),
sessions: stores.sessionStore,
preflightSessionSnapshot: () => true,
refreshCanonicalContinuity: async () => {},
resolveSandboxBoundaryRootSession: async (sessionId) => {
resolvedRootSessionId = sessionId;
resolverStarted.resolve();
await releaseResolver.promise;
return session.id;
},
onSandboxBoundaryGraphWake: async (rootSessionId) => {
assert.equal(rootSessionId, session.id);
wakeNotificationStarted = true;
wakeStarted.resolve();
await releaseWake.promise;
wakeFinished.resolve();
},
onPoison: () => {},
});
const binding = coordinator.bindRun(identity);
const request = sandboxBoundaryEvent({
sessionId: session.id,
requestId: 'boundary_wake_wait',
status: 'pending',
baseRevision: 0,
turnId: identity.turnId,
runId: identity.runId,
expansion: { network: { enabled: true } },
justification: 'Connect to the requested service.',
createdAt: 1,
});
await binding.acceptSandboxBoundaryRequest({
request,
continuation: sandboxBoundaryContinuation(identity, request.requestId),
});

let answerSettled = false;
let answerResult:
| Awaited<ReturnType<(typeof coordinator.handlers)['interaction.answer']>>
| undefined;
const answer = coordinator.handlers['interaction.answer'](
{
sessionId: session.id,
interactionId: request.requestId,
answer: { kind: 'sandbox_boundary', decision: 'allow' },
},
connection(),
);
void answer.then(
(result) => {
answerResult = result;
answerSettled = true;
},
() => {
answerSettled = true;
},
);
let querySettled = false;
let query: ReturnType<(typeof coordinator.handlers)['interaction.query']> | undefined;
try {
await withTimeout(
resolverStarted.promise,
5_000,
'sandbox boundary root-session resolver did not start',
);
query = coordinator.handlers['interaction.query'](
{ sessionId: session.id, interactionId: request.requestId },
connection(),
);
void query.then(
() => {
querySettled = true;
},
() => {
querySettled = true;
},
);
await new Promise<void>((resolve) => setImmediate(resolve));
assert.equal(
querySettled,
false,
'interaction query bypassed the resolver admission lease',
);
releaseResolver.resolve();
await answer;
assert.ok(query);
const queryResult = await query;
assert.equal(queryResult.ok, true);
assert.equal(querySettled, true);
await withTimeout(wakeStarted.promise, 5_000, 'sandbox boundary graph wake did not start');
assert.equal(
answerSettled,
true,
'interaction answer waited for graph wake reconciliation',
);
assert.equal(answerResult?.ok, true);
if (answerResult?.ok) assert.equal(answerResult.result.status, 'answered');
assert.equal(resolvedRootSessionId, session.id);
} finally {
releaseResolver.resolve();
releaseWake.resolve();
if (wakeNotificationStarted) await wakeFinished.promise;
await answer.catch(() => undefined);
}
await binding.close('turn_terminal');
binding.release();
await coordinator.close();
});
});

test('poisons when detached graph wake notification rejects', async () => {
await withStore(async ({ owner, store, stores }) => {
const workspace = join(owner.capability.canonicalPath, 'wake-rejection-workspace');
await mkdir(workspace);
const session = await stores.sessionStore.create({
cwd: workspace,
llmConnectionId: 'cccccccc-cccc-4ccc-8ccc-cccccccccccc',
llmConnectionSlug: 'fake',
model: 'fake-model',
permissionMode: 'ask',
});
const identity = { ...RUN, sessionId: session.id };
const poison: RuntimeInteractionFailStopError[] = [];
const coordinator = new HostInteractionCoordinator({
store,
sandboxBoundaries: stores.sessionStore,
sessionAdmission: new SessionAdmissionGate(),
sessions: stores.sessionStore,
preflightSessionSnapshot: () => true,
refreshCanonicalContinuity: async () => {},
resolveSandboxBoundaryRootSession: async () => session.id,
onSandboxBoundaryGraphWake: async () => {
throw new Error('graph wake notification failed');
},
onPoison: (error) => poison.push(error),
});
const binding = coordinator.bindRun(identity);
const request = sandboxBoundaryEvent({
sessionId: session.id,
requestId: 'boundary_wake_rejection',
status: 'pending',
baseRevision: 0,
turnId: identity.turnId,
runId: identity.runId,
expansion: { network: { enabled: true } },
justification: 'Connect to the requested service.',
createdAt: 1,
});
await binding.acceptSandboxBoundaryRequest({
request,
continuation: sandboxBoundaryContinuation(identity, request.requestId),
});

const answerResult = await coordinator.handlers['interaction.answer'](
{
sessionId: session.id,
interactionId: request.requestId,
answer: { kind: 'sandbox_boundary', decision: 'allow' },
},
connection(),
);
assert.equal(answerResult.ok, true);
await new Promise<void>((resolve) => setImmediate(resolve));
assert.equal(poison.length, 1);
assert.equal(coordinator.isPoisoned(), true);
await assert.rejects(binding.close('turn_terminal'), poison[0]);
await assert.rejects(coordinator.close(), poison[0]);
});
});

test('a queued stop waits for sandbox boundary publication before closing its Run', async () => {
await withStore(async ({ owner, store, stores }) => {
const workspace = join(owner.capability.canonicalPath, 'publication-workspace');
Expand Down Expand Up @@ -417,7 +610,8 @@ describe('HostInteractionCoordinator', () => {
await releaseAdmissionRefresh.promise;
},
onPoison: () => {},
onSandboxBoundarySettled: async () => {},
resolveSandboxBoundaryRootSession: async () => undefined,
onSandboxBoundaryGraphWake: async () => {},
});
const binding = await bindRuntimeInteractionRun(coordinator, identity);
const request = sandboxBoundaryEvent({
Expand Down Expand Up @@ -501,7 +695,8 @@ describe('HostInteractionCoordinator', () => {
preflightSessionSnapshot: () => false,
refreshCanonicalContinuity: async () => {},
onPoison: () => {},
onSandboxBoundarySettled: async () => {},
resolveSandboxBoundaryRootSession: async () => undefined,
onSandboxBoundaryGraphWake: async () => {},
});
const ownerRun = coordinator.bindRun(identity);

Expand Down Expand Up @@ -845,7 +1040,8 @@ function createCoordinator(
preflightSessionSnapshot: () => true,
refreshCanonicalContinuity: async () => {},
onPoison: () => {},
onSandboxBoundarySettled: async () => {},
resolveSandboxBoundaryRootSession: async () => undefined,
onSandboxBoundaryGraphWake: async () => {},
...overrides,
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2717,7 +2717,8 @@ test('hosted linked child roots share admission, message, terminal, and stop aut
onPoison: () => {
drainRequested = true;
},
onSandboxBoundarySettled: async () => {},
resolveSandboxBoundaryRootSession: async () => undefined,
onSandboxBoundaryGraphWake: async () => {},
});
const interactionAuthority: RuntimeInteractionAuthority = {
bindRun: (identity) => {
Expand Down Expand Up @@ -5286,7 +5287,8 @@ async function createFailureFixture(options: {
refreshCanonicalContinuity: (sessionId, admission) =>
requireContinuity(continuity).refreshCanonical(sessionId, admission),
onPoison: requestDrain,
onSandboxBoundarySettled: async () => {},
resolveSandboxBoundaryRootSession: async () => undefined,
onSandboxBoundaryGraphWake: async () => {},
})
: undefined;
const backends = new BackendRegistry();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import assert from 'node:assert/strict';
import { test } from 'node:test';
import { agentGraphIdForRootSession } from '@maka/runtime/stream-graph-coordinator';
import {
notifySandboxBoundaryGraphWake,
resolveSandboxBoundaryRootSession,
sandboxBoundaryGraphWakeRoot,
} from '../server/sandbox-boundary-graph-wake.js';

Expand Down Expand Up @@ -78,9 +78,8 @@ test('rejects graph operator lineage that is not owned by its parent Session', a
);
});

test('reads durable operator lineage before notifying only its root graph', async () => {
test('resolves durable operator lineage for only its root graph', async () => {
const reads: string[] = [];
const wakes: string[] = [];
const headers = new Map([
[
'graph-operator',
Expand All @@ -106,16 +105,17 @@ test('reads durable operator lineage before notifying only its root graph', asyn
return header;
},
};
const notify = async (sessionId: string) => {
wakes.push(sessionId);
};

const graphIds = idsFor('root-session');
await notifySandboxBoundaryGraphWake('graph-operator', reader, graphIds, notify);
await notifySandboxBoundaryGraphWake('ordinary-child', reader, graphIds, notify);
assert.equal(
await resolveSandboxBoundaryRootSession('graph-operator', reader, graphIds),
'root-session',
);
assert.equal(
await resolveSandboxBoundaryRootSession('ordinary-child', reader, graphIds),
undefined,
);

assert.deepEqual(reads, ['graph-operator', 'ordinary-child']);
assert.deepEqual(wakes, ['root-session']);
});

function idsFor(rootSessionId: string) {
Expand Down
22 changes: 12 additions & 10 deletions packages/runtime-host/src/server/execution-composition.ts
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ import { HostPluginPlatform } from './plugin-platform.js';
import { RootAdmissionOwner } from './root-admission-owner.js';
import { RootTurnCoordinator } from './root-turn-coordinator.js';
import { RuntimePolicyActivationGate } from './runtime-policy-activation-gate.js';
import { notifySandboxBoundaryGraphWake } from './sandbox-boundary-graph-wake.js';
import { resolveSandboxBoundaryRootSession } from './sandbox-boundary-graph-wake.js';
import { HostRuntimePolicyCoordinator } from './runtime-policy-coordinator.js';
import { startHostModelMetadataRefresh } from './model-metadata-refresh.js';
import { HostRuntimeResourceCoordinator } from './runtime-resource-coordinator.js';
Expand Down Expand Up @@ -680,17 +680,19 @@ export async function createExecutionRuntimeHostComposition(
beginDrain();
context.requestDrain();
},
onSandboxBoundarySettled: (sessionId) =>
notifySandboxBoundaryGraphWake(
sessionId,
stores.sessionStore,
{
resolveSandboxBoundaryRootSession: async (sessionId) => {
try {
return await resolveSandboxBoundaryRootSession(sessionId, stores.sessionStore, {
listGraphIds: (rootSessionId) =>
requireGraphCoordinator(graphCoordinator).listGraphIds(rootSessionId),
},
(rootSessionId) =>
requireGraphSupervisorWake(graphSupervisorWake).notifyPermissionResponse(rootSessionId),
),
});
} catch (error) {
if (isSessionNotFoundError(error)) return undefined;
throw error;
}
},
onSandboxBoundaryGraphWake: (rootSessionId) =>
requireGraphSupervisorWake(graphSupervisorWake).notifyPermissionResponse(rootSessionId),
});
memory = new HostMemoryCoordinator({
store: memoryStore,
Expand Down
Loading