From 2ec28d7f36c4dc561fbcdde6d24d01c1b3268300 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:49:27 +0000 Subject: [PATCH 01/10] fix(eval): restore log commands for evaluation resources --- src/core/eval.tsx | 42 +++++++-- src/core/evalLogSource.ts | 95 ++++++++++++++++++++ src/handlers/eval/batch-evaluation/index.tsx | 4 +- src/handlers/eval/batch-insights/index.tsx | 4 +- src/handlers/eval/logs/index.tsx | 29 ++++++ src/handlers/eval/online-eval/index.tsx | 4 +- src/handlers/eval/online-insight/index.tsx | 4 +- src/handlers/eval/types.tsx | 26 +++++- src/testing/TestCoreClient.tsx | 39 ++++++-- 9 files changed, 229 insertions(+), 18 deletions(-) create mode 100644 src/core/evalLogSource.ts create mode 100644 src/handlers/eval/logs/index.tsx diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 26cb71a1f6..54d22170ab 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -141,6 +141,7 @@ import type { DeleteOnlineInsightResponse, EvaluateInput, EvaluateResult, + EvalLogsInput, GetBatchEvaluationResult, GetTracesInput, LlmAsAJudgeUpdate, @@ -173,6 +174,8 @@ import { toClientConfig } from "./utils"; import { grantOnlineEvalScope } from "./onlineEvalExecutionRole"; import { accountIdFromArn, deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole"; import { harnessRuntimeFromResponse } from "./harness"; +import { resolveEvalLogSource } from "./evalLogSource"; +import type { LogSource } from "./observability/types"; const DEFAULT_INGESTION_WAIT_MS = 180_000; const DATASET_EXAMPLES_BATCH_LIMIT = 1000; @@ -237,6 +240,14 @@ export class EvalClient implements CoreEvalClient { private readonly cloudWatch: CloudWatchClient = new CloudWatchClient(clients), ) {} + resolveLogSource( + input: EvalLogsInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + return resolveEvalLogSource({ client: this, input, options, signal }); + } + async createEvaluator( request: CreateEvaluatorRequest, options: CoreOptions, @@ -457,11 +468,14 @@ export class EvalClient implements CoreEvalClient { async getBatchEvaluation( id: string, options: CoreOptions, - { includeResults = true }: { includeResults?: boolean } = {}, + { includeResults = true, signal }: { includeResults?: boolean; signal?: AbortSignal } = {}, ): Promise { const job = await this.clients .data(toClientConfig(options)) - .send(new GetBatchEvaluationCommand({ batchEvaluationId: id })); + .send( + new GetBatchEvaluationCommand({ batchEvaluationId: id }), + signal ? { abortSignal: signal } : undefined, + ); const detail: BatchEvaluationDetail = { ...job }; const cw = job.outputConfig?.cloudWatchConfig; @@ -496,8 +510,15 @@ export class EvalClient implements CoreEvalClient { } } - async getBatchInsights(id: string, options: CoreOptions): Promise { - const { detail } = await this.getBatchEvaluation(id, options, { includeResults: false }); + async getBatchInsights( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + const { detail } = await this.getBatchEvaluation(id, options, { + includeResults: false, + ...(signal ? { signal } : {}), + }); if (!EvalClient.isBatchInsights(detail)) { throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); } @@ -1167,10 +1188,15 @@ export class EvalClient implements CoreEvalClient { ); } - async getOnlineInsight(id: string, options: CoreOptions): Promise { + async getOnlineInsight( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { const control = this.clients.control(toClientConfig(options)); const config = await control.send( new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id }), + signal ? { abortSignal: signal } : undefined, ); if ((config.insights?.length ?? 0) === 0) throw new InputValidationError(`"${id}" is not an online-insight config`, { @@ -1307,10 +1333,14 @@ export class EvalClient implements CoreEvalClient { async getOnlineEvaluationConfig( id: string, options: CoreOptions, + signal?: AbortSignal, ): Promise { return this.clients .control(toClientConfig(options)) - .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); + .send( + new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id }), + signal ? { abortSignal: signal } : undefined, + ); } async listOnlineEvaluationConfigs( diff --git a/src/core/evalLogSource.ts b/src/core/evalLogSource.ts new file mode 100644 index 0000000000..be4765bd5d --- /dev/null +++ b/src/core/evalLogSource.ts @@ -0,0 +1,95 @@ +import type { CoreEvalClient, EvalLogsInput } from "../handlers/eval/types"; +import { InputValidationError, MalformedServiceResponseError } from "../errors"; +import { serviceIdFromArn } from "./arn"; +import type { LogSource } from "./observability/types"; +import type { CoreOptions } from "./types"; + +type EvalLogClient = Pick< + CoreEvalClient, + "getOnlineEvaluationConfig" | "getOnlineInsight" | "getBatchEvaluation" | "getBatchInsights" +>; + +export async function resolveEvalLogSource({ + client, + input, + options, + signal, +}: { + client: EvalLogClient; + input: EvalLogsInput; + options: CoreOptions; + signal?: AbortSignal; +}): Promise { + const config = await (async () => { + switch (input.resource) { + case "online-eval": + return client.getOnlineEvaluationConfig(input.id, options, signal); + case "online-insight": + return client.getOnlineInsight(input.id, options, signal); + case "batch-evaluation": + return ( + await client.getBatchEvaluation(input.id, options, { + includeResults: false, + ...(signal ? { signal } : {}), + }) + ).detail; + case "batch-insights": + return client.getBatchInsights(input.id, options, signal); + } + })(); + + const cloudWatchConfig = config.outputConfig?.cloudWatchConfig; + if (cloudWatchConfig?.logGroupName) { + return { logGroupName: cloudWatchConfig.logGroupName }; + } + if (config.outputConfig && !cloudWatchConfig) { + throw new InputValidationError( + `${input.resource} "${input.id}" has an unsupported output destination; ` + + "logs require a CloudWatch output configuration.", + ); + } + if (cloudWatchConfig?.resultDestination !== "SOURCE_LOG_GROUP") { + const isOnline = input.resource === "online-eval" || input.resource === "online-insight"; + const onlineConfigId = + "onlineEvaluationConfigId" in config ? config.onlineEvaluationConfigId : undefined; + return { + logGroupName: isOnline + ? `/aws/bedrock-agentcore/evaluations/results/${onlineConfigId ?? serviceIdFromArn(input.id)}` + : "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + }; + } + + let dataSourceConfig = config.dataSourceConfig; + if (dataSourceConfig && "onlineEvaluationConfigSource" in dataSourceConfig) { + const configArn = dataSourceConfig.onlineEvaluationConfigSource?.onlineEvaluationConfigArn; + if (!configArn) { + throw new MalformedServiceResponseError( + `${input.resource} "${input.id}" returned an online evaluation source without a config ARN.`, + ); + } + dataSourceConfig = ( + await client.getOnlineEvaluationConfig(serviceIdFromArn(configArn), options, signal) + ).dataSourceConfig; + } + + const source = dataSourceConfig?.cloudWatchLogs; + if (source?.logGroupNamePrefixes?.length) { + throw new InputValidationError( + `${input.resource} "${input.id}" writes results to source log groups selected by prefix. ` + + "Read the matching groups directly in CloudWatch Logs, or configure one explicit source log group.", + ); + } + if ((source?.logGroupNames?.length ?? 0) > 1) { + throw new InputValidationError( + `${input.resource} "${input.id}" writes results to multiple source log groups. ` + + "Read each group directly in CloudWatch Logs, or configure one explicit source log group.", + ); + } + const logGroupName = source?.logGroupNames?.[0]; + if (!logGroupName) { + throw new MalformedServiceResponseError( + `${input.resource} "${input.id}" writes results to a source log group but returned no log group name.`, + ); + } + return { logGroupName }; +} diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx index bcf6a5fb56..67b5299e36 100644 --- a/src/handlers/eval/batch-evaluation/index.tsx +++ b/src/handlers/eval/batch-evaluation/index.tsx @@ -8,6 +8,7 @@ import { createListBatchEvaluationsHandler } from "./list"; import { createEvaluateBatchEvaluationHandler } from "./evaluate"; import { createSimulateBatchEvaluationHandler } from "./simulate"; import { createStopBatchEvaluationHandler } from "./stop"; +import { createEvalLogsHandler } from "../logs"; // batch-evaluation supports evaluate + simulate (start jobs), get + list, and // stop. A bare invocation opens the interactive TUI (list → get), matching @@ -22,7 +23,8 @@ export function createBatchEvaluationHandler(core: Core, io: AppIO): Router { .handler(createSimulateBatchEvaluationHandler(core, io)) .handler(createGetBatchEvaluationHandler(core, io)) .handler(createListBatchEvaluationsHandler(core)) - .handler(createStopBatchEvaluationHandler(core)); + .handler(createStopBatchEvaluationHandler(core)) + .handler(createEvalLogsHandler(core, io, "batch-evaluation")); } export { BatchEvaluationScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/batch-insights/index.tsx b/src/handlers/eval/batch-insights/index.tsx index 2d40420374..6f6861601a 100644 --- a/src/handlers/eval/batch-insights/index.tsx +++ b/src/handlers/eval/batch-insights/index.tsx @@ -7,6 +7,7 @@ import { createGetBatchInsightsHandler } from "./get"; import { createListBatchInsightsHandler } from "./list"; import { createRunBatchInsightsHandler } from "./run"; import { createStopBatchInsightsHandler } from "./stop"; +import { createEvalLogsHandler } from "../logs"; export function createBatchInsightsHandler(core: Core, io: AppIO): Router { return new Router("batch-insights", "run and inspect AgentCore batch insights") @@ -16,7 +17,8 @@ export function createBatchInsightsHandler(core: Core, io: AppIO): Router { .handler(createRunBatchInsightsHandler(core, io)) .handler(createGetBatchInsightsHandler(core)) .handler(createListBatchInsightsHandler(core)) - .handler(createStopBatchInsightsHandler(core)); + .handler(createStopBatchInsightsHandler(core)) + .handler(createEvalLogsHandler(core, io, "batch-insights")); } export { BatchInsightsScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/logs/index.tsx b/src/handlers/eval/logs/index.tsx new file mode 100644 index 0000000000..0dec9bb818 --- /dev/null +++ b/src/handlers/eval/logs/index.tsx @@ -0,0 +1,29 @@ +import z from "zod"; +import type { AppIO } from "../../../io"; +import { flag } from "../../../router"; +import { createLogsHandler } from "../../observability/logs"; +import type { Core } from "../../types"; +import { coreOptsFromCtx } from "../../utils"; +import type { EvalLogsInput } from "../types"; + +export const createEvalLogsHandler = (core: Core, io: AppIO, resource: EvalLogsInput["resource"]) => + createLogsHandler(io, { + name: "logs", + description: `stream or search ${resource} logs`, + flags: [flag("id", `the ID of the ${resource} resource`, z.string().min(1))] as const, + read: async (ctx, flags, request, signal) => { + const options = coreOptsFromCtx(ctx); + const source = await core.eval.resolveLogSource({ resource, id: flags.id }, options, signal); + + if (request.mode === "search") { + return { + events: core.observability.searchLogs(source, request.query, options, signal), + }; + } + + return { + events: core.observability.tailLogs(source, request.query, options, signal), + announcement: `Streaming logs for ${resource} ${flags.id}... (Ctrl+C to stop)`, + }; + }, + }); diff --git a/src/handlers/eval/online-eval/index.tsx b/src/handlers/eval/online-eval/index.tsx index a6af130729..3693a8beb6 100644 --- a/src/handlers/eval/online-eval/index.tsx +++ b/src/handlers/eval/online-eval/index.tsx @@ -10,6 +10,7 @@ import { createUpdateOnlineEvalHandler } from "./update"; import { createPauseOnlineEvalHandler } from "./pause"; import { createResumeOnlineEvalHandler } from "./resume"; import { createDeleteOnlineEvalHandler } from "./delete"; +import { createEvalLogsHandler } from "../logs"; export function createOnlineEvalHandler(core: Core, io: AppIO): Router { return new Router("online-eval", "manage AgentCore online evaluation configs") @@ -22,7 +23,8 @@ export function createOnlineEvalHandler(core: Core, io: AppIO): Router { .handler(createUpdateOnlineEvalHandler(core, io)) .handler(createPauseOnlineEvalHandler(core)) .handler(createResumeOnlineEvalHandler(core)) - .handler(createDeleteOnlineEvalHandler(core)); + .handler(createDeleteOnlineEvalHandler(core)) + .handler(createEvalLogsHandler(core, io, "online-eval")); } export { OnlineEvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-insight/index.tsx b/src/handlers/eval/online-insight/index.tsx index b54329b493..87c227f9ab 100644 --- a/src/handlers/eval/online-insight/index.tsx +++ b/src/handlers/eval/online-insight/index.tsx @@ -10,6 +10,7 @@ import { createUpdateOnlineInsightHandler } from "./update"; import { createPauseOnlineInsightHandler } from "./pause"; import { createResumeOnlineInsightHandler } from "./resume"; import { createDeleteOnlineInsightHandler } from "./delete"; +import { createEvalLogsHandler } from "../logs"; export function createOnlineInsightHandler(core: Core, io: AppIO): Router { return new Router("online-insight", "manage AgentCore online insight configs") @@ -22,7 +23,8 @@ export function createOnlineInsightHandler(core: Core, io: AppIO): Router { .handler(createUpdateOnlineInsightHandler(core, io)) .handler(createPauseOnlineInsightHandler(core)) .handler(createResumeOnlineInsightHandler(core)) - .handler(createDeleteOnlineInsightHandler(core)); + .handler(createDeleteOnlineInsightHandler(core)) + .handler(createEvalLogsHandler(core, io, "online-insight")); } export { OnlineInsightScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 0b362d5269..df1d7213e6 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -57,8 +57,14 @@ import type { DataSourceConfig as DataPlaneDataSourceConfig, } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreOptions } from "../../core/types"; +import type { LogSource } from "../../core/observability/types"; import type { EvaluatorModelProvider } from "../../projectSchemas/evaluator"; +export type EvalLogsInput = { + resource: "online-eval" | "online-insight" | "batch-evaluation" | "batch-insights"; + id: string; +}; + // SessionWindow is the resolved explicit time filter for a batch evaluation's // session source (from --start-time/--end-time). Maps directly to the SDK's // SessionFilterConfig. @@ -388,6 +394,11 @@ export type DatasetUpdateProgressEvent = { // and implemented by src/core/eval.tsx (dependency inversion: handlers own the // abstraction). export interface CoreEvalClient { + resolveLogSource( + input: EvalLogsInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; createEvaluator( request: CreateEvaluatorRequest, options: CoreOptions, @@ -432,9 +443,13 @@ export interface CoreEvalClient { getBatchEvaluation( id: string, options: CoreOptions, - opts?: { includeResults?: boolean }, + opts?: { includeResults?: boolean; signal?: AbortSignal }, ): Promise; - getBatchInsights(id: string, options: CoreOptions): Promise; + getBatchInsights( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; listBatchEvaluations( nextToken: string | undefined, maxResults: number | undefined, @@ -509,6 +524,7 @@ export interface CoreEvalClient { getOnlineEvaluationConfig( id: string, options: CoreOptions, + signal?: AbortSignal, ): Promise; listOnlineEvaluationConfigs( nextToken: string | undefined, @@ -534,7 +550,11 @@ export interface CoreEvalClient { update: UpdateOnlineInsightInput, options: CoreOptions, ): Promise; - getOnlineInsight(id: string, options: CoreOptions): Promise; + getOnlineInsight( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; listOnlineInsights( nextToken: string | undefined, maxResults: number | undefined, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 153406ecee..0cfa682732 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -189,6 +189,7 @@ import type { DatasetUpdateProgressEvent, EvaluateInput, EvaluateResult, + EvalLogsInput, InvokeDatasetInput, InvokeDatasetResult, GetBatchEvaluationResult, @@ -202,6 +203,7 @@ import type { UpdateOnlineEvalInput, } from "../handlers/eval/types"; import { isTerminalStatus } from "../core/batchEvaluationResults"; +import { resolveEvalLogSource } from "../core/evalLogSource"; import { abortable } from "../core/abortable"; import type { CoreFetch, CoreOptions, CreateCloudFormationClient } from "../core/types"; import type { ProjectManager } from "../handlers/project/types"; @@ -1629,6 +1631,18 @@ export class TestEvalClient implements CoreEvalClient { // calls records every invocation in order, for assertions. readonly calls: RecordedCall[] = []; + async resolveLogSource( + input: EvalLogsInput, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + this.calls.push({ + method: "resolveLogSource", + args: signal ? [input, options, signal] : [input, options], + }); + return resolveEvalLogSource({ client: this, input, options, signal }); + } + // List responses are keyed by the nextToken that requests them (undefined = // the first page), so tests can serve multi-page listings. private listResponses = new Map(); @@ -2064,7 +2078,7 @@ export class TestEvalClient implements CoreEvalClient { async getBatchEvaluation( id: string, options: CoreOptions, - opts: { includeResults?: boolean } = {}, + opts: { includeResults?: boolean; signal?: AbortSignal } = {}, ): Promise { this.calls.push({ method: "getBatchEvaluation", args: [id, options, opts] }); if (this.error) throw this.error; @@ -2084,8 +2098,15 @@ export class TestEvalClient implements CoreEvalClient { return { detail }; } - async getBatchInsights(id: string, options: CoreOptions): Promise { - this.calls.push({ method: "getBatchInsights", args: [id, options] }); + async getBatchInsights( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + this.calls.push({ + method: "getBatchInsights", + args: signal ? [id, options, signal] : [id, options], + }); if (this.error) throw this.error; const detail: BatchEvaluationDetail = { ...this.batchEvalGetResponse }; @@ -2267,8 +2288,12 @@ export class TestEvalClient implements CoreEvalClient { async getOnlineEvaluationConfig( id: string, options: CoreOptions, + signal?: AbortSignal, ): Promise { - this.calls.push({ method: "getOnlineEvaluationConfig", args: [id, options] }); + this.calls.push({ + method: "getOnlineEvaluationConfig", + args: signal ? [id, options, signal] : [id, options], + }); if (this.error) throw this.error; return this.onlineEvalGetResponse; } @@ -2330,8 +2355,12 @@ export class TestEvalClient implements CoreEvalClient { async getOnlineInsight( id: string, options: CoreOptions, + signal?: AbortSignal, ): Promise { - this.calls.push({ method: "getOnlineInsight", args: [id, options] }); + this.calls.push({ + method: "getOnlineInsight", + args: signal ? [id, options, signal] : [id, options], + }); if (this.error) throw this.error; return this.onlineEvalGetResponse; } From ad5382a82d1eac6fc09453f04aca0d72a1674c59 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:49:27 +0000 Subject: [PATCH 02/10] test(eval): cover log destinations and cancellation --- src/core/evalLogSource.test.ts | 317 ++++++++++++++++++ .../batch-evaluation.test.tsx | 1 + .../batch-insights.screen.test.tsx | 2 +- .../batch-insights/batch-insights.test.tsx | 8 +- src/handlers/eval/logs/logs.test.tsx | 252 ++++++++++++++ .../online-eval/online-eval.screen.test.tsx | 2 +- .../eval/online-eval/online-eval.test.tsx | 1 + .../online-insight.screen.test.tsx | 2 +- .../online-insight/online-insight.test.tsx | 1 + 9 files changed, 582 insertions(+), 4 deletions(-) create mode 100644 src/core/evalLogSource.test.ts create mode 100644 src/handlers/eval/logs/logs.test.tsx diff --git a/src/core/evalLogSource.test.ts b/src/core/evalLogSource.test.ts new file mode 100644 index 0000000000..356e0bd5be --- /dev/null +++ b/src/core/evalLogSource.test.ts @@ -0,0 +1,317 @@ +import { describe, expect, test } from "bun:test"; +import { + GetOnlineEvaluationConfigCommand, + type BedrockAgentCoreControlClient, +} from "@aws-sdk/client-bedrock-agentcore-control"; +import { + GetBatchEvaluationCommand, + type BedrockAgentCoreClient, +} from "@aws-sdk/client-bedrock-agentcore"; +import { EvalClient } from "./eval"; +import type { AwsClients } from "./types"; +import type { EvalLogsInput } from "../handlers/eval/types"; +import { InputValidationError, MalformedServiceResponseError } from "../errors"; + +const OPTIONS = { region: "us-west-2", endpointUrl: "https://agentcore.example.test" }; +const ID = "evaluation-1"; +const SOURCE_GROUP = "/custom/source"; +const ONLINE_CONFIG_ID = "config-1"; +const ONLINE_CONFIG_ARN = `arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/${ONLINE_CONFIG_ID}`; +const SOURCE_OUTPUT = { cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" } }; + +type Metadata = Record | Error; +type MetadataCommand = GetOnlineEvaluationConfigCommand | GetBatchEvaluationCommand; +type SendOptions = { abortSignal?: AbortSignal }; + +function setup( + responses: { + online?: Metadata; + batch?: Metadata; + beforeResponse?: (command: MetadataCommand, options?: SendOptions) => Promise; + } = {}, +) { + const commands: MetadataCommand[] = []; + const sendOptions: (SendOptions | undefined)[] = []; + const online = responses.online ?? { + onlineEvaluationConfigId: ONLINE_CONFIG_ID, + dataSourceConfig: { cloudWatchLogs: { logGroupNames: [SOURCE_GROUP] } }, + }; + const batch = responses.batch ?? { outputConfig: SOURCE_OUTPUT }; + const unexpectedClient = () => { + throw new Error("Unexpected AWS client; resolution must not read evaluation result streams."); + }; + const send = async (command: MetadataCommand, response: Metadata, options?: SendOptions) => { + commands.push(command); + sendOptions.push(options); + await responses.beforeResponse?.(command, options); + if (response instanceof Error) throw response; + return response; + }; + const clients: AwsClients = { + control: (config) => { + expect(config).toEqual({ region: OPTIONS.region, endpoint: OPTIONS.endpointUrl }); + return { + send: (command: GetOnlineEvaluationConfigCommand, options?: SendOptions) => + send(command, online, options), + } as unknown as BedrockAgentCoreControlClient; + }, + data: (config) => { + expect(config).toEqual({ region: OPTIONS.region, endpoint: OPTIONS.endpointUrl }); + return { + send: (command: GetBatchEvaluationCommand, options?: SendOptions) => + send(command, batch, options), + } as unknown as BedrockAgentCoreClient; + }, + logs: unexpectedClient, + iam: unexpectedClient, + xray: unexpectedClient, + applicationSignals: unexpectedClient, + }; + return { client: new EvalClient(clients), commands, sendOptions }; +} + +function waitForAbort(signal?: AbortSignal): Promise { + if (!signal) throw new Error("Metadata lookup did not receive a cancellation signal."); + if (signal.aborted) return Promise.reject(signal.reason); + return new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); +} + +describe("evaluation log source resolution", () => { + test.each([ + { onlineEvaluationConfigId: ONLINE_CONFIG_ID, expectedId: ONLINE_CONFIG_ID }, + { onlineEvaluationConfigId: undefined, expectedId: ID }, + ])("uses the online config ID or requested ID for the default group", async (testCase) => { + const { client } = setup({ + online: { onlineEvaluationConfigId: testCase.onlineEvaluationConfigId }, + }); + + await expect( + client.resolveLogSource({ resource: "online-eval", id: ID }, OPTIONS), + ).resolves.toEqual({ + logGroupName: `/aws/bedrock-agentcore/evaluations/results/${testCase.expectedId}`, + }); + }); + + test.each([ONLINE_CONFIG_ARN, ONLINE_CONFIG_ID])( + "resolves a batch source reference %s to the online config's input group", + async (reference) => { + const { client, commands } = setup({ + online: { + onlineEvaluationConfigId: ONLINE_CONFIG_ID, + dataSourceConfig: { cloudWatchLogs: { logGroupNames: [SOURCE_GROUP] } }, + outputConfig: { cloudWatchConfig: { logGroupName: "/custom/online-results" } }, + }, + batch: { + status: "COMPLETED", + outputConfig: SOURCE_OUTPUT, + dataSourceConfig: { + onlineEvaluationConfigSource: { onlineEvaluationConfigArn: reference }, + }, + }, + }); + + await expect( + client.resolveLogSource({ resource: "batch-evaluation", id: ID }, OPTIONS), + ).resolves.toEqual({ logGroupName: SOURCE_GROUP }); + expect(commands).toHaveLength(2); + expect(commands[0]).toBeInstanceOf(GetBatchEvaluationCommand); + expect(commands[0]?.input).toEqual({ batchEvaluationId: ID }); + expect(commands[1]).toBeInstanceOf(GetOnlineEvaluationConfigCommand); + expect(commands[1]?.input).toEqual({ onlineEvaluationConfigId: ONLINE_CONFIG_ID }); + }, + ); + + describe.each(["online-eval", "batch-evaluation"] as const)( + "%s output configurations", + (resource) => { + const resolve = (metadata: Metadata) => { + const { client } = setup({ online: metadata, batch: metadata }); + return client.resolveLogSource({ resource, id: ID }, OPTIONS); + }; + + test("uses the default for a dedicated destination without an explicit group", async () => { + await expect( + resolve({ + outputConfig: { cloudWatchConfig: { resultDestination: "DEDICATED_LOG_GROUP" } }, + }), + ).resolves.toEqual({ + logGroupName: + resource === "online-eval" + ? `/aws/bedrock-agentcore/evaluations/results/${ID}` + : "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + }); + }); + + test.each([ + { source: { logGroupNames: ["/one", "/two"] }, message: "multiple source log groups" }, + { source: { logGroupNamePrefixes: ["/prefix/"] }, message: "selected by prefix" }, + ])("rejects ambiguous source groups: $message", async ({ source, message }) => { + const result = resolve({ + outputConfig: SOURCE_OUTPUT, + dataSourceConfig: { cloudWatchLogs: source }, + }); + + await expect(result).rejects.toBeInstanceOf(InputValidationError); + await expect(result).rejects.toThrow(message); + await expect(result).rejects.toThrow("configure one explicit source log group"); + }); + + test.each([ + undefined, + { cloudWatchLogs: { logGroupNames: [] } }, + { cloudWatchLogs: { logGroupNames: [""] } }, + { $unknown: ["futureSource", {}] }, + ])( + "rejects a source destination without a usable source group: %j", + async (dataSourceConfig) => { + await expect( + resolve({ outputConfig: SOURCE_OUTPUT, dataSourceConfig }), + ).rejects.toBeInstanceOf(MalformedServiceResponseError); + }, + ); + + test("rejects an unknown output destination instead of assuming the default group", async () => { + const result = resolve({ outputConfig: { $unknown: ["futureOutput", {}] } }); + + await expect(result).rejects.toBeInstanceOf(InputValidationError); + await expect(result).rejects.toThrow("logs require a CloudWatch output configuration"); + }); + }, + ); + + test("rejects a batch source reference that has no ARN", async () => { + const { client, commands } = setup({ + batch: { + outputConfig: SOURCE_OUTPUT, + dataSourceConfig: { onlineEvaluationConfigSource: {} }, + }, + }); + + await expect( + client.resolveLogSource({ resource: "batch-evaluation", id: ID }, OPTIONS), + ).rejects.toBeInstanceOf(MalformedServiceResponseError); + expect(commands).toHaveLength(1); + }); + + test.each([ + "online-eval", + "online-insight", + "batch-evaluation", + "batch-insights", + ] satisfies EvalLogsInput["resource"][])( + "propagates %s metadata lookup failures", + async (resource) => { + const error = new Error("Metadata lookup failed"); + const { client } = setup({ online: error, batch: error }); + + await expect(client.resolveLogSource({ resource, id: ID }, OPTIONS)).rejects.toBe(error); + }, + ); + + test("propagates failures looking up a batch job's online config source", async () => { + const error = new Error("Source lookup failed"); + const { client } = setup({ + online: error, + batch: { + outputConfig: SOURCE_OUTPUT, + dataSourceConfig: { + onlineEvaluationConfigSource: { onlineEvaluationConfigArn: ONLINE_CONFIG_ARN }, + }, + }, + }); + + await expect( + client.resolveLogSource({ resource: "batch-evaluation", id: ID }, OPTIONS), + ).rejects.toBe(error); + }); + + describe.each([ + "online-eval", + "online-insight", + "batch-evaluation", + "batch-insights", + ] satisfies EvalLogsInput["resource"][])("%s cancellation", (resource) => { + test("passes the signal to metadata lookup without changing SDK request input", async () => { + const controller = new AbortController(); + const metadata = { + insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], + outputConfig: { cloudWatchConfig: { logGroupName: "/custom/results" } }, + }; + const { client, commands, sendOptions } = setup({ online: metadata, batch: metadata }); + + await expect( + client.resolveLogSource({ resource, id: ID }, OPTIONS, controller.signal), + ).resolves.toEqual({ logGroupName: "/custom/results" }); + expect(sendOptions).toEqual([{ abortSignal: controller.signal }]); + expect(commands[0]?.input).toEqual( + resource.startsWith("online") + ? { onlineEvaluationConfigId: ID } + : { batchEvaluationId: ID }, + ); + }); + + test("omits SDK cancellation options when no signal is supplied", async () => { + const metadata = { + insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], + outputConfig: { cloudWatchConfig: { logGroupName: "/custom/results" } }, + }; + const { client, sendOptions } = setup({ online: metadata, batch: metadata }); + + await client.resolveLogSource({ resource, id: ID }, OPTIONS); + + expect(sendOptions).toEqual([undefined]); + }); + + test("cancels an in-flight metadata lookup", async () => { + const controller = new AbortController(); + const reason = new DOMException("Cancelled lookup", "AbortError"); + const { client } = setup({ + beforeResponse: (_command, options) => waitForAbort(options?.abortSignal), + }); + const pending = client.resolveLogSource({ resource, id: ID }, OPTIONS, controller.signal); + + controller.abort(reason); + + await expect(pending).rejects.toBe(reason); + }); + }); + + test.each(["batch-evaluation", "batch-insights"] as const)( + "cancels %s during the nested online config source lookup", + async (resource) => { + const controller = new AbortController(); + const reason = new DOMException("Cancelled source lookup", "AbortError"); + let sourceLookupStarted = () => {}; + const started = new Promise((resolve) => { + sourceLookupStarted = resolve; + }); + const { client, sendOptions } = setup({ + batch: { + insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], + outputConfig: SOURCE_OUTPUT, + dataSourceConfig: { + onlineEvaluationConfigSource: { onlineEvaluationConfigArn: ONLINE_CONFIG_ARN }, + }, + }, + beforeResponse: async (command, options) => { + if (command instanceof GetOnlineEvaluationConfigCommand) { + sourceLookupStarted(); + await waitForAbort(options?.abortSignal); + } + }, + }); + const pending = client.resolveLogSource({ resource, id: ID }, OPTIONS, controller.signal); + await started; + + controller.abort(reason); + + await expect(pending).rejects.toBe(reason); + expect(sendOptions).toEqual([ + { abortSignal: controller.signal }, + { abortSignal: controller.signal }, + ]); + }, + ); +}); diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx index b01cde1dde..e24b865d0f 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -71,6 +71,7 @@ describe("eval batch-evaluation command hierarchy", () => { "get", "list", "stop", + "logs", ]); }); diff --git a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx b/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx index 15f961580b..79f8b22431 100644 --- a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx @@ -58,7 +58,7 @@ describe("batch-insights menu", () => { await waitForText(screen.lastFrame, "list batch insights runs"); expect(menuEntries(screen.lastFrame()!)).toEqual({ screens: ["get", "list"], - cliOnly: ["run", "stop"], + cliOnly: ["run", "stop", "logs"], }); }); }); diff --git a/src/handlers/eval/batch-insights/batch-insights.test.tsx b/src/handlers/eval/batch-insights/batch-insights.test.tsx index a8b6afd480..ff97076acc 100644 --- a/src/handlers/eval/batch-insights/batch-insights.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.test.tsx @@ -42,7 +42,13 @@ describe("eval batch-insights command hierarchy", () => { ?.children() .find((child) => child.name() === "batch-insights"); - expect(group?.children().map((child) => child.name())).toEqual(["run", "get", "list", "stop"]); + expect(group?.children().map((child) => child.name())).toEqual([ + "run", + "get", + "list", + "stop", + "logs", + ]); }); test("prints help for a bare invocation without a Core call", async () => { diff --git a/src/handlers/eval/logs/logs.test.tsx b/src/handlers/eval/logs/logs.test.tsx new file mode 100644 index 0000000000..fa33c912b6 --- /dev/null +++ b/src/handlers/eval/logs/logs.test.tsx @@ -0,0 +1,252 @@ +import { describe, expect, test } from "bun:test"; +import type { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; +import type { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; +import { + DescribeLogGroupsCommand, + FilterLogEventsCommand, + StartLiveTailCommand, + type CloudWatchLogsClient, +} from "@aws-sdk/client-cloudwatch-logs"; +import { CoreClient } from "../../../core"; +import { InputValidationError, UserCancellationError } from "../../../errors"; +import { createSilentLogger, TestGlobalConfigAccessor, testIO } from "../../../testing"; +import { createRootHandler } from "../../index"; +import type { EvalLogsInput } from "../types"; + +const SINCE = 1_709_391_000_000; +const ID = "config-AbC123XyZ9"; +const RESOURCES: EvalLogsInput["resource"][] = [ + "online-eval", + "online-insight", + "batch-evaluation", + "batch-insights", +]; +const EVENTS = [ + { timestamp: SINCE + 1_000, message: "first result", logStreamName: "run-a" }, + { timestamp: SINCE + 2_000, message: "second result", logStreamName: "run-b" }, +]; +const SOURCE_GROUP = "/custom/source"; + +function setup( + metadata: Record = {}, + readMetadata?: (signal?: AbortSignal) => Promise>, +) { + const commands: { name: string; input: unknown }[] = []; + const response = { + onlineEvaluationConfigId: ID, + batchEvaluationId: ID, + status: "COMPLETED", + insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], + dataSourceConfig: { cloudWatchLogs: { logGroupNames: [SOURCE_GROUP] } }, + ...metadata, + }; + const sendMetadata = async ( + command: { constructor: { name: string }; input: unknown }, + options?: { abortSignal?: AbortSignal }, + ) => { + commands.push({ name: command.constructor.name, input: command.input }); + if (readMetadata) return readMetadata(options?.abortSignal); + if (command.constructor.name === "GetOnlineEvaluationConfigCommand") { + const { batchEvaluationId: _, ...config } = response; + return config; + } + const { onlineEvaluationConfigId: _, ...job } = response; + return job; + }; + const unusedClient = () => { + throw new Error("Unexpected AWS client"); + }; + const core = new CoreClient({ + createControlClient: () => ({ send: sendMetadata }) as unknown as BedrockAgentCoreControlClient, + createDataClient: () => ({ send: sendMetadata }) as unknown as BedrockAgentCoreClient, + createLogsClient: () => + ({ + send: async (command: unknown) => { + if (command instanceof FilterLogEventsCommand) { + commands.push({ name: command.constructor.name, input: command.input }); + return command.input.nextToken + ? { events: [EVENTS[1]] } + : { events: [EVENTS[0]], nextToken: "page-2" }; + } + if (command instanceof DescribeLogGroupsCommand) { + commands.push({ name: command.constructor.name, input: command.input }); + return { + logGroups: [ + { + logGroupName: command.input.logGroupNamePrefix, + logGroupArn: "arn:aws:logs:us-west-2:123456789012:log-group:test", + }, + ], + }; + } + if (command instanceof StartLiveTailCommand) { + commands.push({ name: command.constructor.name, input: command.input }); + return { + responseStream: (async function* () { + yield { sessionUpdate: { sessionResults: EVENTS } }; + })(), + }; + } + throw new Error("Unexpected CloudWatch command"); + }, + }) as unknown as CloudWatchLogsClient, + createIamClient: unusedClient, + createXrayClient: unusedClient, + createApplicationSignalsClient: unusedClient, + logger: createSilentLogger(), + }); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + return { + commands, + io, + route: (resource: EvalLogsInput["resource"], flags: readonly string[]) => + root.route([ + "node", + "agentcore", + "eval", + resource, + "logs", + ...flags, + "--region", + "us-west-2", + ]), + }; +} + +describe.each(RESOURCES)("eval %s logs", (resource) => { + const defaultGroup = resource.startsWith("online") + ? `/aws/bedrock-agentcore/evaluations/results/${ID}` + : "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default"; + const searchFlags = ["--id", ID, "--since", `${SINCE}`, "--until", `${SINCE + 60_000}`]; + + test.each([ + { destination: "default", outputConfig: undefined, expectedGroup: defaultGroup }, + { + destination: "custom", + outputConfig: { + cloudWatchConfig: { logGroupName: "/custom/results", logStreamName: "run-a" }, + }, + expectedGroup: "/custom/results", + }, + { + destination: "source", + outputConfig: { cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" } }, + expectedGroup: SOURCE_GROUP, + }, + ])("searches the entire $destination group across pages and streams", async (testCase) => { + const { commands, io, route } = setup({ outputConfig: testCase.outputConfig }); + + await route(resource, [...searchFlags, "--json"]); + + expect( + io + .stdout() + .split("\n") + .map((line) => JSON.parse(line)), + ).toEqual( + EVENTS.map((event) => ({ ...event, timestamp: new Date(event.timestamp).toISOString() })), + ); + const searches = commands.filter((command) => command.name === "FilterLogEventsCommand"); + expect(searches.map((command) => command.input)).toEqual([ + { logGroupName: testCase.expectedGroup, startTime: SINCE, endTime: SINCE + 60_000 }, + { + logGroupName: testCase.expectedGroup, + startTime: SINCE, + endTime: SINCE + 60_000, + nextToken: "page-2", + }, + ]); + expect(commands[0]?.name).toBe( + resource.startsWith("online") + ? "GetOnlineEvaluationConfigCommand" + : "GetBatchEvaluationCommand", + ); + expect(io.stderr()).toBe(""); + }); + + test("streams the configured group by default with a stderr announcement", async () => { + const { commands, io, route } = setup({ + outputConfig: { cloudWatchConfig: { logGroupName: "/custom/results" } }, + }); + + await route(resource, ["--id", ID, "--query", "ERROR"]); + + expect(io.stdout()).toBe( + "2024-03-02T14:50:01.000Z first result\n2024-03-02T14:50:02.000Z second result", + ); + expect(io.stderr()).toBe(`Streaming logs for ${resource} ${ID}... (Ctrl+C to stop)`); + expect(commands.find((command) => command.name === "DescribeLogGroupsCommand")?.input).toEqual({ + logGroupNamePrefix: "/custom/results", + }); + expect(commands.find((command) => command.name === "StartLiveTailCommand")?.input).toEqual({ + logGroupIdentifiers: ["arn:aws:logs:us-west-2:123456789012:log-group:test"], + logEventFilterPattern: "ERROR", + }); + }); + + test("Ctrl+C cancels metadata lookup without waiting for a service response", async () => { + let receivedSignal: AbortSignal | undefined; + let releaseMetadata = () => {}; + let metadataStarted = () => {}; + const started = new Promise((resolve) => { + metadataStarted = resolve; + }); + const { commands, io, route } = setup({}, (signal) => { + receivedSignal = signal; + metadataStarted(); + return new Promise((resolve, reject) => { + releaseMetadata = () => resolve({}); + signal?.addEventListener("abort", () => reject(new Error("transport aborted")), { + once: true, + }); + }); + }); + const pending = route(resource, ["--id", ID, "--since", "1h"]); + + try { + await started; + expect(receivedSignal).toBeDefined(); + process.emit("SIGINT", "SIGINT"); + await expect(pending).rejects.toBeInstanceOf(UserCancellationError); + expect(receivedSignal?.aborted).toBe(true); + } finally { + releaseMetadata(); + await pending.catch(() => {}); + } + expect(commands).toHaveLength(1); + expect(io.stdout()).toBe(""); + expect(io.stderr()).toBe(""); + }); + + test.each([ + { flags: [], message: "required option '--id' not specified" }, + { flags: ["--id", ""], message: "Invalid value for option '--id'" }, + { + flags: ["--id", ID, "--tail", "--since", "1h"], + message: "--tail cannot be combined with --since or --until", + }, + ])("validates flags before fetching metadata: $message", async ({ flags, message }) => { + const { commands, route } = setup(); + + await expect(route(resource, flags)).rejects.toThrow(message); + expect(commands).toHaveLength(0); + }); +}); + +test.each(["online-insight", "batch-insights"] as const)( + "rejects a resource without insights through %s logs", + async (resource) => { + const { commands, route } = setup({ insights: [] }); + + await expect(route(resource, ["--id", ID, "--since", "1h"])).rejects.toBeInstanceOf( + InputValidationError, + ); + expect(commands).toHaveLength(1); + }, +); diff --git a/src/handlers/eval/online-eval/online-eval.screen.test.tsx b/src/handlers/eval/online-eval/online-eval.screen.test.tsx index 149deb2261..b9cf4f2272 100644 --- a/src/handlers/eval/online-eval/online-eval.screen.test.tsx +++ b/src/handlers/eval/online-eval/online-eval.screen.test.tsx @@ -67,7 +67,7 @@ describe("online-eval menu", () => { await waitForText(screen.lastFrame, "get an online evaluation config by ID"); expect(menuEntries(screen.lastFrame()!)).toEqual({ screens: ["get", "list"], - cliOnly: ["create", "update", "pause", "resume", "delete"], + cliOnly: ["create", "update", "pause", "resume", "delete", "logs"], }); }); }); diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx index 5473fe0bdc..f82f4d591d 100644 --- a/src/handlers/eval/online-eval/online-eval.test.tsx +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -101,6 +101,7 @@ describe("eval online-eval command hierarchy", () => { "pause", "resume", "delete", + "logs", ]); }); diff --git a/src/handlers/eval/online-insight/online-insight.screen.test.tsx b/src/handlers/eval/online-insight/online-insight.screen.test.tsx index b8b95b7b51..1a0814b385 100644 --- a/src/handlers/eval/online-insight/online-insight.screen.test.tsx +++ b/src/handlers/eval/online-insight/online-insight.screen.test.tsx @@ -69,7 +69,7 @@ describe("online-insight menu", () => { await waitForText(screen.lastFrame, "get an online insight config by ID"); expect(menuEntries(screen.lastFrame()!)).toEqual({ screens: ["get", "list"], - cliOnly: ["create", "update", "pause", "resume", "delete"], + cliOnly: ["create", "update", "pause", "resume", "delete", "logs"], }); }); }); diff --git a/src/handlers/eval/online-insight/online-insight.test.tsx b/src/handlers/eval/online-insight/online-insight.test.tsx index 6c798ccead..0c4429af94 100644 --- a/src/handlers/eval/online-insight/online-insight.test.tsx +++ b/src/handlers/eval/online-insight/online-insight.test.tsx @@ -103,6 +103,7 @@ describe("eval online-insight command hierarchy", () => { "pause", "resume", "delete", + "logs", ]); }); }); From df0612ba0b48689e8bbb25555e28d4baf43cb016 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Mon, 12 Oct 2026 15:49:27 +0000 Subject: [PATCH 03/10] docs(eval): document resource log commands --- command.md | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/command.md b/command.md index 427bde6eeb..fc8ea3dfcf 100644 --- a/command.md +++ b/command.md @@ -152,6 +152,7 @@ This reference was generated from `agentcore --help` for version `1.0.0-rc.5`. - [`agentcore eval online-eval pause`](#agentcore-eval-online-eval-pause) - [`agentcore eval online-eval resume`](#agentcore-eval-online-eval-resume) - [`agentcore eval online-eval delete`](#agentcore-eval-online-eval-delete) + - [`agentcore eval online-eval logs`](#agentcore-eval-online-eval-logs) - [`agentcore eval online-insight`](#agentcore-eval-online-insight) - [`agentcore eval online-insight create`](#agentcore-eval-online-insight-create) - [`agentcore eval online-insight get`](#agentcore-eval-online-insight-get) @@ -160,6 +161,7 @@ This reference was generated from `agentcore --help` for version `1.0.0-rc.5`. - [`agentcore eval online-insight pause`](#agentcore-eval-online-insight-pause) - [`agentcore eval online-insight resume`](#agentcore-eval-online-insight-resume) - [`agentcore eval online-insight delete`](#agentcore-eval-online-insight-delete) + - [`agentcore eval online-insight logs`](#agentcore-eval-online-insight-logs) - [`agentcore eval dataset`](#agentcore-eval-dataset) - [`agentcore eval dataset create`](#agentcore-eval-dataset-create) - [`agentcore eval dataset get`](#agentcore-eval-dataset-get) @@ -173,11 +175,13 @@ This reference was generated from `agentcore --help` for version `1.0.0-rc.5`. - [`agentcore eval batch-evaluation get`](#agentcore-eval-batch-evaluation-get) - [`agentcore eval batch-evaluation list`](#agentcore-eval-batch-evaluation-list) - [`agentcore eval batch-evaluation stop`](#agentcore-eval-batch-evaluation-stop) + - [`agentcore eval batch-evaluation logs`](#agentcore-eval-batch-evaluation-logs) - [`agentcore eval batch-insights`](#agentcore-eval-batch-insights) - [`agentcore eval batch-insights run`](#agentcore-eval-batch-insights-run) - [`agentcore eval batch-insights get`](#agentcore-eval-batch-insights-get) - [`agentcore eval batch-insights list`](#agentcore-eval-batch-insights-list) - [`agentcore eval batch-insights stop`](#agentcore-eval-batch-insights-stop) + - [`agentcore eval batch-insights logs`](#agentcore-eval-batch-insights-logs) - [`agentcore eval ondemand`](#agentcore-eval-ondemand) - [`agentcore eval ondemand evaluate`](#agentcore-eval-ondemand-evaluate) - [`agentcore eval ondemand simulate`](#agentcore-eval-ondemand-simulate) @@ -2300,6 +2304,24 @@ delete an online evaluation config by ID - `--id `: the ID of the online evaluation config to delete (required) +##### `agentcore eval online-eval logs` + +```text +agentcore eval online-eval logs [options] +``` + +stream or search online-eval logs + +**Options** + +- `--id `: the ID of the online-eval resource (required) +- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--tail`: tail new log records (default: false) +- `--level `: filter by log level (error, warn, info, debug) +- `--query `: CloudWatch Logs filter pattern +- `--limit `: maximum number of log records to return in search mode + #### `agentcore eval online-insight` ```text @@ -2414,6 +2436,24 @@ delete an online insight config by ID - `--id `: the ID of the online insight config to delete (required) +##### `agentcore eval online-insight logs` + +```text +agentcore eval online-insight logs [options] +``` + +stream or search online-insight logs + +**Options** + +- `--id `: the ID of the online-insight resource (required) +- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--tail`: tail new log records (default: false) +- `--level `: filter by log level (error, warn, info, debug) +- `--query `: CloudWatch Logs filter pattern +- `--limit `: maximum number of log records to return in search mode + #### `agentcore eval dataset` ```text @@ -2599,6 +2639,24 @@ stop a running batch evaluation - `--id `: the ID of the batch evaluation to stop (required) +##### `agentcore eval batch-evaluation logs` + +```text +agentcore eval batch-evaluation logs [options] +``` + +stream or search batch-evaluation logs + +**Options** + +- `--id `: the ID of the batch-evaluation resource (required) +- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--tail`: tail new log records (default: false) +- `--level `: filter by log level (error, warn, info, debug) +- `--query `: CloudWatch Logs filter pattern +- `--limit `: maximum number of log records to return in search mode + #### `agentcore eval batch-insights` ```text @@ -2667,6 +2725,24 @@ stop a running batch insights run - `--id `: the ID of the batch insights run to stop (required) +##### `agentcore eval batch-insights logs` + +```text +agentcore eval batch-insights logs [options] +``` + +stream or search batch-insights logs + +**Options** + +- `--id `: the ID of the batch-insights resource (required) +- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--tail`: tail new log records (default: false) +- `--level `: filter by log level (error, warn, info, debug) +- `--query `: CloudWatch Logs filter pattern +- `--limit `: maximum number of log records to return in search mode + #### `agentcore eval ondemand` ```text From 6adca4d7ab1963785bde3d957b9d345852edb747 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:01:47 +0000 Subject: [PATCH 04/10] refactor(eval): simplify log source resolution and tests --- command.md | 76 ----- src/core/eval.tsx | 148 +++++++-- src/core/evalLogSource.test.ts | 317 ------------------- src/core/evalLogSource.ts | 95 ------ src/handlers/eval/batch-evaluation/index.tsx | 7 +- src/handlers/eval/batch-insights/index.tsx | 7 +- src/handlers/eval/logs/index.tsx | 16 +- src/handlers/eval/logs/logs.test.tsx | 287 +++++++++-------- src/handlers/eval/online-eval/index.tsx | 7 +- src/handlers/eval/online-insight/index.tsx | 7 +- src/handlers/eval/types.tsx | 39 +-- src/testing/TestCoreClient.tsx | 60 ++-- 12 files changed, 345 insertions(+), 721 deletions(-) delete mode 100644 src/core/evalLogSource.test.ts delete mode 100644 src/core/evalLogSource.ts diff --git a/command.md b/command.md index fc8ea3dfcf..427bde6eeb 100644 --- a/command.md +++ b/command.md @@ -152,7 +152,6 @@ This reference was generated from `agentcore --help` for version `1.0.0-rc.5`. - [`agentcore eval online-eval pause`](#agentcore-eval-online-eval-pause) - [`agentcore eval online-eval resume`](#agentcore-eval-online-eval-resume) - [`agentcore eval online-eval delete`](#agentcore-eval-online-eval-delete) - - [`agentcore eval online-eval logs`](#agentcore-eval-online-eval-logs) - [`agentcore eval online-insight`](#agentcore-eval-online-insight) - [`agentcore eval online-insight create`](#agentcore-eval-online-insight-create) - [`agentcore eval online-insight get`](#agentcore-eval-online-insight-get) @@ -161,7 +160,6 @@ This reference was generated from `agentcore --help` for version `1.0.0-rc.5`. - [`agentcore eval online-insight pause`](#agentcore-eval-online-insight-pause) - [`agentcore eval online-insight resume`](#agentcore-eval-online-insight-resume) - [`agentcore eval online-insight delete`](#agentcore-eval-online-insight-delete) - - [`agentcore eval online-insight logs`](#agentcore-eval-online-insight-logs) - [`agentcore eval dataset`](#agentcore-eval-dataset) - [`agentcore eval dataset create`](#agentcore-eval-dataset-create) - [`agentcore eval dataset get`](#agentcore-eval-dataset-get) @@ -175,13 +173,11 @@ This reference was generated from `agentcore --help` for version `1.0.0-rc.5`. - [`agentcore eval batch-evaluation get`](#agentcore-eval-batch-evaluation-get) - [`agentcore eval batch-evaluation list`](#agentcore-eval-batch-evaluation-list) - [`agentcore eval batch-evaluation stop`](#agentcore-eval-batch-evaluation-stop) - - [`agentcore eval batch-evaluation logs`](#agentcore-eval-batch-evaluation-logs) - [`agentcore eval batch-insights`](#agentcore-eval-batch-insights) - [`agentcore eval batch-insights run`](#agentcore-eval-batch-insights-run) - [`agentcore eval batch-insights get`](#agentcore-eval-batch-insights-get) - [`agentcore eval batch-insights list`](#agentcore-eval-batch-insights-list) - [`agentcore eval batch-insights stop`](#agentcore-eval-batch-insights-stop) - - [`agentcore eval batch-insights logs`](#agentcore-eval-batch-insights-logs) - [`agentcore eval ondemand`](#agentcore-eval-ondemand) - [`agentcore eval ondemand evaluate`](#agentcore-eval-ondemand-evaluate) - [`agentcore eval ondemand simulate`](#agentcore-eval-ondemand-simulate) @@ -2304,24 +2300,6 @@ delete an online evaluation config by ID - `--id `: the ID of the online evaluation config to delete (required) -##### `agentcore eval online-eval logs` - -```text -agentcore eval online-eval logs [options] -``` - -stream or search online-eval logs - -**Options** - -- `--id `: the ID of the online-eval resource (required) -- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--tail`: tail new log records (default: false) -- `--level `: filter by log level (error, warn, info, debug) -- `--query `: CloudWatch Logs filter pattern -- `--limit `: maximum number of log records to return in search mode - #### `agentcore eval online-insight` ```text @@ -2436,24 +2414,6 @@ delete an online insight config by ID - `--id `: the ID of the online insight config to delete (required) -##### `agentcore eval online-insight logs` - -```text -agentcore eval online-insight logs [options] -``` - -stream or search online-insight logs - -**Options** - -- `--id `: the ID of the online-insight resource (required) -- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--tail`: tail new log records (default: false) -- `--level `: filter by log level (error, warn, info, debug) -- `--query `: CloudWatch Logs filter pattern -- `--limit `: maximum number of log records to return in search mode - #### `agentcore eval dataset` ```text @@ -2639,24 +2599,6 @@ stop a running batch evaluation - `--id `: the ID of the batch evaluation to stop (required) -##### `agentcore eval batch-evaluation logs` - -```text -agentcore eval batch-evaluation logs [options] -``` - -stream or search batch-evaluation logs - -**Options** - -- `--id `: the ID of the batch-evaluation resource (required) -- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--tail`: tail new log records (default: false) -- `--level `: filter by log level (error, warn, info, debug) -- `--query `: CloudWatch Logs filter pattern -- `--limit `: maximum number of log records to return in search mode - #### `agentcore eval batch-insights` ```text @@ -2725,24 +2667,6 @@ stop a running batch insights run - `--id `: the ID of the batch insights run to stop (required) -##### `agentcore eval batch-insights logs` - -```text -agentcore eval batch-insights logs [options] -``` - -stream or search batch-insights logs - -**Options** - -- `--id `: the ID of the batch-insights resource (required) -- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--tail`: tail new log records (default: false) -- `--level `: filter by log level (error, warn, info, debug) -- `--query `: CloudWatch Logs filter pattern -- `--limit `: maximum number of log records to return in search mode - #### `agentcore eval ondemand` ```text diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 54d22170ab..f512755317 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -141,7 +141,6 @@ import type { DeleteOnlineInsightResponse, EvaluateInput, EvaluateResult, - EvalLogsInput, GetBatchEvaluationResult, GetTracesInput, LlmAsAJudgeUpdate, @@ -174,7 +173,7 @@ import { toClientConfig } from "./utils"; import { grantOnlineEvalScope } from "./onlineEvalExecutionRole"; import { accountIdFromArn, deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole"; import { harnessRuntimeFromResponse } from "./harness"; -import { resolveEvalLogSource } from "./evalLogSource"; +import { serviceIdFromArn } from "./arn"; import type { LogSource } from "./observability/types"; const DEFAULT_INGESTION_WAIT_MS = 180_000; @@ -240,12 +239,120 @@ export class EvalClient implements CoreEvalClient { private readonly cloudWatch: CloudWatchClient = new CloudWatchClient(clients), ) {} - resolveLogSource( - input: EvalLogsInput, + async resourceOnlineEvaluationLogSource( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + const config = await this.onlineLogConfig(id, options, signal); + return this.evaluationLogSource({ + config, + defaultLogGroup: `/aws/bedrock-agentcore/evaluations/results/${config.onlineEvaluationConfigId ?? serviceIdFromArn(id)}`, + options, + signal, + }); + } + + async resourceOnlineInsightLogSource( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + const config = await this.onlineLogConfig(id, options, signal); + if (!config.insights?.length) { + throw new InputValidationError(`"${id}" is not an online-insight config`); + } + return this.evaluationLogSource({ + config, + defaultLogGroup: `/aws/bedrock-agentcore/evaluations/results/${config.onlineEvaluationConfigId ?? serviceIdFromArn(id)}`, + options, + signal, + }); + } + + async resourceBatchEvaluationLogSource( + id: string, options: CoreOptions, signal?: AbortSignal, ): Promise { - return resolveEvalLogSource({ client: this, input, options, signal }); + const config = await this.batchLogConfig(id, options, signal); + return this.evaluationLogSource({ + config, + defaultLogGroup: "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + options, + signal, + }); + } + + async resourceBatchInsightsLogSource( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise { + const config = await this.batchLogConfig(id, options, signal); + if (!EvalClient.isBatchInsights(config)) { + throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); + } + return this.evaluationLogSource({ + config, + defaultLogGroup: "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + options, + signal, + }); + } + + private onlineLogConfig(id: string, options: CoreOptions, signal?: AbortSignal) { + return this.clients + .control(toClientConfig(options)) + .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id }), { + abortSignal: signal, + }); + } + + private batchLogConfig(id: string, options: CoreOptions, signal?: AbortSignal) { + return this.clients + .data(toClientConfig(options)) + .send(new GetBatchEvaluationCommand({ batchEvaluationId: id }), { abortSignal: signal }); + } + + private async evaluationLogSource({ + config, + defaultLogGroup, + options, + signal, + }: { + config: GetOnlineEvaluationConfigResponse | BatchEvaluationDetail; + defaultLogGroup: string; + options: CoreOptions; + signal?: AbortSignal; + }): Promise { + const output = config.outputConfig?.cloudWatchConfig; + if (output?.logGroupName) return { logGroupName: output.logGroupName }; + if (config.outputConfig && !output) { + throw new InputValidationError("Logs require a CloudWatch output configuration."); + } + if (output?.resultDestination !== "SOURCE_LOG_GROUP") { + return { logGroupName: defaultLogGroup }; + } + let source = config.dataSourceConfig; + if (source && "onlineEvaluationConfigSource" in source) { + const arn = source.onlineEvaluationConfigSource?.onlineEvaluationConfigArn; + if (!arn) throw new MalformedServiceResponseError("Missing source online config ARN."); + source = (await this.onlineLogConfig(serviceIdFromArn(arn), options, signal)) + .dataSourceConfig; + } + const groups = source?.cloudWatchLogs; + if (groups?.logGroupNamePrefixes?.length || (groups?.logGroupNames?.length ?? 0) > 1) { + throw new InputValidationError( + "Results use multiple source log groups or prefixes. " + + "Read them directly in CloudWatch Logs, or configure one explicit source log group.", + ); + } + const logGroupName = groups?.logGroupNames?.[0]; + if (!logGroupName) { + throw new MalformedServiceResponseError("Missing source log group name."); + } + return { logGroupName }; } async createEvaluator( @@ -468,14 +575,11 @@ export class EvalClient implements CoreEvalClient { async getBatchEvaluation( id: string, options: CoreOptions, - { includeResults = true, signal }: { includeResults?: boolean; signal?: AbortSignal } = {}, + { includeResults = true }: { includeResults?: boolean } = {}, ): Promise { const job = await this.clients .data(toClientConfig(options)) - .send( - new GetBatchEvaluationCommand({ batchEvaluationId: id }), - signal ? { abortSignal: signal } : undefined, - ); + .send(new GetBatchEvaluationCommand({ batchEvaluationId: id })); const detail: BatchEvaluationDetail = { ...job }; const cw = job.outputConfig?.cloudWatchConfig; @@ -510,15 +614,8 @@ export class EvalClient implements CoreEvalClient { } } - async getBatchInsights( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const { detail } = await this.getBatchEvaluation(id, options, { - includeResults: false, - ...(signal ? { signal } : {}), - }); + async getBatchInsights(id: string, options: CoreOptions): Promise { + const { detail } = await this.getBatchEvaluation(id, options, { includeResults: false }); if (!EvalClient.isBatchInsights(detail)) { throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); } @@ -1188,15 +1285,10 @@ export class EvalClient implements CoreEvalClient { ); } - async getOnlineInsight( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { + async getOnlineInsight(id: string, options: CoreOptions): Promise { const control = this.clients.control(toClientConfig(options)); const config = await control.send( new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id }), - signal ? { abortSignal: signal } : undefined, ); if ((config.insights?.length ?? 0) === 0) throw new InputValidationError(`"${id}" is not an online-insight config`, { @@ -1333,14 +1425,10 @@ export class EvalClient implements CoreEvalClient { async getOnlineEvaluationConfig( id: string, options: CoreOptions, - signal?: AbortSignal, ): Promise { return this.clients .control(toClientConfig(options)) - .send( - new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id }), - signal ? { abortSignal: signal } : undefined, - ); + .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); } async listOnlineEvaluationConfigs( diff --git a/src/core/evalLogSource.test.ts b/src/core/evalLogSource.test.ts deleted file mode 100644 index 356e0bd5be..0000000000 --- a/src/core/evalLogSource.test.ts +++ /dev/null @@ -1,317 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { - GetOnlineEvaluationConfigCommand, - type BedrockAgentCoreControlClient, -} from "@aws-sdk/client-bedrock-agentcore-control"; -import { - GetBatchEvaluationCommand, - type BedrockAgentCoreClient, -} from "@aws-sdk/client-bedrock-agentcore"; -import { EvalClient } from "./eval"; -import type { AwsClients } from "./types"; -import type { EvalLogsInput } from "../handlers/eval/types"; -import { InputValidationError, MalformedServiceResponseError } from "../errors"; - -const OPTIONS = { region: "us-west-2", endpointUrl: "https://agentcore.example.test" }; -const ID = "evaluation-1"; -const SOURCE_GROUP = "/custom/source"; -const ONLINE_CONFIG_ID = "config-1"; -const ONLINE_CONFIG_ARN = `arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/${ONLINE_CONFIG_ID}`; -const SOURCE_OUTPUT = { cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" } }; - -type Metadata = Record | Error; -type MetadataCommand = GetOnlineEvaluationConfigCommand | GetBatchEvaluationCommand; -type SendOptions = { abortSignal?: AbortSignal }; - -function setup( - responses: { - online?: Metadata; - batch?: Metadata; - beforeResponse?: (command: MetadataCommand, options?: SendOptions) => Promise; - } = {}, -) { - const commands: MetadataCommand[] = []; - const sendOptions: (SendOptions | undefined)[] = []; - const online = responses.online ?? { - onlineEvaluationConfigId: ONLINE_CONFIG_ID, - dataSourceConfig: { cloudWatchLogs: { logGroupNames: [SOURCE_GROUP] } }, - }; - const batch = responses.batch ?? { outputConfig: SOURCE_OUTPUT }; - const unexpectedClient = () => { - throw new Error("Unexpected AWS client; resolution must not read evaluation result streams."); - }; - const send = async (command: MetadataCommand, response: Metadata, options?: SendOptions) => { - commands.push(command); - sendOptions.push(options); - await responses.beforeResponse?.(command, options); - if (response instanceof Error) throw response; - return response; - }; - const clients: AwsClients = { - control: (config) => { - expect(config).toEqual({ region: OPTIONS.region, endpoint: OPTIONS.endpointUrl }); - return { - send: (command: GetOnlineEvaluationConfigCommand, options?: SendOptions) => - send(command, online, options), - } as unknown as BedrockAgentCoreControlClient; - }, - data: (config) => { - expect(config).toEqual({ region: OPTIONS.region, endpoint: OPTIONS.endpointUrl }); - return { - send: (command: GetBatchEvaluationCommand, options?: SendOptions) => - send(command, batch, options), - } as unknown as BedrockAgentCoreClient; - }, - logs: unexpectedClient, - iam: unexpectedClient, - xray: unexpectedClient, - applicationSignals: unexpectedClient, - }; - return { client: new EvalClient(clients), commands, sendOptions }; -} - -function waitForAbort(signal?: AbortSignal): Promise { - if (!signal) throw new Error("Metadata lookup did not receive a cancellation signal."); - if (signal.aborted) return Promise.reject(signal.reason); - return new Promise((_resolve, reject) => { - signal.addEventListener("abort", () => reject(signal.reason), { once: true }); - }); -} - -describe("evaluation log source resolution", () => { - test.each([ - { onlineEvaluationConfigId: ONLINE_CONFIG_ID, expectedId: ONLINE_CONFIG_ID }, - { onlineEvaluationConfigId: undefined, expectedId: ID }, - ])("uses the online config ID or requested ID for the default group", async (testCase) => { - const { client } = setup({ - online: { onlineEvaluationConfigId: testCase.onlineEvaluationConfigId }, - }); - - await expect( - client.resolveLogSource({ resource: "online-eval", id: ID }, OPTIONS), - ).resolves.toEqual({ - logGroupName: `/aws/bedrock-agentcore/evaluations/results/${testCase.expectedId}`, - }); - }); - - test.each([ONLINE_CONFIG_ARN, ONLINE_CONFIG_ID])( - "resolves a batch source reference %s to the online config's input group", - async (reference) => { - const { client, commands } = setup({ - online: { - onlineEvaluationConfigId: ONLINE_CONFIG_ID, - dataSourceConfig: { cloudWatchLogs: { logGroupNames: [SOURCE_GROUP] } }, - outputConfig: { cloudWatchConfig: { logGroupName: "/custom/online-results" } }, - }, - batch: { - status: "COMPLETED", - outputConfig: SOURCE_OUTPUT, - dataSourceConfig: { - onlineEvaluationConfigSource: { onlineEvaluationConfigArn: reference }, - }, - }, - }); - - await expect( - client.resolveLogSource({ resource: "batch-evaluation", id: ID }, OPTIONS), - ).resolves.toEqual({ logGroupName: SOURCE_GROUP }); - expect(commands).toHaveLength(2); - expect(commands[0]).toBeInstanceOf(GetBatchEvaluationCommand); - expect(commands[0]?.input).toEqual({ batchEvaluationId: ID }); - expect(commands[1]).toBeInstanceOf(GetOnlineEvaluationConfigCommand); - expect(commands[1]?.input).toEqual({ onlineEvaluationConfigId: ONLINE_CONFIG_ID }); - }, - ); - - describe.each(["online-eval", "batch-evaluation"] as const)( - "%s output configurations", - (resource) => { - const resolve = (metadata: Metadata) => { - const { client } = setup({ online: metadata, batch: metadata }); - return client.resolveLogSource({ resource, id: ID }, OPTIONS); - }; - - test("uses the default for a dedicated destination without an explicit group", async () => { - await expect( - resolve({ - outputConfig: { cloudWatchConfig: { resultDestination: "DEDICATED_LOG_GROUP" } }, - }), - ).resolves.toEqual({ - logGroupName: - resource === "online-eval" - ? `/aws/bedrock-agentcore/evaluations/results/${ID}` - : "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", - }); - }); - - test.each([ - { source: { logGroupNames: ["/one", "/two"] }, message: "multiple source log groups" }, - { source: { logGroupNamePrefixes: ["/prefix/"] }, message: "selected by prefix" }, - ])("rejects ambiguous source groups: $message", async ({ source, message }) => { - const result = resolve({ - outputConfig: SOURCE_OUTPUT, - dataSourceConfig: { cloudWatchLogs: source }, - }); - - await expect(result).rejects.toBeInstanceOf(InputValidationError); - await expect(result).rejects.toThrow(message); - await expect(result).rejects.toThrow("configure one explicit source log group"); - }); - - test.each([ - undefined, - { cloudWatchLogs: { logGroupNames: [] } }, - { cloudWatchLogs: { logGroupNames: [""] } }, - { $unknown: ["futureSource", {}] }, - ])( - "rejects a source destination without a usable source group: %j", - async (dataSourceConfig) => { - await expect( - resolve({ outputConfig: SOURCE_OUTPUT, dataSourceConfig }), - ).rejects.toBeInstanceOf(MalformedServiceResponseError); - }, - ); - - test("rejects an unknown output destination instead of assuming the default group", async () => { - const result = resolve({ outputConfig: { $unknown: ["futureOutput", {}] } }); - - await expect(result).rejects.toBeInstanceOf(InputValidationError); - await expect(result).rejects.toThrow("logs require a CloudWatch output configuration"); - }); - }, - ); - - test("rejects a batch source reference that has no ARN", async () => { - const { client, commands } = setup({ - batch: { - outputConfig: SOURCE_OUTPUT, - dataSourceConfig: { onlineEvaluationConfigSource: {} }, - }, - }); - - await expect( - client.resolveLogSource({ resource: "batch-evaluation", id: ID }, OPTIONS), - ).rejects.toBeInstanceOf(MalformedServiceResponseError); - expect(commands).toHaveLength(1); - }); - - test.each([ - "online-eval", - "online-insight", - "batch-evaluation", - "batch-insights", - ] satisfies EvalLogsInput["resource"][])( - "propagates %s metadata lookup failures", - async (resource) => { - const error = new Error("Metadata lookup failed"); - const { client } = setup({ online: error, batch: error }); - - await expect(client.resolveLogSource({ resource, id: ID }, OPTIONS)).rejects.toBe(error); - }, - ); - - test("propagates failures looking up a batch job's online config source", async () => { - const error = new Error("Source lookup failed"); - const { client } = setup({ - online: error, - batch: { - outputConfig: SOURCE_OUTPUT, - dataSourceConfig: { - onlineEvaluationConfigSource: { onlineEvaluationConfigArn: ONLINE_CONFIG_ARN }, - }, - }, - }); - - await expect( - client.resolveLogSource({ resource: "batch-evaluation", id: ID }, OPTIONS), - ).rejects.toBe(error); - }); - - describe.each([ - "online-eval", - "online-insight", - "batch-evaluation", - "batch-insights", - ] satisfies EvalLogsInput["resource"][])("%s cancellation", (resource) => { - test("passes the signal to metadata lookup without changing SDK request input", async () => { - const controller = new AbortController(); - const metadata = { - insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], - outputConfig: { cloudWatchConfig: { logGroupName: "/custom/results" } }, - }; - const { client, commands, sendOptions } = setup({ online: metadata, batch: metadata }); - - await expect( - client.resolveLogSource({ resource, id: ID }, OPTIONS, controller.signal), - ).resolves.toEqual({ logGroupName: "/custom/results" }); - expect(sendOptions).toEqual([{ abortSignal: controller.signal }]); - expect(commands[0]?.input).toEqual( - resource.startsWith("online") - ? { onlineEvaluationConfigId: ID } - : { batchEvaluationId: ID }, - ); - }); - - test("omits SDK cancellation options when no signal is supplied", async () => { - const metadata = { - insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], - outputConfig: { cloudWatchConfig: { logGroupName: "/custom/results" } }, - }; - const { client, sendOptions } = setup({ online: metadata, batch: metadata }); - - await client.resolveLogSource({ resource, id: ID }, OPTIONS); - - expect(sendOptions).toEqual([undefined]); - }); - - test("cancels an in-flight metadata lookup", async () => { - const controller = new AbortController(); - const reason = new DOMException("Cancelled lookup", "AbortError"); - const { client } = setup({ - beforeResponse: (_command, options) => waitForAbort(options?.abortSignal), - }); - const pending = client.resolveLogSource({ resource, id: ID }, OPTIONS, controller.signal); - - controller.abort(reason); - - await expect(pending).rejects.toBe(reason); - }); - }); - - test.each(["batch-evaluation", "batch-insights"] as const)( - "cancels %s during the nested online config source lookup", - async (resource) => { - const controller = new AbortController(); - const reason = new DOMException("Cancelled source lookup", "AbortError"); - let sourceLookupStarted = () => {}; - const started = new Promise((resolve) => { - sourceLookupStarted = resolve; - }); - const { client, sendOptions } = setup({ - batch: { - insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], - outputConfig: SOURCE_OUTPUT, - dataSourceConfig: { - onlineEvaluationConfigSource: { onlineEvaluationConfigArn: ONLINE_CONFIG_ARN }, - }, - }, - beforeResponse: async (command, options) => { - if (command instanceof GetOnlineEvaluationConfigCommand) { - sourceLookupStarted(); - await waitForAbort(options?.abortSignal); - } - }, - }); - const pending = client.resolveLogSource({ resource, id: ID }, OPTIONS, controller.signal); - await started; - - controller.abort(reason); - - await expect(pending).rejects.toBe(reason); - expect(sendOptions).toEqual([ - { abortSignal: controller.signal }, - { abortSignal: controller.signal }, - ]); - }, - ); -}); diff --git a/src/core/evalLogSource.ts b/src/core/evalLogSource.ts deleted file mode 100644 index be4765bd5d..0000000000 --- a/src/core/evalLogSource.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { CoreEvalClient, EvalLogsInput } from "../handlers/eval/types"; -import { InputValidationError, MalformedServiceResponseError } from "../errors"; -import { serviceIdFromArn } from "./arn"; -import type { LogSource } from "./observability/types"; -import type { CoreOptions } from "./types"; - -type EvalLogClient = Pick< - CoreEvalClient, - "getOnlineEvaluationConfig" | "getOnlineInsight" | "getBatchEvaluation" | "getBatchInsights" ->; - -export async function resolveEvalLogSource({ - client, - input, - options, - signal, -}: { - client: EvalLogClient; - input: EvalLogsInput; - options: CoreOptions; - signal?: AbortSignal; -}): Promise { - const config = await (async () => { - switch (input.resource) { - case "online-eval": - return client.getOnlineEvaluationConfig(input.id, options, signal); - case "online-insight": - return client.getOnlineInsight(input.id, options, signal); - case "batch-evaluation": - return ( - await client.getBatchEvaluation(input.id, options, { - includeResults: false, - ...(signal ? { signal } : {}), - }) - ).detail; - case "batch-insights": - return client.getBatchInsights(input.id, options, signal); - } - })(); - - const cloudWatchConfig = config.outputConfig?.cloudWatchConfig; - if (cloudWatchConfig?.logGroupName) { - return { logGroupName: cloudWatchConfig.logGroupName }; - } - if (config.outputConfig && !cloudWatchConfig) { - throw new InputValidationError( - `${input.resource} "${input.id}" has an unsupported output destination; ` + - "logs require a CloudWatch output configuration.", - ); - } - if (cloudWatchConfig?.resultDestination !== "SOURCE_LOG_GROUP") { - const isOnline = input.resource === "online-eval" || input.resource === "online-insight"; - const onlineConfigId = - "onlineEvaluationConfigId" in config ? config.onlineEvaluationConfigId : undefined; - return { - logGroupName: isOnline - ? `/aws/bedrock-agentcore/evaluations/results/${onlineConfigId ?? serviceIdFromArn(input.id)}` - : "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", - }; - } - - let dataSourceConfig = config.dataSourceConfig; - if (dataSourceConfig && "onlineEvaluationConfigSource" in dataSourceConfig) { - const configArn = dataSourceConfig.onlineEvaluationConfigSource?.onlineEvaluationConfigArn; - if (!configArn) { - throw new MalformedServiceResponseError( - `${input.resource} "${input.id}" returned an online evaluation source without a config ARN.`, - ); - } - dataSourceConfig = ( - await client.getOnlineEvaluationConfig(serviceIdFromArn(configArn), options, signal) - ).dataSourceConfig; - } - - const source = dataSourceConfig?.cloudWatchLogs; - if (source?.logGroupNamePrefixes?.length) { - throw new InputValidationError( - `${input.resource} "${input.id}" writes results to source log groups selected by prefix. ` + - "Read the matching groups directly in CloudWatch Logs, or configure one explicit source log group.", - ); - } - if ((source?.logGroupNames?.length ?? 0) > 1) { - throw new InputValidationError( - `${input.resource} "${input.id}" writes results to multiple source log groups. ` + - "Read each group directly in CloudWatch Logs, or configure one explicit source log group.", - ); - } - const logGroupName = source?.logGroupNames?.[0]; - if (!logGroupName) { - throw new MalformedServiceResponseError( - `${input.resource} "${input.id}" writes results to a source log group but returned no log group name.`, - ); - } - return { logGroupName }; -} diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx index 67b5299e36..d967f49820 100644 --- a/src/handlers/eval/batch-evaluation/index.tsx +++ b/src/handlers/eval/batch-evaluation/index.tsx @@ -24,7 +24,12 @@ export function createBatchEvaluationHandler(core: Core, io: AppIO): Router { .handler(createGetBatchEvaluationHandler(core, io)) .handler(createListBatchEvaluationsHandler(core)) .handler(createStopBatchEvaluationHandler(core)) - .handler(createEvalLogsHandler(core, io, "batch-evaluation")); + .handler( + createEvalLogsHandler(core, io, { + resource: "batch-evaluation", + logSource: core.eval.resourceBatchEvaluationLogSource.bind(core.eval), + }), + ); } export { BatchEvaluationScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/batch-insights/index.tsx b/src/handlers/eval/batch-insights/index.tsx index 6f6861601a..babb3e08ed 100644 --- a/src/handlers/eval/batch-insights/index.tsx +++ b/src/handlers/eval/batch-insights/index.tsx @@ -18,7 +18,12 @@ export function createBatchInsightsHandler(core: Core, io: AppIO): Router { .handler(createGetBatchInsightsHandler(core)) .handler(createListBatchInsightsHandler(core)) .handler(createStopBatchInsightsHandler(core)) - .handler(createEvalLogsHandler(core, io, "batch-insights")); + .handler( + createEvalLogsHandler(core, io, { + resource: "batch-insights", + logSource: core.eval.resourceBatchInsightsLogSource.bind(core.eval), + }), + ); } export { BatchInsightsScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/logs/index.tsx b/src/handlers/eval/logs/index.tsx index 0dec9bb818..a662e4a800 100644 --- a/src/handlers/eval/logs/index.tsx +++ b/src/handlers/eval/logs/index.tsx @@ -4,16 +4,26 @@ import { flag } from "../../../router"; import { createLogsHandler } from "../../observability/logs"; import type { Core } from "../../types"; import { coreOptsFromCtx } from "../../utils"; -import type { EvalLogsInput } from "../types"; +import type { CoreEvalClient } from "../types"; -export const createEvalLogsHandler = (core: Core, io: AppIO, resource: EvalLogsInput["resource"]) => +export const createEvalLogsHandler = ( + core: Core, + io: AppIO, + { + resource, + logSource, + }: { + resource: string; + logSource: CoreEvalClient["resourceBatchEvaluationLogSource"]; + }, +) => createLogsHandler(io, { name: "logs", description: `stream or search ${resource} logs`, flags: [flag("id", `the ID of the ${resource} resource`, z.string().min(1))] as const, read: async (ctx, flags, request, signal) => { const options = coreOptsFromCtx(ctx); - const source = await core.eval.resolveLogSource({ resource, id: flags.id }, options, signal); + const source = await logSource(flags.id, options, signal); if (request.mode === "search") { return { diff --git a/src/handlers/eval/logs/logs.test.tsx b/src/handlers/eval/logs/logs.test.tsx index fa33c912b6..fc589cf036 100644 --- a/src/handlers/eval/logs/logs.test.tsx +++ b/src/handlers/eval/logs/logs.test.tsx @@ -1,5 +1,8 @@ import { describe, expect, test } from "bun:test"; -import type { BedrockAgentCoreControlClient } from "@aws-sdk/client-bedrock-agentcore-control"; +import { + type BedrockAgentCoreControlClient, + type GetOnlineEvaluationConfigCommand, +} from "@aws-sdk/client-bedrock-agentcore-control"; import type { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; import { DescribeLogGroupsCommand, @@ -11,79 +14,97 @@ import { CoreClient } from "../../../core"; import { InputValidationError, UserCancellationError } from "../../../errors"; import { createSilentLogger, TestGlobalConfigAccessor, testIO } from "../../../testing"; import { createRootHandler } from "../../index"; -import type { EvalLogsInput } from "../types"; +type Resource = "online-eval" | "online-insight" | "batch-evaluation" | "batch-insights"; const SINCE = 1_709_391_000_000; const ID = "config-AbC123XyZ9"; -const RESOURCES: EvalLogsInput["resource"][] = [ - "online-eval", - "online-insight", - "batch-evaluation", - "batch-insights", -]; +const ONLINE_GROUP = `/aws/bedrock-agentcore/evaluations/results/${ID}`; +const BATCH_GROUP = "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default"; +const SOURCE_GROUP = "/custom/source"; const EVENTS = [ { timestamp: SINCE + 1_000, message: "first result", logStreamName: "run-a" }, { timestamp: SINCE + 2_000, message: "second result", logStreamName: "run-b" }, ]; -const SOURCE_GROUP = "/custom/source"; +const SOURCE_OUTPUT = { cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" } }; +const CUSTOM_OUTPUT = { + cloudWatchConfig: { logGroupName: "/custom/results", logStreamName: "run-a" }, +}; +const CUSTOM_CONFIG = { outputConfig: CUSTOM_OUTPUT }; +const JSON_EVENTS = EVENTS.map((event) => ({ + ...event, + timestamp: new Date(event.timestamp).toISOString(), +})); function setup( - metadata: Record = {}, - readMetadata?: (signal?: AbortSignal) => Promise>, + options: { + online?: Record; + batch?: Record; + logGroup?: string; + onlineId?: string; + readMetadata?: (signal?: AbortSignal) => Promise>; + } = {}, ) { - const commands: { name: string; input: unknown }[] = []; - const response = { - onlineEvaluationConfigId: ID, - batchEvaluationId: ID, - status: "COMPLETED", - insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], - dataSourceConfig: { cloudWatchLogs: { logGroupNames: [SOURCE_GROUP] } }, - ...metadata, - }; - const sendMetadata = async ( - command: { constructor: { name: string }; input: unknown }, - options?: { abortSignal?: AbortSignal }, - ) => { - commands.push({ name: command.constructor.name, input: command.input }); - if (readMetadata) return readMetadata(options?.abortSignal); - if (command.constructor.name === "GetOnlineEvaluationConfigCommand") { - const { batchEvaluationId: _, ...config } = response; - return config; - } - const { onlineEvaluationConfigId: _, ...job } = response; - return job; - }; + const logGroup = options.logGroup ?? ONLINE_GROUP; + const onlineId = options.onlineId ?? ID; + const insights = [{ insightId: "Builtin.Insight.FailureAnalysis" }]; + const logGroupArn = `arn:aws:logs:us-west-2:123456789012:log-group:${logGroup}`; const unusedClient = () => { throw new Error("Unexpected AWS client"); }; const core = new CoreClient({ - createControlClient: () => ({ send: sendMetadata }) as unknown as BedrockAgentCoreControlClient, - createDataClient: () => ({ send: sendMetadata }) as unknown as BedrockAgentCoreClient, + createControlClient: () => + ({ + send: async ( + command: GetOnlineEvaluationConfigCommand, + sendOptions?: { abortSignal?: AbortSignal }, + ) => { + if (options.readMetadata) return options.readMetadata(sendOptions?.abortSignal); + if (command.input.onlineEvaluationConfigId !== onlineId) { + throw new Error("Online evaluation configuration not found"); + } + return { + onlineEvaluationConfigId: onlineId, + insights, + dataSourceConfig: { cloudWatchLogs: { logGroupNames: [SOURCE_GROUP] } }, + ...options.online, + }; + }, + }) as unknown as BedrockAgentCoreControlClient, + createDataClient: () => + ({ + send: async () => ({ + batchEvaluationId: ID, + status: "COMPLETED", + insights, + ...options.batch, + }), + }) as unknown as BedrockAgentCoreClient, createLogsClient: () => ({ send: async (command: unknown) => { if (command instanceof FilterLogEventsCommand) { - commands.push({ name: command.constructor.name, input: command.input }); - return command.input.nextToken - ? { events: [EVENTS[1]] } - : { events: [EVENTS[0]], nextToken: "page-2" }; + const events = (command.input.logGroupName === logGroup ? EVENTS : []).filter( + (event) => + !command.input.logStreamNames || + command.input.logStreamNames.includes(event.logStreamName), + ); + const offset = Number(command.input.nextToken ?? 0); + return { + events: events.slice(offset, offset + 1), + nextToken: offset + 1 < events.length ? `${offset + 1}` : undefined, + }; } if (command instanceof DescribeLogGroupsCommand) { - commands.push({ name: command.constructor.name, input: command.input }); return { - logGroups: [ - { - logGroupName: command.input.logGroupNamePrefix, - logGroupArn: "arn:aws:logs:us-west-2:123456789012:log-group:test", - }, - ], + logGroups: [{ logGroupName: logGroup, logGroupArn }], }; } if (command instanceof StartLiveTailCommand) { - commands.push({ name: command.constructor.name, input: command.input }); return { responseStream: (async function* () { - yield { sessionUpdate: { sessionResults: EVENTS } }; + if (command.input.logGroupIdentifiers?.includes(logGroupArn)) { + yield { sessionUpdate: { sessionResults: EVENTS } }; + } })(), }; } @@ -103,9 +124,8 @@ function setup( }); return { - commands, io, - route: (resource: EvalLogsInput["resource"], flags: readonly string[]) => + route: (resource: Resource, flags: readonly string[]) => root.route([ "node", "agentcore", @@ -119,134 +139,111 @@ function setup( }; } -describe.each(RESOURCES)("eval %s logs", (resource) => { - const defaultGroup = resource.startsWith("online") - ? `/aws/bedrock-agentcore/evaluations/results/${ID}` - : "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default"; - const searchFlags = ["--id", ID, "--since", `${SINCE}`, "--until", `${SINCE + 60_000}`]; - +describe("evaluation logs", () => { test.each([ - { destination: "default", outputConfig: undefined, expectedGroup: defaultGroup }, - { - destination: "custom", - outputConfig: { - cloudWatchConfig: { logGroupName: "/custom/results", logStreamName: "run-a" }, + ["online-eval", "default", { logGroup: ONLINE_GROUP }], + ["online-insight", "default", { logGroup: ONLINE_GROUP }], + ["batch-evaluation", "default", { logGroup: BATCH_GROUP }], + ["batch-insights", "default", { logGroup: BATCH_GROUP }], + ["online-eval", "custom", { logGroup: "/custom/results", online: CUSTOM_CONFIG }], + ["batch-evaluation", "custom", { logGroup: "/custom/results", batch: CUSTOM_CONFIG }], + ["online-eval", "source", { logGroup: SOURCE_GROUP, online: { outputConfig: SOURCE_OUTPUT } }], + [ + "batch-evaluation", + "referenced online source", + { + logGroup: SOURCE_GROUP, + onlineId: "source-config", + online: CUSTOM_CONFIG, + batch: { + outputConfig: SOURCE_OUTPUT, + dataSourceConfig: { + onlineEvaluationConfigSource: { + onlineEvaluationConfigArn: + "arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/source-config", + }, + }, + }, }, - expectedGroup: "/custom/results", - }, - { - destination: "source", - outputConfig: { cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" } }, - expectedGroup: SOURCE_GROUP, - }, - ])("searches the entire $destination group across pages and streams", async (testCase) => { - const { commands, io, route } = setup({ outputConfig: testCase.outputConfig }); + ], + ] as const)("%s %s includes all pages and streams", async (resource, _destination, options) => { + const { io, route } = setup(options); - await route(resource, [...searchFlags, "--json"]); + await route(resource, ["--id", ID, "--since", `${SINCE}`, "--json"]); expect( io .stdout() .split("\n") .map((line) => JSON.parse(line)), - ).toEqual( - EVENTS.map((event) => ({ ...event, timestamp: new Date(event.timestamp).toISOString() })), - ); - const searches = commands.filter((command) => command.name === "FilterLogEventsCommand"); - expect(searches.map((command) => command.input)).toEqual([ - { logGroupName: testCase.expectedGroup, startTime: SINCE, endTime: SINCE + 60_000 }, - { - logGroupName: testCase.expectedGroup, - startTime: SINCE, - endTime: SINCE + 60_000, - nextToken: "page-2", - }, - ]); - expect(commands[0]?.name).toBe( - resource.startsWith("online") - ? "GetOnlineEvaluationConfigCommand" - : "GetBatchEvaluationCommand", - ); + ).toEqual(JSON_EVENTS); expect(io.stderr()).toBe(""); }); - test("streams the configured group by default with a stderr announcement", async () => { - const { commands, io, route } = setup({ - outputConfig: { cloudWatchConfig: { logGroupName: "/custom/results" } }, - }); + test("tails by default and announces the stream on stderr", async () => { + const { io, route } = setup(); - await route(resource, ["--id", ID, "--query", "ERROR"]); + await route("online-eval", ["--id", ID]); expect(io.stdout()).toBe( "2024-03-02T14:50:01.000Z first result\n2024-03-02T14:50:02.000Z second result", ); - expect(io.stderr()).toBe(`Streaming logs for ${resource} ${ID}... (Ctrl+C to stop)`); - expect(commands.find((command) => command.name === "DescribeLogGroupsCommand")?.input).toEqual({ - logGroupNamePrefix: "/custom/results", - }); - expect(commands.find((command) => command.name === "StartLiveTailCommand")?.input).toEqual({ - logGroupIdentifiers: ["arn:aws:logs:us-west-2:123456789012:log-group:test"], - logEventFilterPattern: "ERROR", - }); + expect(io.stderr()).toBe(`Streaming logs for online-eval ${ID}... (Ctrl+C to stop)`); }); - test("Ctrl+C cancels metadata lookup without waiting for a service response", async () => { - let receivedSignal: AbortSignal | undefined; - let releaseMetadata = () => {}; - let metadataStarted = () => {}; - const started = new Promise((resolve) => { - metadataStarted = resolve; - }); - const { commands, io, route } = setup({}, (signal) => { - receivedSignal = signal; - metadataStarted(); - return new Promise((resolve, reject) => { - releaseMetadata = () => resolve({}); - signal?.addEventListener("abort", () => reject(new Error("transport aborted")), { - once: true, - }); - }); + test("Ctrl+C cancels a pending metadata lookup", async () => { + const started = Promise.withResolvers(); + const metadata = Promise.withResolvers>(); + const { io, route } = setup({ + readMetadata: (signal) => { + signal?.addEventListener("abort", () => metadata.reject(new Error("transport aborted"))); + started.resolve(); + return metadata.promise; + }, }); - const pending = route(resource, ["--id", ID, "--since", "1h"]); + const pending = route("online-eval", ["--id", ID, "--since", "1h"]); try { - await started; - expect(receivedSignal).toBeDefined(); + await started.promise; process.emit("SIGINT", "SIGINT"); await expect(pending).rejects.toBeInstanceOf(UserCancellationError); - expect(receivedSignal?.aborted).toBe(true); } finally { - releaseMetadata(); + metadata.resolve({}); await pending.catch(() => {}); } - expect(commands).toHaveLength(1); expect(io.stdout()).toBe(""); expect(io.stderr()).toBe(""); }); - test.each([ - { flags: [], message: "required option '--id' not specified" }, - { flags: ["--id", ""], message: "Invalid value for option '--id'" }, - { - flags: ["--id", ID, "--tail", "--since", "1h"], - message: "--tail cannot be combined with --since or --until", - }, - ])("validates flags before fetching metadata: $message", async ({ flags, message }) => { - const { commands, route } = setup(); + test("requires an explicit resource ID", async () => { + const { route } = setup(); - await expect(route(resource, flags)).rejects.toThrow(message); - expect(commands).toHaveLength(0); + await expect(route("online-eval", ["--since", "1h"])).rejects.toThrow( + "required option '--id' not specified", + ); }); -}); -test.each(["online-insight", "batch-insights"] as const)( - "rejects a resource without insights through %s logs", - async (resource) => { - const { commands, route } = setup({ insights: [] }); + test.each(["online-insight", "batch-insights"] as const)( + "%s rejects a resource without insights", + async (resource) => { + const { route } = setup({ online: { insights: [] }, batch: { insights: [] } }); - await expect(route(resource, ["--id", ID, "--since", "1h"])).rejects.toBeInstanceOf( - InputValidationError, + await expect(route(resource, ["--id", ID, "--since", "1h"])).rejects.toBeInstanceOf( + InputValidationError, + ); + }, + ); + + test("explains how to read results when the source has multiple groups", async () => { + const { route } = setup({ + online: { + outputConfig: SOURCE_OUTPUT, + dataSourceConfig: { cloudWatchLogs: { logGroupNames: ["/one", "/two"] } }, + }, + }); + + await expect(route("online-eval", ["--id", ID, "--since", "1h"])).rejects.toThrow( + "configure one explicit source log group", ); - expect(commands).toHaveLength(1); - }, -); + }); +}); diff --git a/src/handlers/eval/online-eval/index.tsx b/src/handlers/eval/online-eval/index.tsx index 3693a8beb6..d6090a24aa 100644 --- a/src/handlers/eval/online-eval/index.tsx +++ b/src/handlers/eval/online-eval/index.tsx @@ -24,7 +24,12 @@ export function createOnlineEvalHandler(core: Core, io: AppIO): Router { .handler(createPauseOnlineEvalHandler(core)) .handler(createResumeOnlineEvalHandler(core)) .handler(createDeleteOnlineEvalHandler(core)) - .handler(createEvalLogsHandler(core, io, "online-eval")); + .handler( + createEvalLogsHandler(core, io, { + resource: "online-eval", + logSource: core.eval.resourceOnlineEvaluationLogSource.bind(core.eval), + }), + ); } export { OnlineEvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-insight/index.tsx b/src/handlers/eval/online-insight/index.tsx index 87c227f9ab..d4735452c0 100644 --- a/src/handlers/eval/online-insight/index.tsx +++ b/src/handlers/eval/online-insight/index.tsx @@ -24,7 +24,12 @@ export function createOnlineInsightHandler(core: Core, io: AppIO): Router { .handler(createPauseOnlineInsightHandler(core)) .handler(createResumeOnlineInsightHandler(core)) .handler(createDeleteOnlineInsightHandler(core)) - .handler(createEvalLogsHandler(core, io, "online-insight")); + .handler( + createEvalLogsHandler(core, io, { + resource: "online-insight", + logSource: core.eval.resourceOnlineInsightLogSource.bind(core.eval), + }), + ); } export { OnlineInsightScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index df1d7213e6..2883ee11c3 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -60,11 +60,6 @@ import type { CoreOptions } from "../../core/types"; import type { LogSource } from "../../core/observability/types"; import type { EvaluatorModelProvider } from "../../projectSchemas/evaluator"; -export type EvalLogsInput = { - resource: "online-eval" | "online-insight" | "batch-evaluation" | "batch-insights"; - id: string; -}; - // SessionWindow is the resolved explicit time filter for a batch evaluation's // session source (from --start-time/--end-time). Maps directly to the SDK's // SessionFilterConfig. @@ -394,8 +389,23 @@ export type DatasetUpdateProgressEvent = { // and implemented by src/core/eval.tsx (dependency inversion: handlers own the // abstraction). export interface CoreEvalClient { - resolveLogSource( - input: EvalLogsInput, + resourceOnlineEvaluationLogSource( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; + resourceOnlineInsightLogSource( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; + resourceBatchEvaluationLogSource( + id: string, + options: CoreOptions, + signal?: AbortSignal, + ): Promise; + resourceBatchInsightsLogSource( + id: string, options: CoreOptions, signal?: AbortSignal, ): Promise; @@ -443,13 +453,9 @@ export interface CoreEvalClient { getBatchEvaluation( id: string, options: CoreOptions, - opts?: { includeResults?: boolean; signal?: AbortSignal }, + opts?: { includeResults?: boolean }, ): Promise; - getBatchInsights( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise; + getBatchInsights(id: string, options: CoreOptions): Promise; listBatchEvaluations( nextToken: string | undefined, maxResults: number | undefined, @@ -524,7 +530,6 @@ export interface CoreEvalClient { getOnlineEvaluationConfig( id: string, options: CoreOptions, - signal?: AbortSignal, ): Promise; listOnlineEvaluationConfigs( nextToken: string | undefined, @@ -550,11 +555,7 @@ export interface CoreEvalClient { update: UpdateOnlineInsightInput, options: CoreOptions, ): Promise; - getOnlineInsight( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise; + getOnlineInsight(id: string, options: CoreOptions): Promise; listOnlineInsights( nextToken: string | undefined, maxResults: number | undefined, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 0cfa682732..1cd64021ea 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -189,7 +189,6 @@ import type { DatasetUpdateProgressEvent, EvaluateInput, EvaluateResult, - EvalLogsInput, InvokeDatasetInput, InvokeDatasetResult, GetBatchEvaluationResult, @@ -203,7 +202,6 @@ import type { UpdateOnlineEvalInput, } from "../handlers/eval/types"; import { isTerminalStatus } from "../core/batchEvaluationResults"; -import { resolveEvalLogSource } from "../core/evalLogSource"; import { abortable } from "../core/abortable"; import type { CoreFetch, CoreOptions, CreateCloudFormationClient } from "../core/types"; import type { ProjectManager } from "../handlers/project/types"; @@ -1630,17 +1628,30 @@ export class TestPaymentClient implements CorePaymentClient { export class TestEvalClient implements CoreEvalClient { // calls records every invocation in order, for assertions. readonly calls: RecordedCall[] = []; + logSource: LogSource = { logGroupName: "/test/eval/results" }; - async resolveLogSource( - input: EvalLogsInput, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - this.calls.push({ - method: "resolveLogSource", - args: signal ? [input, options, signal] : [input, options], - }); - return resolveEvalLogSource({ client: this, input, options, signal }); + async resourceOnlineEvaluationLogSource(id: string, options: CoreOptions, signal?: AbortSignal) { + this.calls.push({ method: "resourceOnlineEvaluationLogSource", args: [id, options, signal] }); + if (this.error) throw this.error; + return this.logSource; + } + + async resourceOnlineInsightLogSource(id: string, options: CoreOptions, signal?: AbortSignal) { + this.calls.push({ method: "resourceOnlineInsightLogSource", args: [id, options, signal] }); + if (this.error) throw this.error; + return this.logSource; + } + + async resourceBatchEvaluationLogSource(id: string, options: CoreOptions, signal?: AbortSignal) { + this.calls.push({ method: "resourceBatchEvaluationLogSource", args: [id, options, signal] }); + if (this.error) throw this.error; + return this.logSource; + } + + async resourceBatchInsightsLogSource(id: string, options: CoreOptions, signal?: AbortSignal) { + this.calls.push({ method: "resourceBatchInsightsLogSource", args: [id, options, signal] }); + if (this.error) throw this.error; + return this.logSource; } // List responses are keyed by the nextToken that requests them (undefined = @@ -2078,7 +2089,7 @@ export class TestEvalClient implements CoreEvalClient { async getBatchEvaluation( id: string, options: CoreOptions, - opts: { includeResults?: boolean; signal?: AbortSignal } = {}, + opts: { includeResults?: boolean } = {}, ): Promise { this.calls.push({ method: "getBatchEvaluation", args: [id, options, opts] }); if (this.error) throw this.error; @@ -2098,15 +2109,8 @@ export class TestEvalClient implements CoreEvalClient { return { detail }; } - async getBatchInsights( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - this.calls.push({ - method: "getBatchInsights", - args: signal ? [id, options, signal] : [id, options], - }); + async getBatchInsights(id: string, options: CoreOptions): Promise { + this.calls.push({ method: "getBatchInsights", args: [id, options] }); if (this.error) throw this.error; const detail: BatchEvaluationDetail = { ...this.batchEvalGetResponse }; @@ -2288,12 +2292,8 @@ export class TestEvalClient implements CoreEvalClient { async getOnlineEvaluationConfig( id: string, options: CoreOptions, - signal?: AbortSignal, ): Promise { - this.calls.push({ - method: "getOnlineEvaluationConfig", - args: signal ? [id, options, signal] : [id, options], - }); + this.calls.push({ method: "getOnlineEvaluationConfig", args: [id, options] }); if (this.error) throw this.error; return this.onlineEvalGetResponse; } @@ -2355,12 +2355,8 @@ export class TestEvalClient implements CoreEvalClient { async getOnlineInsight( id: string, options: CoreOptions, - signal?: AbortSignal, ): Promise { - this.calls.push({ - method: "getOnlineInsight", - args: signal ? [id, options, signal] : [id, options], - }); + this.calls.push({ method: "getOnlineInsight", args: [id, options] }); if (this.error) throw this.error; return this.onlineEvalGetResponse; } From ed6441f7582731557b20f9959799cd15d3a85289 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Mon, 12 Oct 2026 16:14:08 +0000 Subject: [PATCH 05/10] refactor(eval): reuse existing getters for log sources --- src/core/eval.tsx | 62 ++++---------------- src/handlers/eval/batch-evaluation/index.tsx | 2 +- src/handlers/eval/batch-insights/index.tsx | 2 +- src/handlers/eval/logs/index.tsx | 6 +- src/handlers/eval/logs/logs.test.tsx | 23 ++++---- src/handlers/eval/online-eval/index.tsx | 2 +- src/handlers/eval/online-insight/index.tsx | 2 +- src/handlers/eval/types.tsx | 24 ++------ src/testing/TestCoreClient.tsx | 16 ++--- 9 files changed, 40 insertions(+), 99 deletions(-) diff --git a/src/core/eval.tsx b/src/core/eval.tsx index f512755317..e0933aa1b6 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -239,92 +239,52 @@ export class EvalClient implements CoreEvalClient { private readonly cloudWatch: CloudWatchClient = new CloudWatchClient(clients), ) {} - async resourceOnlineEvaluationLogSource( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const config = await this.onlineLogConfig(id, options, signal); + async resourceOnlineEvaluationLogSource(id: string, options: CoreOptions): Promise { + const config = await this.getOnlineEvaluationConfig(id, options); return this.evaluationLogSource({ config, defaultLogGroup: `/aws/bedrock-agentcore/evaluations/results/${config.onlineEvaluationConfigId ?? serviceIdFromArn(id)}`, options, - signal, }); } - async resourceOnlineInsightLogSource( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const config = await this.onlineLogConfig(id, options, signal); - if (!config.insights?.length) { - throw new InputValidationError(`"${id}" is not an online-insight config`); - } + async resourceOnlineInsightLogSource(id: string, options: CoreOptions): Promise { + const config = await this.getOnlineInsight(id, options); return this.evaluationLogSource({ config, defaultLogGroup: `/aws/bedrock-agentcore/evaluations/results/${config.onlineEvaluationConfigId ?? serviceIdFromArn(id)}`, options, - signal, }); } - async resourceBatchEvaluationLogSource( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const config = await this.batchLogConfig(id, options, signal); + async resourceBatchEvaluationLogSource(id: string, options: CoreOptions): Promise { + const { detail: config } = await this.getBatchEvaluation(id, options, { + includeResults: false, + }); return this.evaluationLogSource({ config, defaultLogGroup: "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", options, - signal, }); } - async resourceBatchInsightsLogSource( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise { - const config = await this.batchLogConfig(id, options, signal); - if (!EvalClient.isBatchInsights(config)) { - throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); - } + async resourceBatchInsightsLogSource(id: string, options: CoreOptions): Promise { + const config = await this.getBatchInsights(id, options); return this.evaluationLogSource({ config, defaultLogGroup: "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", options, - signal, }); } - private onlineLogConfig(id: string, options: CoreOptions, signal?: AbortSignal) { - return this.clients - .control(toClientConfig(options)) - .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id }), { - abortSignal: signal, - }); - } - - private batchLogConfig(id: string, options: CoreOptions, signal?: AbortSignal) { - return this.clients - .data(toClientConfig(options)) - .send(new GetBatchEvaluationCommand({ batchEvaluationId: id }), { abortSignal: signal }); - } - private async evaluationLogSource({ config, defaultLogGroup, options, - signal, }: { config: GetOnlineEvaluationConfigResponse | BatchEvaluationDetail; defaultLogGroup: string; options: CoreOptions; - signal?: AbortSignal; }): Promise { const output = config.outputConfig?.cloudWatchConfig; if (output?.logGroupName) return { logGroupName: output.logGroupName }; @@ -338,7 +298,7 @@ export class EvalClient implements CoreEvalClient { if (source && "onlineEvaluationConfigSource" in source) { const arn = source.onlineEvaluationConfigSource?.onlineEvaluationConfigArn; if (!arn) throw new MalformedServiceResponseError("Missing source online config ARN."); - source = (await this.onlineLogConfig(serviceIdFromArn(arn), options, signal)) + source = (await this.getOnlineEvaluationConfig(serviceIdFromArn(arn), options)) .dataSourceConfig; } const groups = source?.cloudWatchLogs; diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx index d967f49820..aef1b1f137 100644 --- a/src/handlers/eval/batch-evaluation/index.tsx +++ b/src/handlers/eval/batch-evaluation/index.tsx @@ -27,7 +27,7 @@ export function createBatchEvaluationHandler(core: Core, io: AppIO): Router { .handler( createEvalLogsHandler(core, io, { resource: "batch-evaluation", - logSource: core.eval.resourceBatchEvaluationLogSource.bind(core.eval), + resolveLogSource: core.eval.resourceBatchEvaluationLogSource.bind(core.eval), }), ); } diff --git a/src/handlers/eval/batch-insights/index.tsx b/src/handlers/eval/batch-insights/index.tsx index babb3e08ed..370aa7d061 100644 --- a/src/handlers/eval/batch-insights/index.tsx +++ b/src/handlers/eval/batch-insights/index.tsx @@ -21,7 +21,7 @@ export function createBatchInsightsHandler(core: Core, io: AppIO): Router { .handler( createEvalLogsHandler(core, io, { resource: "batch-insights", - logSource: core.eval.resourceBatchInsightsLogSource.bind(core.eval), + resolveLogSource: core.eval.resourceBatchInsightsLogSource.bind(core.eval), }), ); } diff --git a/src/handlers/eval/logs/index.tsx b/src/handlers/eval/logs/index.tsx index a662e4a800..8baa1ea076 100644 --- a/src/handlers/eval/logs/index.tsx +++ b/src/handlers/eval/logs/index.tsx @@ -11,10 +11,10 @@ export const createEvalLogsHandler = ( io: AppIO, { resource, - logSource, + resolveLogSource, }: { resource: string; - logSource: CoreEvalClient["resourceBatchEvaluationLogSource"]; + resolveLogSource: CoreEvalClient["resourceBatchEvaluationLogSource"]; }, ) => createLogsHandler(io, { @@ -23,7 +23,7 @@ export const createEvalLogsHandler = ( flags: [flag("id", `the ID of the ${resource} resource`, z.string().min(1))] as const, read: async (ctx, flags, request, signal) => { const options = coreOptsFromCtx(ctx); - const source = await logSource(flags.id, options, signal); + const source = await resolveLogSource(flags.id, options); if (request.mode === "search") { return { diff --git a/src/handlers/eval/logs/logs.test.tsx b/src/handlers/eval/logs/logs.test.tsx index fc589cf036..d1c2814911 100644 --- a/src/handlers/eval/logs/logs.test.tsx +++ b/src/handlers/eval/logs/logs.test.tsx @@ -41,7 +41,7 @@ function setup( batch?: Record; logGroup?: string; onlineId?: string; - readMetadata?: (signal?: AbortSignal) => Promise>; + readLogs?: (signal?: AbortSignal) => Promise>; } = {}, ) { const logGroup = options.logGroup ?? ONLINE_GROUP; @@ -54,11 +54,7 @@ function setup( const core = new CoreClient({ createControlClient: () => ({ - send: async ( - command: GetOnlineEvaluationConfigCommand, - sendOptions?: { abortSignal?: AbortSignal }, - ) => { - if (options.readMetadata) return options.readMetadata(sendOptions?.abortSignal); + send: async (command: GetOnlineEvaluationConfigCommand) => { if (command.input.onlineEvaluationConfigId !== onlineId) { throw new Error("Online evaluation configuration not found"); } @@ -81,8 +77,9 @@ function setup( }) as unknown as BedrockAgentCoreClient, createLogsClient: () => ({ - send: async (command: unknown) => { + send: async (command: unknown, sendOptions?: { abortSignal?: AbortSignal }) => { if (command instanceof FilterLogEventsCommand) { + if (options.readLogs) return options.readLogs(sendOptions?.abortSignal); const events = (command.input.logGroupName === logGroup ? EVENTS : []).filter( (event) => !command.input.logStreamNames || @@ -191,14 +188,14 @@ describe("evaluation logs", () => { expect(io.stderr()).toBe(`Streaming logs for online-eval ${ID}... (Ctrl+C to stop)`); }); - test("Ctrl+C cancels a pending metadata lookup", async () => { + test("Ctrl+C cancels a pending log search", async () => { const started = Promise.withResolvers(); - const metadata = Promise.withResolvers>(); + const search = Promise.withResolvers>(); const { io, route } = setup({ - readMetadata: (signal) => { - signal?.addEventListener("abort", () => metadata.reject(new Error("transport aborted"))); + readLogs: (signal) => { + signal?.addEventListener("abort", () => search.reject(new Error("transport aborted"))); started.resolve(); - return metadata.promise; + return search.promise; }, }); const pending = route("online-eval", ["--id", ID, "--since", "1h"]); @@ -208,7 +205,7 @@ describe("evaluation logs", () => { process.emit("SIGINT", "SIGINT"); await expect(pending).rejects.toBeInstanceOf(UserCancellationError); } finally { - metadata.resolve({}); + search.resolve({}); await pending.catch(() => {}); } expect(io.stdout()).toBe(""); diff --git a/src/handlers/eval/online-eval/index.tsx b/src/handlers/eval/online-eval/index.tsx index d6090a24aa..c30e90916d 100644 --- a/src/handlers/eval/online-eval/index.tsx +++ b/src/handlers/eval/online-eval/index.tsx @@ -27,7 +27,7 @@ export function createOnlineEvalHandler(core: Core, io: AppIO): Router { .handler( createEvalLogsHandler(core, io, { resource: "online-eval", - logSource: core.eval.resourceOnlineEvaluationLogSource.bind(core.eval), + resolveLogSource: core.eval.resourceOnlineEvaluationLogSource.bind(core.eval), }), ); } diff --git a/src/handlers/eval/online-insight/index.tsx b/src/handlers/eval/online-insight/index.tsx index d4735452c0..a89084d174 100644 --- a/src/handlers/eval/online-insight/index.tsx +++ b/src/handlers/eval/online-insight/index.tsx @@ -27,7 +27,7 @@ export function createOnlineInsightHandler(core: Core, io: AppIO): Router { .handler( createEvalLogsHandler(core, io, { resource: "online-insight", - logSource: core.eval.resourceOnlineInsightLogSource.bind(core.eval), + resolveLogSource: core.eval.resourceOnlineInsightLogSource.bind(core.eval), }), ); } diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 2883ee11c3..0369de86f0 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -389,26 +389,10 @@ export type DatasetUpdateProgressEvent = { // and implemented by src/core/eval.tsx (dependency inversion: handlers own the // abstraction). export interface CoreEvalClient { - resourceOnlineEvaluationLogSource( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise; - resourceOnlineInsightLogSource( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise; - resourceBatchEvaluationLogSource( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise; - resourceBatchInsightsLogSource( - id: string, - options: CoreOptions, - signal?: AbortSignal, - ): Promise; + resourceOnlineEvaluationLogSource(id: string, options: CoreOptions): Promise; + resourceOnlineInsightLogSource(id: string, options: CoreOptions): Promise; + resourceBatchEvaluationLogSource(id: string, options: CoreOptions): Promise; + resourceBatchInsightsLogSource(id: string, options: CoreOptions): Promise; createEvaluator( request: CreateEvaluatorRequest, options: CoreOptions, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 1cd64021ea..d7b37f402a 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -1630,26 +1630,26 @@ export class TestEvalClient implements CoreEvalClient { readonly calls: RecordedCall[] = []; logSource: LogSource = { logGroupName: "/test/eval/results" }; - async resourceOnlineEvaluationLogSource(id: string, options: CoreOptions, signal?: AbortSignal) { - this.calls.push({ method: "resourceOnlineEvaluationLogSource", args: [id, options, signal] }); + async resourceOnlineEvaluationLogSource(id: string, options: CoreOptions) { + this.calls.push({ method: "resourceOnlineEvaluationLogSource", args: [id, options] }); if (this.error) throw this.error; return this.logSource; } - async resourceOnlineInsightLogSource(id: string, options: CoreOptions, signal?: AbortSignal) { - this.calls.push({ method: "resourceOnlineInsightLogSource", args: [id, options, signal] }); + async resourceOnlineInsightLogSource(id: string, options: CoreOptions) { + this.calls.push({ method: "resourceOnlineInsightLogSource", args: [id, options] }); if (this.error) throw this.error; return this.logSource; } - async resourceBatchEvaluationLogSource(id: string, options: CoreOptions, signal?: AbortSignal) { - this.calls.push({ method: "resourceBatchEvaluationLogSource", args: [id, options, signal] }); + async resourceBatchEvaluationLogSource(id: string, options: CoreOptions) { + this.calls.push({ method: "resourceBatchEvaluationLogSource", args: [id, options] }); if (this.error) throw this.error; return this.logSource; } - async resourceBatchInsightsLogSource(id: string, options: CoreOptions, signal?: AbortSignal) { - this.calls.push({ method: "resourceBatchInsightsLogSource", args: [id, options, signal] }); + async resourceBatchInsightsLogSource(id: string, options: CoreOptions) { + this.calls.push({ method: "resourceBatchInsightsLogSource", args: [id, options] }); if (this.error) throw this.error; return this.logSource; } From c4363f11d93a0a5088a7a6cae0e77be8beacd476 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:40:58 +0000 Subject: [PATCH 06/10] refactor(eval): limit log commands to evaluation resources --- src/core/eval.tsx | 88 +++++++++---------- .../batch-insights.screen.test.tsx | 2 +- .../batch-insights/batch-insights.test.tsx | 8 +- src/handlers/eval/batch-insights/index.tsx | 9 +- src/handlers/eval/logs/logs.test.tsx | 20 +---- src/handlers/eval/online-insight/index.tsx | 9 +- .../online-insight.screen.test.tsx | 2 +- .../online-insight/online-insight.test.tsx | 1 - src/handlers/eval/types.tsx | 2 - src/testing/TestCoreClient.tsx | 12 --- 10 files changed, 50 insertions(+), 103 deletions(-) diff --git a/src/core/eval.tsx b/src/core/eval.tsx index e0933aa1b6..50307ca161 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -239,80 +239,78 @@ export class EvalClient implements CoreEvalClient { private readonly cloudWatch: CloudWatchClient = new CloudWatchClient(clients), ) {} - async resourceOnlineEvaluationLogSource(id: string, options: CoreOptions): Promise { - const config = await this.getOnlineEvaluationConfig(id, options); - return this.evaluationLogSource({ - config, - defaultLogGroup: `/aws/bedrock-agentcore/evaluations/results/${config.onlineEvaluationConfigId ?? serviceIdFromArn(id)}`, + async resourceOnlineEvaluationLogSource( + onlineEvaluationConfigId: string, + options: CoreOptions, + ): Promise { + const onlineEvaluationConfig = await this.getOnlineEvaluationConfig( + onlineEvaluationConfigId, options, - }); - } - - async resourceOnlineInsightLogSource(id: string, options: CoreOptions): Promise { - const config = await this.getOnlineInsight(id, options); + ); return this.evaluationLogSource({ - config, - defaultLogGroup: `/aws/bedrock-agentcore/evaluations/results/${config.onlineEvaluationConfigId ?? serviceIdFromArn(id)}`, + evaluationMetadata: onlineEvaluationConfig, + defaultLogGroupName: `/aws/bedrock-agentcore/evaluations/results/${onlineEvaluationConfig.onlineEvaluationConfigId ?? serviceIdFromArn(onlineEvaluationConfigId)}`, options, }); } - async resourceBatchEvaluationLogSource(id: string, options: CoreOptions): Promise { - const { detail: config } = await this.getBatchEvaluation(id, options, { + async resourceBatchEvaluationLogSource( + batchEvaluationId: string, + options: CoreOptions, + ): Promise { + const { detail: batchEvaluation } = await this.getBatchEvaluation(batchEvaluationId, options, { includeResults: false, }); return this.evaluationLogSource({ - config, - defaultLogGroup: "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", - options, - }); - } - - async resourceBatchInsightsLogSource(id: string, options: CoreOptions): Promise { - const config = await this.getBatchInsights(id, options); - return this.evaluationLogSource({ - config, - defaultLogGroup: "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + evaluationMetadata: batchEvaluation, + defaultLogGroupName: "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", options, }); } private async evaluationLogSource({ - config, - defaultLogGroup, + evaluationMetadata, + defaultLogGroupName, options, }: { - config: GetOnlineEvaluationConfigResponse | BatchEvaluationDetail; - defaultLogGroup: string; + evaluationMetadata: GetOnlineEvaluationConfigResponse | BatchEvaluationDetail; + defaultLogGroupName: string; options: CoreOptions; }): Promise { - const output = config.outputConfig?.cloudWatchConfig; - if (output?.logGroupName) return { logGroupName: output.logGroupName }; - if (config.outputConfig && !output) { + const cloudWatchOutputConfig = evaluationMetadata.outputConfig?.cloudWatchConfig; + if (cloudWatchOutputConfig?.logGroupName) + return { logGroupName: cloudWatchOutputConfig.logGroupName }; + if (evaluationMetadata.outputConfig && !cloudWatchOutputConfig) { throw new InputValidationError("Logs require a CloudWatch output configuration."); } - if (output?.resultDestination !== "SOURCE_LOG_GROUP") { - return { logGroupName: defaultLogGroup }; + if (cloudWatchOutputConfig?.resultDestination !== "SOURCE_LOG_GROUP") { + return { logGroupName: defaultLogGroupName }; } - let source = config.dataSourceConfig; - if (source && "onlineEvaluationConfigSource" in source) { - const arn = source.onlineEvaluationConfigSource?.onlineEvaluationConfigArn; - if (!arn) throw new MalformedServiceResponseError("Missing source online config ARN."); - source = (await this.getOnlineEvaluationConfig(serviceIdFromArn(arn), options)) - .dataSourceConfig; + let dataSourceConfig = evaluationMetadata.dataSourceConfig; + if (dataSourceConfig && "onlineEvaluationConfigSource" in dataSourceConfig) { + const onlineEvaluationConfigArn = + dataSourceConfig.onlineEvaluationConfigSource?.onlineEvaluationConfigArn; + if (!onlineEvaluationConfigArn) + throw new MalformedServiceResponseError("Missing source online config ARN."); + dataSourceConfig = ( + await this.getOnlineEvaluationConfig(serviceIdFromArn(onlineEvaluationConfigArn), options) + ).dataSourceConfig; } - const groups = source?.cloudWatchLogs; - if (groups?.logGroupNamePrefixes?.length || (groups?.logGroupNames?.length ?? 0) > 1) { + const cloudWatchSourceConfig = dataSourceConfig?.cloudWatchLogs; + if ( + cloudWatchSourceConfig?.logGroupNamePrefixes?.length || + (cloudWatchSourceConfig?.logGroupNames?.length ?? 0) > 1 + ) { throw new InputValidationError( "Results use multiple source log groups or prefixes. " + "Read them directly in CloudWatch Logs, or configure one explicit source log group.", ); } - const logGroupName = groups?.logGroupNames?.[0]; - if (!logGroupName) { + const sourceLogGroupName = cloudWatchSourceConfig?.logGroupNames?.[0]; + if (!sourceLogGroupName) { throw new MalformedServiceResponseError("Missing source log group name."); } - return { logGroupName }; + return { logGroupName: sourceLogGroupName }; } async createEvaluator( diff --git a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx b/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx index 79f8b22431..15f961580b 100644 --- a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx @@ -58,7 +58,7 @@ describe("batch-insights menu", () => { await waitForText(screen.lastFrame, "list batch insights runs"); expect(menuEntries(screen.lastFrame()!)).toEqual({ screens: ["get", "list"], - cliOnly: ["run", "stop", "logs"], + cliOnly: ["run", "stop"], }); }); }); diff --git a/src/handlers/eval/batch-insights/batch-insights.test.tsx b/src/handlers/eval/batch-insights/batch-insights.test.tsx index ff97076acc..a8b6afd480 100644 --- a/src/handlers/eval/batch-insights/batch-insights.test.tsx +++ b/src/handlers/eval/batch-insights/batch-insights.test.tsx @@ -42,13 +42,7 @@ describe("eval batch-insights command hierarchy", () => { ?.children() .find((child) => child.name() === "batch-insights"); - expect(group?.children().map((child) => child.name())).toEqual([ - "run", - "get", - "list", - "stop", - "logs", - ]); + expect(group?.children().map((child) => child.name())).toEqual(["run", "get", "list", "stop"]); }); test("prints help for a bare invocation without a Core call", async () => { diff --git a/src/handlers/eval/batch-insights/index.tsx b/src/handlers/eval/batch-insights/index.tsx index 370aa7d061..2d40420374 100644 --- a/src/handlers/eval/batch-insights/index.tsx +++ b/src/handlers/eval/batch-insights/index.tsx @@ -7,7 +7,6 @@ import { createGetBatchInsightsHandler } from "./get"; import { createListBatchInsightsHandler } from "./list"; import { createRunBatchInsightsHandler } from "./run"; import { createStopBatchInsightsHandler } from "./stop"; -import { createEvalLogsHandler } from "../logs"; export function createBatchInsightsHandler(core: Core, io: AppIO): Router { return new Router("batch-insights", "run and inspect AgentCore batch insights") @@ -17,13 +16,7 @@ export function createBatchInsightsHandler(core: Core, io: AppIO): Router { .handler(createRunBatchInsightsHandler(core, io)) .handler(createGetBatchInsightsHandler(core)) .handler(createListBatchInsightsHandler(core)) - .handler(createStopBatchInsightsHandler(core)) - .handler( - createEvalLogsHandler(core, io, { - resource: "batch-insights", - resolveLogSource: core.eval.resourceBatchInsightsLogSource.bind(core.eval), - }), - ); + .handler(createStopBatchInsightsHandler(core)); } export { BatchInsightsScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/logs/logs.test.tsx b/src/handlers/eval/logs/logs.test.tsx index d1c2814911..07889ffa89 100644 --- a/src/handlers/eval/logs/logs.test.tsx +++ b/src/handlers/eval/logs/logs.test.tsx @@ -11,11 +11,11 @@ import { type CloudWatchLogsClient, } from "@aws-sdk/client-cloudwatch-logs"; import { CoreClient } from "../../../core"; -import { InputValidationError, UserCancellationError } from "../../../errors"; +import { UserCancellationError } from "../../../errors"; import { createSilentLogger, TestGlobalConfigAccessor, testIO } from "../../../testing"; import { createRootHandler } from "../../index"; -type Resource = "online-eval" | "online-insight" | "batch-evaluation" | "batch-insights"; +type Resource = "online-eval" | "batch-evaluation"; const SINCE = 1_709_391_000_000; const ID = "config-AbC123XyZ9"; const ONLINE_GROUP = `/aws/bedrock-agentcore/evaluations/results/${ID}`; @@ -46,7 +46,6 @@ function setup( ) { const logGroup = options.logGroup ?? ONLINE_GROUP; const onlineId = options.onlineId ?? ID; - const insights = [{ insightId: "Builtin.Insight.FailureAnalysis" }]; const logGroupArn = `arn:aws:logs:us-west-2:123456789012:log-group:${logGroup}`; const unusedClient = () => { throw new Error("Unexpected AWS client"); @@ -60,7 +59,6 @@ function setup( } return { onlineEvaluationConfigId: onlineId, - insights, dataSourceConfig: { cloudWatchLogs: { logGroupNames: [SOURCE_GROUP] } }, ...options.online, }; @@ -71,7 +69,6 @@ function setup( send: async () => ({ batchEvaluationId: ID, status: "COMPLETED", - insights, ...options.batch, }), }) as unknown as BedrockAgentCoreClient, @@ -139,9 +136,7 @@ function setup( describe("evaluation logs", () => { test.each([ ["online-eval", "default", { logGroup: ONLINE_GROUP }], - ["online-insight", "default", { logGroup: ONLINE_GROUP }], ["batch-evaluation", "default", { logGroup: BATCH_GROUP }], - ["batch-insights", "default", { logGroup: BATCH_GROUP }], ["online-eval", "custom", { logGroup: "/custom/results", online: CUSTOM_CONFIG }], ["batch-evaluation", "custom", { logGroup: "/custom/results", batch: CUSTOM_CONFIG }], ["online-eval", "source", { logGroup: SOURCE_GROUP, online: { outputConfig: SOURCE_OUTPUT } }], @@ -220,17 +215,6 @@ describe("evaluation logs", () => { ); }); - test.each(["online-insight", "batch-insights"] as const)( - "%s rejects a resource without insights", - async (resource) => { - const { route } = setup({ online: { insights: [] }, batch: { insights: [] } }); - - await expect(route(resource, ["--id", ID, "--since", "1h"])).rejects.toBeInstanceOf( - InputValidationError, - ); - }, - ); - test("explains how to read results when the source has multiple groups", async () => { const { route } = setup({ online: { diff --git a/src/handlers/eval/online-insight/index.tsx b/src/handlers/eval/online-insight/index.tsx index a89084d174..b54329b493 100644 --- a/src/handlers/eval/online-insight/index.tsx +++ b/src/handlers/eval/online-insight/index.tsx @@ -10,7 +10,6 @@ import { createUpdateOnlineInsightHandler } from "./update"; import { createPauseOnlineInsightHandler } from "./pause"; import { createResumeOnlineInsightHandler } from "./resume"; import { createDeleteOnlineInsightHandler } from "./delete"; -import { createEvalLogsHandler } from "../logs"; export function createOnlineInsightHandler(core: Core, io: AppIO): Router { return new Router("online-insight", "manage AgentCore online insight configs") @@ -23,13 +22,7 @@ export function createOnlineInsightHandler(core: Core, io: AppIO): Router { .handler(createUpdateOnlineInsightHandler(core, io)) .handler(createPauseOnlineInsightHandler(core)) .handler(createResumeOnlineInsightHandler(core)) - .handler(createDeleteOnlineInsightHandler(core)) - .handler( - createEvalLogsHandler(core, io, { - resource: "online-insight", - resolveLogSource: core.eval.resourceOnlineInsightLogSource.bind(core.eval), - }), - ); + .handler(createDeleteOnlineInsightHandler(core)); } export { OnlineInsightScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-insight/online-insight.screen.test.tsx b/src/handlers/eval/online-insight/online-insight.screen.test.tsx index 1a0814b385..b8b95b7b51 100644 --- a/src/handlers/eval/online-insight/online-insight.screen.test.tsx +++ b/src/handlers/eval/online-insight/online-insight.screen.test.tsx @@ -69,7 +69,7 @@ describe("online-insight menu", () => { await waitForText(screen.lastFrame, "get an online insight config by ID"); expect(menuEntries(screen.lastFrame()!)).toEqual({ screens: ["get", "list"], - cliOnly: ["create", "update", "pause", "resume", "delete", "logs"], + cliOnly: ["create", "update", "pause", "resume", "delete"], }); }); }); diff --git a/src/handlers/eval/online-insight/online-insight.test.tsx b/src/handlers/eval/online-insight/online-insight.test.tsx index 0c4429af94..6c798ccead 100644 --- a/src/handlers/eval/online-insight/online-insight.test.tsx +++ b/src/handlers/eval/online-insight/online-insight.test.tsx @@ -103,7 +103,6 @@ describe("eval online-insight command hierarchy", () => { "pause", "resume", "delete", - "logs", ]); }); }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 0369de86f0..9b9320b5c7 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -390,9 +390,7 @@ export type DatasetUpdateProgressEvent = { // abstraction). export interface CoreEvalClient { resourceOnlineEvaluationLogSource(id: string, options: CoreOptions): Promise; - resourceOnlineInsightLogSource(id: string, options: CoreOptions): Promise; resourceBatchEvaluationLogSource(id: string, options: CoreOptions): Promise; - resourceBatchInsightsLogSource(id: string, options: CoreOptions): Promise; createEvaluator( request: CreateEvaluatorRequest, options: CoreOptions, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index d7b37f402a..c52888b70b 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -1636,24 +1636,12 @@ export class TestEvalClient implements CoreEvalClient { return this.logSource; } - async resourceOnlineInsightLogSource(id: string, options: CoreOptions) { - this.calls.push({ method: "resourceOnlineInsightLogSource", args: [id, options] }); - if (this.error) throw this.error; - return this.logSource; - } - async resourceBatchEvaluationLogSource(id: string, options: CoreOptions) { this.calls.push({ method: "resourceBatchEvaluationLogSource", args: [id, options] }); if (this.error) throw this.error; return this.logSource; } - async resourceBatchInsightsLogSource(id: string, options: CoreOptions) { - this.calls.push({ method: "resourceBatchInsightsLogSource", args: [id, options] }); - if (this.error) throw this.error; - return this.logSource; - } - // List responses are keyed by the nextToken that requests them (undefined = // the first page), so tests can serve multi-page listings. private listResponses = new Map(); From 4ddb953fbd50542bbbddbc9e99b9a5db60bd35e4 Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:52:20 +0000 Subject: [PATCH 07/10] test(eval): simplify log handler coverage --- src/handlers/eval/logs/logs.test.tsx | 223 +++++---------------------- 1 file changed, 40 insertions(+), 183 deletions(-) diff --git a/src/handlers/eval/logs/logs.test.tsx b/src/handlers/eval/logs/logs.test.tsx index 07889ffa89..355f2a41b1 100644 --- a/src/handlers/eval/logs/logs.test.tsx +++ b/src/handlers/eval/logs/logs.test.tsx @@ -1,125 +1,34 @@ import { describe, expect, test } from "bun:test"; import { - type BedrockAgentCoreControlClient, - type GetOnlineEvaluationConfigCommand, -} from "@aws-sdk/client-bedrock-agentcore-control"; -import type { BedrockAgentCoreClient } from "@aws-sdk/client-bedrock-agentcore"; -import { - DescribeLogGroupsCommand, - FilterLogEventsCommand, - StartLiveTailCommand, - type CloudWatchLogsClient, -} from "@aws-sdk/client-cloudwatch-logs"; -import { CoreClient } from "../../../core"; -import { UserCancellationError } from "../../../errors"; -import { createSilentLogger, TestGlobalConfigAccessor, testIO } from "../../../testing"; + createSilentLogger, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../testing"; import { createRootHandler } from "../../index"; -type Resource = "online-eval" | "batch-evaluation"; +const ID = "evaluation-1"; const SINCE = 1_709_391_000_000; -const ID = "config-AbC123XyZ9"; -const ONLINE_GROUP = `/aws/bedrock-agentcore/evaluations/results/${ID}`; -const BATCH_GROUP = "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default"; -const SOURCE_GROUP = "/custom/source"; +const LOG_SOURCE = { logGroupName: "/custom/evaluation-results" }; const EVENTS = [ - { timestamp: SINCE + 1_000, message: "first result", logStreamName: "run-a" }, - { timestamp: SINCE + 2_000, message: "second result", logStreamName: "run-b" }, + { timestamp: new Date(SINCE), message: "first result", logStreamName: "run-a" }, + { timestamp: new Date(SINCE + 1_000), message: "second result", logStreamName: "run-b" }, ]; -const SOURCE_OUTPUT = { cloudWatchConfig: { resultDestination: "SOURCE_LOG_GROUP" } }; -const CUSTOM_OUTPUT = { - cloudWatchConfig: { logGroupName: "/custom/results", logStreamName: "run-a" }, -}; -const CUSTOM_CONFIG = { outputConfig: CUSTOM_OUTPUT }; -const JSON_EVENTS = EVENTS.map((event) => ({ - ...event, - timestamp: new Date(event.timestamp).toISOString(), -})); -function setup( - options: { - online?: Record; - batch?: Record; - logGroup?: string; - onlineId?: string; - readLogs?: (signal?: AbortSignal) => Promise>; - } = {}, -) { - const logGroup = options.logGroup ?? ONLINE_GROUP; - const onlineId = options.onlineId ?? ID; - const logGroupArn = `arn:aws:logs:us-west-2:123456789012:log-group:${logGroup}`; - const unusedClient = () => { - throw new Error("Unexpected AWS client"); - }; - const core = new CoreClient({ - createControlClient: () => - ({ - send: async (command: GetOnlineEvaluationConfigCommand) => { - if (command.input.onlineEvaluationConfigId !== onlineId) { - throw new Error("Online evaluation configuration not found"); - } - return { - onlineEvaluationConfigId: onlineId, - dataSourceConfig: { cloudWatchLogs: { logGroupNames: [SOURCE_GROUP] } }, - ...options.online, - }; - }, - }) as unknown as BedrockAgentCoreControlClient, - createDataClient: () => - ({ - send: async () => ({ - batchEvaluationId: ID, - status: "COMPLETED", - ...options.batch, - }), - }) as unknown as BedrockAgentCoreClient, - createLogsClient: () => - ({ - send: async (command: unknown, sendOptions?: { abortSignal?: AbortSignal }) => { - if (command instanceof FilterLogEventsCommand) { - if (options.readLogs) return options.readLogs(sendOptions?.abortSignal); - const events = (command.input.logGroupName === logGroup ? EVENTS : []).filter( - (event) => - !command.input.logStreamNames || - command.input.logStreamNames.includes(event.logStreamName), - ); - const offset = Number(command.input.nextToken ?? 0); - return { - events: events.slice(offset, offset + 1), - nextToken: offset + 1 < events.length ? `${offset + 1}` : undefined, - }; - } - if (command instanceof DescribeLogGroupsCommand) { - return { - logGroups: [{ logGroupName: logGroup, logGroupArn }], - }; - } - if (command instanceof StartLiveTailCommand) { - return { - responseStream: (async function* () { - if (command.input.logGroupIdentifiers?.includes(logGroupArn)) { - yield { sessionUpdate: { sessionResults: EVENTS } }; - } - })(), - }; - } - throw new Error("Unexpected CloudWatch command"); - }, - }) as unknown as CloudWatchLogsClient, - createIamClient: unusedClient, - createXrayClient: unusedClient, - createApplicationSignalsClient: unusedClient, - logger: createSilentLogger(), - }); +function setup(resource: "online-eval" | "batch-evaluation") { + const core = new TestCoreClient(); + core.eval.logSource = LOG_SOURCE; + core.observability.logEvents = EVENTS; const io = testIO(); const root = createRootHandler(core, { io: io.io, logger: createSilentLogger(), globalConfigAccessor: new TestGlobalConfigAccessor(), }); - return { + core, io, - route: (resource: Resource, flags: readonly string[]) => + route: (flags: string[]) => root.route([ "node", "agentcore", @@ -133,98 +42,46 @@ function setup( }; } -describe("evaluation logs", () => { - test.each([ - ["online-eval", "default", { logGroup: ONLINE_GROUP }], - ["batch-evaluation", "default", { logGroup: BATCH_GROUP }], - ["online-eval", "custom", { logGroup: "/custom/results", online: CUSTOM_CONFIG }], - ["batch-evaluation", "custom", { logGroup: "/custom/results", batch: CUSTOM_CONFIG }], - ["online-eval", "source", { logGroup: SOURCE_GROUP, online: { outputConfig: SOURCE_OUTPUT } }], - [ - "batch-evaluation", - "referenced online source", - { - logGroup: SOURCE_GROUP, - onlineId: "source-config", - online: CUSTOM_CONFIG, - batch: { - outputConfig: SOURCE_OUTPUT, - dataSourceConfig: { - onlineEvaluationConfigSource: { - onlineEvaluationConfigArn: - "arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/source-config", - }, - }, - }, - }, - ], - ] as const)("%s %s includes all pages and streams", async (resource, _destination, options) => { - const { io, route } = setup(options); +describe.each([ + ["online-eval", "resourceOnlineEvaluationLogSource"], + ["batch-evaluation", "resourceBatchEvaluationLogSource"], +] as const)("%s logs", (resource, sourceMethod) => { + test("searches the resolved group and prints every record as JSON", async () => { + const { core, io, route } = setup(resource); - await route(resource, ["--id", ID, "--since", `${SINCE}`, "--json"]); + await route(["--id", ID, "--since", `${SINCE}`, "--json"]); + expect(core.eval.calls[0]).toEqual({ + method: sourceMethod, + args: [ID, { region: "us-west-2" }], + }); + expect(core.observability.calls[0]?.method).toBe("searchLogs"); + expect(core.observability.calls[0]?.args[0]).toEqual(LOG_SOURCE); expect( io .stdout() .split("\n") .map((line) => JSON.parse(line)), - ).toEqual(JSON_EVENTS); + ).toEqual(EVENTS.map((event) => ({ ...event, timestamp: event.timestamp.toISOString() }))); expect(io.stderr()).toBe(""); }); - test("tails by default and announces the stream on stderr", async () => { - const { io, route } = setup(); + test("tails the resolved group by default and announces it on stderr", async () => { + const { core, io, route } = setup(resource); - await route("online-eval", ["--id", ID]); + await route(["--id", ID]); + expect(core.observability.calls[0]?.method).toBe("tailLogs"); + expect(core.observability.calls[0]?.args[0]).toEqual(LOG_SOURCE); expect(io.stdout()).toBe( - "2024-03-02T14:50:01.000Z first result\n2024-03-02T14:50:02.000Z second result", - ); - expect(io.stderr()).toBe(`Streaming logs for online-eval ${ID}... (Ctrl+C to stop)`); - }); - - test("Ctrl+C cancels a pending log search", async () => { - const started = Promise.withResolvers(); - const search = Promise.withResolvers>(); - const { io, route } = setup({ - readLogs: (signal) => { - signal?.addEventListener("abort", () => search.reject(new Error("transport aborted"))); - started.resolve(); - return search.promise; - }, - }); - const pending = route("online-eval", ["--id", ID, "--since", "1h"]); - - try { - await started.promise; - process.emit("SIGINT", "SIGINT"); - await expect(pending).rejects.toBeInstanceOf(UserCancellationError); - } finally { - search.resolve({}); - await pending.catch(() => {}); - } - expect(io.stdout()).toBe(""); - expect(io.stderr()).toBe(""); - }); - - test("requires an explicit resource ID", async () => { - const { route } = setup(); - - await expect(route("online-eval", ["--since", "1h"])).rejects.toThrow( - "required option '--id' not specified", + EVENTS.map((event) => `${event.timestamp.toISOString()} ${event.message}`).join("\n"), ); + expect(io.stderr()).toBe(`Streaming logs for ${resource} ${ID}... (Ctrl+C to stop)`); }); +}); - test("explains how to read results when the source has multiple groups", async () => { - const { route } = setup({ - online: { - outputConfig: SOURCE_OUTPUT, - dataSourceConfig: { cloudWatchLogs: { logGroupNames: ["/one", "/two"] } }, - }, - }); +test("requires an explicit resource ID", async () => { + const { route } = setup("online-eval"); - await expect(route("online-eval", ["--id", ID, "--since", "1h"])).rejects.toThrow( - "configure one explicit source log group", - ); - }); + await expect(route(["--since", "1h"])).rejects.toThrow("required option '--id' not specified"); }); From a0d76e31a81bd95dc14c687379a65d639d80263e Mon Sep 17 00:00:00 2001 From: Hweinstock <42325418+Hweinstock@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:10:57 +0000 Subject: [PATCH 08/10] test(eval): cover log sources with recorded fixtures --- src/handlers/eval/logs/logs.test.tsx | 37 ++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/handlers/eval/logs/logs.test.tsx b/src/handlers/eval/logs/logs.test.tsx index 355f2a41b1..a5993cbdca 100644 --- a/src/handlers/eval/logs/logs.test.tsx +++ b/src/handlers/eval/logs/logs.test.tsx @@ -1,6 +1,9 @@ import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../core"; import { createSilentLogger, + fixtureFactories, TestCoreClient, TestGlobalConfigAccessor, testIO, @@ -85,3 +88,37 @@ test("requires an explicit resource ID", async () => { await expect(route(["--since", "1h"])).rejects.toThrow("required option '--id' not specified"); }); + +test.each([ + [ + "online configured output", + "online-eval", + "resourceOnlineEvaluationLogSource", + "agentcore_cli_online_eval_role_warn-tSVCZiBf4l", + "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-tSVCZiBf4l", + ], + [ + "online source output", + "online-eval", + "resourceOnlineEvaluationLogSource", + "agentcore_cli_online_eval_fixture-5gZw987afd", + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT", + ], + [ + "batch configured output", + "batch-evaluation", + "resourceBatchEvaluationLogSource", + "GTProbe2_1786034545579-8ffefc851e", + "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + ], +] as const)( + "%s resolves its recorded log group", + async (_name, resource, method, id, logGroupName) => { + const core = new CoreClient({ + ...fixtureFactories(join(import.meta.dir, "..", resource, "__fixtures__")), + logger: createSilentLogger(), + }); + + expect(await core.eval[method](id, { region: "us-west-2" })).toEqual({ logGroupName }); + }, +); From 60778727f1f2da1700823b2b7576e6f2e3dcc52f Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 13 Oct 2026 16:36:30 +0000 Subject: [PATCH 09/10] fix(eval): include batch evaluation errors in get --- command.md | 21 +++- src/core/batchEvaluationResults.test.ts | 90 ++++++++++++++ src/core/batchEvaluationResults.tsx | 24 ++-- src/core/eval.tsx | 50 +------- .../batch-evaluation.screen.test.tsx | 23 ++++ .../batch-evaluation.test.tsx | 1 - ...tchEvaluationCommand.44cbc27686231142.json | 62 ++++++++++ ...tchEvaluationCommand.7e6d701230e33025.json | 52 +++++++++ ...tchEvaluationCommand.9722e50585569cfc.json | 59 ++++++++++ ...tchEvaluationCommand.cca0b9b391e38aa4.json | 61 ++++++++++ .../GetLogEventsCommand.19472bb2137a9b1b.json | 5 + .../GetLogEventsCommand.204d8eebd0586459.json | 5 + .../GetLogEventsCommand.24332c6f0fa347f9.json | 16 +++ .../GetLogEventsCommand.3163c4675fc54d15.json | 5 + .../GetLogEventsCommand.78b264dd2e58203d.json | 16 +++ .../GetLogEventsCommand.a8ae9aee2feab3ce.json | 5 + .../GetLogEventsCommand.b1d3d9333f277b7d.json | 5 + .../GetLogEventsCommand.ca64738ae7fe600.json | 5 + .../GetLogEventsCommand.ddd7a682ad7886d4.json | 16 +++ .../GetLogEventsCommand.df3ce8073f0f9f9c.json | 5 + .../GetLogEventsCommand.e7a494ca2d983fe1.json | 5 + .../GetLogEventsCommand.f0d38df552f516cf.json | 16 +++ .../errors-metadata-only.golden.json | 48 ++++++++ .../get/__fixtures__/errors.golden.json | 62 ++++++++++ .../__fixtures__/missing-events.golden.json | 73 ++++++++++++ .../mixed-custom-output.golden.json | 76 ++++++++++++ .../get/__fixtures__/success.golden.json | 74 ++++++++++++ .../batch-evaluation/get/get.fixture.test.tsx | 110 ++++++++++++++++++ .../eval/batch-evaluation/get/index.tsx | 3 +- src/handlers/eval/batch-evaluation/index.tsx | 9 +- src/handlers/eval/logs/index.tsx | 39 ------- src/handlers/eval/online-eval/index.tsx | 9 +- src/handlers/eval/online-eval/logs/index.tsx | 28 +++++ .../eval/{ => online-eval}/logs/logs.test.tsx | 55 ++++----- src/handlers/eval/types.tsx | 5 +- src/testing/TestCoreClient.tsx | 6 - 36 files changed, 993 insertions(+), 151 deletions(-) create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.44cbc27686231142.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.7e6d701230e33025.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.9722e50585569cfc.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.cca0b9b391e38aa4.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.19472bb2137a9b1b.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.204d8eebd0586459.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.24332c6f0fa347f9.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.3163c4675fc54d15.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.78b264dd2e58203d.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.a8ae9aee2feab3ce.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.b1d3d9333f277b7d.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ca64738ae7fe600.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ddd7a682ad7886d4.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.df3ce8073f0f9f9c.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.e7a494ca2d983fe1.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.f0d38df552f516cf.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/errors-metadata-only.golden.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/errors.golden.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/missing-events.golden.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/mixed-custom-output.golden.json create mode 100644 src/handlers/eval/batch-evaluation/get/__fixtures__/success.golden.json create mode 100644 src/handlers/eval/batch-evaluation/get/get.fixture.test.tsx delete mode 100644 src/handlers/eval/logs/index.tsx create mode 100644 src/handlers/eval/online-eval/logs/index.tsx rename src/handlers/eval/{ => online-eval}/logs/logs.test.tsx (66%) diff --git a/command.md b/command.md index 27277623c9..1153864d20 100644 --- a/command.md +++ b/command.md @@ -153,6 +153,7 @@ This reference was generated from `agentcore --help` for version `1.0.0-rc.5`. - [`agentcore eval online-eval pause`](#agentcore-eval-online-eval-pause) - [`agentcore eval online-eval resume`](#agentcore-eval-online-eval-resume) - [`agentcore eval online-eval delete`](#agentcore-eval-online-eval-delete) + - [`agentcore eval online-eval logs`](#agentcore-eval-online-eval-logs) - [`agentcore eval online-insight`](#agentcore-eval-online-insight) - [`agentcore eval online-insight create`](#agentcore-eval-online-insight-create) - [`agentcore eval online-insight get`](#agentcore-eval-online-insight-get) @@ -2309,6 +2310,24 @@ delete an online evaluation config by ID - `--id `: the ID of the online evaluation config to delete (required) +##### `agentcore eval online-eval logs` + +```text +agentcore eval online-eval logs [options] +``` + +stream or search online-eval logs + +**Options** + +- `--id `: the ID of the online-eval resource (required) +- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" +- `--tail`: tail new log records (default: false) +- `--level `: filter by log level (error, warn, info, debug) +- `--query `: CloudWatch Logs filter pattern +- `--limit `: maximum number of log records to return in search mode + #### `agentcore eval online-insight` ```text @@ -2576,7 +2595,7 @@ replay a dataset against a Runtime, then batch-evaluate the resulting sessions agentcore eval batch-evaluation get [options] ``` -get a batch evaluation by ID, with CloudWatch-backed results when available +get a batch evaluation by ID, with CloudWatch-backed results and errors when available **Options** diff --git a/src/core/batchEvaluationResults.test.ts b/src/core/batchEvaluationResults.test.ts index d3e77d229b..16a2bfb5f3 100644 --- a/src/core/batchEvaluationResults.test.ts +++ b/src/core/batchEvaluationResults.test.ts @@ -109,3 +109,93 @@ test("parseEvaluationLogEvent warns on and skips an unparseable line", () => { expect(warnings).toHaveLength(1); expect(warnings[0]).toContain("unparseable"); }); + +test.each([ + [ + "standard error without evaluator", + { + "error.type": "LogEventMissingException", + "error.message": "Session span data is incomplete.", + }, + { errorType: "LogEventMissingException", error: "Session span data is incomplete." }, + ], + [ + "standard error with evaluator", + { + "gen_ai.evaluation.name": "Builtin.Helpfulness", + "error.type": "EvaluationException", + "error.message": "Evaluation failed.", + }, + { + evaluatorId: "Builtin.Helpfulness", + errorType: "EvaluationException", + error: "Evaluation failed.", + }, + ], + [ + "legacy error", + { + "gen_ai.evaluation.name": "Builtin.Helpfulness", + "gen_ai.evaluation.error": "Evaluator failed.", + }, + { evaluatorId: "Builtin.Helpfulness", error: "Evaluator failed." }, + ], + [ + "unnamed legacy error", + { "gen_ai.evaluation.error": "Evaluator failed." }, + { error: "Evaluator failed." }, + ], + ["type only", { "error.type": "EvaluationException" }, { errorType: "EvaluationException" }], + ["message only", { "error.message": "Evaluation failed." }, { error: "Evaluation failed." }], + [ + "standard message takes precedence", + { "error.message": "Standard message.", "gen_ai.evaluation.error": "Legacy message." }, + { error: "Standard message." }, + ], +] as const)("parseEvaluationLogEvent retains %s and scope", (_name, attrs, expected) => { + const result = parseEvaluationLogEvent( + JSON.stringify({ + name: "gen_ai.evaluation.result", + traceId: "trace-1", + spanId: "span-1", + attributes: { + ...attrs, + "session.id": "session-1", + "aws.bedrock_agentcore.evaluation_level": "Trace", + "gen_ai.tool.name": "lookup", + }, + }), + createSilentLogger(), + ); + expect(result).toEqual({ + evaluatorId: undefined, + score: undefined, + label: undefined, + explanation: undefined, + error: undefined, + errorType: undefined, + ...expected, + traceId: "trace-1", + spanId: "span-1", + sessionId: "session-1", + level: "Trace", + toolName: "lookup", + }); +}); + +test.each( + [ + null, + [], + "control message", + { attributes: null }, + { attributes: [] }, + { attributes: "invalid" }, + { name: "application.error", attributes: { "error.message": "Unrelated error." } }, + { attributes: { "error.type": "UnrelatedException" } }, + { name: "gen_ai.evaluation.result", attributes: { "error.message": 42, "error.type": {} } }, + { name: "gen_ai.evaluation.result", attributes: {} }, + ].map((value) => ({ value })), +)("parseEvaluationLogEvent skips non-evaluation or malformed data: %j", ({ value }) => { + expect(parseEvaluationLogEvent(JSON.stringify(value), createSilentLogger())).toBeNull(); +}); diff --git a/src/core/batchEvaluationResults.tsx b/src/core/batchEvaluationResults.tsx index 3cbbe70865..43a2b3175f 100644 --- a/src/core/batchEvaluationResults.tsx +++ b/src/core/batchEvaluationResults.tsx @@ -43,9 +43,9 @@ export function parseEvaluationLogEvent( message: string, logger: Logger, ): BatchEvaluationResultEntry | null { - let parsed: Record; + let value: unknown; try { - parsed = JSON.parse(message) as Record; + value = JSON.parse(message); } catch { // Swallow rather than throw: CloudWatch result streams interleave non-JSON // control lines with the evaluation records, so one unparseable line is @@ -54,11 +54,20 @@ export function parseEvaluationLogEvent( logger.warn("skipping unparseable batch-evaluation result log line"); return null; } - const attrs = (parsed["attributes"] ?? {}) as Record; - const evaluatorId = attrs["gen_ai.evaluation.name"] as string | undefined; - if (!evaluatorId) return null; - + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const parsed = value as Record; + const attributes = parsed["attributes"]; + if (!attributes || typeof attributes !== "object" || Array.isArray(attributes)) return null; + const attrs = attributes as Record; const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined); + const evaluatorId = str(attrs["gen_ai.evaluation.name"]); + const error = str(attrs["error.message"]) ?? str(attrs["gen_ai.evaluation.error"]); + const errorType = str(attrs["error.type"]); + // Failures before scoring may have no evaluator name. Only accept unnamed + // errors from evaluation result records, not unrelated application errors. + if (!evaluatorId && !(parsed["name"] === "gen_ai.evaluation.result" && (error || errorType))) { + return null; + } // Keys are read verbatim as the recorded stream emits them: scores, level, // session id, and tool name under `attributes`; traceId/spanId at the top // level. If AgentCore renames one, the corresponding field goes undefined and @@ -73,6 +82,7 @@ export function parseEvaluationLogEvent( score: attrs["gen_ai.evaluation.score.value"] as number | undefined, label: str(attrs["gen_ai.evaluation.score.label"]), explanation: str(attrs["gen_ai.evaluation.explanation"]), - error: str(attrs["gen_ai.evaluation.error"]), + error, + errorType, }; } diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 9b03f0b918..49304b0740 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -235,56 +235,18 @@ export class EvalClient implements CoreEvalClient { onlineEvaluationConfigId, options, ); - return this.evaluationLogSource({ - evaluationMetadata: onlineEvaluationConfig, - defaultLogGroupName: `/aws/bedrock-agentcore/evaluations/results/${onlineEvaluationConfig.onlineEvaluationConfigId ?? serviceIdFromArn(onlineEvaluationConfigId)}`, - options, - }); - } - - async resourceBatchEvaluationLogSource( - batchEvaluationId: string, - options: CoreOptions, - ): Promise { - const { detail: batchEvaluation } = await this.getBatchEvaluation(batchEvaluationId, options, { - includeResults: false, - }); - return this.evaluationLogSource({ - evaluationMetadata: batchEvaluation, - defaultLogGroupName: "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", - options, - }); - } - - private async evaluationLogSource({ - evaluationMetadata, - defaultLogGroupName, - options, - }: { - evaluationMetadata: GetOnlineEvaluationConfigResponse | BatchEvaluationDetail; - defaultLogGroupName: string; - options: CoreOptions; - }): Promise { - const cloudWatchOutputConfig = evaluationMetadata.outputConfig?.cloudWatchConfig; + const cloudWatchOutputConfig = onlineEvaluationConfig.outputConfig?.cloudWatchConfig; if (cloudWatchOutputConfig?.logGroupName) return { logGroupName: cloudWatchOutputConfig.logGroupName }; - if (evaluationMetadata.outputConfig && !cloudWatchOutputConfig) { + if (onlineEvaluationConfig.outputConfig && !cloudWatchOutputConfig) { throw new InputValidationError("Logs require a CloudWatch output configuration."); } if (cloudWatchOutputConfig?.resultDestination !== "SOURCE_LOG_GROUP") { - return { logGroupName: defaultLogGroupName }; - } - let dataSourceConfig = evaluationMetadata.dataSourceConfig; - if (dataSourceConfig && "onlineEvaluationConfigSource" in dataSourceConfig) { - const onlineEvaluationConfigArn = - dataSourceConfig.onlineEvaluationConfigSource?.onlineEvaluationConfigArn; - if (!onlineEvaluationConfigArn) - throw new MalformedServiceResponseError("Missing source online config ARN."); - dataSourceConfig = ( - await this.getOnlineEvaluationConfig(serviceIdFromArn(onlineEvaluationConfigArn), options) - ).dataSourceConfig; + return { + logGroupName: `/aws/bedrock-agentcore/evaluations/results/${onlineEvaluationConfig.onlineEvaluationConfigId ?? serviceIdFromArn(onlineEvaluationConfigId)}`, + }; } - const cloudWatchSourceConfig = dataSourceConfig?.cloudWatchLogs; + const cloudWatchSourceConfig = onlineEvaluationConfig.dataSourceConfig?.cloudWatchLogs; if ( cloudWatchSourceConfig?.logGroupNamePrefixes?.length || (cloudWatchSourceConfig?.logGroupNames?.length ?? 0) > 1 diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.screen.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.screen.test.tsx index 5f88afe636..2323107b82 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.screen.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.screen.test.tsx @@ -145,6 +145,29 @@ describe("batch-evaluation picker", () => { }); describe("batch-evaluation detail (raw JSON)", () => { + test("renders evaluation errors without an evaluator name alongside scores", async () => { + const core = new TestCoreClient(); + core.eval.setBatchEvalGetResponse({ + batchEvaluationId: "be-1", + batchEvaluationArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/be-1", + batchEvaluationName: "mixed", + createdAt: new Date("2026-10-13T00:00:00Z"), + status: "COMPLETED_WITH_ERRORS", + outputConfig: { cloudWatchConfig: { logGroupName: "results", logStreamName: "run-be-1" } }, + }); + core.eval.setBatchEvalResults([ + { evaluatorId: "Builtin.Helpfulness", score: 0.83 }, + { sessionId: "s2", errorType: "LogEventMissingException", error: "Incomplete span data." }, + ]); + const screen = renderScreen("/agentcore/eval/batch-evaluation/get/be-1", { core }); + await screen.resize(120, 40); + await waitForText(screen.lastFrame, "LogEventMissingException"); + const frame = screen.lastFrame()!; + expect(frame).toContain("COMPLETED_WITH_ERRORS"); + expect(frame).toContain("Builtin.Helpfulness"); + expect(frame).toContain("Incomplete span data."); + }); + test("renders the full response, including merged results", async () => { const core = new TestCoreClient(); core.eval.setBatchEvalGetResponse(getResponse()); diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx index e24b865d0f..b01cde1dde 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.test.tsx @@ -71,7 +71,6 @@ describe("eval batch-evaluation command hierarchy", () => { "get", "list", "stop", - "logs", ]); }); diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.44cbc27686231142.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.44cbc27686231142.json new file mode 100644 index 0000000000..8df54bcc38 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.44cbc27686231142.json @@ -0,0 +1,62 @@ +{ + "batchEvaluationId": "cli_logs_success_20261013-ab043c87dc", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc", + "batchEvaluationName": "cli_logs_success_20261013", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-10-13T16:26:05.707Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "ExampleAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/ExampleAgent-abc123def4-DEFAULT" + ], + "filterConfig": { + "timeRange": { + "startTime": { + "$date": "2026-09-22T21:11:00.000Z" + }, + "endTime": { + "$date": "2026-09-22T21:13:43.770Z" + } + } + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_success_20261013-ab043c87dc", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 2, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 0, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 2, + "totalFailed": 0 + } + ] + }, + "updatedAt": { + "$date": "2026-10-13T16:27:10.308Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.7e6d701230e33025.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.7e6d701230e33025.json new file mode 100644 index 0000000000..c68b0140e8 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.7e6d701230e33025.json @@ -0,0 +1,52 @@ +{ + "batchEvaluationId": "cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationName": "cli_logs_errors_20261013", + "status": "FAILED", + "createdAt": { + "$date": "2026-10-13T16:26:49.738Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_errors_20261013-0ce8bc1c19", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": { + "$date": "2026-10-13T16:27:52.679Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.9722e50585569cfc.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.9722e50585569cfc.json new file mode 100644 index 0000000000..7bb7dffd86 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.9722e50585569cfc.json @@ -0,0 +1,59 @@ +{ + "batchEvaluationId": "cli_logs_missing_spans_20261013-c5290ec8b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4", + "batchEvaluationName": "cli_logs_missing_spans_20261013", + "status": "FAILED", + "createdAt": { + "$date": "2026-10-13T16:28:39.096Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_missing_spans_20261013-c5290ec8b4", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": {}, + "totalEvaluated": 0, + "totalFailed": 2 + } + ] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": { + "$date": "2026-10-13T16:29:42.078Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.cca0b9b391e38aa4.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.cca0b9b391e38aa4.json new file mode 100644 index 0000000000..3e0f951cd6 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.cca0b9b391e38aa4.json @@ -0,0 +1,61 @@ +{ + "batchEvaluationId": "cli_logs_mixed_retry_20261013-356d56c3e2", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2", + "batchEvaluationName": "cli_logs_mixed_retry_20261013", + "status": "COMPLETED_WITH_ERRORS", + "createdAt": { + "$date": "2026-10-13T16:30:57.517Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-success-session", + "cli-error-session-1" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/cli-eval-custom-results-20261013", + "logStreamName": "run-cli_logs_mixed_retry_20261013-356d56c3e2", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 1, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 1, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 1, + "totalFailed": 1 + } + ] + }, + "errorDetails": [ + "1 of 2 sessions failed during batch evaluation." + ], + "updatedAt": { + "$date": "2026-10-13T16:32:00.790Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.19472bb2137a9b1b.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.19472bb2137a9b1b.json new file mode 100644 index 0000000000..9a52102887 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.19472bb2137a9b1b.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960907581903186934337743831143601170456856218530971647/s", + "nextBackwardToken": "b/39960907581903186934337743831143601170456856218530971647/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.204d8eebd0586459.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.204d8eebd0586459.json new file mode 100644 index 0000000000..d6e6fc25e1 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.204d8eebd0586459.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960904401237102248709087646470246815711648066568159231/s", + "nextBackwardToken": "b/39960904401237102248709087646470246815711648066568159231/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.24332c6f0fa347f9.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.24332c6f0fa347f9.json new file mode 100644 index 0000000000..cf57d4a063 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.24332c6f0fa347f9.json @@ -0,0 +1,16 @@ +{ + "events": [ + { + "timestamp": 1791908816416, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"unknown\",\"service.name\":\"unknown\"}},\"timeUnixNano\":1791908816416864651,\"observedTimeUnixNano\":1791908816416864651,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"session.id\":\"cli-error-session-1\",\"error.type\":\"JobProcessorCustomerException\",\"error.message\":\"No spans were found for the session. Verify sessionId and that spans exist in CloudWatch Logs within the configured time range. sessionId=cli-error-session-1\",\"error\":1,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_errors_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_errors_20261013-0ce8bc1c19\",\"aws.bedrock_agentcore.evaluation_level\":\"Session\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19\",\"runId\":\"cli_logs_errors_20261013-0ce8bc1c19\"}", + "ingestionTime": 1791908816434 + }, + { + "timestamp": 1791908816990, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"unknown\",\"service.name\":\"unknown\"}},\"timeUnixNano\":1791908816990527629,\"observedTimeUnixNano\":1791908816990527629,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"session.id\":\"cli-error-session-2\",\"error.type\":\"JobProcessorCustomerException\",\"error.message\":\"No spans were found for the session. Verify sessionId and that spans exist in CloudWatch Logs within the configured time range. sessionId=cli-error-session-2\",\"error\":1,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_errors_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_errors_20261013-0ce8bc1c19\",\"aws.bedrock_agentcore.evaluation_level\":\"Session\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19\",\"runId\":\"cli_logs_errors_20261013-0ce8bc1c19\"}", + "ingestionTime": 1791908817009 + } + ], + "nextForwardToken": "f/39960901946694431602003071097048778476685763281800921088/s", + "nextBackwardToken": "b/39960901933893803858046493413112439451357795578818527232/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.3163c4675fc54d15.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.3163c4675fc54d15.json new file mode 100644 index 0000000000..aa5782a679 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.3163c4675fc54d15.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960901946705581972436051573341415136669738910833672191/s", + "nextBackwardToken": "b/39960901946705581972436051573341415136669738910833672191/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.78b264dd2e58203d.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.78b264dd2e58203d.json new file mode 100644 index 0000000000..3ac8a69102 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.78b264dd2e58203d.json @@ -0,0 +1,16 @@ +{ + "events": [ + { + "timestamp": 1791908778022, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"ExampleAgent.DEFAULT\",\"service.name\":\"ExampleAgent.DEFAULT\"}},\"traceId\":\"11111111111111111111111111111111\",\"timeUnixNano\":1790111530829538485,\"observedTimeUnixNano\":1791908778022597517,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-success-session-1\",\"gen_ai.response.id\":\"11111111111111111111111111111111\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user asked to use the add_numbers tool to calculate 19 + 23 and reply with the result. The tool was called and returned '42' as the result. The assistant faithfully reported this result to the user, directly answering their request. Per instructions, the tool output takes priority over the assistant's own knowledge, so even though 19+23 mathematically equals 42 (actually this is correct), the key point is the assistant correctly relayed the tool's output without alteration. This directly satisfies the user's goal of getting the calculated result reported back clearly and concisely. The response is clear, correct relative to the tool output, and fully completes the user's request with no unnecessary detours or missing information.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_success_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_success_20261013-ab043c87dc\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc\",\"runId\":\"cli_logs_success_20261013-ab043c87dc\",\"service.name\":\"ExampleAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1790111530829,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1791908778043 + }, + { + "timestamp": 1791908778319, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"ExampleAgent.DEFAULT\",\"service.name\":\"ExampleAgent.DEFAULT\"}},\"traceId\":\"22222222222222222222222222222222\",\"timeUnixNano\":1790111537006994281,\"observedTimeUnixNano\":1791908778319268111,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-success-session-2\",\"gen_ai.response.id\":\"22222222222222222222222222222222\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user asked for the result of 19 + 23 using the add_numbers tool. The tool returned 42, and the assistant faithfully reported this result clearly and concisely. Per instructions, tool output takes priority over the assistant's own knowledge, so the response correctly relays the tool's result without second-guessing it. The response directly answers the user's request in a clear, simple format, fully satisfying the user's goal of obtaining the sum. There is no unnecessary detour or confusion, and the response is appropriately concise for the simple task at hand. It doesn't exceed the scope of the request (which only asked for the result), so it doesn't reach 'Above and Beyond' territory, but it fully and correctly moves the user to their goal.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_success_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_success_20261013-ab043c87dc\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc\",\"runId\":\"cli_logs_success_20261013-ab043c87dc\",\"service.name\":\"ExampleAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1790111537006,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1791908778337 + } + ], + "nextForwardToken": "f/39960901084302314029625343543969326199500319855255027712/s", + "nextBackwardToken": "b/39960901077678992705661748470577813917483628836431921152/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.a8ae9aee2feab3ce.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.a8ae9aee2feab3ce.json new file mode 100644 index 0000000000..af950bed09 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.a8ae9aee2feab3ce.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960901946705581972436051573341415136669738910833672191/s", + "nextBackwardToken": "b/39960901946694431602003071097048778476685763281800921088/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.b1d3d9333f277b7d.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.b1d3d9333f277b7d.json new file mode 100644 index 0000000000..ae478c6b00 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.b1d3d9333f277b7d.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960901084313464400058324067013653815084951113065005055/s", + "nextBackwardToken": "b/39960901084302314029625343543969326199500319855255027712/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ca64738ae7fe600.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ca64738ae7fe600.json new file mode 100644 index 0000000000..a7ca73d78b --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ca64738ae7fe600.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960907581903186934337743831143601170456856218530971647/s", + "nextBackwardToken": "b/39960907581892036563904763660330687294661243643001307136/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ddd7a682ad7886d4.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ddd7a682ad7886d4.json new file mode 100644 index 0000000000..762097a1ab --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ddd7a682ad7886d4.json @@ -0,0 +1,16 @@ +{ + "events": [ + { + "timestamp": 1791909065712, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"CliEvalBugbash.DEFAULT\",\"service.name\":\"CliEvalBugbash.DEFAULT\"}},\"traceId\":\"11111111111111111111111111111111\",\"timeUnixNano\":1791908920718596452,\"observedTimeUnixNano\":1791909065712495301,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-error-session-1\",\"gen_ai.response.id\":\"11111111111111111111111111111111\",\"error.type\":\"LogEventMissingException\",\"error.message\":\"Session span data is incomplete. Span with ID: 1111111111111111 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data\",\"error\":1,\"throttle\":0,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_mixed_retry_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_mixed_retry_20261013-356d56c3e2\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2\",\"runId\":\"cli_logs_mixed_retry_20261013-356d56c3e2\"}", + "ingestionTime": 1791909065731 + }, + { + "timestamp": 1791909069681, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"CliEvalBugbash.DEFAULT\",\"service.name\":\"CliEvalBugbash.DEFAULT\"}},\"traceId\":\"22222222222222222222222222222222\",\"timeUnixNano\":1791908957275183718,\"observedTimeUnixNano\":1791909069681539038,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-success-session\",\"gen_ai.response.id\":\"22222222222222222222222222222222\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user asked to use add_numbers to calculate 19 + 23 and reply with the result. The tool returned 42 as the output. The assistant faithfully relayed the tool's result, directly answering the user's request in a clear and concise manner. Per instructions, tool output takes priority over the assistant's own knowledge, so the correctness of 19+23=42 mathematically is not to be judged against the tool result; the assistant correctly reported what the tool returned. This directly satisfies the user's goal of getting the result communicated back to them, with no unnecessary detours or confusion. It is a straightforward, complete answer to a simple request, though it doesn't go beyond what was asked (no extra insight or anticipation of further needs), so it fits 'Very Helpful' rather than 'Above and Beyond'.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_mixed_retry_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_mixed_retry_20261013-356d56c3e2\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2\",\"runId\":\"cli_logs_mixed_retry_20261013-356d56c3e2\",\"service.name\":\"CliEvalBugbash.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1791908957275,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1791909069696 + } + ], + "nextForwardToken": "f/39960907581892036563904763660330687294661243643001307136/s", + "nextBackwardToken": "b/39960907493380378870936720406782127828441216943887810560/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.df3ce8073f0f9f9c.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.df3ce8073f0f9f9c.json new file mode 100644 index 0000000000..9d0a95be5b --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.df3ce8073f0f9f9c.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960904401237102248709087646470246815711648066568159231/s", + "nextBackwardToken": "b/39960904401225951878276107303233082821806418104132501504/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.e7a494ca2d983fe1.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.e7a494ca2d983fe1.json new file mode 100644 index 0000000000..2e27446642 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.e7a494ca2d983fe1.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960901084313464400058324067013653815084951113065005055/s", + "nextBackwardToken": "b/39960901084313464400058324067013653815084951113065005055/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.f0d38df552f516cf.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.f0d38df552f516cf.json new file mode 100644 index 0000000000..cc09734bc8 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.f0d38df552f516cf.json @@ -0,0 +1,16 @@ +{ + "events": [ + { + "timestamp": 1791908926062, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"CliEvalBugbash.DEFAULT\",\"service.name\":\"CliEvalBugbash.DEFAULT\"}},\"traceId\":\"11111111111111111111111111111111\",\"timeUnixNano\":1790111530829538485,\"observedTimeUnixNano\":1791908926062092209,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-error-session-1\",\"gen_ai.response.id\":\"11111111111111111111111111111111\",\"error.type\":\"LogEventMissingException\",\"error.message\":\"Session span data is incomplete. Span with ID: 1111111111111111 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data\",\"error\":1,\"throttle\":0,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_missing_spans_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_missing_spans_20261013-c5290ec8b4\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4\",\"runId\":\"cli_logs_missing_spans_20261013-c5290ec8b4\"}", + "ingestionTime": 1791908926089 + }, + { + "timestamp": 1791908927055, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"CliEvalBugbash.DEFAULT\",\"service.name\":\"CliEvalBugbash.DEFAULT\"}},\"traceId\":\"22222222222222222222222222222222\",\"timeUnixNano\":1790111537006994281,\"observedTimeUnixNano\":1791908927055462115,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-error-session-2\",\"gen_ai.response.id\":\"22222222222222222222222222222222\",\"error.type\":\"LogEventMissingException\",\"error.message\":\"Session span data is incomplete. Span with ID: 2222222222222222 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data\",\"error\":1,\"throttle\":0,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_missing_spans_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_missing_spans_20261013-c5290ec8b4\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4\",\"runId\":\"cli_logs_missing_spans_20261013-c5290ec8b4\"}", + "ingestionTime": 1791908927070 + } + ], + "nextForwardToken": "f/39960904401225951878276107303233082821806418104132501504/s", + "nextBackwardToken": "b/39960904379081311896135198522502265348562750423972249600/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/errors-metadata-only.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/errors-metadata-only.golden.json new file mode 100644 index 0000000000..0fd0aa9228 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/errors-metadata-only.golden.json @@ -0,0 +1,48 @@ +{ + "batchEvaluationId": "cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationName": "cli_logs_errors_20261013", + "status": "FAILED", + "createdAt": "2026-10-13T16:26:49.738Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_errors_20261013-0ce8bc1c19", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": "2026-10-13T16:27:52.679Z" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/errors.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/errors.golden.json new file mode 100644 index 0000000000..0b4f18203c --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/errors.golden.json @@ -0,0 +1,62 @@ +{ + "batchEvaluationId": "cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationName": "cli_logs_errors_20261013", + "status": "FAILED", + "createdAt": "2026-10-13T16:26:49.738Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_errors_20261013-0ce8bc1c19", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": "2026-10-13T16:27:52.679Z", + "results": [ + { + "level": "Session", + "sessionId": "cli-error-session-1", + "error": "No spans were found for the session. Verify sessionId and that spans exist in CloudWatch Logs within the configured time range. sessionId=cli-error-session-1", + "errorType": "JobProcessorCustomerException" + }, + { + "level": "Session", + "sessionId": "cli-error-session-2", + "error": "No spans were found for the session. Verify sessionId and that spans exist in CloudWatch Logs within the configured time range. sessionId=cli-error-session-2", + "errorType": "JobProcessorCustomerException" + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/missing-events.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/missing-events.golden.json new file mode 100644 index 0000000000..df07adad70 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/missing-events.golden.json @@ -0,0 +1,73 @@ +{ + "batchEvaluationId": "cli_logs_missing_spans_20261013-c5290ec8b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4", + "batchEvaluationName": "cli_logs_missing_spans_20261013", + "status": "FAILED", + "createdAt": "2026-10-13T16:28:39.096Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_missing_spans_20261013-c5290ec8b4", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": {}, + "totalEvaluated": 0, + "totalFailed": 2 + } + ] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": "2026-10-13T16:29:42.078Z", + "results": [ + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-error-session-1", + "traceId": "11111111111111111111111111111111", + "error": "Session span data is incomplete. Span with ID: 1111111111111111 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data", + "errorType": "LogEventMissingException" + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-error-session-2", + "traceId": "22222222222222222222222222222222", + "error": "Session span data is incomplete. Span with ID: 2222222222222222 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data", + "errorType": "LogEventMissingException" + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/mixed-custom-output.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/mixed-custom-output.golden.json new file mode 100644 index 0000000000..8f0089ac7a --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/mixed-custom-output.golden.json @@ -0,0 +1,76 @@ +{ + "batchEvaluationId": "cli_logs_mixed_retry_20261013-356d56c3e2", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2", + "batchEvaluationName": "cli_logs_mixed_retry_20261013", + "status": "COMPLETED_WITH_ERRORS", + "createdAt": "2026-10-13T16:30:57.517Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-success-session", + "cli-error-session-1" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/cli-eval-custom-results-20261013", + "logStreamName": "run-cli_logs_mixed_retry_20261013-356d56c3e2", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 1, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 1, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 1, + "totalFailed": 1 + } + ] + }, + "errorDetails": [ + "1 of 2 sessions failed during batch evaluation." + ], + "updatedAt": "2026-10-13T16:32:00.790Z", + "results": [ + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-error-session-1", + "traceId": "11111111111111111111111111111111", + "error": "Session span data is incomplete. Span with ID: 1111111111111111 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data", + "errorType": "LogEventMissingException" + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-success-session", + "traceId": "22222222222222222222222222222222", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user asked to use add_numbers to calculate 19 + 23 and reply with the result. The tool returned 42 as the output. The assistant faithfully relayed the tool's result, directly answering the user's request in a clear and concise manner. Per instructions, tool output takes priority over the assistant's own knowledge, so the correctness of 19+23=42 mathematically is not to be judged against the tool result; the assistant correctly reported what the tool returned. This directly satisfies the user's goal of getting the result communicated back to them, with no unnecessary detours or confusion. It is a straightforward, complete answer to a simple request, though it doesn't go beyond what was asked (no extra insight or anticipation of further needs), so it fits 'Very Helpful' rather than 'Above and Beyond'." + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/success.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/success.golden.json new file mode 100644 index 0000000000..546c69cf00 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/success.golden.json @@ -0,0 +1,74 @@ +{ + "batchEvaluationId": "cli_logs_success_20261013-ab043c87dc", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc", + "batchEvaluationName": "cli_logs_success_20261013", + "status": "COMPLETED", + "createdAt": "2026-10-13T16:26:05.707Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "ExampleAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/ExampleAgent-abc123def4-DEFAULT" + ], + "filterConfig": { + "timeRange": { + "startTime": "2026-09-22T21:11:00.000Z", + "endTime": "2026-09-22T21:13:43.770Z" + } + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_success_20261013-ab043c87dc", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 2, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 0, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 2, + "totalFailed": 0 + } + ] + }, + "updatedAt": "2026-10-13T16:27:10.308Z", + "results": [ + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-success-session-1", + "traceId": "11111111111111111111111111111111", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user asked to use the add_numbers tool to calculate 19 + 23 and reply with the result. The tool was called and returned '42' as the result. The assistant faithfully reported this result to the user, directly answering their request. Per instructions, the tool output takes priority over the assistant's own knowledge, so even though 19+23 mathematically equals 42 (actually this is correct), the key point is the assistant correctly relayed the tool's output without alteration. This directly satisfies the user's goal of getting the calculated result reported back clearly and concisely. The response is clear, correct relative to the tool output, and fully completes the user's request with no unnecessary detours or missing information." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-success-session-2", + "traceId": "22222222222222222222222222222222", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user asked for the result of 19 + 23 using the add_numbers tool. The tool returned 42, and the assistant faithfully reported this result clearly and concisely. Per instructions, tool output takes priority over the assistant's own knowledge, so the response correctly relays the tool's result without second-guessing it. The response directly answers the user's request in a clear, simple format, fully satisfying the user's goal of obtaining the sum. There is no unnecessary detour or confusion, and the response is appropriately concise for the simple task at hand. It doesn't exceed the scope of the request (which only asked for the result), so it doesn't reach 'Above and Beyond' territory, but it fully and correctly moves the user to their goal." + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/get.fixture.test.tsx b/src/handlers/eval/batch-evaluation/get/get.fixture.test.tsx new file mode 100644 index 0000000000..cb4036cdf7 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/get.fixture.test.tsx @@ -0,0 +1,110 @@ +import { expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../../testing"; +import { createRootHandler } from "../../../index"; + +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Recorded against real batch jobs: a complete arithmetic-agent trace and two +// deliberately incomplete traces whose missing log events cause service errors. +// RECORD=1 refreshes the SDK responses and golden output. These jobs and their +// output streams must still exist; use fresh jobs if their logs have expired. +// Redact account identifiers in recordings before publishing them. +test.each([ + ["success", "cli_logs_success_20261013-ab043c87dc", "COMPLETED", 2, 0, undefined], + [ + "errors", + "cli_logs_errors_20261013-0ce8bc1c19", + "FAILED", + 0, + 2, + "JobProcessorCustomerException", + ], + [ + "missing-events", + "cli_logs_missing_spans_20261013-c5290ec8b4", + "FAILED", + 0, + 2, + "LogEventMissingException", + ], + [ + "mixed-custom-output", + "cli_logs_mixed_retry_20261013-356d56c3e2", + "COMPLETED_WITH_ERRORS", + 1, + 1, + "LogEventMissingException", + ], + ["errors-metadata-only", "cli_logs_errors_20261013-0ce8bc1c19", "FAILED", 0, 0, undefined], +] as const)( + "get retains recorded %s output", + async (name, id, status, scores, errors, errorType) => { + const metadataOnly = name === "errors-metadata-only"; + const io = testIO(); + const core = new CoreClient({ + ...fixtureFactories(FIXTURES), + logger: createSilentLogger(), + }); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route([ + "node", + "agentcore", + "eval", + "batch-evaluation", + "get", + "--id", + id, + "--region", + "us-east-1", + "--json", + ...(metadataOnly ? ["--disable-cw-results"] : []), + ]); + + expect(io.stderr()).toBe(""); + const detail = JSON.parse(io.stdout()); + expect(detail.batchEvaluationId).toBe(id); + expect(detail.status).toBe(status); + if (metadataOnly) { + expect(detail.results).toBeUndefined(); + } else { + expect(detail.results).toHaveLength(scores + errors); + const scored = detail.results.filter( + (result: { score?: number }) => typeof result.score === "number", + ); + const failed = detail.results.filter( + (result: { error?: string }) => result.error !== undefined, + ); + expect(scored).toHaveLength(scores); + expect(failed).toHaveLength(errors); + for (const result of scored) { + expect(result.evaluatorId).toBe("Builtin.Helpfulness"); + expect(result.error).toBeUndefined(); + } + for (const result of failed) { + expect(result.errorType).toBe(errorType); + expect(result.sessionId).toStartWith("cli-error-session-"); + if (errorType === "LogEventMissingException") { + expect(result.error).toContain("missing a corresponding log event"); + expect(result.evaluatorId).toBe("Builtin.Helpfulness"); + } else { + expect(result.error).toContain("No spans were found for the session"); + expect(result.evaluatorId).toBeUndefined(); + } + } + } + matchGolden(FIXTURES, `${name}.golden.json`, io.stdout()); + }, +); diff --git a/src/handlers/eval/batch-evaluation/get/index.tsx b/src/handlers/eval/batch-evaluation/get/index.tsx index 7d70c0b3a1..19fb3f5e7d 100644 --- a/src/handlers/eval/batch-evaluation/get/index.tsx +++ b/src/handlers/eval/batch-evaluation/get/index.tsx @@ -10,7 +10,8 @@ import { coreOptsFromCtx } from "../../../utils"; export const createGetBatchEvaluationHandler = (core: Core, io: AppIO) => createHandler({ name: "get", - description: "get a batch evaluation by ID, with CloudWatch-backed results when available", + description: + "get a batch evaluation by ID, with CloudWatch-backed results and errors when available", flags: [ flag("id", "the ID of the batch evaluation", z.string().min(1)), flag( diff --git a/src/handlers/eval/batch-evaluation/index.tsx b/src/handlers/eval/batch-evaluation/index.tsx index aef1b1f137..bcf6a5fb56 100644 --- a/src/handlers/eval/batch-evaluation/index.tsx +++ b/src/handlers/eval/batch-evaluation/index.tsx @@ -8,7 +8,6 @@ import { createListBatchEvaluationsHandler } from "./list"; import { createEvaluateBatchEvaluationHandler } from "./evaluate"; import { createSimulateBatchEvaluationHandler } from "./simulate"; import { createStopBatchEvaluationHandler } from "./stop"; -import { createEvalLogsHandler } from "../logs"; // batch-evaluation supports evaluate + simulate (start jobs), get + list, and // stop. A bare invocation opens the interactive TUI (list → get), matching @@ -23,13 +22,7 @@ export function createBatchEvaluationHandler(core: Core, io: AppIO): Router { .handler(createSimulateBatchEvaluationHandler(core, io)) .handler(createGetBatchEvaluationHandler(core, io)) .handler(createListBatchEvaluationsHandler(core)) - .handler(createStopBatchEvaluationHandler(core)) - .handler( - createEvalLogsHandler(core, io, { - resource: "batch-evaluation", - resolveLogSource: core.eval.resourceBatchEvaluationLogSource.bind(core.eval), - }), - ); + .handler(createStopBatchEvaluationHandler(core)); } export { BatchEvaluationScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/logs/index.tsx b/src/handlers/eval/logs/index.tsx deleted file mode 100644 index 8baa1ea076..0000000000 --- a/src/handlers/eval/logs/index.tsx +++ /dev/null @@ -1,39 +0,0 @@ -import z from "zod"; -import type { AppIO } from "../../../io"; -import { flag } from "../../../router"; -import { createLogsHandler } from "../../observability/logs"; -import type { Core } from "../../types"; -import { coreOptsFromCtx } from "../../utils"; -import type { CoreEvalClient } from "../types"; - -export const createEvalLogsHandler = ( - core: Core, - io: AppIO, - { - resource, - resolveLogSource, - }: { - resource: string; - resolveLogSource: CoreEvalClient["resourceBatchEvaluationLogSource"]; - }, -) => - createLogsHandler(io, { - name: "logs", - description: `stream or search ${resource} logs`, - flags: [flag("id", `the ID of the ${resource} resource`, z.string().min(1))] as const, - read: async (ctx, flags, request, signal) => { - const options = coreOptsFromCtx(ctx); - const source = await resolveLogSource(flags.id, options); - - if (request.mode === "search") { - return { - events: core.observability.searchLogs(source, request.query, options, signal), - }; - } - - return { - events: core.observability.tailLogs(source, request.query, options, signal), - announcement: `Streaming logs for ${resource} ${flags.id}... (Ctrl+C to stop)`, - }; - }, - }); diff --git a/src/handlers/eval/online-eval/index.tsx b/src/handlers/eval/online-eval/index.tsx index c30e90916d..2facc05fbe 100644 --- a/src/handlers/eval/online-eval/index.tsx +++ b/src/handlers/eval/online-eval/index.tsx @@ -10,7 +10,7 @@ import { createUpdateOnlineEvalHandler } from "./update"; import { createPauseOnlineEvalHandler } from "./pause"; import { createResumeOnlineEvalHandler } from "./resume"; import { createDeleteOnlineEvalHandler } from "./delete"; -import { createEvalLogsHandler } from "../logs"; +import { createOnlineEvalLogsHandler } from "./logs"; export function createOnlineEvalHandler(core: Core, io: AppIO): Router { return new Router("online-eval", "manage AgentCore online evaluation configs") @@ -24,12 +24,7 @@ export function createOnlineEvalHandler(core: Core, io: AppIO): Router { .handler(createPauseOnlineEvalHandler(core)) .handler(createResumeOnlineEvalHandler(core)) .handler(createDeleteOnlineEvalHandler(core)) - .handler( - createEvalLogsHandler(core, io, { - resource: "online-eval", - resolveLogSource: core.eval.resourceOnlineEvaluationLogSource.bind(core.eval), - }), - ); + .handler(createOnlineEvalLogsHandler(core, io)); } export { OnlineEvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-eval/logs/index.tsx b/src/handlers/eval/online-eval/logs/index.tsx new file mode 100644 index 0000000000..1dd85a06b8 --- /dev/null +++ b/src/handlers/eval/online-eval/logs/index.tsx @@ -0,0 +1,28 @@ +import z from "zod"; +import type { AppIO } from "../../../../io"; +import { flag } from "../../../../router"; +import { createLogsHandler } from "../../../observability/logs"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createOnlineEvalLogsHandler = (core: Core, io: AppIO) => + createLogsHandler(io, { + name: "logs", + description: "stream or search online-eval logs", + flags: [flag("id", "the ID of the online-eval resource", z.string().min(1))] as const, + read: async (ctx, flags, request, signal) => { + const options = coreOptsFromCtx(ctx); + const source = await core.eval.resourceOnlineEvaluationLogSource(flags.id, options); + + if (request.mode === "search") { + return { + events: core.observability.searchLogs(source, request.query, options, signal), + }; + } + + return { + events: core.observability.tailLogs(source, request.query, options, signal), + announcement: `Streaming logs for online-eval ${flags.id}... (Ctrl+C to stop)`, + }; + }, + }); diff --git a/src/handlers/eval/logs/logs.test.tsx b/src/handlers/eval/online-eval/logs/logs.test.tsx similarity index 66% rename from src/handlers/eval/logs/logs.test.tsx rename to src/handlers/eval/online-eval/logs/logs.test.tsx index a5993cbdca..9d957b95b5 100644 --- a/src/handlers/eval/logs/logs.test.tsx +++ b/src/handlers/eval/online-eval/logs/logs.test.tsx @@ -1,14 +1,14 @@ import { describe, expect, test } from "bun:test"; import { join } from "node:path"; -import { CoreClient } from "../../../core"; +import { CoreClient } from "../../../../core"; import { createSilentLogger, fixtureFactories, TestCoreClient, TestGlobalConfigAccessor, testIO, -} from "../../../testing"; -import { createRootHandler } from "../../index"; +} from "../../../../testing"; +import { createRootHandler } from "../../../index"; const ID = "evaluation-1"; const SINCE = 1_709_391_000_000; @@ -18,7 +18,7 @@ const EVENTS = [ { timestamp: new Date(SINCE + 1_000), message: "second result", logStreamName: "run-b" }, ]; -function setup(resource: "online-eval" | "batch-evaluation") { +function setup() { const core = new TestCoreClient(); core.eval.logSource = LOG_SOURCE; core.observability.logEvents = EVENTS; @@ -36,7 +36,7 @@ function setup(resource: "online-eval" | "batch-evaluation") { "node", "agentcore", "eval", - resource, + "online-eval", "logs", ...flags, "--region", @@ -45,17 +45,14 @@ function setup(resource: "online-eval" | "batch-evaluation") { }; } -describe.each([ - ["online-eval", "resourceOnlineEvaluationLogSource"], - ["batch-evaluation", "resourceBatchEvaluationLogSource"], -] as const)("%s logs", (resource, sourceMethod) => { +describe("online-eval logs", () => { test("searches the resolved group and prints every record as JSON", async () => { - const { core, io, route } = setup(resource); + const { core, io, route } = setup(); await route(["--id", ID, "--since", `${SINCE}`, "--json"]); expect(core.eval.calls[0]).toEqual({ - method: sourceMethod, + method: "resourceOnlineEvaluationLogSource", args: [ID, { region: "us-west-2" }], }); expect(core.observability.calls[0]?.method).toBe("searchLogs"); @@ -70,7 +67,7 @@ describe.each([ }); test("tails the resolved group by default and announces it on stderr", async () => { - const { core, io, route } = setup(resource); + const { core, io, route } = setup(); await route(["--id", ID]); @@ -79,12 +76,12 @@ describe.each([ expect(io.stdout()).toBe( EVENTS.map((event) => `${event.timestamp.toISOString()} ${event.message}`).join("\n"), ); - expect(io.stderr()).toBe(`Streaming logs for ${resource} ${ID}... (Ctrl+C to stop)`); + expect(io.stderr()).toBe(`Streaming logs for online-eval ${ID}... (Ctrl+C to stop)`); }); }); test("requires an explicit resource ID", async () => { - const { route } = setup("online-eval"); + const { route } = setup(); await expect(route(["--since", "1h"])).rejects.toThrow("required option '--id' not specified"); }); @@ -92,33 +89,21 @@ test("requires an explicit resource ID", async () => { test.each([ [ "online configured output", - "online-eval", - "resourceOnlineEvaluationLogSource", "agentcore_cli_online_eval_role_warn-tSVCZiBf4l", "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-tSVCZiBf4l", ], [ "online source output", - "online-eval", - "resourceOnlineEvaluationLogSource", "agentcore_cli_online_eval_fixture-5gZw987afd", "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT", ], - [ - "batch configured output", - "batch-evaluation", - "resourceBatchEvaluationLogSource", - "GTProbe2_1786034545579-8ffefc851e", - "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", - ], -] as const)( - "%s resolves its recorded log group", - async (_name, resource, method, id, logGroupName) => { - const core = new CoreClient({ - ...fixtureFactories(join(import.meta.dir, "..", resource, "__fixtures__")), - logger: createSilentLogger(), - }); +] as const)("%s resolves its recorded log group", async (_name, id, logGroupName) => { + const core = new CoreClient({ + ...fixtureFactories(join(import.meta.dir, "..", "__fixtures__")), + logger: createSilentLogger(), + }); - expect(await core.eval[method](id, { region: "us-west-2" })).toEqual({ logGroupName }); - }, -); + expect(await core.eval.resourceOnlineEvaluationLogSource(id, { region: "us-west-2" })).toEqual({ + logGroupName, + }); +}); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 0d149dab14..92470edaad 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -89,7 +89,8 @@ export type SessionSourceValue = // indistinguishable list — this keeps `level` and the id fields so callers can // tell a SESSION result from a TRACE or TOOL_CALL one, and group by session. export type BatchEvaluationResultEntry = { - evaluatorId: string; + // Session/trace failures can occur before an evaluator is selected. + evaluatorId?: string; // The scope the score applies to, read from the result log record's // `aws.bedrock_agentcore.evaluation_level` attribute (Title-case, e.g. "Trace" // / "Session"). The trustworthy discriminator — do not infer it from which id @@ -103,6 +104,7 @@ export type BatchEvaluationResultEntry = { label?: string; explanation?: string; error?: string; + errorType?: string; }; // BatchEvaluationDetail is a GetBatchEvaluation response augmented with the @@ -343,7 +345,6 @@ export type DatasetUpdateProgressEvent = { // abstraction). export interface CoreEvalClient { resourceOnlineEvaluationLogSource(id: string, options: CoreOptions): Promise; - resourceBatchEvaluationLogSource(id: string, options: CoreOptions): Promise; createEvaluator( request: CreateEvaluatorRequest, options: CoreOptions, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 52d6239028..38a5441204 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -1633,12 +1633,6 @@ export class TestEvalClient implements CoreEvalClient { return this.logSource; } - async resourceBatchEvaluationLogSource(id: string, options: CoreOptions) { - this.calls.push({ method: "resourceBatchEvaluationLogSource", args: [id, options] }); - if (this.error) throw this.error; - return this.logSource; - } - // List responses are keyed by the nextToken that requests them (undefined = // the first page), so tests can serve multi-page listings. private listResponses = new Map(); From 7cc950b6542c989ada66b9c5f66c9e50da7e54b2 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Tue, 13 Oct 2026 17:54:41 +0000 Subject: [PATCH 10/10] test(eval): record online log output as golden tests --- command.md | 21 +------- .../batch-evaluation.screen.test.tsx | 23 -------- ...lterLogEventsCommand.18535fcf55ae5a0c.json | 13 +++++ ...luationConfigCommand.27452cfd63d8c546.json | 43 +++++++++++++++ .../__fixtures__/logs-search-json.golden.txt | 6 +++ .../__fixtures__/logs-search-text.golden.txt | 1 + .../eval/online-eval/logs/logs.test.tsx | 54 +++++++++++++------ 7 files changed, 103 insertions(+), 58 deletions(-) create mode 100644 src/handlers/eval/online-eval/logs/__fixtures__/FilterLogEventsCommand.18535fcf55ae5a0c.json create mode 100644 src/handlers/eval/online-eval/logs/__fixtures__/GetOnlineEvaluationConfigCommand.27452cfd63d8c546.json create mode 100644 src/handlers/eval/online-eval/logs/__fixtures__/logs-search-json.golden.txt create mode 100644 src/handlers/eval/online-eval/logs/__fixtures__/logs-search-text.golden.txt diff --git a/command.md b/command.md index 1153864d20..27277623c9 100644 --- a/command.md +++ b/command.md @@ -153,7 +153,6 @@ This reference was generated from `agentcore --help` for version `1.0.0-rc.5`. - [`agentcore eval online-eval pause`](#agentcore-eval-online-eval-pause) - [`agentcore eval online-eval resume`](#agentcore-eval-online-eval-resume) - [`agentcore eval online-eval delete`](#agentcore-eval-online-eval-delete) - - [`agentcore eval online-eval logs`](#agentcore-eval-online-eval-logs) - [`agentcore eval online-insight`](#agentcore-eval-online-insight) - [`agentcore eval online-insight create`](#agentcore-eval-online-insight-create) - [`agentcore eval online-insight get`](#agentcore-eval-online-insight-get) @@ -2310,24 +2309,6 @@ delete an online evaluation config by ID - `--id `: the ID of the online evaluation config to delete (required) -##### `agentcore eval online-eval logs` - -```text -agentcore eval online-eval logs [options] -``` - -stream or search online-eval logs - -**Options** - -- `--id `: the ID of the online-eval resource (required) -- `--since `: search window start: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--until `: search window end: "5m", "1h", ISO 8601, epoch ms, or "now" -- `--tail`: tail new log records (default: false) -- `--level `: filter by log level (error, warn, info, debug) -- `--query `: CloudWatch Logs filter pattern -- `--limit `: maximum number of log records to return in search mode - #### `agentcore eval online-insight` ```text @@ -2595,7 +2576,7 @@ replay a dataset against a Runtime, then batch-evaluate the resulting sessions agentcore eval batch-evaluation get [options] ``` -get a batch evaluation by ID, with CloudWatch-backed results and errors when available +get a batch evaluation by ID, with CloudWatch-backed results when available **Options** diff --git a/src/handlers/eval/batch-evaluation/batch-evaluation.screen.test.tsx b/src/handlers/eval/batch-evaluation/batch-evaluation.screen.test.tsx index 2323107b82..5f88afe636 100644 --- a/src/handlers/eval/batch-evaluation/batch-evaluation.screen.test.tsx +++ b/src/handlers/eval/batch-evaluation/batch-evaluation.screen.test.tsx @@ -145,29 +145,6 @@ describe("batch-evaluation picker", () => { }); describe("batch-evaluation detail (raw JSON)", () => { - test("renders evaluation errors without an evaluator name alongside scores", async () => { - const core = new TestCoreClient(); - core.eval.setBatchEvalGetResponse({ - batchEvaluationId: "be-1", - batchEvaluationArn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/be-1", - batchEvaluationName: "mixed", - createdAt: new Date("2026-10-13T00:00:00Z"), - status: "COMPLETED_WITH_ERRORS", - outputConfig: { cloudWatchConfig: { logGroupName: "results", logStreamName: "run-be-1" } }, - }); - core.eval.setBatchEvalResults([ - { evaluatorId: "Builtin.Helpfulness", score: 0.83 }, - { sessionId: "s2", errorType: "LogEventMissingException", error: "Incomplete span data." }, - ]); - const screen = renderScreen("/agentcore/eval/batch-evaluation/get/be-1", { core }); - await screen.resize(120, 40); - await waitForText(screen.lastFrame, "LogEventMissingException"); - const frame = screen.lastFrame()!; - expect(frame).toContain("COMPLETED_WITH_ERRORS"); - expect(frame).toContain("Builtin.Helpfulness"); - expect(frame).toContain("Incomplete span data."); - }); - test("renders the full response, including merged results", async () => { const core = new TestCoreClient(); core.eval.setBatchEvalGetResponse(getResponse()); diff --git a/src/handlers/eval/online-eval/logs/__fixtures__/FilterLogEventsCommand.18535fcf55ae5a0c.json b/src/handlers/eval/online-eval/logs/__fixtures__/FilterLogEventsCommand.18535fcf55ae5a0c.json new file mode 100644 index 0000000000..f722a87b30 --- /dev/null +++ b/src/handlers/eval/online-eval/logs/__fixtures__/FilterLogEventsCommand.18535fcf55ae5a0c.json @@ -0,0 +1,13 @@ +{ + "events": [ + { + "logStreamName": "evaluation-results-bugbash725_MyOnlineEval-x7cbMK8W2z", + "timestamp": 1781115098820, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"bugbash725_MyAgent.DEFAULT\",\"service.name\":\"bugbash725_MyAgent.DEFAULT\"}},\"traceId\":\"0123456789abcdef0123456789abcdef\",\"timeUnixNano\":1781114199904331579,\"observedTimeUnixNano\":1781115098820963246,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.response.id\":\"0123456789abcdef0123456789abcdef\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The question asks what 10 times 7 is. The correct answer is 70. The candidate response states '10 times 7 is 70.' which is factually correct.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.online_evaluation_config.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/bugbash725_MyOnlineEval-x7cbMK8W2z\",\"aws.bedrock_agentcore.online_evaluation_config.name\":\"bugbash725_MyOnlineEval\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"onlineEvaluationConfigId\":\"bugbash725_MyOnlineEval-x7cbMK8W2z\",\"service.name\":\"bugbash725_MyAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1781114199904,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"onlineEvaluationConfigId\"],[\"label\",\"service.name\",\"onlineEvaluationConfigId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1781115098889, + "eventId": "39720193988040511357685805728500488452731795775606226944" + } + ], + "searchedLogStreams": [], + "nextToken": "Bxkq6kVGFtq2y_MoigeqscPOdhXVbhiVtLoAmXb5jCrRcpxvY0x9YUCxuwMruX42nCD2vfgcEZ8-w6UkBc3rJUj_43m6SYNpFrMlWtnITL0uATy60abGwUvdvO7YXfSbaCbiOfqQN0LCvGYJDRJr26lt0cMuO0GN0qdWRFMH0aS9AuFpCL9jYmMapb8WA_SDuq-Fw9OABOYW05vGFkEMzJXDFcv5q_K0TEg68FjoBUQehvVSOTZ82QA_lIHqaLzSMdI1FANF1ve7Q4OFWrjOXQVMuLCH7CvQx454AmcFtZA3qATL_KyPkWpHeitbYXd3hMrTN73A3z87n1pu99xxeg" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/logs/__fixtures__/GetOnlineEvaluationConfigCommand.27452cfd63d8c546.json b/src/handlers/eval/online-eval/logs/__fixtures__/GetOnlineEvaluationConfigCommand.27452cfd63d8c546.json new file mode 100644 index 0000000000..64519f5302 --- /dev/null +++ b/src/handlers/eval/online-eval/logs/__fixtures__/GetOnlineEvaluationConfigCommand.27452cfd63d8c546.json @@ -0,0 +1,43 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/bugbash725_MyOnlineEval-x7cbMK8W2z", + "onlineEvaluationConfigId": "bugbash725_MyOnlineEval-x7cbMK8W2z", + "onlineEvaluationConfigName": "bugbash725_MyOnlineEval", + "rule": { + "samplingConfig": { + "samplingPercentage": 100 + }, + "sessionConfig": { + "sessionTimeoutMinutes": 5 + } + }, + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "bugbash725_MyAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/bugbash725_MyAgent-w4LhmF4ztO-DEFAULT" + ] + } + }, + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-10T18:05:53.375Z" + }, + "updatedAt": { + "$date": "2026-06-10T18:06:03.710Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/bugbash725_MyOnlineEval-x7cbMK8W2z", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationExecutionRoleArn": "arn:aws:iam::123456789012:role/AgentCoreEvaluationFixtureRole" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-json.golden.txt b/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-json.golden.txt new file mode 100644 index 0000000000..eafb6c9dbe --- /dev/null +++ b/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-json.golden.txt @@ -0,0 +1,6 @@ +{ + "timestamp": "2026-06-10T18:11:38.820Z", + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"bugbash725_MyAgent.DEFAULT\",\"service.name\":\"bugbash725_MyAgent.DEFAULT\"}},\"traceId\":\"0123456789abcdef0123456789abcdef\",\"timeUnixNano\":1781114199904331579,\"observedTimeUnixNano\":1781115098820963246,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.response.id\":\"0123456789abcdef0123456789abcdef\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The question asks what 10 times 7 is. The correct answer is 70. The candidate response states '10 times 7 is 70.' which is factually correct.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.online_evaluation_config.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/bugbash725_MyOnlineEval-x7cbMK8W2z\",\"aws.bedrock_agentcore.online_evaluation_config.name\":\"bugbash725_MyOnlineEval\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"onlineEvaluationConfigId\":\"bugbash725_MyOnlineEval-x7cbMK8W2z\",\"service.name\":\"bugbash725_MyAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1781114199904,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"onlineEvaluationConfigId\"],[\"label\",\"service.name\",\"onlineEvaluationConfigId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": "2026-06-10T18:11:38.889Z", + "logStreamName": "evaluation-results-bugbash725_MyOnlineEval-x7cbMK8W2z" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-text.golden.txt b/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-text.golden.txt new file mode 100644 index 0000000000..4cf73b31ff --- /dev/null +++ b/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-text.golden.txt @@ -0,0 +1 @@ +2026-06-10T18:11:38.820Z {"resource":{"attributes":{"aws.service.type":"gen_ai_agent","aws.local.service":"bugbash725_MyAgent.DEFAULT","service.name":"bugbash725_MyAgent.DEFAULT"}},"traceId":"0123456789abcdef0123456789abcdef","timeUnixNano":1781114199904331579,"observedTimeUnixNano":1781115098820963246,"severityNumber":9,"name":"gen_ai.evaluation.result","attributes":{"gen_ai.evaluation.name":"Builtin.Correctness","session.id":"00000000-0000-4000-8000-000000000001","gen_ai.response.id":"0123456789abcdef0123456789abcdef","gen_ai.evaluation.score.value":1.0,"gen_ai.evaluation.explanation":"The question asks what 10 times 7 is. The correct answer is 70. The candidate response states '10 times 7 is 70.' which is factually correct.","gen_ai.evaluation.score.label":"Perfectly Correct","aws.bedrock_agentcore.online_evaluation_config.arn":"arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/bugbash725_MyOnlineEval-x7cbMK8W2z","aws.bedrock_agentcore.online_evaluation_config.name":"bugbash725_MyOnlineEval","aws.bedrock_agentcore.evaluator.arn":"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness","aws.bedrock_agentcore.evaluator.rating_scale":"Numerical","aws.bedrock_agentcore.evaluation_level":"Trace"},"onlineEvaluationConfigId":"bugbash725_MyOnlineEval-x7cbMK8W2z","service.name":"bugbash725_MyAgent.DEFAULT","label":"Perfectly Correct","_aws":{"Timestamp":1781114199904,"CloudWatchMetrics":[{"Namespace":"Bedrock-AgentCore/Evaluations","Dimensions":[["service.name"],["label","service.name"],["service.name","onlineEvaluationConfigId"],["label","service.name","onlineEvaluationConfigId"]],"Metrics":[{"Name":"Builtin.Correctness","Unit":"None"}]}]},"Builtin.Correctness":1.0} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/logs/logs.test.tsx b/src/handlers/eval/online-eval/logs/logs.test.tsx index 9d957b95b5..b56b98c764 100644 --- a/src/handlers/eval/online-eval/logs/logs.test.tsx +++ b/src/handlers/eval/online-eval/logs/logs.test.tsx @@ -4,6 +4,7 @@ import { CoreClient } from "../../../../core"; import { createSilentLogger, fixtureFactories, + matchGolden, TestCoreClient, TestGlobalConfigAccessor, testIO, @@ -46,24 +47,47 @@ function setup() { } describe("online-eval logs", () => { - test("searches the resolved group and prints every record as JSON", async () => { - const { core, io, route } = setup(); + // Pin a real online evaluation result and its search window. Re-recording + // requires the configuration and retained CloudWatch event to still exist. + // Redact account identifiers in recordings before publishing them. + test.each([ + ["text", false], + ["json", true], + ] as const)("searches recorded online evaluation logs as %s", async (format, json) => { + const fixtures = join(import.meta.dir, "__fixtures__"); + const core = new CoreClient({ + ...fixtureFactories(fixtures), + logger: createSilentLogger(), + }); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); - await route(["--id", ID, "--since", `${SINCE}`, "--json"]); + await root.route([ + "node", + "agentcore", + "eval", + "online-eval", + "logs", + "--id", + "bugbash725_MyOnlineEval-x7cbMK8W2z", + "--since", + "1781115098000", + "--until", + "1781115100000", + "--limit", + "1", + "--region", + "us-east-1", + ...(json ? ["--json"] : []), + ]); - expect(core.eval.calls[0]).toEqual({ - method: "resourceOnlineEvaluationLogSource", - args: [ID, { region: "us-west-2" }], - }); - expect(core.observability.calls[0]?.method).toBe("searchLogs"); - expect(core.observability.calls[0]?.args[0]).toEqual(LOG_SOURCE); - expect( - io - .stdout() - .split("\n") - .map((line) => JSON.parse(line)), - ).toEqual(EVENTS.map((event) => ({ ...event, timestamp: event.timestamp.toISOString() }))); expect(io.stderr()).toBe(""); + expect(io.stdout()).toContain("Builtin.Correctness"); + matchGolden(fixtures, `logs-search-${format}.golden.txt`, io.stdout()); }); test("tails the resolved group by default and announces it on stderr", async () => {