diff --git a/src/core/batchEvaluationResults.test.ts b/src/core/batchEvaluationResults.test.ts index d3e77d229b..16a2bfb5f3 100644 --- a/src/core/batchEvaluationResults.test.ts +++ b/src/core/batchEvaluationResults.test.ts @@ -109,3 +109,93 @@ test("parseEvaluationLogEvent warns on and skips an unparseable line", () => { expect(warnings).toHaveLength(1); expect(warnings[0]).toContain("unparseable"); }); + +test.each([ + [ + "standard error without evaluator", + { + "error.type": "LogEventMissingException", + "error.message": "Session span data is incomplete.", + }, + { errorType: "LogEventMissingException", error: "Session span data is incomplete." }, + ], + [ + "standard error with evaluator", + { + "gen_ai.evaluation.name": "Builtin.Helpfulness", + "error.type": "EvaluationException", + "error.message": "Evaluation failed.", + }, + { + evaluatorId: "Builtin.Helpfulness", + errorType: "EvaluationException", + error: "Evaluation failed.", + }, + ], + [ + "legacy error", + { + "gen_ai.evaluation.name": "Builtin.Helpfulness", + "gen_ai.evaluation.error": "Evaluator failed.", + }, + { evaluatorId: "Builtin.Helpfulness", error: "Evaluator failed." }, + ], + [ + "unnamed legacy error", + { "gen_ai.evaluation.error": "Evaluator failed." }, + { error: "Evaluator failed." }, + ], + ["type only", { "error.type": "EvaluationException" }, { errorType: "EvaluationException" }], + ["message only", { "error.message": "Evaluation failed." }, { error: "Evaluation failed." }], + [ + "standard message takes precedence", + { "error.message": "Standard message.", "gen_ai.evaluation.error": "Legacy message." }, + { error: "Standard message." }, + ], +] as const)("parseEvaluationLogEvent retains %s and scope", (_name, attrs, expected) => { + const result = parseEvaluationLogEvent( + JSON.stringify({ + name: "gen_ai.evaluation.result", + traceId: "trace-1", + spanId: "span-1", + attributes: { + ...attrs, + "session.id": "session-1", + "aws.bedrock_agentcore.evaluation_level": "Trace", + "gen_ai.tool.name": "lookup", + }, + }), + createSilentLogger(), + ); + expect(result).toEqual({ + evaluatorId: undefined, + score: undefined, + label: undefined, + explanation: undefined, + error: undefined, + errorType: undefined, + ...expected, + traceId: "trace-1", + spanId: "span-1", + sessionId: "session-1", + level: "Trace", + toolName: "lookup", + }); +}); + +test.each( + [ + null, + [], + "control message", + { attributes: null }, + { attributes: [] }, + { attributes: "invalid" }, + { name: "application.error", attributes: { "error.message": "Unrelated error." } }, + { attributes: { "error.type": "UnrelatedException" } }, + { name: "gen_ai.evaluation.result", attributes: { "error.message": 42, "error.type": {} } }, + { name: "gen_ai.evaluation.result", attributes: {} }, + ].map((value) => ({ value })), +)("parseEvaluationLogEvent skips non-evaluation or malformed data: %j", ({ value }) => { + expect(parseEvaluationLogEvent(JSON.stringify(value), createSilentLogger())).toBeNull(); +}); diff --git a/src/core/batchEvaluationResults.tsx b/src/core/batchEvaluationResults.tsx index 3cbbe70865..43a2b3175f 100644 --- a/src/core/batchEvaluationResults.tsx +++ b/src/core/batchEvaluationResults.tsx @@ -43,9 +43,9 @@ export function parseEvaluationLogEvent( message: string, logger: Logger, ): BatchEvaluationResultEntry | null { - let parsed: Record; + let value: unknown; try { - parsed = JSON.parse(message) as Record; + value = JSON.parse(message); } catch { // Swallow rather than throw: CloudWatch result streams interleave non-JSON // control lines with the evaluation records, so one unparseable line is @@ -54,11 +54,20 @@ export function parseEvaluationLogEvent( logger.warn("skipping unparseable batch-evaluation result log line"); return null; } - const attrs = (parsed["attributes"] ?? {}) as Record; - const evaluatorId = attrs["gen_ai.evaluation.name"] as string | undefined; - if (!evaluatorId) return null; - + if (!value || typeof value !== "object" || Array.isArray(value)) return null; + const parsed = value as Record; + const attributes = parsed["attributes"]; + if (!attributes || typeof attributes !== "object" || Array.isArray(attributes)) return null; + const attrs = attributes as Record; const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined); + const evaluatorId = str(attrs["gen_ai.evaluation.name"]); + const error = str(attrs["error.message"]) ?? str(attrs["gen_ai.evaluation.error"]); + const errorType = str(attrs["error.type"]); + // Failures before scoring may have no evaluator name. Only accept unnamed + // errors from evaluation result records, not unrelated application errors. + if (!evaluatorId && !(parsed["name"] === "gen_ai.evaluation.result" && (error || errorType))) { + return null; + } // Keys are read verbatim as the recorded stream emits them: scores, level, // session id, and tool name under `attributes`; traceId/spanId at the top // level. If AgentCore renames one, the corresponding field goes undefined and @@ -73,6 +82,7 @@ export function parseEvaluationLogEvent( score: attrs["gen_ai.evaluation.score.value"] as number | undefined, label: str(attrs["gen_ai.evaluation.score.label"]), explanation: str(attrs["gen_ai.evaluation.explanation"]), - error: str(attrs["gen_ai.evaluation.error"]), + error, + errorType, }; } diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 58bec5abdc..49304b0740 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -164,6 +164,8 @@ import { toClientConfig } from "./utils"; import { grantOnlineEvalScope } from "./onlineEvalExecutionRole"; import { accountIdFromArn, deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole"; import { harnessRuntimeFromResponse } from "./harness"; +import { serviceIdFromArn } from "./arn"; +import type { LogSource } from "./observability/types"; const DEFAULT_INGESTION_WAIT_MS = 180_000; const DATASET_EXAMPLES_BATCH_LIMIT = 1000; @@ -225,6 +227,42 @@ export class EvalClient implements CoreEvalClient { private readonly cloudWatch: CloudWatchClient = new CloudWatchClient(clients), ) {} + async resourceOnlineEvaluationLogSource( + onlineEvaluationConfigId: string, + options: CoreOptions, + ): Promise { + const onlineEvaluationConfig = await this.getOnlineEvaluationConfig( + onlineEvaluationConfigId, + options, + ); + const cloudWatchOutputConfig = onlineEvaluationConfig.outputConfig?.cloudWatchConfig; + if (cloudWatchOutputConfig?.logGroupName) + return { logGroupName: cloudWatchOutputConfig.logGroupName }; + if (onlineEvaluationConfig.outputConfig && !cloudWatchOutputConfig) { + throw new InputValidationError("Logs require a CloudWatch output configuration."); + } + if (cloudWatchOutputConfig?.resultDestination !== "SOURCE_LOG_GROUP") { + return { + logGroupName: `/aws/bedrock-agentcore/evaluations/results/${onlineEvaluationConfig.onlineEvaluationConfigId ?? serviceIdFromArn(onlineEvaluationConfigId)}`, + }; + } + const cloudWatchSourceConfig = onlineEvaluationConfig.dataSourceConfig?.cloudWatchLogs; + if ( + cloudWatchSourceConfig?.logGroupNamePrefixes?.length || + (cloudWatchSourceConfig?.logGroupNames?.length ?? 0) > 1 + ) { + throw new InputValidationError( + "Results use multiple source log groups or prefixes. " + + "Read them directly in CloudWatch Logs, or configure one explicit source log group.", + ); + } + const sourceLogGroupName = cloudWatchSourceConfig?.logGroupNames?.[0]; + if (!sourceLogGroupName) { + throw new MalformedServiceResponseError("Missing source log group name."); + } + return { logGroupName: sourceLogGroupName }; + } + async createEvaluator( request: CreateEvaluatorRequest, options: CoreOptions, diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.44cbc27686231142.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.44cbc27686231142.json new file mode 100644 index 0000000000..8df54bcc38 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.44cbc27686231142.json @@ -0,0 +1,62 @@ +{ + "batchEvaluationId": "cli_logs_success_20261013-ab043c87dc", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc", + "batchEvaluationName": "cli_logs_success_20261013", + "status": "COMPLETED", + "createdAt": { + "$date": "2026-10-13T16:26:05.707Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "ExampleAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/ExampleAgent-abc123def4-DEFAULT" + ], + "filterConfig": { + "timeRange": { + "startTime": { + "$date": "2026-09-22T21:11:00.000Z" + }, + "endTime": { + "$date": "2026-09-22T21:13:43.770Z" + } + } + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_success_20261013-ab043c87dc", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 2, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 0, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 2, + "totalFailed": 0 + } + ] + }, + "updatedAt": { + "$date": "2026-10-13T16:27:10.308Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.7e6d701230e33025.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.7e6d701230e33025.json new file mode 100644 index 0000000000..c68b0140e8 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.7e6d701230e33025.json @@ -0,0 +1,52 @@ +{ + "batchEvaluationId": "cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationName": "cli_logs_errors_20261013", + "status": "FAILED", + "createdAt": { + "$date": "2026-10-13T16:26:49.738Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_errors_20261013-0ce8bc1c19", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": { + "$date": "2026-10-13T16:27:52.679Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.9722e50585569cfc.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.9722e50585569cfc.json new file mode 100644 index 0000000000..7bb7dffd86 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.9722e50585569cfc.json @@ -0,0 +1,59 @@ +{ + "batchEvaluationId": "cli_logs_missing_spans_20261013-c5290ec8b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4", + "batchEvaluationName": "cli_logs_missing_spans_20261013", + "status": "FAILED", + "createdAt": { + "$date": "2026-10-13T16:28:39.096Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_missing_spans_20261013-c5290ec8b4", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": {}, + "totalEvaluated": 0, + "totalFailed": 2 + } + ] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": { + "$date": "2026-10-13T16:29:42.078Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.cca0b9b391e38aa4.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.cca0b9b391e38aa4.json new file mode 100644 index 0000000000..3e0f951cd6 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.cca0b9b391e38aa4.json @@ -0,0 +1,61 @@ +{ + "batchEvaluationId": "cli_logs_mixed_retry_20261013-356d56c3e2", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2", + "batchEvaluationName": "cli_logs_mixed_retry_20261013", + "status": "COMPLETED_WITH_ERRORS", + "createdAt": { + "$date": "2026-10-13T16:30:57.517Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-success-session", + "cli-error-session-1" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/cli-eval-custom-results-20261013", + "logStreamName": "run-cli_logs_mixed_retry_20261013-356d56c3e2", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 1, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 1, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 1, + "totalFailed": 1 + } + ] + }, + "errorDetails": [ + "1 of 2 sessions failed during batch evaluation." + ], + "updatedAt": { + "$date": "2026-10-13T16:32:00.790Z" + } +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.19472bb2137a9b1b.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.19472bb2137a9b1b.json new file mode 100644 index 0000000000..9a52102887 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.19472bb2137a9b1b.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960907581903186934337743831143601170456856218530971647/s", + "nextBackwardToken": "b/39960907581903186934337743831143601170456856218530971647/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.204d8eebd0586459.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.204d8eebd0586459.json new file mode 100644 index 0000000000..d6e6fc25e1 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.204d8eebd0586459.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960904401237102248709087646470246815711648066568159231/s", + "nextBackwardToken": "b/39960904401237102248709087646470246815711648066568159231/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.24332c6f0fa347f9.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.24332c6f0fa347f9.json new file mode 100644 index 0000000000..cf57d4a063 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.24332c6f0fa347f9.json @@ -0,0 +1,16 @@ +{ + "events": [ + { + "timestamp": 1791908816416, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"unknown\",\"service.name\":\"unknown\"}},\"timeUnixNano\":1791908816416864651,\"observedTimeUnixNano\":1791908816416864651,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"session.id\":\"cli-error-session-1\",\"error.type\":\"JobProcessorCustomerException\",\"error.message\":\"No spans were found for the session. Verify sessionId and that spans exist in CloudWatch Logs within the configured time range. sessionId=cli-error-session-1\",\"error\":1,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_errors_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_errors_20261013-0ce8bc1c19\",\"aws.bedrock_agentcore.evaluation_level\":\"Session\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19\",\"runId\":\"cli_logs_errors_20261013-0ce8bc1c19\"}", + "ingestionTime": 1791908816434 + }, + { + "timestamp": 1791908816990, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"unknown\",\"service.name\":\"unknown\"}},\"timeUnixNano\":1791908816990527629,\"observedTimeUnixNano\":1791908816990527629,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"session.id\":\"cli-error-session-2\",\"error.type\":\"JobProcessorCustomerException\",\"error.message\":\"No spans were found for the session. Verify sessionId and that spans exist in CloudWatch Logs within the configured time range. sessionId=cli-error-session-2\",\"error\":1,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_errors_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_errors_20261013-0ce8bc1c19\",\"aws.bedrock_agentcore.evaluation_level\":\"Session\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19\",\"runId\":\"cli_logs_errors_20261013-0ce8bc1c19\"}", + "ingestionTime": 1791908817009 + } + ], + "nextForwardToken": "f/39960901946694431602003071097048778476685763281800921088/s", + "nextBackwardToken": "b/39960901933893803858046493413112439451357795578818527232/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.3163c4675fc54d15.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.3163c4675fc54d15.json new file mode 100644 index 0000000000..aa5782a679 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.3163c4675fc54d15.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960901946705581972436051573341415136669738910833672191/s", + "nextBackwardToken": "b/39960901946705581972436051573341415136669738910833672191/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.78b264dd2e58203d.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.78b264dd2e58203d.json new file mode 100644 index 0000000000..3ac8a69102 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.78b264dd2e58203d.json @@ -0,0 +1,16 @@ +{ + "events": [ + { + "timestamp": 1791908778022, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"ExampleAgent.DEFAULT\",\"service.name\":\"ExampleAgent.DEFAULT\"}},\"traceId\":\"11111111111111111111111111111111\",\"timeUnixNano\":1790111530829538485,\"observedTimeUnixNano\":1791908778022597517,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-success-session-1\",\"gen_ai.response.id\":\"11111111111111111111111111111111\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user asked to use the add_numbers tool to calculate 19 + 23 and reply with the result. The tool was called and returned '42' as the result. The assistant faithfully reported this result to the user, directly answering their request. Per instructions, the tool output takes priority over the assistant's own knowledge, so even though 19+23 mathematically equals 42 (actually this is correct), the key point is the assistant correctly relayed the tool's output without alteration. This directly satisfies the user's goal of getting the calculated result reported back clearly and concisely. The response is clear, correct relative to the tool output, and fully completes the user's request with no unnecessary detours or missing information.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_success_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_success_20261013-ab043c87dc\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc\",\"runId\":\"cli_logs_success_20261013-ab043c87dc\",\"service.name\":\"ExampleAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1790111530829,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1791908778043 + }, + { + "timestamp": 1791908778319, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"ExampleAgent.DEFAULT\",\"service.name\":\"ExampleAgent.DEFAULT\"}},\"traceId\":\"22222222222222222222222222222222\",\"timeUnixNano\":1790111537006994281,\"observedTimeUnixNano\":1791908778319268111,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-success-session-2\",\"gen_ai.response.id\":\"22222222222222222222222222222222\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user asked for the result of 19 + 23 using the add_numbers tool. The tool returned 42, and the assistant faithfully reported this result clearly and concisely. Per instructions, tool output takes priority over the assistant's own knowledge, so the response correctly relays the tool's result without second-guessing it. The response directly answers the user's request in a clear, simple format, fully satisfying the user's goal of obtaining the sum. There is no unnecessary detour or confusion, and the response is appropriately concise for the simple task at hand. It doesn't exceed the scope of the request (which only asked for the result), so it doesn't reach 'Above and Beyond' territory, but it fully and correctly moves the user to their goal.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_success_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_success_20261013-ab043c87dc\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc\",\"runId\":\"cli_logs_success_20261013-ab043c87dc\",\"service.name\":\"ExampleAgent.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1790111537006,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1791908778337 + } + ], + "nextForwardToken": "f/39960901084302314029625343543969326199500319855255027712/s", + "nextBackwardToken": "b/39960901077678992705661748470577813917483628836431921152/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.a8ae9aee2feab3ce.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.a8ae9aee2feab3ce.json new file mode 100644 index 0000000000..af950bed09 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.a8ae9aee2feab3ce.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960901946705581972436051573341415136669738910833672191/s", + "nextBackwardToken": "b/39960901946694431602003071097048778476685763281800921088/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.b1d3d9333f277b7d.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.b1d3d9333f277b7d.json new file mode 100644 index 0000000000..ae478c6b00 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.b1d3d9333f277b7d.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960901084313464400058324067013653815084951113065005055/s", + "nextBackwardToken": "b/39960901084302314029625343543969326199500319855255027712/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ca64738ae7fe600.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ca64738ae7fe600.json new file mode 100644 index 0000000000..a7ca73d78b --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ca64738ae7fe600.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960907581903186934337743831143601170456856218530971647/s", + "nextBackwardToken": "b/39960907581892036563904763660330687294661243643001307136/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ddd7a682ad7886d4.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ddd7a682ad7886d4.json new file mode 100644 index 0000000000..762097a1ab --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.ddd7a682ad7886d4.json @@ -0,0 +1,16 @@ +{ + "events": [ + { + "timestamp": 1791909065712, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"CliEvalBugbash.DEFAULT\",\"service.name\":\"CliEvalBugbash.DEFAULT\"}},\"traceId\":\"11111111111111111111111111111111\",\"timeUnixNano\":1791908920718596452,\"observedTimeUnixNano\":1791909065712495301,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-error-session-1\",\"gen_ai.response.id\":\"11111111111111111111111111111111\",\"error.type\":\"LogEventMissingException\",\"error.message\":\"Session span data is incomplete. Span with ID: 1111111111111111 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data\",\"error\":1,\"throttle\":0,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_mixed_retry_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_mixed_retry_20261013-356d56c3e2\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2\",\"runId\":\"cli_logs_mixed_retry_20261013-356d56c3e2\"}", + "ingestionTime": 1791909065731 + }, + { + "timestamp": 1791909069681, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"CliEvalBugbash.DEFAULT\",\"service.name\":\"CliEvalBugbash.DEFAULT\"}},\"traceId\":\"22222222222222222222222222222222\",\"timeUnixNano\":1791908957275183718,\"observedTimeUnixNano\":1791909069681539038,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-success-session\",\"gen_ai.response.id\":\"22222222222222222222222222222222\",\"gen_ai.evaluation.score.value\":0.83,\"gen_ai.evaluation.explanation\":\"The user asked to use add_numbers to calculate 19 + 23 and reply with the result. The tool returned 42 as the output. The assistant faithfully relayed the tool's result, directly answering the user's request in a clear and concise manner. Per instructions, tool output takes priority over the assistant's own knowledge, so the correctness of 19+23=42 mathematically is not to be judged against the tool result; the assistant correctly reported what the tool returned. This directly satisfies the user's goal of getting the result communicated back to them, with no unnecessary detours or confusion. It is a straightforward, complete answer to a simple request, though it doesn't go beyond what was asked (no extra insight or anticipation of further needs), so it fits 'Very Helpful' rather than 'Above and Beyond'.\",\"gen_ai.evaluation.score.label\":\"Very Helpful\",\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_mixed_retry_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_mixed_retry_20261013-356d56c3e2\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2\",\"runId\":\"cli_logs_mixed_retry_20261013-356d56c3e2\",\"service.name\":\"CliEvalBugbash.DEFAULT\",\"label\":\"Very Helpful\",\"_aws\":{\"Timestamp\":1791908957275,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"evaluationJobId\"],[\"label\",\"service.name\",\"evaluationJobId\"]],\"Metrics\":[{\"Name\":\"Builtin.Helpfulness\",\"Unit\":\"None\"}]}]},\"Builtin.Helpfulness\":0.83}", + "ingestionTime": 1791909069696 + } + ], + "nextForwardToken": "f/39960907581892036563904763660330687294661243643001307136/s", + "nextBackwardToken": "b/39960907493380378870936720406782127828441216943887810560/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.df3ce8073f0f9f9c.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.df3ce8073f0f9f9c.json new file mode 100644 index 0000000000..9d0a95be5b --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.df3ce8073f0f9f9c.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960904401237102248709087646470246815711648066568159231/s", + "nextBackwardToken": "b/39960904401225951878276107303233082821806418104132501504/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.e7a494ca2d983fe1.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.e7a494ca2d983fe1.json new file mode 100644 index 0000000000..2e27446642 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.e7a494ca2d983fe1.json @@ -0,0 +1,5 @@ +{ + "events": [], + "nextForwardToken": "f/39960901084313464400058324067013653815084951113065005055/s", + "nextBackwardToken": "b/39960901084313464400058324067013653815084951113065005055/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.f0d38df552f516cf.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.f0d38df552f516cf.json new file mode 100644 index 0000000000..cc09734bc8 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/GetLogEventsCommand.f0d38df552f516cf.json @@ -0,0 +1,16 @@ +{ + "events": [ + { + "timestamp": 1791908926062, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"CliEvalBugbash.DEFAULT\",\"service.name\":\"CliEvalBugbash.DEFAULT\"}},\"traceId\":\"11111111111111111111111111111111\",\"timeUnixNano\":1790111530829538485,\"observedTimeUnixNano\":1791908926062092209,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-error-session-1\",\"gen_ai.response.id\":\"11111111111111111111111111111111\",\"error.type\":\"LogEventMissingException\",\"error.message\":\"Session span data is incomplete. Span with ID: 1111111111111111 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data\",\"error\":1,\"throttle\":0,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_missing_spans_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_missing_spans_20261013-c5290ec8b4\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4\",\"runId\":\"cli_logs_missing_spans_20261013-c5290ec8b4\"}", + "ingestionTime": 1791908926089 + }, + { + "timestamp": 1791908927055, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"CliEvalBugbash.DEFAULT\",\"service.name\":\"CliEvalBugbash.DEFAULT\"}},\"traceId\":\"22222222222222222222222222222222\",\"timeUnixNano\":1790111537006994281,\"observedTimeUnixNano\":1791908927055462115,\"severityNumber\":17,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Helpfulness\",\"session.id\":\"cli-error-session-2\",\"gen_ai.response.id\":\"22222222222222222222222222222222\",\"error.type\":\"LogEventMissingException\",\"error.message\":\"Session span data is incomplete. Span with ID: 2222222222222222 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data\",\"error\":1,\"throttle\":0,\"aws.bedrock_agentcore.evaluation_job.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4\",\"aws.bedrock_agentcore.evaluation_job.name\":\"cli_logs_missing_spans_20261013\",\"aws.bedrock_agentcore.evaluation_job.run_id\":\"cli_logs_missing_spans_20261013-c5290ec8b4\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"evaluationJobId\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4\",\"runId\":\"cli_logs_missing_spans_20261013-c5290ec8b4\"}", + "ingestionTime": 1791908927070 + } + ], + "nextForwardToken": "f/39960904401225951878276107303233082821806418104132501504/s", + "nextBackwardToken": "b/39960904379081311896135198522502265348562750423972249600/s" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/errors-metadata-only.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/errors-metadata-only.golden.json new file mode 100644 index 0000000000..0fd0aa9228 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/errors-metadata-only.golden.json @@ -0,0 +1,48 @@ +{ + "batchEvaluationId": "cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationName": "cli_logs_errors_20261013", + "status": "FAILED", + "createdAt": "2026-10-13T16:26:49.738Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_errors_20261013-0ce8bc1c19", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": "2026-10-13T16:27:52.679Z" +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/errors.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/errors.golden.json new file mode 100644 index 0000000000..0b4f18203c --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/errors.golden.json @@ -0,0 +1,62 @@ +{ + "batchEvaluationId": "cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19", + "batchEvaluationName": "cli_logs_errors_20261013", + "status": "FAILED", + "createdAt": "2026-10-13T16:26:49.738Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_errors_20261013-0ce8bc1c19", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": "2026-10-13T16:27:52.679Z", + "results": [ + { + "level": "Session", + "sessionId": "cli-error-session-1", + "error": "No spans were found for the session. Verify sessionId and that spans exist in CloudWatch Logs within the configured time range. sessionId=cli-error-session-1", + "errorType": "JobProcessorCustomerException" + }, + { + "level": "Session", + "sessionId": "cli-error-session-2", + "error": "No spans were found for the session. Verify sessionId and that spans exist in CloudWatch Logs within the configured time range. sessionId=cli-error-session-2", + "errorType": "JobProcessorCustomerException" + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/missing-events.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/missing-events.golden.json new file mode 100644 index 0000000000..df07adad70 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/missing-events.golden.json @@ -0,0 +1,73 @@ +{ + "batchEvaluationId": "cli_logs_missing_spans_20261013-c5290ec8b4", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4", + "batchEvaluationName": "cli_logs_missing_spans_20261013", + "status": "FAILED", + "createdAt": "2026-10-13T16:28:39.096Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-error-session-1", + "cli-error-session-2" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_missing_spans_20261013-c5290ec8b4", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 0, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 2, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": {}, + "totalEvaluated": 0, + "totalFailed": 2 + } + ] + }, + "errorDetails": [ + "All 2 sessions failed during batch evaluation." + ], + "updatedAt": "2026-10-13T16:29:42.078Z", + "results": [ + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-error-session-1", + "traceId": "11111111111111111111111111111111", + "error": "Session span data is incomplete. Span with ID: 1111111111111111 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data", + "errorType": "LogEventMissingException" + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-error-session-2", + "traceId": "22222222222222222222222222222222", + "error": "Session span data is incomplete. Span with ID: 2222222222222222 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data", + "errorType": "LogEventMissingException" + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/mixed-custom-output.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/mixed-custom-output.golden.json new file mode 100644 index 0000000000..8f0089ac7a --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/mixed-custom-output.golden.json @@ -0,0 +1,76 @@ +{ + "batchEvaluationId": "cli_logs_mixed_retry_20261013-356d56c3e2", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_mixed_retry_20261013-356d56c3e2", + "batchEvaluationName": "cli_logs_mixed_retry_20261013", + "status": "COMPLETED_WITH_ERRORS", + "createdAt": "2026-10-13T16:30:57.517Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "CliEvalBugbash.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/cli-eval-errors-20261013" + ], + "filterConfig": { + "sessionIds": [ + "cli-success-session", + "cli-error-session-1" + ] + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/cli-eval-custom-results-20261013", + "logStreamName": "run-cli_logs_mixed_retry_20261013-356d56c3e2", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 1, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 1, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 1, + "totalFailed": 1 + } + ] + }, + "errorDetails": [ + "1 of 2 sessions failed during batch evaluation." + ], + "updatedAt": "2026-10-13T16:32:00.790Z", + "results": [ + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-error-session-1", + "traceId": "11111111111111111111111111111111", + "error": "Session span data is incomplete. Span with ID: 1111111111111111 and name: invoke_agent Strands Agents is missing a corresponding log event. Please retry the request with the complete span data", + "errorType": "LogEventMissingException" + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-success-session", + "traceId": "22222222222222222222222222222222", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user asked to use add_numbers to calculate 19 + 23 and reply with the result. The tool returned 42 as the output. The assistant faithfully relayed the tool's result, directly answering the user's request in a clear and concise manner. Per instructions, tool output takes priority over the assistant's own knowledge, so the correctness of 19+23=42 mathematically is not to be judged against the tool result; the assistant correctly reported what the tool returned. This directly satisfies the user's goal of getting the result communicated back to them, with no unnecessary detours or confusion. It is a straightforward, complete answer to a simple request, though it doesn't go beyond what was asked (no extra insight or anticipation of further needs), so it fits 'Very Helpful' rather than 'Above and Beyond'." + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/__fixtures__/success.golden.json b/src/handlers/eval/batch-evaluation/get/__fixtures__/success.golden.json new file mode 100644 index 0000000000..546c69cf00 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/__fixtures__/success.golden.json @@ -0,0 +1,74 @@ +{ + "batchEvaluationId": "cli_logs_success_20261013-ab043c87dc", + "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc", + "batchEvaluationName": "cli_logs_success_20261013", + "status": "COMPLETED", + "createdAt": "2026-10-13T16:26:05.707Z", + "evaluators": [ + { + "evaluatorId": "Builtin.Helpfulness" + } + ], + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "ExampleAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/ExampleAgent-abc123def4-DEFAULT" + ], + "filterConfig": { + "timeRange": { + "startTime": "2026-09-22T21:11:00.000Z", + "endTime": "2026-09-22T21:13:43.770Z" + } + } + } + }, + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", + "logStreamName": "run-cli_logs_success_20261013-ab043c87dc", + "metricsNamespace": "Bedrock-AgentCore/Evaluations", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationResults": { + "numberOfSessionsCompleted": 2, + "numberOfSessionsInProgress": 0, + "numberOfSessionsFailed": 0, + "totalNumberOfSessions": 2, + "numberOfSessionsIgnored": 0, + "evaluatorSummaries": [ + { + "evaluatorId": "Builtin.Helpfulness", + "statistics": { + "averageScore": 0.83 + }, + "totalEvaluated": 2, + "totalFailed": 0 + } + ] + }, + "updatedAt": "2026-10-13T16:27:10.308Z", + "results": [ + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-success-session-1", + "traceId": "11111111111111111111111111111111", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user asked to use the add_numbers tool to calculate 19 + 23 and reply with the result. The tool was called and returned '42' as the result. The assistant faithfully reported this result to the user, directly answering their request. Per instructions, the tool output takes priority over the assistant's own knowledge, so even though 19+23 mathematically equals 42 (actually this is correct), the key point is the assistant correctly relayed the tool's output without alteration. This directly satisfies the user's goal of getting the calculated result reported back clearly and concisely. The response is clear, correct relative to the tool output, and fully completes the user's request with no unnecessary detours or missing information." + }, + { + "evaluatorId": "Builtin.Helpfulness", + "level": "Trace", + "sessionId": "cli-success-session-2", + "traceId": "22222222222222222222222222222222", + "score": 0.83, + "label": "Very Helpful", + "explanation": "The user asked for the result of 19 + 23 using the add_numbers tool. The tool returned 42, and the assistant faithfully reported this result clearly and concisely. Per instructions, tool output takes priority over the assistant's own knowledge, so the response correctly relays the tool's result without second-guessing it. The response directly answers the user's request in a clear, simple format, fully satisfying the user's goal of obtaining the sum. There is no unnecessary detour or confusion, and the response is appropriately concise for the simple task at hand. It doesn't exceed the scope of the request (which only asked for the result), so it doesn't reach 'Above and Beyond' territory, but it fully and correctly moves the user to their goal." + } + ] +} \ No newline at end of file diff --git a/src/handlers/eval/batch-evaluation/get/get.fixture.test.tsx b/src/handlers/eval/batch-evaluation/get/get.fixture.test.tsx new file mode 100644 index 0000000000..cb4036cdf7 --- /dev/null +++ b/src/handlers/eval/batch-evaluation/get/get.fixture.test.tsx @@ -0,0 +1,110 @@ +import { expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestGlobalConfigAccessor, + testIO, +} from "../../../../testing"; +import { createRootHandler } from "../../../index"; + +const FIXTURES = join(import.meta.dir, "__fixtures__"); + +// Recorded against real batch jobs: a complete arithmetic-agent trace and two +// deliberately incomplete traces whose missing log events cause service errors. +// RECORD=1 refreshes the SDK responses and golden output. These jobs and their +// output streams must still exist; use fresh jobs if their logs have expired. +// Redact account identifiers in recordings before publishing them. +test.each([ + ["success", "cli_logs_success_20261013-ab043c87dc", "COMPLETED", 2, 0, undefined], + [ + "errors", + "cli_logs_errors_20261013-0ce8bc1c19", + "FAILED", + 0, + 2, + "JobProcessorCustomerException", + ], + [ + "missing-events", + "cli_logs_missing_spans_20261013-c5290ec8b4", + "FAILED", + 0, + 2, + "LogEventMissingException", + ], + [ + "mixed-custom-output", + "cli_logs_mixed_retry_20261013-356d56c3e2", + "COMPLETED_WITH_ERRORS", + 1, + 1, + "LogEventMissingException", + ], + ["errors-metadata-only", "cli_logs_errors_20261013-0ce8bc1c19", "FAILED", 0, 0, undefined], +] as const)( + "get retains recorded %s output", + async (name, id, status, scores, errors, errorType) => { + const metadataOnly = name === "errors-metadata-only"; + const io = testIO(); + const core = new CoreClient({ + ...fixtureFactories(FIXTURES), + logger: createSilentLogger(), + }); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route([ + "node", + "agentcore", + "eval", + "batch-evaluation", + "get", + "--id", + id, + "--region", + "us-east-1", + "--json", + ...(metadataOnly ? ["--disable-cw-results"] : []), + ]); + + expect(io.stderr()).toBe(""); + const detail = JSON.parse(io.stdout()); + expect(detail.batchEvaluationId).toBe(id); + expect(detail.status).toBe(status); + if (metadataOnly) { + expect(detail.results).toBeUndefined(); + } else { + expect(detail.results).toHaveLength(scores + errors); + const scored = detail.results.filter( + (result: { score?: number }) => typeof result.score === "number", + ); + const failed = detail.results.filter( + (result: { error?: string }) => result.error !== undefined, + ); + expect(scored).toHaveLength(scores); + expect(failed).toHaveLength(errors); + for (const result of scored) { + expect(result.evaluatorId).toBe("Builtin.Helpfulness"); + expect(result.error).toBeUndefined(); + } + for (const result of failed) { + expect(result.errorType).toBe(errorType); + expect(result.sessionId).toStartWith("cli-error-session-"); + if (errorType === "LogEventMissingException") { + expect(result.error).toContain("missing a corresponding log event"); + expect(result.evaluatorId).toBe("Builtin.Helpfulness"); + } else { + expect(result.error).toContain("No spans were found for the session"); + expect(result.evaluatorId).toBeUndefined(); + } + } + } + matchGolden(FIXTURES, `${name}.golden.json`, io.stdout()); + }, +); diff --git a/src/handlers/eval/batch-evaluation/get/index.tsx b/src/handlers/eval/batch-evaluation/get/index.tsx index 7d70c0b3a1..19fb3f5e7d 100644 --- a/src/handlers/eval/batch-evaluation/get/index.tsx +++ b/src/handlers/eval/batch-evaluation/get/index.tsx @@ -10,7 +10,8 @@ import { coreOptsFromCtx } from "../../../utils"; export const createGetBatchEvaluationHandler = (core: Core, io: AppIO) => createHandler({ name: "get", - description: "get a batch evaluation by ID, with CloudWatch-backed results when available", + description: + "get a batch evaluation by ID, with CloudWatch-backed results and errors when available", flags: [ flag("id", "the ID of the batch evaluation", z.string().min(1)), flag( diff --git a/src/handlers/eval/online-eval/index.tsx b/src/handlers/eval/online-eval/index.tsx index a6af130729..2facc05fbe 100644 --- a/src/handlers/eval/online-eval/index.tsx +++ b/src/handlers/eval/online-eval/index.tsx @@ -10,6 +10,7 @@ import { createUpdateOnlineEvalHandler } from "./update"; import { createPauseOnlineEvalHandler } from "./pause"; import { createResumeOnlineEvalHandler } from "./resume"; import { createDeleteOnlineEvalHandler } from "./delete"; +import { createOnlineEvalLogsHandler } from "./logs"; export function createOnlineEvalHandler(core: Core, io: AppIO): Router { return new Router("online-eval", "manage AgentCore online evaluation configs") @@ -22,7 +23,8 @@ export function createOnlineEvalHandler(core: Core, io: AppIO): Router { .handler(createUpdateOnlineEvalHandler(core, io)) .handler(createPauseOnlineEvalHandler(core)) .handler(createResumeOnlineEvalHandler(core)) - .handler(createDeleteOnlineEvalHandler(core)); + .handler(createDeleteOnlineEvalHandler(core)) + .handler(createOnlineEvalLogsHandler(core, io)); } export { OnlineEvalScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-eval/logs/__fixtures__/FilterLogEventsCommand.18535fcf55ae5a0c.json b/src/handlers/eval/online-eval/logs/__fixtures__/FilterLogEventsCommand.18535fcf55ae5a0c.json new file mode 100644 index 0000000000..f722a87b30 --- /dev/null +++ b/src/handlers/eval/online-eval/logs/__fixtures__/FilterLogEventsCommand.18535fcf55ae5a0c.json @@ -0,0 +1,13 @@ +{ + "events": [ + { + "logStreamName": "evaluation-results-bugbash725_MyOnlineEval-x7cbMK8W2z", + "timestamp": 1781115098820, + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"bugbash725_MyAgent.DEFAULT\",\"service.name\":\"bugbash725_MyAgent.DEFAULT\"}},\"traceId\":\"0123456789abcdef0123456789abcdef\",\"timeUnixNano\":1781114199904331579,\"observedTimeUnixNano\":1781115098820963246,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.response.id\":\"0123456789abcdef0123456789abcdef\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The question asks what 10 times 7 is. The correct answer is 70. The candidate response states '10 times 7 is 70.' which is factually correct.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.online_evaluation_config.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/bugbash725_MyOnlineEval-x7cbMK8W2z\",\"aws.bedrock_agentcore.online_evaluation_config.name\":\"bugbash725_MyOnlineEval\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"onlineEvaluationConfigId\":\"bugbash725_MyOnlineEval-x7cbMK8W2z\",\"service.name\":\"bugbash725_MyAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1781114199904,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"onlineEvaluationConfigId\"],[\"label\",\"service.name\",\"onlineEvaluationConfigId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": 1781115098889, + "eventId": "39720193988040511357685805728500488452731795775606226944" + } + ], + "searchedLogStreams": [], + "nextToken": "Bxkq6kVGFtq2y_MoigeqscPOdhXVbhiVtLoAmXb5jCrRcpxvY0x9YUCxuwMruX42nCD2vfgcEZ8-w6UkBc3rJUj_43m6SYNpFrMlWtnITL0uATy60abGwUvdvO7YXfSbaCbiOfqQN0LCvGYJDRJr26lt0cMuO0GN0qdWRFMH0aS9AuFpCL9jYmMapb8WA_SDuq-Fw9OABOYW05vGFkEMzJXDFcv5q_K0TEg68FjoBUQehvVSOTZ82QA_lIHqaLzSMdI1FANF1ve7Q4OFWrjOXQVMuLCH7CvQx454AmcFtZA3qATL_KyPkWpHeitbYXd3hMrTN73A3z87n1pu99xxeg" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/logs/__fixtures__/GetOnlineEvaluationConfigCommand.27452cfd63d8c546.json b/src/handlers/eval/online-eval/logs/__fixtures__/GetOnlineEvaluationConfigCommand.27452cfd63d8c546.json new file mode 100644 index 0000000000..64519f5302 --- /dev/null +++ b/src/handlers/eval/online-eval/logs/__fixtures__/GetOnlineEvaluationConfigCommand.27452cfd63d8c546.json @@ -0,0 +1,43 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/bugbash725_MyOnlineEval-x7cbMK8W2z", + "onlineEvaluationConfigId": "bugbash725_MyOnlineEval-x7cbMK8W2z", + "onlineEvaluationConfigName": "bugbash725_MyOnlineEval", + "rule": { + "samplingConfig": { + "samplingPercentage": 100 + }, + "sessionConfig": { + "sessionTimeoutMinutes": 5 + } + }, + "dataSourceConfig": { + "cloudWatchLogs": { + "serviceNames": [ + "bugbash725_MyAgent.DEFAULT" + ], + "logGroupNames": [ + "/aws/bedrock-agentcore/runtimes/bugbash725_MyAgent-w4LhmF4ztO-DEFAULT" + ] + } + }, + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-10T18:05:53.375Z" + }, + "updatedAt": { + "$date": "2026-06-10T18:06:03.710Z" + }, + "evaluators": [ + { + "evaluatorId": "Builtin.Correctness" + } + ], + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/bugbash725_MyOnlineEval-x7cbMK8W2z", + "resultDestination": "DEDICATED_LOG_GROUP" + } + }, + "evaluationExecutionRoleArn": "arn:aws:iam::123456789012:role/AgentCoreEvaluationFixtureRole" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-json.golden.txt b/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-json.golden.txt new file mode 100644 index 0000000000..eafb6c9dbe --- /dev/null +++ b/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-json.golden.txt @@ -0,0 +1,6 @@ +{ + "timestamp": "2026-06-10T18:11:38.820Z", + "message": "{\"resource\":{\"attributes\":{\"aws.service.type\":\"gen_ai_agent\",\"aws.local.service\":\"bugbash725_MyAgent.DEFAULT\",\"service.name\":\"bugbash725_MyAgent.DEFAULT\"}},\"traceId\":\"0123456789abcdef0123456789abcdef\",\"timeUnixNano\":1781114199904331579,\"observedTimeUnixNano\":1781115098820963246,\"severityNumber\":9,\"name\":\"gen_ai.evaluation.result\",\"attributes\":{\"gen_ai.evaluation.name\":\"Builtin.Correctness\",\"session.id\":\"00000000-0000-4000-8000-000000000001\",\"gen_ai.response.id\":\"0123456789abcdef0123456789abcdef\",\"gen_ai.evaluation.score.value\":1.0,\"gen_ai.evaluation.explanation\":\"The question asks what 10 times 7 is. The correct answer is 70. The candidate response states '10 times 7 is 70.' which is factually correct.\",\"gen_ai.evaluation.score.label\":\"Perfectly Correct\",\"aws.bedrock_agentcore.online_evaluation_config.arn\":\"arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/bugbash725_MyOnlineEval-x7cbMK8W2z\",\"aws.bedrock_agentcore.online_evaluation_config.name\":\"bugbash725_MyOnlineEval\",\"aws.bedrock_agentcore.evaluator.arn\":\"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness\",\"aws.bedrock_agentcore.evaluator.rating_scale\":\"Numerical\",\"aws.bedrock_agentcore.evaluation_level\":\"Trace\"},\"onlineEvaluationConfigId\":\"bugbash725_MyOnlineEval-x7cbMK8W2z\",\"service.name\":\"bugbash725_MyAgent.DEFAULT\",\"label\":\"Perfectly Correct\",\"_aws\":{\"Timestamp\":1781114199904,\"CloudWatchMetrics\":[{\"Namespace\":\"Bedrock-AgentCore/Evaluations\",\"Dimensions\":[[\"service.name\"],[\"label\",\"service.name\"],[\"service.name\",\"onlineEvaluationConfigId\"],[\"label\",\"service.name\",\"onlineEvaluationConfigId\"]],\"Metrics\":[{\"Name\":\"Builtin.Correctness\",\"Unit\":\"None\"}]}]},\"Builtin.Correctness\":1.0}", + "ingestionTime": "2026-06-10T18:11:38.889Z", + "logStreamName": "evaluation-results-bugbash725_MyOnlineEval-x7cbMK8W2z" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-text.golden.txt b/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-text.golden.txt new file mode 100644 index 0000000000..4cf73b31ff --- /dev/null +++ b/src/handlers/eval/online-eval/logs/__fixtures__/logs-search-text.golden.txt @@ -0,0 +1 @@ +2026-06-10T18:11:38.820Z {"resource":{"attributes":{"aws.service.type":"gen_ai_agent","aws.local.service":"bugbash725_MyAgent.DEFAULT","service.name":"bugbash725_MyAgent.DEFAULT"}},"traceId":"0123456789abcdef0123456789abcdef","timeUnixNano":1781114199904331579,"observedTimeUnixNano":1781115098820963246,"severityNumber":9,"name":"gen_ai.evaluation.result","attributes":{"gen_ai.evaluation.name":"Builtin.Correctness","session.id":"00000000-0000-4000-8000-000000000001","gen_ai.response.id":"0123456789abcdef0123456789abcdef","gen_ai.evaluation.score.value":1.0,"gen_ai.evaluation.explanation":"The question asks what 10 times 7 is. The correct answer is 70. The candidate response states '10 times 7 is 70.' which is factually correct.","gen_ai.evaluation.score.label":"Perfectly Correct","aws.bedrock_agentcore.online_evaluation_config.arn":"arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/bugbash725_MyOnlineEval-x7cbMK8W2z","aws.bedrock_agentcore.online_evaluation_config.name":"bugbash725_MyOnlineEval","aws.bedrock_agentcore.evaluator.arn":"arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness","aws.bedrock_agentcore.evaluator.rating_scale":"Numerical","aws.bedrock_agentcore.evaluation_level":"Trace"},"onlineEvaluationConfigId":"bugbash725_MyOnlineEval-x7cbMK8W2z","service.name":"bugbash725_MyAgent.DEFAULT","label":"Perfectly Correct","_aws":{"Timestamp":1781114199904,"CloudWatchMetrics":[{"Namespace":"Bedrock-AgentCore/Evaluations","Dimensions":[["service.name"],["label","service.name"],["service.name","onlineEvaluationConfigId"],["label","service.name","onlineEvaluationConfigId"]],"Metrics":[{"Name":"Builtin.Correctness","Unit":"None"}]}]},"Builtin.Correctness":1.0} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/logs/index.tsx b/src/handlers/eval/online-eval/logs/index.tsx new file mode 100644 index 0000000000..1dd85a06b8 --- /dev/null +++ b/src/handlers/eval/online-eval/logs/index.tsx @@ -0,0 +1,28 @@ +import z from "zod"; +import type { AppIO } from "../../../../io"; +import { flag } from "../../../../router"; +import { createLogsHandler } from "../../../observability/logs"; +import type { Core } from "../../../types"; +import { coreOptsFromCtx } from "../../../utils"; + +export const createOnlineEvalLogsHandler = (core: Core, io: AppIO) => + createLogsHandler(io, { + name: "logs", + description: "stream or search online-eval logs", + flags: [flag("id", "the ID of the online-eval resource", z.string().min(1))] as const, + read: async (ctx, flags, request, signal) => { + const options = coreOptsFromCtx(ctx); + const source = await core.eval.resourceOnlineEvaluationLogSource(flags.id, options); + + if (request.mode === "search") { + return { + events: core.observability.searchLogs(source, request.query, options, signal), + }; + } + + return { + events: core.observability.tailLogs(source, request.query, options, signal), + announcement: `Streaming logs for online-eval ${flags.id}... (Ctrl+C to stop)`, + }; + }, + }); diff --git a/src/handlers/eval/online-eval/logs/logs.test.tsx b/src/handlers/eval/online-eval/logs/logs.test.tsx new file mode 100644 index 0000000000..b56b98c764 --- /dev/null +++ b/src/handlers/eval/online-eval/logs/logs.test.tsx @@ -0,0 +1,133 @@ +import { describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { CoreClient } from "../../../../core"; +import { + createSilentLogger, + fixtureFactories, + matchGolden, + TestCoreClient, + TestGlobalConfigAccessor, + testIO, +} from "../../../../testing"; +import { createRootHandler } from "../../../index"; + +const ID = "evaluation-1"; +const SINCE = 1_709_391_000_000; +const LOG_SOURCE = { logGroupName: "/custom/evaluation-results" }; +const EVENTS = [ + { timestamp: new Date(SINCE), message: "first result", logStreamName: "run-a" }, + { timestamp: new Date(SINCE + 1_000), message: "second result", logStreamName: "run-b" }, +]; + +function setup() { + const core = new TestCoreClient(); + core.eval.logSource = LOG_SOURCE; + core.observability.logEvents = EVENTS; + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + return { + core, + io, + route: (flags: string[]) => + root.route([ + "node", + "agentcore", + "eval", + "online-eval", + "logs", + ...flags, + "--region", + "us-west-2", + ]), + }; +} + +describe("online-eval logs", () => { + // Pin a real online evaluation result and its search window. Re-recording + // requires the configuration and retained CloudWatch event to still exist. + // Redact account identifiers in recordings before publishing them. + test.each([ + ["text", false], + ["json", true], + ] as const)("searches recorded online evaluation logs as %s", async (format, json) => { + const fixtures = join(import.meta.dir, "__fixtures__"); + const core = new CoreClient({ + ...fixtureFactories(fixtures), + logger: createSilentLogger(), + }); + const io = testIO(); + const root = createRootHandler(core, { + io: io.io, + logger: createSilentLogger(), + globalConfigAccessor: new TestGlobalConfigAccessor(), + }); + + await root.route([ + "node", + "agentcore", + "eval", + "online-eval", + "logs", + "--id", + "bugbash725_MyOnlineEval-x7cbMK8W2z", + "--since", + "1781115098000", + "--until", + "1781115100000", + "--limit", + "1", + "--region", + "us-east-1", + ...(json ? ["--json"] : []), + ]); + + expect(io.stderr()).toBe(""); + expect(io.stdout()).toContain("Builtin.Correctness"); + matchGolden(fixtures, `logs-search-${format}.golden.txt`, io.stdout()); + }); + + test("tails the resolved group by default and announces it on stderr", async () => { + const { core, io, route } = setup(); + + await route(["--id", ID]); + + expect(core.observability.calls[0]?.method).toBe("tailLogs"); + expect(core.observability.calls[0]?.args[0]).toEqual(LOG_SOURCE); + expect(io.stdout()).toBe( + EVENTS.map((event) => `${event.timestamp.toISOString()} ${event.message}`).join("\n"), + ); + expect(io.stderr()).toBe(`Streaming logs for online-eval ${ID}... (Ctrl+C to stop)`); + }); +}); + +test("requires an explicit resource ID", async () => { + const { route } = setup(); + + await expect(route(["--since", "1h"])).rejects.toThrow("required option '--id' not specified"); +}); + +test.each([ + [ + "online configured output", + "agentcore_cli_online_eval_role_warn-tSVCZiBf4l", + "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-tSVCZiBf4l", + ], + [ + "online source output", + "agentcore_cli_online_eval_fixture-5gZw987afd", + "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT", + ], +] as const)("%s resolves its recorded log group", async (_name, id, logGroupName) => { + const core = new CoreClient({ + ...fixtureFactories(join(import.meta.dir, "..", "__fixtures__")), + logger: createSilentLogger(), + }); + + expect(await core.eval.resourceOnlineEvaluationLogSource(id, { region: "us-west-2" })).toEqual({ + logGroupName, + }); +}); diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx index 5473fe0bdc..f82f4d591d 100644 --- a/src/handlers/eval/online-eval/online-eval.test.tsx +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -101,6 +101,7 @@ describe("eval online-eval command hierarchy", () => { "pause", "resume", "delete", + "logs", ]); }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index d84fc8e506..92470edaad 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -57,6 +57,7 @@ import type { DataSourceConfig as DataPlaneDataSourceConfig, } from "@aws-sdk/client-bedrock-agentcore"; import type { CoreOptions } from "../../core/types"; +import type { LogSource } from "../../core/observability/types"; import type { EvaluatorModelProvider } from "../../projectSchemas/evaluator"; // SessionWindow is the resolved explicit time filter for a batch evaluation's @@ -88,7 +89,8 @@ export type SessionSourceValue = // indistinguishable list — this keeps `level` and the id fields so callers can // tell a SESSION result from a TRACE or TOOL_CALL one, and group by session. export type BatchEvaluationResultEntry = { - evaluatorId: string; + // Session/trace failures can occur before an evaluator is selected. + evaluatorId?: string; // The scope the score applies to, read from the result log record's // `aws.bedrock_agentcore.evaluation_level` attribute (Title-case, e.g. "Trace" // / "Session"). The trustworthy discriminator — do not infer it from which id @@ -102,6 +104,7 @@ export type BatchEvaluationResultEntry = { label?: string; explanation?: string; error?: string; + errorType?: string; }; // BatchEvaluationDetail is a GetBatchEvaluation response augmented with the @@ -341,6 +344,7 @@ export type DatasetUpdateProgressEvent = { // and implemented by src/core/eval.tsx (dependency inversion: handlers own the // abstraction). export interface CoreEvalClient { + resourceOnlineEvaluationLogSource(id: string, options: CoreOptions): Promise; createEvaluator( request: CreateEvaluatorRequest, options: CoreOptions, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 6cb2832f27..38a5441204 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -1625,6 +1625,13 @@ export class TestPaymentClient implements CorePaymentClient { export class TestEvalClient implements CoreEvalClient { // calls records every invocation in order, for assertions. readonly calls: RecordedCall[] = []; + logSource: LogSource = { logGroupName: "/test/eval/results" }; + + async resourceOnlineEvaluationLogSource(id: string, options: CoreOptions) { + this.calls.push({ method: "resourceOnlineEvaluationLogSource", args: [id, options] }); + if (this.error) throw this.error; + return this.logSource; + } // List responses are keyed by the nextToken that requests them (undefined = // the first page), so tests can serve multi-page listings.