fix(eval): add online logs and include batch evaluation errors in get

src/core/batchEvaluationResults.test.ts

@@ -109,3 +109,93 @@ test("parseEvaluationLogEvent warns on and skips an unparseable line", () => {
   expect(warnings).toHaveLength(1);
   expect(warnings[0]).toContain("unparseable");
 });
+
+test.each([
+  [
+    "standard error without evaluator",
+    {
+      "error.type": "LogEventMissingException",
+      "error.message": "Session span data is incomplete.",
+    },
+    { errorType: "LogEventMissingException", error: "Session span data is incomplete." },
+  ],
+  [
+    "standard error with evaluator",
+    {
+      "gen_ai.evaluation.name": "Builtin.Helpfulness",
+      "error.type": "EvaluationException",
+      "error.message": "Evaluation failed.",
+    },
+    {
+      evaluatorId: "Builtin.Helpfulness",
+      errorType: "EvaluationException",
+      error: "Evaluation failed.",
+    },
+  ],
+  [
+    "legacy error",
+    {
+      "gen_ai.evaluation.name": "Builtin.Helpfulness",
+      "gen_ai.evaluation.error": "Evaluator failed.",
+    },
+    { evaluatorId: "Builtin.Helpfulness", error: "Evaluator failed." },
+  ],
+  [
+    "unnamed legacy error",
+    { "gen_ai.evaluation.error": "Evaluator failed." },
+    { error: "Evaluator failed." },
+  ],
+  ["type only", { "error.type": "EvaluationException" }, { errorType: "EvaluationException" }],
+  ["message only", { "error.message": "Evaluation failed." }, { error: "Evaluation failed." }],
+  [
+    "standard message takes precedence",
+    { "error.message": "Standard message.", "gen_ai.evaluation.error": "Legacy message." },
+    { error: "Standard message." },
+  ],
+] as const)("parseEvaluationLogEvent retains %s and scope", (_name, attrs, expected) => {
+  const result = parseEvaluationLogEvent(
+    JSON.stringify({
+      name: "gen_ai.evaluation.result",
+      traceId: "trace-1",
+      spanId: "span-1",
+      attributes: {
+        ...attrs,
+        "session.id": "session-1",
+        "aws.bedrock_agentcore.evaluation_level": "Trace",
+        "gen_ai.tool.name": "lookup",
+      },
+    }),
+    createSilentLogger(),
+  );
+  expect(result).toEqual({
+    evaluatorId: undefined,
+    score: undefined,
+    label: undefined,
+    explanation: undefined,
+    error: undefined,
+    errorType: undefined,
+    ...expected,
+    traceId: "trace-1",
+    spanId: "span-1",
+    sessionId: "session-1",
+    level: "Trace",
+    toolName: "lookup",
+  });
+});
+
+test.each(
+  [
+    null,
+    [],
+    "control message",
+    { attributes: null },
+    { attributes: [] },
+    { attributes: "invalid" },
+    { name: "application.error", attributes: { "error.message": "Unrelated error." } },
+    { attributes: { "error.type": "UnrelatedException" } },
+    { name: "gen_ai.evaluation.result", attributes: { "error.message": 42, "error.type": {} } },
+    { name: "gen_ai.evaluation.result", attributes: {} },
+  ].map((value) => ({ value })),
+)("parseEvaluationLogEvent skips non-evaluation or malformed data: %j", ({ value }) => {
+  expect(parseEvaluationLogEvent(JSON.stringify(value), createSilentLogger())).toBeNull();
+});

src/core/batchEvaluationResults.tsx

@@ -43,9 +43,9 @@ export function parseEvaluationLogEvent(
   message: string,
   logger: Logger,
 ): BatchEvaluationResultEntry | null {
-  let parsed: Record<string, unknown>;
+  let value: unknown;
   try {
-    parsed = JSON.parse(message) as Record<string, unknown>;
+    value = JSON.parse(message);
   } catch {
     // Swallow rather than throw: CloudWatch result streams interleave non-JSON
     // control lines with the evaluation records, so one unparseable line is
@@ -54,11 +54,20 @@ export function parseEvaluationLogEvent(
     logger.warn("skipping unparseable batch-evaluation result log line");
     return null;
   }
-  const attrs = (parsed["attributes"] ?? {}) as Record<string, unknown>;
-  const evaluatorId = attrs["gen_ai.evaluation.name"] as string | undefined;
-  if (!evaluatorId) return null;
-
+  if (!value || typeof value !== "object" || Array.isArray(value)) return null;
+  const parsed = value as Record<string, unknown>;
+  const attributes = parsed["attributes"];
+  if (!attributes || typeof attributes !== "object" || Array.isArray(attributes)) return null;
+  const attrs = attributes as Record<string, unknown>;
   const str = (v: unknown): string | undefined => (typeof v === "string" ? v : undefined);
+  const evaluatorId = str(attrs["gen_ai.evaluation.name"]);
+  const error = str(attrs["error.message"]) ?? str(attrs["gen_ai.evaluation.error"]);
+  const errorType = str(attrs["error.type"]);
+  // Failures before scoring may have no evaluator name. Only accept unnamed
+  // errors from evaluation result records, not unrelated application errors.
+  if (!evaluatorId && !(parsed["name"] === "gen_ai.evaluation.result" && (error || errorType))) {
+    return null;
+  }
   // Keys are read verbatim as the recorded stream emits them: scores, level,
   // session id, and tool name under `attributes`; traceId/spanId at the top
   // level. If AgentCore renames one, the corresponding field goes undefined and
@@ -73,6 +82,7 @@ export function parseEvaluationLogEvent(
     score: attrs["gen_ai.evaluation.score.value"] as number | undefined,
     label: str(attrs["gen_ai.evaluation.score.label"]),
     explanation: str(attrs["gen_ai.evaluation.explanation"]),
-    error: str(attrs["gen_ai.evaluation.error"]),
+    error,
+    errorType,
   };
 }

src/core/eval.tsx

@@ -164,6 +164,8 @@ import { toClientConfig } from "./utils";
 import { grantOnlineEvalScope } from "./onlineEvalExecutionRole";
 import { accountIdFromArn, deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole";
 import { harnessRuntimeFromResponse } from "./harness";
+import { serviceIdFromArn } from "./arn";
+import type { LogSource } from "./observability/types";
 
 const DEFAULT_INGESTION_WAIT_MS = 180_000;
 const DATASET_EXAMPLES_BATCH_LIMIT = 1000;
@@ -225,6 +227,42 @@ export class EvalClient implements CoreEvalClient {
     private readonly cloudWatch: CloudWatchClient = new CloudWatchClient(clients),
   ) {}
 
+  async resourceOnlineEvaluationLogSource(
+    onlineEvaluationConfigId: string,
+    options: CoreOptions,
+  ): Promise<LogSource> {
+    const onlineEvaluationConfig = await this.getOnlineEvaluationConfig(
+      onlineEvaluationConfigId,
+      options,
+    );
+    const cloudWatchOutputConfig = onlineEvaluationConfig.outputConfig?.cloudWatchConfig;
+    if (cloudWatchOutputConfig?.logGroupName)
+      return { logGroupName: cloudWatchOutputConfig.logGroupName };
+    if (onlineEvaluationConfig.outputConfig && !cloudWatchOutputConfig) {
+      throw new InputValidationError("Logs require a CloudWatch output configuration.");
+    }
+    if (cloudWatchOutputConfig?.resultDestination !== "SOURCE_LOG_GROUP") {
+      return {
+        logGroupName: `/aws/bedrock-agentcore/evaluations/results/${onlineEvaluationConfig.onlineEvaluationConfigId ?? serviceIdFromArn(onlineEvaluationConfigId)}`,
+      };
+    }
+    const cloudWatchSourceConfig = onlineEvaluationConfig.dataSourceConfig?.cloudWatchLogs;
+    if (
+      cloudWatchSourceConfig?.logGroupNamePrefixes?.length ||
+      (cloudWatchSourceConfig?.logGroupNames?.length ?? 0) > 1
+    ) {
+      throw new InputValidationError(
+        "Results use multiple source log groups or prefixes. " +
+          "Read them directly in CloudWatch Logs, or configure one explicit source log group.",
+      );
+    }
+    const sourceLogGroupName = cloudWatchSourceConfig?.logGroupNames?.[0];
+    if (!sourceLogGroupName) {
+      throw new MalformedServiceResponseError("Missing source log group name.");
+    }
+    return { logGroupName: sourceLogGroupName };
+  }
+
   async createEvaluator(
     request: CreateEvaluatorRequest,
     options: CoreOptions,

src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.44cbc27686231142.json

@@ -0,0 +1,62 @@
+{
+  "batchEvaluationId": "cli_logs_success_20261013-ab043c87dc",
+  "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_success_20261013-ab043c87dc",
+  "batchEvaluationName": "cli_logs_success_20261013",
+  "status": "COMPLETED",
+  "createdAt": {
+    "$date": "2026-10-13T16:26:05.707Z"
+  },
+  "evaluators": [
+    {
+      "evaluatorId": "Builtin.Helpfulness"
+    }
+  ],
+  "dataSourceConfig": {
+    "cloudWatchLogs": {
+      "serviceNames": [
+        "ExampleAgent.DEFAULT"
+      ],
+      "logGroupNames": [
+        "/aws/bedrock-agentcore/runtimes/ExampleAgent-abc123def4-DEFAULT"
+      ],
+      "filterConfig": {
+        "timeRange": {
+          "startTime": {
+            "$date": "2026-09-22T21:11:00.000Z"
+          },
+          "endTime": {
+            "$date": "2026-09-22T21:13:43.770Z"
+          }
+        }
+      }
+    }
+  },
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default",
+      "logStreamName": "run-cli_logs_success_20261013-ab043c87dc",
+      "metricsNamespace": "Bedrock-AgentCore/Evaluations",
+      "resultDestination": "DEDICATED_LOG_GROUP"
+    }
+  },
+  "evaluationResults": {
+    "numberOfSessionsCompleted": 2,
+    "numberOfSessionsInProgress": 0,
+    "numberOfSessionsFailed": 0,
+    "totalNumberOfSessions": 2,
+    "numberOfSessionsIgnored": 0,
+    "evaluatorSummaries": [
+      {
+        "evaluatorId": "Builtin.Helpfulness",
+        "statistics": {
+          "averageScore": 0.83
+        },
+        "totalEvaluated": 2,
+        "totalFailed": 0
+      }
+    ]
+  },
+  "updatedAt": {
+    "$date": "2026-10-13T16:27:10.308Z"
+  }
+}

src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.7e6d701230e33025.json

@@ -0,0 +1,52 @@
+{
+  "batchEvaluationId": "cli_logs_errors_20261013-0ce8bc1c19",
+  "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_errors_20261013-0ce8bc1c19",
+  "batchEvaluationName": "cli_logs_errors_20261013",
+  "status": "FAILED",
+  "createdAt": {
+    "$date": "2026-10-13T16:26:49.738Z"
+  },
+  "evaluators": [
+    {
+      "evaluatorId": "Builtin.Helpfulness"
+    }
+  ],
+  "dataSourceConfig": {
+    "cloudWatchLogs": {
+      "serviceNames": [
+        "CliEvalBugbash.DEFAULT"
+      ],
+      "logGroupNames": [
+        "/aws/bedrock-agentcore/cli-eval-errors-20261013"
+      ],
+      "filterConfig": {
+        "sessionIds": [
+          "cli-error-session-1",
+          "cli-error-session-2"
+        ]
+      }
+    }
+  },
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default",
+      "logStreamName": "run-cli_logs_errors_20261013-0ce8bc1c19",
+      "metricsNamespace": "Bedrock-AgentCore/Evaluations",
+      "resultDestination": "DEDICATED_LOG_GROUP"
+    }
+  },
+  "evaluationResults": {
+    "numberOfSessionsCompleted": 0,
+    "numberOfSessionsInProgress": 0,
+    "numberOfSessionsFailed": 2,
+    "totalNumberOfSessions": 2,
+    "numberOfSessionsIgnored": 0,
+    "evaluatorSummaries": []
+  },
+  "errorDetails": [
+    "All 2 sessions failed during batch evaluation."
+  ],
+  "updatedAt": {
+    "$date": "2026-10-13T16:27:52.679Z"
+  }
+}

src/handlers/eval/batch-evaluation/get/__fixtures__/GetBatchEvaluationCommand.9722e50585569cfc.json

@@ -0,0 +1,59 @@
+{
+  "batchEvaluationId": "cli_logs_missing_spans_20261013-c5290ec8b4",
+  "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/cli_logs_missing_spans_20261013-c5290ec8b4",
+  "batchEvaluationName": "cli_logs_missing_spans_20261013",
+  "status": "FAILED",
+  "createdAt": {
+    "$date": "2026-10-13T16:28:39.096Z"
+  },
+  "evaluators": [
+    {
+      "evaluatorId": "Builtin.Helpfulness"
+    }
+  ],
+  "dataSourceConfig": {
+    "cloudWatchLogs": {
+      "serviceNames": [
+        "CliEvalBugbash.DEFAULT"
+      ],
+      "logGroupNames": [
+        "/aws/bedrock-agentcore/cli-eval-errors-20261013"
+      ],
+      "filterConfig": {
+        "sessionIds": [
+          "cli-error-session-1",
+          "cli-error-session-2"
+        ]
+      }
+    }
+  },
+  "outputConfig": {
+    "cloudWatchConfig": {
+      "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default",
+      "logStreamName": "run-cli_logs_missing_spans_20261013-c5290ec8b4",
+      "metricsNamespace": "Bedrock-AgentCore/Evaluations",
+      "resultDestination": "DEDICATED_LOG_GROUP"
+    }
+  },
+  "evaluationResults": {
+    "numberOfSessionsCompleted": 0,
+    "numberOfSessionsInProgress": 0,
+    "numberOfSessionsFailed": 2,
+    "totalNumberOfSessions": 2,
+    "numberOfSessionsIgnored": 0,
+    "evaluatorSummaries": [
+      {
+        "evaluatorId": "Builtin.Helpfulness",
+        "statistics": {},
+        "totalEvaluated": 0,
+        "totalFailed": 2
+      }
+    ]
+  },
+  "errorDetails": [
+    "All 2 sessions failed during batch evaluation."
+  ],
+  "updatedAt": {
+    "$date": "2026-10-13T16:29:42.078Z"
+  }
+}