From 952b73bb8aa3c7b50b837c49b721a27622d0cd16 Mon Sep 17 00:00:00 2001 From: jariy17 Date: Mon, 12 Oct 2026 17:37:13 +0000 Subject: [PATCH 1/2] refactor(eval)!: remove batch insights and online insights Remove the eval batch-insights and eval online-insight command groups, the add/remove online-insight project resource, their TUI screens and pickers, and the insights and clusteringConfig fields of online evaluation configs, which now require at least one evaluator. --- src/components/BatchInsightsPicker.tsx | 79 --- src/components/OnlineInsightPicker.tsx | 93 --- src/components/Root.tsx | 50 -- src/components/wizard/fields.tsx | 10 +- src/core/batchInsights.test.tsx | 167 ----- src/core/dev/inspector/resources.ts | 1 - src/core/eval.tsx | 212 ------- src/core/filteredPaginator.test.ts | 2 +- src/core/project/manager.tsx | 4 +- ...tAgentRuntimeCommand.9f77333d1b9dcf5d.json | 47 -- ...tAgentRuntimeCommand.aea9a32f8da3ce46.json | 43 -- ...tchEvaluationCommand.78dd8df2fc94e379.json | 36 -- ...chEvaluationsCommand.23f97c9dcdd6350b.json | 110 ---- ...tchEvaluationCommand.7af4b4e0e2cc1c3d.json | 21 - ...tchEvaluationCommand.bf8946c07edd8624.json | 22 - ...tchEvaluationCommand.72d5c5e77b44a747.json | 5 - .../__fixtures__/get.golden.json | 32 - .../__fixtures__/list.golden.json | 18 - .../__fixtures__/run.golden.json | 19 - .../__fixtures__/stop.golden.json | 5 - .../batch-insights.fixture.test.tsx | 136 ----- .../batch-insights.screen.test.tsx | 135 ----- .../batch-insights/batch-insights.test.tsx | 276 --------- .../eval/batch-insights/get/index.tsx | 21 - .../eval/batch-insights/get/screen.tsx | 30 - src/handlers/eval/batch-insights/index.tsx | 22 - .../eval/batch-insights/list/index.tsx | 26 - .../eval/batch-insights/list/screen.tsx | 18 - .../eval/batch-insights/run/index.tsx | 65 -- src/handlers/eval/batch-insights/screen.tsx | 6 - .../eval/batch-insights/stop/index.tsx | 17 - src/handlers/eval/index.tsx | 6 - ...luationConfigCommand.38066f8427f9fba0.json | 14 - ...luationConfigCommand.39087bc3e00af960.json | 5 - ...tAgentRuntimeCommand.80bb567edcaa4bb1.json | 46 -- ...luationConfigCommand.39087bc3e00af960.json | 43 -- ...luationConfigCommand.859097834b7e85cf.json | 6 - ...luationConfigCommand.d3a07848de1b01eb.json | 42 -- ...uationConfigsCommand.23f97c9dcdd6350b.json | 447 -------------- ...luationConfigCommand.3ab70a72f531453e.json | 9 - ...aluationConfigCommand.a3122d8485a8784.json | 9 - ...luationConfigCommand.db059fecaf31eb68.json | 9 - .../__fixtures__/create.golden.json | 12 - .../__fixtures__/delete.golden.json | 5 - .../__fixtures__/get.golden.json | 39 -- .../__fixtures__/list.golden.json | 57 -- .../__fixtures__/pause.golden.json | 7 - .../__fixtures__/resume.golden.json | 7 - .../__fixtures__/update.golden.json | 7 - .../eval/online-insight/create/index.tsx | 115 ---- .../eval/online-insight/delete/index.tsx | 17 - .../eval/online-insight/get/index.tsx | 17 - .../eval/online-insight/get/screen.tsx | 78 --- src/handlers/eval/online-insight/index.tsx | 28 - .../eval/online-insight/list/index.tsx | 23 - .../eval/online-insight/list/screen.tsx | 18 - .../online-insight.screen.test.tsx | 195 ------ .../online-insight/online-insight.test.tsx | 392 ------------ .../eval/online-insight/pause/index.tsx | 23 - .../eval/online-insight/resume/index.tsx | 23 - src/handlers/eval/online-insight/screen.tsx | 6 - .../eval/online-insight/update/index.tsx | 106 ---- src/handlers/eval/types.tsx | 79 --- src/handlers/project/add/add.screen.test.tsx | 1 - src/handlers/project/add/index.ts | 3 - .../project/add/online-insight/index.test.ts | 281 --------- .../project/add/online-insight/index.ts | 167 ----- .../online-insight.screen.test.tsx | 276 --------- .../project/add/online-insight/screen.tsx | 568 ------------------ src/handlers/project/remove/index.test.ts | 20 - src/handlers/project/remove/index.ts | 1 - .../project/remove/remove.screen.test.tsx | 10 - src/handlers/project/remove/screen.tsx | 14 +- src/handlers/project/types.ts | 5 - src/handlers/root.test.tsx | 1 - src/projectSchemas/online-eval-config.test.ts | 23 +- src/projectSchemas/online-eval-config.ts | 29 +- src/testing/TestCoreClient.tsx | 108 ---- 78 files changed, 12 insertions(+), 5113 deletions(-) delete mode 100644 src/components/BatchInsightsPicker.tsx delete mode 100644 src/components/OnlineInsightPicker.tsx delete mode 100644 src/core/batchInsights.test.tsx delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.aea9a32f8da3ce46.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/GetBatchEvaluationCommand.78dd8df2fc94e379.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.7af4b4e0e2cc1c3d.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.bf8946c07edd8624.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/StopBatchEvaluationCommand.72d5c5e77b44a747.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/get.golden.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/list.golden.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/run.golden.json delete mode 100644 src/handlers/eval/batch-insights/__fixtures__/stop.golden.json delete mode 100644 src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx delete mode 100644 src/handlers/eval/batch-insights/batch-insights.screen.test.tsx delete mode 100644 src/handlers/eval/batch-insights/batch-insights.test.tsx delete mode 100644 src/handlers/eval/batch-insights/get/index.tsx delete mode 100644 src/handlers/eval/batch-insights/get/screen.tsx delete mode 100644 src/handlers/eval/batch-insights/index.tsx delete mode 100644 src/handlers/eval/batch-insights/list/index.tsx delete mode 100644 src/handlers/eval/batch-insights/list/screen.tsx delete mode 100644 src/handlers/eval/batch-insights/run/index.tsx delete mode 100644 src/handlers/eval/batch-insights/screen.tsx delete mode 100644 src/handlers/eval/batch-insights/stop/index.tsx delete mode 100644 src/handlers/eval/online-insight/__fixtures__/CreateOnlineEvaluationConfigCommand.38066f8427f9fba0.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/DeleteOnlineEvaluationConfigCommand.39087bc3e00af960.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.39087bc3e00af960.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.d3a07848de1b01eb.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.3ab70a72f531453e.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.a3122d8485a8784.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.db059fecaf31eb68.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/create.golden.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/delete.golden.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/get.golden.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/list.golden.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/pause.golden.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/resume.golden.json delete mode 100644 src/handlers/eval/online-insight/__fixtures__/update.golden.json delete mode 100644 src/handlers/eval/online-insight/create/index.tsx delete mode 100644 src/handlers/eval/online-insight/delete/index.tsx delete mode 100644 src/handlers/eval/online-insight/get/index.tsx delete mode 100644 src/handlers/eval/online-insight/get/screen.tsx delete mode 100644 src/handlers/eval/online-insight/index.tsx delete mode 100644 src/handlers/eval/online-insight/list/index.tsx delete mode 100644 src/handlers/eval/online-insight/list/screen.tsx delete mode 100644 src/handlers/eval/online-insight/online-insight.screen.test.tsx delete mode 100644 src/handlers/eval/online-insight/online-insight.test.tsx delete mode 100644 src/handlers/eval/online-insight/pause/index.tsx delete mode 100644 src/handlers/eval/online-insight/resume/index.tsx delete mode 100644 src/handlers/eval/online-insight/screen.tsx delete mode 100644 src/handlers/eval/online-insight/update/index.tsx delete mode 100644 src/handlers/project/add/online-insight/index.test.ts delete mode 100644 src/handlers/project/add/online-insight/index.ts delete mode 100644 src/handlers/project/add/online-insight/online-insight.screen.test.tsx delete mode 100644 src/handlers/project/add/online-insight/screen.tsx diff --git a/src/components/BatchInsightsPicker.tsx b/src/components/BatchInsightsPicker.tsx deleted file mode 100644 index 649a146a0a..0000000000 --- a/src/components/BatchInsightsPicker.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import type { BatchEvaluationSummary } from "@aws-sdk/client-bedrock-agentcore"; -import { useNavigate } from "react-router"; -import type { ScreenProps } from "../handlers/types"; -import { coreOptsFromCtx } from "../handlers/utils"; -import { formatTimestamp } from "./formatTimestamp"; -import { PaginatedTablePicker } from "./PaginatedTablePicker"; -import type { DataTableColumn } from "./ui/data-table"; - -interface BatchInsightsRow extends Record { - batchEvaluationId: string; - name: string; - status: string; - updatedAt: string; -} - -const batchInsightsColumns = [ - { key: "name", header: "name", flex: true }, - { key: "status", header: "status", width: 22 }, - { - key: "updatedAt", - header: "updated UTC", - width: 16, - render: formatTimestamp, - }, -] satisfies DataTableColumn[]; - -function toRow(summary: BatchEvaluationSummary): BatchInsightsRow { - const id = summary.batchEvaluationId ?? ""; - return { - batchEvaluationId: id, - name: summary.batchEvaluationName ?? id, - status: summary.status ?? "-", - updatedAt: summary.updatedAt?.toISOString() ?? "-", - }; -} - -export interface BatchInsightsPickerProps extends ScreenProps { - breadcrumb: string[]; - description?: string; - onSelect: (batchEvaluationId: string) => void; - onEscape?: () => void; -} - -export function BatchInsightsPicker({ - ctx, - core, - breadcrumb, - description, - onSelect, - onEscape, -}: BatchInsightsPickerProps) { - const opts = coreOptsFromCtx(ctx); - const navigate = useNavigate(); - const goBack = onEscape ?? (() => navigate("/" + breadcrumb.slice(0, -1).join("/"))); - - return ( - { - const response = await core.eval.listBatchInsights(token, pageSize, opts); - return { - items: response.batchEvaluations ?? [], - nextToken: response.nextToken, - }; - }} - toRow={toRow} - columns={batchInsightsColumns} - getValue={(row) => row.batchEvaluationId} - onSelect={onSelect} - onBack={goBack} - loadingMessage="loading batch insights…" - errorMessage={(error) => `Error: ${error.message}`} - emptyMessage="No batch insights found in this Region." - emptyPageMessage="No batch insights on this page." - /> - ); -} diff --git a/src/components/OnlineInsightPicker.tsx b/src/components/OnlineInsightPicker.tsx deleted file mode 100644 index 2b5f57b999..0000000000 --- a/src/components/OnlineInsightPicker.tsx +++ /dev/null @@ -1,93 +0,0 @@ -import type { OnlineEvaluationConfigSummary } from "@aws-sdk/client-bedrock-agentcore-control"; -import { useNavigate } from "react-router"; -import type { ScreenProps } from "../handlers/types"; -import { coreOptsFromCtx } from "../handlers/utils"; -import { formatTimestamp } from "./formatTimestamp"; -import { PaginatedTablePicker } from "./PaginatedTablePicker"; -import type { DataTableColumn } from "./ui/data-table"; - -// OnlineInsightRow is the flat, display-ready shape the table renders. It also -// satisfies DataTable's `T extends Record` constraint, which the -// SDK's OnlineEvaluationConfigSummary interface does not. ListOnlineInsights -// returns the same summary type as the online-eval list; richer detail like -// sampling rate and clustering comes from GetOnlineInsight. -interface OnlineInsightRow extends Record { - configId: string; - configName: string; - status: string; - executionStatus: string; - updatedAt: string; -} - -const onlineInsightColumns = [ - { key: "configName", header: "name", flex: true }, - { key: "status", header: "status", width: 12 }, - { key: "executionStatus", header: "execution", width: 11 }, - { - key: "updatedAt", - header: "updated UTC", - width: 16, - render: formatTimestamp, - }, -] satisfies DataTableColumn[]; - -function toRow(config: OnlineEvaluationConfigSummary): OnlineInsightRow { - const id = config.onlineEvaluationConfigId ?? ""; - return { - configId: id, - configName: config.onlineEvaluationConfigName ?? id, - status: config.status ?? "-", - executionStatus: config.executionStatus ?? "-", - updatedAt: config.updatedAt?.toISOString() ?? "-", - }; -} - -export interface OnlineInsightPickerProps extends ScreenProps { - breadcrumb: string[]; - description?: string; - onSelect: (configId: string) => void; - onEscape?: () => void; -} - -/** - * Fetches the caller's online insight configs and renders them as a navigable - * table. The shared body of every "pick an insight config" screen. Esc returns to - * the parent menu derived from the breadcrumb unless a host supplies its own - * onEscape. - */ -export function OnlineInsightPicker({ - ctx, - core, - breadcrumb, - description, - onSelect, - onEscape, -}: OnlineInsightPickerProps) { - const opts = coreOptsFromCtx(ctx); - const navigate = useNavigate(); - const goBack = onEscape ?? (() => navigate("/" + breadcrumb.slice(0, -1).join("/"))); - - return ( - { - const response = await core.eval.listOnlineInsights(token, pageSize, opts); - return { - items: response.onlineEvaluationConfigs ?? [], - nextToken: response.nextToken, - }; - }} - toRow={toRow} - columns={onlineInsightColumns} - getValue={(row) => row.configId} - onSelect={onSelect} - onBack={goBack} - loadingMessage="loading online insight configs…" - errorMessage={(error) => `Error: ${error.message}`} - emptyMessage="No online insight configs found in this Region." - emptyPageMessage="No online insight configs on this page." - /> - ); -} diff --git a/src/components/Root.tsx b/src/components/Root.tsx index 9aeea9fe81..a8d55c98e4 100644 --- a/src/components/Root.tsx +++ b/src/components/Root.tsx @@ -59,21 +59,12 @@ import { OnlineEvalGetScreen, OnlineEvalGetJsonScreen, } from "../handlers/eval/online-eval/get/screen.tsx"; -import { OnlineInsightScreen } from "../handlers/eval/online-insight/screen.tsx"; -import { OnlineInsightListScreen } from "../handlers/eval/online-insight/list/screen.tsx"; -import { - OnlineInsightGetScreen, - OnlineInsightGetJsonScreen, -} from "../handlers/eval/online-insight/get/screen.tsx"; import { BatchEvaluationScreen } from "../handlers/eval/batch-evaluation/screen.tsx"; import { BatchEvaluationListScreen } from "../handlers/eval/batch-evaluation/list/screen.tsx"; import { BatchEvaluationGetJsonScreen } from "../handlers/eval/batch-evaluation/get/screen.tsx"; import { RecommendationScreen } from "../handlers/eval/recommendation/screen.tsx"; import { RecommendationListScreen } from "../handlers/eval/recommendation/list/screen.tsx"; import { RecommendationGetJsonScreen } from "../handlers/eval/recommendation/get/screen.tsx"; -import { BatchInsightsScreen } from "../handlers/eval/batch-insights/screen.tsx"; -import { BatchInsightsListScreen } from "../handlers/eval/batch-insights/list/screen.tsx"; -import { BatchInsightsGetJsonScreen } from "../handlers/eval/batch-insights/get/screen.tsx"; import { DatasetScreen } from "../handlers/eval/dataset/screen.tsx"; import { DatasetListScreen } from "../handlers/eval/dataset/list/screen.tsx"; import { DatasetGetScreen, DatasetGetJsonScreen } from "../handlers/eval/dataset/get/screen.tsx"; @@ -136,7 +127,6 @@ import { AddGatewayScreen } from "../handlers/project/add/gateway/screen.tsx"; import { AddGatewayTargetScreen } from "../handlers/project/add/gateway-target/screen.tsx"; import { AddGatewayConnectorScreen } from "../handlers/project/add/gateway-connector/screen.tsx"; import { AddOnlineEvalScreen } from "../handlers/project/add/online-eval/screen.tsx"; -import { AddOnlineInsightScreen } from "../handlers/project/add/online-insight/screen.tsx"; import { AddHarnessScreen } from "../handlers/project/add/harness/screen.tsx"; import { AddConfigBundleScreen } from "../handlers/project/add/config-bundle/screen.tsx"; import { AddPolicyEngineScreen } from "../handlers/project/add/policy-engine/screen.tsx"; @@ -665,26 +655,6 @@ function RouteTable({ ctx, core }: ScreenProps) { path="agentcore/eval/online-eval/get/:configId/json" element={} /> - } - /> - } - /> - } - /> - } - /> - } - /> } /> } /> - } - /> - } - /> - } - /> - } - /> } /> } /> - } - /> } /> Promise): EvalClient { - return new EvalClient({ - data: () => ({ send: mock(send) }) as never, - } as unknown as AwsClients); -} - -function getEvalClient(send: (command: GetBatchEvaluationCommand) => Promise): EvalClient { - return new EvalClient({ - data: () => ({ send: mock(send) }) as never, - } as unknown as AwsClients); -} - -describe("EvalClient.getBatchInsights", () => { - test("gets an Insights job without reading CloudWatch evaluation results", async () => { - const job = { - batchEvaluationId: "insights-1", - batchEvaluationArn: - "arn:aws:bedrock-agentcore:us-west-2:123456789012:batch-evaluate/insights-1", - batchEvaluationName: "insights-1", - status: "COMPLETED" as const, - createdAt: new Date("2026-08-25T12:00:00.000Z"), - insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], - outputConfig: { - cloudWatchConfig: { - logGroupName: "/aws/example", - logStreamName: "results", - }, - }, - }; - const client = getEvalClient(async (command) => { - expect(command).toBeInstanceOf(GetBatchEvaluationCommand); - expect(command.input).toEqual({ batchEvaluationId: "insights-1" }); - return job; - }); - - await expect(client.getBatchInsights("insights-1", options)).resolves.toEqual(job); - }); - - test("rejects an evaluator-only Batch Evaluation", async () => { - const client = getEvalClient(async () => evaluation("evaluation-1")); - - await expect(client.getBatchInsights("evaluation-1", options)).rejects.toThrow( - 'batch evaluation "evaluation-1" is not a batch insights run', - ); - }); -}); - -describe("EvalClient.listBatchInsights", () => { - test("filters the final Batch Evaluation page", async () => { - const insightsJob = insight("insights-1"); - const client = evalClient(async (command) => { - expect(command).toBeInstanceOf(ListBatchEvaluationsCommand); - expect(command.input).toEqual({ nextToken: "page-2", maxResults: undefined }); - return { batchEvaluations: [evaluation("evaluation-1"), insightsJob] }; - }); - - await expect(client.listBatchInsights("page-2", 10, options)).resolves.toEqual({ - batchEvaluations: [insightsJob], - nextToken: undefined, - }); - }); -}); - -describe("EvalClient.startBatchInsights", () => { - test("resolves an online evaluation ID to its ARN and omits analysis configuration", async () => { - const arn = - "arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/config-1"; - const client = new EvalClient({ - control: () => - ({ - send: mock(async (command: GetOnlineEvaluationConfigCommand) => { - expect(command).toBeInstanceOf(GetOnlineEvaluationConfigCommand); - expect(command.input).toEqual({ onlineEvaluationConfigId: "config-1" }); - return { onlineEvaluationConfigArn: arn }; - }), - }) as never, - data: () => - ({ - send: mock(async (command: StartBatchEvaluationCommand) => { - expect(command).toBeInstanceOf(StartBatchEvaluationCommand); - expect(command.input).toEqual({ - batchEvaluationName: "insights-1", - dataSourceConfig: { - onlineEvaluationConfigSource: { - onlineEvaluationConfigArn: arn, - timeRange: undefined, - }, - }, - description: undefined, - evaluators: undefined, - insights: undefined, - kmsKeyArn: undefined, - }); - return {}; - }), - }) as never, - } as unknown as AwsClients); - - await client.startBatchInsights( - { - name: "insights-1", - source: { origin: "online-eval", onlineEvaluationConfigId: "config-1" }, - }, - options, - ); - }); - - test("uses an online evaluation ARN without a control-plane lookup", async () => { - const arn = - "arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/config-1"; - const client = new EvalClient({ - control: () => - ({ - send: mock(async () => { - throw new Error("unexpected control-plane lookup"); - }), - }) as never, - data: () => - ({ - send: mock(async (command: StartBatchEvaluationCommand) => { - expect(command.input.dataSourceConfig).toEqual({ - onlineEvaluationConfigSource: { - onlineEvaluationConfigArn: arn, - timeRange: undefined, - }, - }); - return {}; - }), - }) as never, - } as unknown as AwsClients); - - await client.startBatchInsights( - { - name: "insights-1", - source: { origin: "online-eval", onlineEvaluationConfigId: arn }, - }, - options, - ); - }); -}); diff --git a/src/core/dev/inspector/resources.ts b/src/core/dev/inspector/resources.ts index 1c20b3257f..a893bc6483 100644 --- a/src/core/dev/inspector/resources.ts +++ b/src/core/dev/inspector/resources.ts @@ -56,7 +56,6 @@ export function handleResources(deps: InspectorDeps): HttpResponse { name: config.name, agent: config.agent, evaluators: config.evaluators, - insights: config.insights, samplingRate: config.samplingRate, description: config.description, logGroupNames: config.logGroupNames, diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 26cb71a1f6..58bec5abdc 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -132,13 +132,6 @@ import type { CreateTargetBasedABTestInput, CreateDatasetInput, CreateOnlineEvalInput, - CreateOnlineInsightInput, - CreateOnlineInsightResponse, - UpdateOnlineInsightInput, - GetOnlineInsightResponse, - ListOnlineInsightsResponse, - UpdateOnlineInsightResponse, - DeleteOnlineInsightResponse, EvaluateInput, EvaluateResult, GetBatchEvaluationResult, @@ -149,7 +142,6 @@ import type { InvokeDatasetInput, InvokeDatasetResult, SpanRecord, - StartBatchInsightsInput, StartBatchEvaluationInput, StartRecommendationInput, UpdateConfigurationBundleInput, @@ -168,7 +160,6 @@ import type { Addition } from "./datasetDiff"; import type { AwsClients, CoreFetch, CoreOptions } from "./types"; import { isValidEvaluatorModelId, type EvaluatorModelProvider } from "../projectSchemas/evaluator"; import type { Logger } from "../logging"; -import { FilteredPaginator } from "./filteredPaginator"; import { toClientConfig } from "./utils"; import { grantOnlineEvalScope } from "./onlineEvalExecutionRole"; import { accountIdFromArn, deleteAbTestRole, provisionAbTestRole } from "./abTestExecutionRole"; @@ -209,7 +200,6 @@ const EVAL_INSIGHTS_ROW_LIMIT: InsightsRowLimit = { ), }; -const DEFAULT_BATCH_INSIGHTS_PAGE_SIZE = 50; const BATCH_EVALUATION_RESULT_MAX_PAGES = 100; // noopLogger is the default for the optional logger arg so callers that don't @@ -223,8 +213,6 @@ const noopLogger: Logger = { child: () => noopLogger, }; -const DEFAULT_ONLINE_INSIGHT_PAGE_SIZE = 100; - export class EvalClient implements CoreEvalClient { constructor( private readonly clients: AwsClients, @@ -496,14 +484,6 @@ export class EvalClient implements CoreEvalClient { } } - async getBatchInsights(id: string, options: CoreOptions): Promise { - const { detail } = await this.getBatchEvaluation(id, options, { includeResults: false }); - if (!EvalClient.isBatchInsights(detail)) { - throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); - } - return detail; - } - async listBatchEvaluations( nextToken: string | undefined, maxResults: number | undefined, @@ -694,25 +674,6 @@ export class EvalClient implements CoreEvalClient { ); } - async listBatchInsights( - nextToken: string | undefined, - maxResults: number | undefined, - options: CoreOptions, - ): Promise { - const page = await FilteredPaginator.paginate({ - fetchPage: async (token, size) => { - const r = await this.listBatchEvaluations(token, size, options); - return { items: r.batchEvaluations ?? [], nextToken: r.nextToken }; - }, - predicate: EvalClient.isBatchInsights, - nextToken, - maxResults, - defaultPageSize: DEFAULT_BATCH_INSIGHTS_PAGE_SIZE, - resourceLabel: "Batch Insights", - }); - return { batchEvaluations: page.items, nextToken: page.nextToken }; - } - async startBatchEvaluation( input: StartBatchEvaluationInput, options: CoreOptions, @@ -731,23 +692,6 @@ export class EvalClient implements CoreEvalClient { ); } - async startBatchInsights( - input: StartBatchInsightsInput, - options: CoreOptions, - ): Promise { - const dataSourceConfig = await this.dataSourceConfigForSource(input.source, options); - return this.clients.data(toClientConfig(options)).send( - new StartBatchEvaluationCommand({ - batchEvaluationName: input.name, - description: input.description, - insights: input.insightIds?.map((insightId) => ({ insightId })), - evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), - dataSourceConfig, - kmsKeyArn: input.kmsKeyArn, - }), - ); - } - // dataSourceConfigForSource maps a resolved SessionSourceValue to the data-plane // dataSourceConfig union. The agent arm reuses the same runtime resolution + // log-group derivation the control-plane agentDataSource uses, then attaches the @@ -795,10 +739,6 @@ export class EvalClient implements CoreEvalClient { }; } - private static isBatchInsights(job: { insights?: unknown[] }): boolean { - return Boolean(job.insights?.length); - } - async getTracesForAgent(input: GetTracesInput, options: CoreOptions): Promise { const qualifier = input.endpoint ?? DEFAULT_ENDPOINT_QUALIFIER; @@ -1075,158 +1015,6 @@ export class EvalClient implements CoreEvalClient { : retryWhileRolePropagates(() => control.send(command)); } - async createOnlineInsight( - input: CreateOnlineInsightInput, - options: CoreOptions, - ): Promise { - const dataSourceConfig = - input.agent !== undefined - ? await agentDataSource(input.agent, input.endpoint, this.clients, options) - : input.dataSourceConfig; - const control = this.clients.control(toClientConfig(options)); - - const command = new CreateOnlineEvaluationConfigCommand({ - onlineEvaluationConfigName: input.name, - description: input.description, - rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters), - dataSourceConfig, - insights: input.insightIds.map((insightId) => ({ insightId })), - clusteringConfig: input.clusteringConfig, - evaluationExecutionRoleArn: input.evaluationExecutionRoleArn, - enableOnCreate: input.enableOnCreate ?? true, - }); - - return control.send(command); - } - - async updateOnlineInsight( - id: string, - update: UpdateOnlineInsightInput, - options: CoreOptions, - ): Promise { - const control = this.clients.control(toClientConfig(options)); - // This GET is both the merge base (UpdateOnlineEvaluationConfig replaces the - // whole rule/insights, so unset fields must be carried over) and the guard - // that rejects a plain online-EVAL config, which carries no insights. - const current = await control.send( - new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id }), - ); - if ((current.insights?.length ?? 0) === 0) - throw new InputValidationError(`"${id}" is not an online-insight config`, { - meta: { onlineEvaluationConfigId: id }, - }); - - const samplingPercentage = - update.samplingRate ?? current.rule?.samplingConfig?.samplingPercentage; - const sessionTimeoutMinutes = - update.sessionTimeoutMinutes ?? current.rule?.sessionConfig?.sessionTimeoutMinutes; - const filters = update.filters ?? current.rule?.filters; - - const insights = - update.insightIds !== undefined - ? update.insightIds.map((insightId) => ({ insightId })) - : current.insights; - const clusteringConfig = update.clusteringConfig ?? current.clusteringConfig; - - let dataSourceConfig = current.dataSourceConfig; - if (update.dataSourceConfig !== undefined) { - dataSourceConfig = update.dataSourceConfig; - } else if (update.agent !== undefined) { - dataSourceConfig = await agentDataSource( - update.agent, - update.clearEndpoint ? DEFAULT_ENDPOINT_QUALIFIER : update.endpoint, - this.clients, - options, - ); - } else if (update.clearEndpoint || update.endpoint !== undefined) { - const currentLogGroup = - current.dataSourceConfig && "cloudWatchLogs" in current.dataSourceConfig - ? current.dataSourceConfig.cloudWatchLogs?.logGroupNames?.[0] - : undefined; - const runtimeId = currentLogGroup ? runtimeIdFromLogGroup(currentLogGroup) : undefined; - if (!runtimeId) { - throw new InputValidationError( - `Online insight config "${id}" was not created from an agent; ` + - `pass --agent or --data-source-config to repoint it`, - { meta: { onlineEvaluationConfigId: id } }, - ); - } - const endpoint = update.clearEndpoint ? DEFAULT_ENDPOINT_QUALIFIER : update.endpoint; - dataSourceConfig = await agentDataSource(runtimeId, endpoint, this.clients, options); - } - - return control.send( - new UpdateOnlineEvaluationConfigCommand({ - onlineEvaluationConfigId: id, - rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), - dataSourceConfig, - insights, - clusteringConfig, - evaluationExecutionRoleArn: update.evaluationExecutionRoleArn, - }), - ); - } - - async getOnlineInsight(id: string, options: CoreOptions): Promise { - const control = this.clients.control(toClientConfig(options)); - const config = await control.send( - new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id }), - ); - if ((config.insights?.length ?? 0) === 0) - throw new InputValidationError(`"${id}" is not an online-insight config`, { - meta: { onlineEvaluationConfigId: id }, - }); - return config; - } - - async listOnlineInsights( - nextToken: string | undefined, - maxResults: number | undefined, - options: CoreOptions, - ): Promise { - const page = await FilteredPaginator.paginate({ - fetchPage: async (token, size) => { - const r = await this.listOnlineEvaluationConfigs(token, size, options); - return { items: r.onlineEvaluationConfigs ?? [], nextToken: r.nextToken }; - }, - predicate: (c) => (c.insights?.length ?? 0) > 0, - nextToken, - maxResults, - defaultPageSize: DEFAULT_ONLINE_INSIGHT_PAGE_SIZE, - resourceLabel: "Online insight", - }); - return { onlineEvaluationConfigs: page.items, nextToken: page.nextToken }; - } - - async setOnlineInsightExecutionStatus( - id: string, - executionStatus: "ENABLED" | "DISABLED", - options: CoreOptions, - ): Promise { - const config = await this.clients - .control(toClientConfig(options)) - .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); - if ((config.insights?.length ?? 0) === 0) - throw new InputValidationError(`"${id}" is not an online-insight config`, { - meta: { onlineEvaluationConfigId: id }, - }); - return this.setOnlineEvaluationExecutionStatus(id, executionStatus, options); - } - - async deleteOnlineInsight( - id: string, - options: CoreOptions, - ): Promise { - const config = await this.clients - .control(toClientConfig(options)) - .send(new GetOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); - if ((config.insights?.length ?? 0) === 0) - throw new InputValidationError(`"${id}" is not an online-insight config`, { - meta: { onlineEvaluationConfigId: id }, - }); - return this.deleteOnlineEvaluationConfig(id, options); - } - // updateOnlineEvaluationConfig fetches the current config and merges the // provided fields over it, because UpdateOnlineEvaluationConfig replaces the // whole `rule` (and, when endpoint changes, `dataSourceConfig`) rather than diff --git a/src/core/filteredPaginator.test.ts b/src/core/filteredPaginator.test.ts index 0c07086f5a..cd7f5c67a1 100644 --- a/src/core/filteredPaginator.test.ts +++ b/src/core/filteredPaginator.test.ts @@ -140,7 +140,7 @@ describe("FilteredPaginator", () => { nextToken: undefined, maxResults: 1, defaultPageSize: 10, - resourceLabel: "Online insight", + resourceLabel: "Gateway target", }), ).rejects.toBeInstanceOf(ResultTruncationError); }); diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index 68b2db702c..97f1ed9c90 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -779,8 +779,7 @@ export class FsProjectManager implements ProjectManager { projectSpec.configBundles.push(parseResource(ConfigBundleSchema, input.resourceConfig)); break; } - case "online-eval": - case "online-insight": { + case "online-eval": { projectSpec.onlineEvalConfigs.push( parseResource(OnlineEvalConfigSchema, input.resourceConfig), ); @@ -1782,7 +1781,6 @@ function toProjectSpecKey(resourceType: ProjectResource) { case "config-bundle": return "configBundles"; case "online-eval": - case "online-insight": return "onlineEvalConfigs"; case "memory": return "memories"; diff --git a/src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json b/src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json deleted file mode 100644 index 2d3b5e7136..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.9f77333d1b9dcf5d.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:runtime/asdf_MyAgent-3s5axvBC6Q", - "agentRuntimeName": "asdf_MyAgent", - "agentRuntimeId": "asdf_MyAgent-3s5axvBC6Q", - "agentRuntimeVersion": "1", - "createdAt": { - "$date": "2026-04-23T21:17:21.895Z" - }, - "lastUpdatedAt": { - "$date": "2026-04-23T21:17:35.159Z" - }, - "roleArn": "arn:aws:iam::685197708687:role/AgentCore-asdf-default-ApplicationAgentMyAgentRunti-KdyUbgImzDRK", - "networkConfiguration": { - "networkMode": "PUBLIC" - }, - "status": "READY", - "lifecycleConfiguration": { - "idleRuntimeSessionTimeout": 900, - "maxLifetime": 28800 - }, - "description": "AgentCore Runtime: asdf_MyAgent", - "workloadIdentityDetails": { - "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:workload-identity-directory/default/workload-identity/asdf_MyAgent-3s5axvBC6Q" - }, - "agentRuntimeArtifact": { - "codeConfiguration": { - "code": { - "s3": { - "bucket": "cdk-hnb659fds-assets-685197708687-us-west-2", - "prefix": "a07977786dda1e2e5be304cb7485237a19ed24d5e05b02e73ca91a43fd2e7280.zip" - } - }, - "runtime": "PYTHON_3_13", - "entryPoint": [ - "opentelemetry-instrument", - "main.py" - ] - } - }, - "environmentVariables": { - "AGENTCORE_GATEWAY_BUGBASHGW1776978672_AUTH_TYPE": "NONE", - "AGENTCORE_GATEWAY_BUGBASHGW1776978672_URL": "https://bugbashgw1776978672-zsy8cbqwts.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp" - }, - "metadataConfiguration": { - "requireMMDSV2": true - } -} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.aea9a32f8da3ce46.json b/src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.aea9a32f8da3ce46.json deleted file mode 100644 index 4ac5722e8b..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/GetAgentRuntimeCommand.aea9a32f8da3ce46.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/demoEval_demoAgent-qGEpAAE68u", - "agentRuntimeName": "demoEval_demoAgent", - "agentRuntimeId": "demoEval_demoAgent-qGEpAAE68u", - "agentRuntimeVersion": "1", - "createdAt": { - "$date": "2026-07-16T17:25:43.408Z" - }, - "lastUpdatedAt": { - "$date": "2026-07-16T17:25:56.842Z" - }, - "roleArn": "arn:aws:iam::123456789012:role/AgentCore-demoEval-defaul-ApplicationAgentDemoAgent-dvtfHphOLpXL", - "status": "READY", - "lifecycleConfiguration": { - "idleRuntimeSessionTimeout": 900, - "maxLifetime": 28800 - }, - "networkConfiguration": { - "networkMode": "PUBLIC" - }, - "description": "AgentCore Runtime: demoEval_demoAgent", - "workloadIdentityDetails": { - "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:123456789012:workload-identity-directory/default/workload-identity/demoEval_demoAgent-qGEpAAE68u" - }, - "agentRuntimeArtifact": { - "codeConfiguration": { - "code": { - "s3": { - "bucket": "cdk-hnb659fds-assets-123456789012-us-west-2", - "prefix": "57d7f4c7b51efde95d53b6c1c2d8aea9fa5d116ab4a0eb0dc53a6656a143ab30.zip" - } - }, - "runtime": "PYTHON_3_14", - "entryPoint": [ - "opentelemetry-instrument", - "main.py" - ] - } - }, - "metadataConfiguration": { - "requireMMDSV2": true - } -} diff --git a/src/handlers/eval/batch-insights/__fixtures__/GetBatchEvaluationCommand.78dd8df2fc94e379.json b/src/handlers/eval/batch-insights/__fixtures__/GetBatchEvaluationCommand.78dd8df2fc94e379.json deleted file mode 100644 index 78e411a1a7..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/GetBatchEvaluationCommand.78dd8df2fc94e379.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", - "batchEvaluationName": "golden_batch_insights_fixture", - "status": "COMPLETED", - "createdAt": { - "$date": "2026-08-21T19:27:51.136Z" - }, - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "dataSourceConfig": { - "cloudWatchLogs": { - "serviceNames": [ - "asdf_MyAgent.DEFAULT" - ], - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT" - ] - } - }, - "evaluationResults": { - "numberOfSessionsCompleted": 2, - "numberOfSessionsInProgress": 0, - "numberOfSessionsFailed": 0, - "totalNumberOfSessions": 2, - "numberOfSessionsIgnored": 0, - "evaluatorSummaries": [] - }, - "description": "Golden batch insights fixture", - "updatedAt": { - "$date": "2026-08-21T19:28:56.756Z" - } -} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/batch-insights/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json deleted file mode 100644 index 1e421c9131..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/ListBatchEvaluationsCommand.23f97c9dcdd6350b.json +++ /dev/null @@ -1,110 +0,0 @@ -{ - "batchEvaluations": [ - { - "batchEvaluationId": "golden_batch_evaluate-b957bb900a", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_evaluate-b957bb900a", - "batchEvaluationName": "golden_batch_evaluate", - "status": "COMPLETED", - "createdAt": { - "$date": "2026-08-11T21:18:32.724Z" - }, - "evaluators": [ - { - "evaluatorId": "Builtin.Helpfulness" - } - ], - "updatedAt": { - "$date": "2026-08-11T21:18:35.488Z" - } - }, - { - "batchEvaluationId": "golden_batch_evaluate_fixture685-d2967f9a21", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_evaluate_fixture685-d2967f9a21", - "batchEvaluationName": "golden_batch_evaluate_fixture685", - "status": "COMPLETED", - "createdAt": { - "$date": "2026-08-11T21:19:58.199Z" - }, - "evaluators": [ - { - "evaluatorId": "Builtin.Helpfulness" - } - ], - "updatedAt": { - "$date": "2026-08-11T21:20:01.334Z" - } - }, - { - "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", - "batchEvaluationName": "golden_batch_insights_fixture", - "status": "COMPLETED", - "createdAt": { - "$date": "2026-08-21T19:27:51.136Z" - }, - "description": "Golden batch insights fixture", - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "updatedAt": { - "$date": "2026-08-21T19:28:56.756Z" - } - }, - { - "batchEvaluationId": "sim_live_1f9e-cbcc074b38", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/sim_live_1f9e-cbcc074b38", - "batchEvaluationName": "sim_live_1f9e", - "status": "COMPLETED", - "createdAt": { - "$date": "2026-08-13T22:26:21.948Z" - }, - "evaluators": [ - { - "evaluatorId": "Builtin.Helpfulness" - } - ], - "updatedAt": { - "$date": "2026-08-13T22:27:25.296Z" - } - }, - { - "batchEvaluationId": "sim_live_6275-c9338a9ec5", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/sim_live_6275-c9338a9ec5", - "batchEvaluationName": "sim_live_6275", - "status": "FAILED", - "createdAt": { - "$date": "2026-08-13T22:18:02.147Z" - }, - "evaluators": [ - { - "evaluatorId": "Builtin.Helpfulness" - } - ], - "errorDetails": [ - "All 2 sessions failed during batch evaluation." - ], - "updatedAt": { - "$date": "2026-08-13T22:19:05.784Z" - } - }, - { - "batchEvaluationId": "sim_mt_2755-7517683065", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/sim_mt_2755-7517683065", - "batchEvaluationName": "sim_mt_2755", - "status": "COMPLETED", - "createdAt": { - "$date": "2026-08-14T19:49:59.317Z" - }, - "evaluators": [ - { - "evaluatorId": "Builtin.Helpfulness" - } - ], - "updatedAt": { - "$date": "2026-08-14T19:51:02.750Z" - } - } - ] -} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.7af4b4e0e2cc1c3d.json b/src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.7af4b4e0e2cc1c3d.json deleted file mode 100644 index 506ae0f3fe..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.7af4b4e0e2cc1c3d.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", - "batchEvaluationName": "golden_batch_insights_fixture", - "status": "PENDING", - "createdAt": { - "$date": "2026-08-21T19:27:51.136Z" - }, - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", - "logStreamName": "run-golden_batch_insights_fixture-cd634815b4" - } - }, - "description": "Golden batch insights fixture" -} diff --git a/src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.bf8946c07edd8624.json b/src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.bf8946c07edd8624.json deleted file mode 100644 index 279a5a0a1f..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/StartBatchEvaluationCommand.bf8946c07edd8624.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "batchEvaluationId": "golden_batch_insights_stop_fixture_20260917_1-4f20901895", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:123456789012:batch-evaluate/golden_batch_insights_stop_fixture_20260917_1-4f20901895", - "batchEvaluationName": "golden_batch_insights_stop_fixture_20260917_1", - "status": "PENDING", - "createdAt": { - "$date": "2026-09-17T21:46:48.837Z" - }, - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", - "logStreamName": "run-golden_batch_insights_stop_fixture_20260917_1-4f20901895", - "metricsNamespace": "Bedrock-AgentCore/Evaluations", - "resultDestination": "DEDICATED_LOG_GROUP" - } - } -} diff --git a/src/handlers/eval/batch-insights/__fixtures__/StopBatchEvaluationCommand.72d5c5e77b44a747.json b/src/handlers/eval/batch-insights/__fixtures__/StopBatchEvaluationCommand.72d5c5e77b44a747.json deleted file mode 100644 index ecdbe06c7c..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/StopBatchEvaluationCommand.72d5c5e77b44a747.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "batchEvaluationId": "golden_batch_insights_stop_fixture_20260917_1-4f20901895", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:123456789012:batch-evaluate/golden_batch_insights_stop_fixture_20260917_1-4f20901895", - "status": "STOPPING" -} diff --git a/src/handlers/eval/batch-insights/__fixtures__/get.golden.json b/src/handlers/eval/batch-insights/__fixtures__/get.golden.json deleted file mode 100644 index 1291f49766..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/get.golden.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", - "batchEvaluationName": "golden_batch_insights_fixture", - "status": "COMPLETED", - "createdAt": "2026-08-21T19:27:51.136Z", - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "dataSourceConfig": { - "cloudWatchLogs": { - "serviceNames": [ - "asdf_MyAgent.DEFAULT" - ], - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/asdf_MyAgent-3s5axvBC6Q-DEFAULT" - ] - } - }, - "evaluationResults": { - "numberOfSessionsCompleted": 2, - "numberOfSessionsInProgress": 0, - "numberOfSessionsFailed": 0, - "totalNumberOfSessions": 2, - "numberOfSessionsIgnored": 0, - "evaluatorSummaries": [] - }, - "description": "Golden batch insights fixture", - "updatedAt": "2026-08-21T19:28:56.756Z" -} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/list.golden.json b/src/handlers/eval/batch-insights/__fixtures__/list.golden.json deleted file mode 100644 index 288e5fcbc6..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/list.golden.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "batchEvaluations": [ - { - "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", - "batchEvaluationName": "golden_batch_insights_fixture", - "status": "COMPLETED", - "createdAt": "2026-08-21T19:27:51.136Z", - "description": "Golden batch insights fixture", - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "updatedAt": "2026-08-21T19:28:56.756Z" - } - ] -} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/run.golden.json b/src/handlers/eval/batch-insights/__fixtures__/run.golden.json deleted file mode 100644 index d071718937..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/run.golden.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "batchEvaluationId": "golden_batch_insights_fixture-cd634815b4", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:batch-evaluate/golden_batch_insights_fixture-cd634815b4", - "batchEvaluationName": "golden_batch_insights_fixture", - "status": "PENDING", - "createdAt": "2026-08-21T19:27:51.136Z", - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/batch-evaluations/results/default", - "logStreamName": "run-golden_batch_insights_fixture-cd634815b4" - } - }, - "description": "Golden batch insights fixture" -} \ No newline at end of file diff --git a/src/handlers/eval/batch-insights/__fixtures__/stop.golden.json b/src/handlers/eval/batch-insights/__fixtures__/stop.golden.json deleted file mode 100644 index ecdbe06c7c..0000000000 --- a/src/handlers/eval/batch-insights/__fixtures__/stop.golden.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "batchEvaluationId": "golden_batch_insights_stop_fixture_20260917_1-4f20901895", - "batchEvaluationArn": "arn:aws:bedrock-agentcore:us-west-2:123456789012:batch-evaluate/golden_batch_insights_stop_fixture_20260917_1-4f20901895", - "status": "STOPPING" -} diff --git a/src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx b/src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx deleted file mode 100644 index 40f7d44237..0000000000 --- a/src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx +++ /dev/null @@ -1,136 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { join } from "node:path"; -import { CoreClient } from "../../../core"; -import { - createSilentLogger, - fixtureFactories, - matchGolden, - settle, - TestGlobalConfigAccessor, - testIO, -} from "../../../testing"; -import { createRootHandler } from "../../index"; - -const REGION = "us-west-2"; -const FIXTURES = join(import.meta.dir, "__fixtures__"); -const FIXTURE_JOB_ID = "golden_batch_insights_fixture-cd634815b4"; -const FIXTURE_NAME = "golden_batch_insights_fixture"; -const FIXTURE_STOP_NAME = "golden_batch_insights_stop_fixture_20260917_1"; -const FIXTURE_AGENT = "asdf_MyAgent-3s5axvBC6Q"; -const FIXTURE_STOP_AGENT = "demoEval_demoAgent-qGEpAAE68u"; - -// Record with: -// RECORD=1 bun test src/handlers/eval/batch-insights/batch-insights.fixture.test.tsx -// -// `get` pins a pre-existing completed insights job. `run` and `stop` create -// durable service jobs, so use unique fixture names before re-recording. Do not -// repeat record mode with the same names: the recorded conflict replaces the -// successful StartBatchEvaluation fixtures. -function createFixtureCore(): CoreClient { - const { - createControlClient, - createDataClient, - createIamClient, - createLogsClient, - createXrayClient, - createApplicationSignalsClient, - } = fixtureFactories(FIXTURES); - return new CoreClient({ - createControlClient, - createDataClient, - createIamClient, - createLogsClient, - createXrayClient, - createApplicationSignalsClient, - logger: createSilentLogger(), - }); -} - -async function run(args: string[]): Promise { - const io = testIO(); - const root = createRootHandler(createFixtureCore(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - await root.route(["bun", "agentcore", ...args, "--region", REGION]); - return io.stdout(); -} - -describe("eval batch-insights (fixture-backed)", () => { - test("get returns the service-side insights job directly", async () => { - const stdout = await run(["eval", "batch-insights", "get", "--id", FIXTURE_JOB_ID, "--json"]); - - matchGolden(FIXTURES, "get.golden.json", stdout); - const detail = JSON.parse(stdout); - expect(detail.status).toBe("COMPLETED"); - expect(detail.insights).toEqual([{ insightId: "Builtin.Insight.FailureAnalysis" }]); - expect(detail.results).toBeUndefined(); - }); - - test("list filters evaluator-only jobs from the shared service page", async () => { - const stdout = await run(["eval", "batch-insights", "list", "--json"]); - - matchGolden(FIXTURES, "list.golden.json", stdout); - const page = JSON.parse(stdout); - expect(page.batchEvaluations.length).toBeGreaterThan(0); - expect( - page.batchEvaluations.every((job: { insights?: unknown[] }) => job.insights?.length), - ).toBe(true); - expect(page.batchEvaluations[0].batchEvaluationId).toBe(FIXTURE_JOB_ID); - }); - - test("run submits an insights batch job", async () => { - const stdout = await run([ - "eval", - "batch-insights", - "run", - "--name", - FIXTURE_NAME, - "--description", - "Golden batch insights fixture", - "--agent", - FIXTURE_AGENT, - "--json", - ]); - - matchGolden(FIXTURES, "run.golden.json", stdout); - const job = JSON.parse(stdout); - expect(job.batchEvaluationId).toBeTruthy(); - expect(job.insights).toEqual([{ insightId: "Builtin.Insight.FailureAnalysis" }]); - }); - - test("stop transitions a running batch insights job", async () => { - const started = JSON.parse( - await run([ - "eval", - "batch-insights", - "run", - "--name", - FIXTURE_STOP_NAME, - "--agent", - FIXTURE_STOP_AGENT, - "--json", - ]), - ); - await settle(1_000); - - const stdout = await run([ - "eval", - "batch-insights", - "stop", - "--id", - started.batchEvaluationId, - "--json", - ]); - - matchGolden(FIXTURES, "stop.golden.json", stdout); - const stopped = JSON.parse(stdout); - expect(stopped.batchEvaluationId).toBe(started.batchEvaluationId); - expect(["STOPPING", "STOPPED"]).toContain(stopped.status); - }, 180_000); - - test("stop requires --id before making an AWS call", async () => { - await expect(run(["eval", "batch-insights", "stop", "--json"])).rejects.toThrow(/--id/); - }); -}); diff --git a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx b/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx deleted file mode 100644 index 2f38973197..0000000000 --- a/src/handlers/eval/batch-insights/batch-insights.screen.test.tsx +++ /dev/null @@ -1,135 +0,0 @@ -import { afterEach, describe, expect, test } from "bun:test"; -import type { - BatchEvaluationSummary, - GetBatchEvaluationResponse, -} from "@aws-sdk/client-bedrock-agentcore"; -import { - cleanupScreens, - renderScreen, - TestCoreClient, - waitFor, - waitForText, - menuEntries, -} from "../../../testing"; - -afterEach(cleanupScreens); - -function summary(overrides: Partial = {}): BatchEvaluationSummary { - return { - batchEvaluationArn: - "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/insights-1", - batchEvaluationId: "insights-1", - batchEvaluationName: "failure_analysis", - status: "COMPLETED", - createdAt: new Date("2026-08-20T01:02:03.000Z"), - updatedAt: new Date("2026-08-21T12:34:56.000Z"), - insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], - ...overrides, - }; -} - -function detail(overrides: Partial = {}): GetBatchEvaluationResponse { - return { - batchEvaluationArn: - "arn:aws:bedrock-agentcore:us-east-1:123456789012:batch-evaluate/insights-1", - batchEvaluationId: "insights-1", - batchEvaluationName: "failure_analysis", - status: "COMPLETED", - createdAt: new Date("2026-08-20T01:02:03.000Z"), - updatedAt: new Date("2026-08-21T12:34:56.000Z"), - insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], - failureAnalysisResult: { - failures: [], - }, - ...overrides, - } as GetBatchEvaluationResponse; -} - -function coreWithBatchEvaluations(items: BatchEvaluationSummary[]): TestCoreClient { - const core = new TestCoreClient(); - core.eval.setBatchInsightsListResponse({ batchEvaluations: items }); - return core; -} - -describe("batch-insights menu", () => { - test("lists the read-only commands, then the rest as command line only", async () => { - const screen = renderScreen("/agentcore/eval/batch-insights"); - - await waitForText(screen.lastFrame, "list batch insights runs"); - expect(menuEntries(screen.lastFrame()!)).toEqual({ - screens: ["get", "list"], - cliOnly: [], - }); - }); -}); - -describe("batch-insights picker", () => { - test("loads Insights from the dedicated Core facade", async () => { - const core = coreWithBatchEvaluations([summary()]); - const screen = renderScreen("/agentcore/eval/batch-insights/list", { core }); - - await waitForText(screen.lastFrame, "failure_analysis"); - expect(core.eval.calls[0]?.method).toBe("listBatchInsights"); - }); - - test("bare get redirects to the filtered picker", async () => { - const core = coreWithBatchEvaluations([summary()]); - const screen = renderScreen("/agentcore/eval/batch-insights/get", { core }); - - await waitForText(screen.lastFrame, "failure_analysis"); - expect(core.eval.calls[0]?.method).toBe("listBatchInsights"); - }); - - test("selection opens the matching insights JSON", async () => { - const core = coreWithBatchEvaluations([summary()]); - core.eval.setBatchEvalGetResponse(detail()); - const screen = renderScreen("/agentcore/eval/batch-insights/list", { core }); - - await waitForText(screen.lastFrame, "failure_analysis"); - await screen.press("return"); - await waitForText(screen.lastFrame, "agentcore → eval → batch-insights → get → insights-1"); - await waitFor(() => - core.eval.calls.some( - (call) => call.method === "getBatchInsights" && call.args[0] === "insights-1", - ), - ); - }); - - test("shows the Insights-specific empty state", async () => { - const core = coreWithBatchEvaluations([]); - const screen = renderScreen("/agentcore/eval/batch-insights/list", { core }); - - await waitForText(screen.lastFrame, "No batch insights found in this Region."); - }); -}); - -describe("batch-insights detail", () => { - test("renders service-side reports without requesting CloudWatch results", async () => { - const core = new TestCoreClient(); - core.eval.setBatchEvalGetResponse(detail()); - const screen = renderScreen("/agentcore/eval/batch-insights/get/insights-1", { core }); - - await waitForText(screen.lastFrame, "failure_analysis"); - const frame = screen.lastFrame()!; - expect(frame).toContain('"failureAnalysisResult"'); - expect(core.eval.calls.find((call) => call.method === "getBatchInsights")).toEqual({ - method: "getBatchInsights", - args: ["insights-1", { region: "us-east-1" }], - }); - }); - - test("rejects direct navigation to an evaluator-only job", async () => { - const core = new TestCoreClient(); - core.eval.setBatchEvalGetResponse( - detail({ - batchEvaluationId: "evaluation-1", - insights: undefined, - evaluators: [{ evaluatorId: "Builtin.Correctness" }], - failureAnalysisResult: undefined, - }), - ); - const screen = renderScreen("/agentcore/eval/batch-insights/get/evaluation-1", { core }); - - await waitForText(screen.lastFrame, "is not a batch insights run"); - }); -}); diff --git a/src/handlers/eval/batch-insights/batch-insights.test.tsx b/src/handlers/eval/batch-insights/batch-insights.test.tsx deleted file mode 100644 index a8b6afd480..0000000000 --- a/src/handlers/eval/batch-insights/batch-insights.test.tsx +++ /dev/null @@ -1,276 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import type { - GetBatchEvaluationResponse, - ListBatchEvaluationsResponse, -} from "@aws-sdk/client-bedrock-agentcore"; -import { - createSilentLogger, - expectError, - TestCoreClient, - TestGlobalConfigAccessor, - testIO, -} from "../../../testing"; -import { createRootHandler } from "../../index"; -import { InputValidationError } from "../../../errors"; - -const REGION = "us-west-2"; - -async function run(args: string[], configure?: (core: TestCoreClient) => void) { - const core = new TestCoreClient(); - configure?.(core); - const io = testIO(); - const root = createRootHandler(core, { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - await root.route(["bun", "agentcore", ...args, "--region", REGION]); - return { core, stdout: io.stdout(), stderr: io.stderr() }; -} - -describe("eval batch-insights command hierarchy", () => { - test("registers batch-insights commands", () => { - const io = testIO(); - const root = createRootHandler(new TestCoreClient(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - const group = root - .children() - .find((child) => child.name() === "eval") - ?.children() - .find((child) => child.name() === "batch-insights"); - - expect(group?.children().map((child) => child.name())).toEqual(["run", "get", "list", "stop"]); - }); - - test("prints help for a bare invocation without a Core call", async () => { - const { core, stdout } = await run(["eval", "batch-insights", "--json"]); - expect(stdout).toContain("Usage: agentcore eval batch-insights"); - expect(core.eval.calls).toHaveLength(0); - }); -}); - -describe("eval batch-insights run", () => { - test("requires --name and exactly one session source", async () => { - await expectError( - run(["eval", "batch-insights", "run", "--agent", "agent-1"]), - /--name/, - InputValidationError, - ); - await expect(run(["eval", "batch-insights", "run", "--name", "insights_run"])).rejects.toThrow( - /specify exactly one of --agent, --online-eval, --data-source-config/, - ); - await expect( - run([ - "eval", - "batch-insights", - "run", - "--name", - "insights_run", - "--agent", - "agent-1", - "--online-eval", - "online-1", - ]), - ).rejects.toThrow(/specify exactly one of --agent, --online-eval, --data-source-config/); - }); - - test("uses failure analysis by default and passes the resolved agent source", async () => { - const { core } = await run([ - "eval", - "batch-insights", - "run", - "--name", - "insights_run", - "--agent", - "agent-1", - "--endpoint", - "prod", - "--start-time", - "2026-08-01T00:00:00Z", - "--end-time", - "2026-08-02T00:00:00Z", - "--session-ids", - "s1", - "s2", - "--description", - "Analyze failures", - "--kms-key-arn", - "arn:aws:kms:us-west-2:123:key/abc", - "--json", - ]); - - expect(core.eval.calls).toEqual([ - { - method: "startBatchInsights", - args: [ - { - name: "insights_run", - description: "Analyze failures", - insightIds: ["Builtin.Insight.FailureAnalysis"], - evaluatorIds: undefined, - source: { - origin: "agent", - agent: "agent-1", - endpoint: "prod", - window: { - startTime: new Date("2026-08-01T00:00:00Z"), - endTime: new Date("2026-08-02T00:00:00Z"), - }, - sessionIds: ["s1", "s2"], - }, - kmsKeyArn: "arn:aws:kms:us-west-2:123:key/abc", - }, - { region: REGION }, - ], - }, - ]); - }); - - test("omits analysis configuration when an online evaluation is the source", async () => { - const { core } = await run([ - "eval", - "batch-insights", - "run", - "--name", - "insights_run", - "--online-eval", - "online-1", - "--json", - ]); - - expect(core.eval.calls[0]?.method).toBe("startBatchInsights"); - expect(core.eval.calls[0]?.args[0]).toMatchObject({ - insightIds: undefined, - evaluatorIds: undefined, - source: { - origin: "online-eval", - onlineEvaluationConfigId: "online-1", - }, - }); - }); - - test("rejects explicit analysis configuration with an online evaluation source", async () => { - await expect( - run([ - "eval", - "batch-insights", - "run", - "--name", - "insights_run", - "--online-eval", - "online-1", - "--insight", - "Builtin.Insight.UserIntent", - ]), - ).rejects.toThrow(/--insight and --evaluators cannot be used with --online-eval/); - - await expect( - run([ - "eval", - "batch-insights", - "run", - "--name", - "insights_run", - "--online-eval", - "online-1", - "--evaluators", - "Builtin.Helpfulness", - ]), - ).rejects.toThrow(/--insight and --evaluators cannot be used with --online-eval/); - }); - - test("accepts explicit insights and optional evaluator chaining for an agent source", async () => { - const { core } = await run([ - "eval", - "batch-insights", - "run", - "--name", - "insights_run", - "--agent", - "agent-1", - "--insight", - "Builtin.Insight.UserIntent", - "Builtin.Insight.ExecutionSummary", - "--evaluators", - "Builtin.Helpfulness", - "--json", - ]); - - expect(core.eval.calls[0]?.args[0]).toMatchObject({ - insightIds: ["Builtin.Insight.UserIntent", "Builtin.Insight.ExecutionSummary"], - evaluatorIds: ["Builtin.Helpfulness"], - source: { origin: "agent", agent: "agent-1" }, - }); - }); -}); - -describe("eval batch-insights get", () => { - test("returns insight reports without reading CloudWatch scores", async () => { - const response = { - batchEvaluationId: "bi-1", - status: "COMPLETED", - insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], - failureAnalysisResult: { failures: [] }, - } as unknown as GetBatchEvaluationResponse; - const { core, stdout } = await run( - ["eval", "batch-insights", "get", "--id", "bi-1", "--json"], - (client) => client.eval.setBatchEvalGetResponse(response), - ); - - expect(JSON.parse(stdout)).toMatchObject({ - batchEvaluationId: "bi-1", - failureAnalysisResult: { failures: [] }, - }); - expect(JSON.parse(stdout).consoleUrl).toBeUndefined(); - expect(core.eval.calls[0]).toEqual({ - method: "getBatchInsights", - args: ["bi-1", { region: REGION }], - }); - }); - - test("rejects an evaluator-only batch evaluation", async () => { - const response = { - batchEvaluationId: "be-1", - evaluators: [{ evaluatorId: "Builtin.Helpfulness" }], - } as unknown as GetBatchEvaluationResponse; - - await expect( - run(["eval", "batch-insights", "get", "--id", "be-1"], (client) => - client.eval.setBatchEvalGetResponse(response), - ), - ).rejects.toThrow(/not a batch insights run/); - }); -}); - -describe("eval batch-insights list", () => { - test("returns the logical Insights page from Core", async () => { - const response = { - batchEvaluations: [ - { - batchEvaluationId: "bi-1", - insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }], - }, - { batchEvaluationId: "bi-2", insights: [{ insightId: "Builtin.Insight.UserIntent" }] }, - ], - nextToken: "next-page", - } as unknown as ListBatchEvaluationsResponse; - const { core, stdout } = await run( - ["eval", "batch-insights", "list", "--next-token", "page-1", "--max-results", "20", "--json"], - (client) => client.eval.setBatchInsightsListResponse(response, "page-1"), - ); - const output = JSON.parse(stdout); - - expect(output.nextToken).toBe("next-page"); - expect( - output.batchEvaluations.map((item: { batchEvaluationId: string }) => item.batchEvaluationId), - ).toEqual(["bi-1", "bi-2"]); - expect(output.batchEvaluations[0].consoleUrl).toBeUndefined(); - expect(core.eval.calls[0]).toEqual({ - method: "listBatchInsights", - args: ["page-1", 20, { region: REGION }], - }); - }); -}); diff --git a/src/handlers/eval/batch-insights/get/index.tsx b/src/handlers/eval/batch-insights/get/index.tsx deleted file mode 100644 index 3b13194dac..0000000000 --- a/src/handlers/eval/batch-insights/get/index.tsx +++ /dev/null @@ -1,21 +0,0 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; -import type { Core } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -export const createGetBatchInsightsHandler = (core: Core) => - createHandler({ - name: "get", - description: "get a batch insights run and its reports by ID", - flags: [flag("id", "the ID of the batch insights run", z.string().min(1))], - handle: async (ctx, flags) => { - const id = flags["id"]; - - const detail = await core.eval.getBatchInsights(id, coreOptsFromCtx(ctx)); - - ctx.require(JsonRendererKey).renderJson(detail); - }, - }); - -export { BatchInsightsGetJsonScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/batch-insights/get/screen.tsx b/src/handlers/eval/batch-insights/get/screen.tsx deleted file mode 100644 index 03340dbdfb..0000000000 --- a/src/handlers/eval/batch-insights/get/screen.tsx +++ /dev/null @@ -1,30 +0,0 @@ -import { useQuery } from "@tanstack/react-query"; -import { useParams } from "react-router"; -import { JsonDetail } from "../../../../components/JsonDetail"; -import type { ScreenProps } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -function useBatchInsightsDetail({ ctx, core }: ScreenProps, id: string | undefined) { - const opts = coreOptsFromCtx(ctx); - return useQuery({ - queryKey: ["batch-insights", opts.region, id], - queryFn: () => core.eval.getBatchInsights(id!, opts), - enabled: id !== undefined, - }); -} - -export function BatchInsightsGetJsonScreen(props: ScreenProps) { - const { batchEvaluationId } = useParams(); - const query = useBatchInsightsDetail(props, batchEvaluationId); - - return ( - void query.refetch()} - /> - ); -} diff --git a/src/handlers/eval/batch-insights/index.tsx b/src/handlers/eval/batch-insights/index.tsx deleted file mode 100644 index 2d40420374..0000000000 --- a/src/handlers/eval/batch-insights/index.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import type { AppIO } from "../../../io"; -import { Router } from "../../../router"; -import { withTuiOnEmptyFlagsAndArgs } from "../../../middleware"; -import { renderTui } from "../../../tui"; -import type { Core } from "../../types"; -import { createGetBatchInsightsHandler } from "./get"; -import { createListBatchInsightsHandler } from "./list"; -import { createRunBatchInsightsHandler } from "./run"; -import { createStopBatchInsightsHandler } from "./stop"; - -export function createBatchInsightsHandler(core: Core, io: AppIO): Router { - return new Router("batch-insights", "run and inspect AgentCore batch insights") - .use(withTuiOnEmptyFlagsAndArgs(core, io)) - .default(renderTui(core, io)) - .supportedTuiCommands("get", "list") - .handler(createRunBatchInsightsHandler(core, io)) - .handler(createGetBatchInsightsHandler(core)) - .handler(createListBatchInsightsHandler(core)) - .handler(createStopBatchInsightsHandler(core)); -} - -export { BatchInsightsScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/batch-insights/list/index.tsx b/src/handlers/eval/batch-insights/list/index.tsx deleted file mode 100644 index 37db28c12b..0000000000 --- a/src/handlers/eval/batch-insights/list/index.tsx +++ /dev/null @@ -1,26 +0,0 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; -import type { Core } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -export const createListBatchInsightsHandler = (core: Core) => - createHandler({ - name: "list", - description: "list batch insights runs", - flags: [ - flag("next-token", "pagination token returned by a previous request", z.string().optional()), - flag("max-results", "maximum number of batch insights runs to return", z.number().optional()), - ], - handle: async (ctx, flags) => { - const response = await core.eval.listBatchInsights( - flags["next-token"], - flags["max-results"], - coreOptsFromCtx(ctx), - ); - - ctx.require(JsonRendererKey).renderJson(response); - }, - }); - -export { BatchInsightsListScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/batch-insights/list/screen.tsx b/src/handlers/eval/batch-insights/list/screen.tsx deleted file mode 100644 index 728015051a..0000000000 --- a/src/handlers/eval/batch-insights/list/screen.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { useNavigate } from "react-router"; -import { BatchInsightsPicker } from "../../../../components/BatchInsightsPicker"; -import type { ScreenProps } from "../../../types"; - -export function BatchInsightsListScreen(props: ScreenProps) { - const navigate = useNavigate(); - - return ( - - navigate(`/agentcore/eval/batch-insights/get/${encodeURIComponent(batchEvaluationId)}`) - } - /> - ); -} diff --git a/src/handlers/eval/batch-insights/run/index.tsx b/src/handlers/eval/batch-insights/run/index.tsx deleted file mode 100644 index 775e43d584..0000000000 --- a/src/handlers/eval/batch-insights/run/index.tsx +++ /dev/null @@ -1,65 +0,0 @@ -import z from "zod"; -import { InputValidationError } from "../../../../errors"; -import { SourceResolver, type AppIO } from "../../../../io"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; -import type { Core } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; -import { SessionSource } from "../../sessionSource"; - -const CONFIGURATION = "Configuration:"; -const ANALYSIS = "Analysis:"; - -const DEFAULT_INSIGHT = "Builtin.Insight.FailureAnalysis"; - -export const createRunBatchInsightsHandler = (core: Core, io: AppIO) => - createHandler({ - name: "run", - description: "start an asynchronous batch insights run over existing sessions", - flags: [ - flag("name", "batch insights name (must be unique in the account)", z.string().min(1), { - group: CONFIGURATION, - }), - flag("description", "optional description", z.string().optional(), { - group: CONFIGURATION, - }), - flag("kms-key-arn", "KMS key to encrypt insights data at rest", z.string().optional(), { - group: CONFIGURATION, - }), - ...SessionSource.flags, - flag("insight", "insight ID(s) to run", z.array(z.string()).optional(), { - group: ANALYSIS, - }), - flag( - "evaluators", - "optional evaluator ID(s) to run alongside the insights", - z.array(z.string()).optional(), - { group: ANALYSIS }, - ), - ], - handle: async (ctx, flags) => { - const resolver = new SourceResolver({ stdin: io.stdin }); - const source = await SessionSource.resolve(flags, resolver); - if ( - source.origin === "online-eval" && - (flags["insight"]?.length || flags["evaluators"]?.length) - ) { - throw new InputValidationError( - "--insight and --evaluators cannot be used with --online-eval", - ); - } - const response = await core.eval.startBatchInsights( - { - name: flags["name"], - description: flags["description"], - insightIds: - source.origin === "online-eval" ? undefined : (flags["insight"] ?? [DEFAULT_INSIGHT]), - evaluatorIds: flags["evaluators"], - source, - kmsKeyArn: flags["kms-key-arn"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); diff --git a/src/handlers/eval/batch-insights/screen.tsx b/src/handlers/eval/batch-insights/screen.tsx deleted file mode 100644 index dcc3f41186..0000000000 --- a/src/handlers/eval/batch-insights/screen.tsx +++ /dev/null @@ -1,6 +0,0 @@ -import { RouterScreen } from "../../../components/RouterScreen"; -import type { ScreenProps } from "../../types"; - -export function BatchInsightsScreen(props: ScreenProps) { - return ; -} diff --git a/src/handlers/eval/batch-insights/stop/index.tsx b/src/handlers/eval/batch-insights/stop/index.tsx deleted file mode 100644 index 75c98a6466..0000000000 --- a/src/handlers/eval/batch-insights/stop/index.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; -import type { Core } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -export const createStopBatchInsightsHandler = (core: Core) => - createHandler({ - name: "stop", - description: "stop a running batch insights run", - flags: [flag("id", "the ID of the batch insights run to stop", z.string())], - handle: async (ctx, flags) => { - ctx - .require(JsonRendererKey) - .renderJson(await core.eval.stopBatchEvaluation(flags["id"], coreOptsFromCtx(ctx))); - }, - }); diff --git a/src/handlers/eval/index.tsx b/src/handlers/eval/index.tsx index a6482c9051..b7dd3bdcd5 100644 --- a/src/handlers/eval/index.tsx +++ b/src/handlers/eval/index.tsx @@ -5,10 +5,8 @@ import type { AppIO } from "../../io"; import type { Core } from "../types"; import { createEvaluatorHandler } from "./evaluator"; import { createOnlineEvalHandler } from "./online-eval"; -import { createOnlineInsightHandler } from "./online-insight"; import { createDatasetHandler } from "./dataset"; import { createBatchEvaluationHandler } from "./batch-evaluation"; -import { createBatchInsightsHandler } from "./batch-insights"; import { createOnDemandHandler } from "./ondemand"; import { createConfigBundleHandler } from "./config-bundle"; import { createAbTestHandler } from "./ab-test"; @@ -23,20 +21,16 @@ export function createEvalHandler(core: Core, io: AppIO): Router { .supportedTuiCommands( "evaluator", "online-eval", - "online-insight", "dataset", "batch-evaluation", - "batch-insights", "config-bundle", "ab-test", "recommendation", ) .handler(createEvaluatorHandler(core, io)) .handler(createOnlineEvalHandler(core, io)) - .handler(createOnlineInsightHandler(core, io)) .handler(createDatasetHandler(core, io)) .handler(createBatchEvaluationHandler(core, io)) - .handler(createBatchInsightsHandler(core, io)) .handler(createOnDemandHandler(core, io)) .handler(createConfigBundleHandler(core, io)) .handler(createAbTestHandler(core, io)) diff --git a/src/handlers/eval/online-insight/__fixtures__/CreateOnlineEvaluationConfigCommand.38066f8427f9fba0.json b/src/handlers/eval/online-insight/__fixtures__/CreateOnlineEvaluationConfigCommand.38066f8427f9fba0.json deleted file mode 100644 index 6a62fba2f1..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/CreateOnlineEvaluationConfigCommand.38066f8427f9fba0.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "createdAt": { - "$date": "2026-08-25T19:22:09.493Z" - }, - "status": "CREATING", - "executionStatus": "DISABLED", - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-NPzirZ4e60" - } - } -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/DeleteOnlineEvaluationConfigCommand.39087bc3e00af960.json b/src/handlers/eval/online-insight/__fixtures__/DeleteOnlineEvaluationConfigCommand.39087bc3e00af960.json deleted file mode 100644 index 78913e4575..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/DeleteOnlineEvaluationConfigCommand.39087bc3e00af960.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "status": "DELETING" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json b/src/handlers/eval/online-insight/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json deleted file mode 100644 index 5e8b390353..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/GetAgentRuntimeCommand.80bb567edcaa4bb1.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:runtime/testAgent_Agent-wm9hYBD93Y", - "agentRuntimeName": "testAgent_Agent", - "agentRuntimeId": "testAgent_Agent-wm9hYBD93Y", - "agentRuntimeVersion": "3", - "createdAt": { - "$date": "2026-03-24T16:02:00.301Z" - }, - "lastUpdatedAt": { - "$date": "2026-03-24T16:13:08.198Z" - }, - "roleArn": "arn:aws:iam::725476964917:role/AgentCore-myimport-defaul-ApplicationAgentTestAgent-tJFjyd6jLIOn", - "networkConfiguration": { - "networkMode": "PUBLIC" - }, - "status": "READY", - "lifecycleConfiguration": { - "idleRuntimeSessionTimeout": 900, - "maxLifetime": 28800 - }, - "description": "AgentCore Runtime: myimport_testAgent_Agent", - "workloadIdentityDetails": { - "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/testAgent_Agent-wm9hYBD93Y" - }, - "agentRuntimeArtifact": { - "codeConfiguration": { - "code": { - "s3": { - "bucket": "cdk-hnb659fds-assets-725476964917-us-west-2", - "prefix": "5f58024b207272eeb8560702468b402cb3aeea1c8ee36d3c5f348fcb471bb58f.zip" - } - }, - "runtime": "PYTHON_3_10", - "entryPoint": [ - "opentelemetry-instrument", - "main.py" - ] - } - }, - "environmentVariables": { - "MEMORY_TESTAGENT_AGENT_MEM_ID": "testAgent_Agent_mem-17a3Lg8yrL" - }, - "metadataConfiguration": { - "requireMMDSV2": true - } -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.39087bc3e00af960.json b/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.39087bc3e00af960.json deleted file mode 100644 index 9cb3c8d784..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.39087bc3e00af960.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigName": "agentcore_cli_online_insight_fixture", - "rule": { - "samplingConfig": { - "samplingPercentage": 25 - }, - "sessionConfig": { - "sessionTimeoutMinutes": 30 - } - }, - "dataSourceConfig": { - "cloudWatchLogs": { - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" - ], - "serviceNames": [ - "testAgent_Agent.DEFAULT" - ] - } - }, - "status": "ACTIVE", - "executionStatus": "DISABLED", - "createdAt": { - "$date": "2026-08-25T19:22:09.493Z" - }, - "updatedAt": { - "$date": "2026-08-25T19:22:25.112Z" - }, - "evaluators": [], - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-NPzirZ4e60" - } - }, - "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json b/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json deleted file mode 100644 index 845409c3be..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.859097834b7e85cf.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "$error": { - "name": "ResourceNotFoundException", - "message": "Online evaluation configuration not found" - } -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.d3a07848de1b01eb.json b/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.d3a07848de1b01eb.json deleted file mode 100644 index a10774b6f6..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/GetOnlineEvaluationConfigCommand.d3a07848de1b01eb.json +++ /dev/null @@ -1,42 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", - "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", - "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", - "rule": { - "samplingConfig": { - "samplingPercentage": 100 - }, - "sessionConfig": { - "sessionTimeoutMinutes": 5 - } - }, - "dataSourceConfig": { - "cloudWatchLogs": { - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/ABVfyLatest_ABVfyLatest-PFLr353QVA-prod" - ], - "serviceNames": [ - "ABVfyLatest_ABVfyLatest.prod" - ] - } - }, - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-17T22:09:59.115Z" - }, - "updatedAt": { - "$date": "2026-06-17T22:10:11.792Z" - }, - "evaluators": [ - { - "evaluatorId": "Builtin.Helpfulness" - } - ], - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/ABVfyLatest_ProdEval-2vqlCb2UiG" - } - }, - "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCore-ABVfyLatest-def-ApplicationOnlineEvalProd-EllKhDcWAWG4" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json deleted file mode 100644 index 98c0eeddf0..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json +++ /dev/null @@ -1,447 +0,0 @@ -{ - "onlineEvaluationConfigs": [ - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_ProdEval-2vqlCb2UiG", - "onlineEvaluationConfigId": "ABVfyLatest_ProdEval-2vqlCb2UiG", - "onlineEvaluationConfigName": "ABVfyLatest_ProdEval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-17T22:09:59.115Z" - }, - "updatedAt": { - "$date": "2026-06-17T22:10:11.792Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", - "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", - "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-17T22:09:59.058Z" - }, - "updatedAt": { - "$date": "2026-06-17T22:10:12.095Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_ProdEval-MtuOYB2GZP", - "onlineEvaluationConfigId": "ABVfyPrerelease_ProdEval-MtuOYB2GZP", - "onlineEvaluationConfigName": "ABVfyPrerelease_ProdEval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-17T20:28:10.738Z" - }, - "updatedAt": { - "$date": "2026-06-17T20:28:26.527Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyPrerelease_StagingEval-fH26hNBHL6", - "onlineEvaluationConfigId": "ABVfyPrerelease_StagingEval-fH26hNBHL6", - "onlineEvaluationConfigName": "ABVfyPrerelease_StagingEval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-17T20:28:09.163Z" - }, - "updatedAt": { - "$date": "2026-06-17T20:28:26.799Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/EvoBugBash_abOnlineEval-Q0Iv1k2cdo", - "onlineEvaluationConfigId": "EvoBugBash_abOnlineEval-Q0Iv1k2cdo", - "onlineEvaluationConfigName": "EvoBugBash_abOnlineEval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-12T22:27:27.466Z" - }, - "updatedAt": { - "$date": "2026-06-12T22:27:40.636Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_ctrlEval-T97o5LAtRS", - "onlineEvaluationConfigId": "GwFilterVfy_ctrlEval-T97o5LAtRS", - "onlineEvaluationConfigName": "GwFilterVfy_ctrlEval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T18:27:10.496Z" - }, - "updatedAt": { - "$date": "2026-06-15T18:27:26.982Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/GwFilterVfy_treatEval-bTv4u14rUQ", - "onlineEvaluationConfigId": "GwFilterVfy_treatEval-bTv4u14rUQ", - "onlineEvaluationConfigName": "GwFilterVfy_treatEval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T18:27:10.370Z" - }, - "updatedAt": { - "$date": "2026-06-15T18:27:27.240Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", - "onlineEvaluationConfigId": "OhioTripPlanner_ProdEvalStagin-8wGIBCFzOE", - "onlineEvaluationConfigName": "OhioTripPlanner_ProdEvalStagin", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-16T00:09:44.834Z" - }, - "updatedAt": { - "$date": "2026-06-16T00:09:56.722Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/OhioTripPlanner_StagingOnly-A3o7OsFV5a", - "onlineEvaluationConfigId": "OhioTripPlanner_StagingOnly-A3o7OsFV5a", - "onlineEvaluationConfigName": "OhioTripPlanner_StagingOnly", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-16T00:22:22.611Z" - }, - "updatedAt": { - "$date": "2026-06-16T00:22:44.675Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeBundle-UekzTY3B12", - "onlineEvaluationConfigId": "PromoteT2_oeBundle-UekzTY3B12", - "onlineEvaluationConfigName": "PromoteT2_oeBundle", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T17:06:17.370Z" - }, - "updatedAt": { - "$date": "2026-06-15T17:06:45.741Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeCtrl-Vwz1F69rtu", - "onlineEvaluationConfigId": "PromoteT2_oeCtrl-Vwz1F69rtu", - "onlineEvaluationConfigName": "PromoteT2_oeCtrl", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T17:06:17.504Z" - }, - "updatedAt": { - "$date": "2026-06-15T17:06:44.609Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2c-2CHeFHBYfA", - "onlineEvaluationConfigId": "PromoteT2_oeT2c-2CHeFHBYfA", - "onlineEvaluationConfigName": "PromoteT2_oeT2c", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T17:06:17.603Z" - }, - "updatedAt": { - "$date": "2026-06-15T17:06:45.174Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeT2t-pKLwdZDXyw", - "onlineEvaluationConfigId": "PromoteT2_oeT2t-pKLwdZDXyw", - "onlineEvaluationConfigName": "PromoteT2_oeT2t", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T17:06:17.784Z" - }, - "updatedAt": { - "$date": "2026-06-15T17:06:45.466Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/PromoteT2_oeTreat-bC8FNM8MEN", - "onlineEvaluationConfigId": "PromoteT2_oeTreat-bC8FNM8MEN", - "onlineEvaluationConfigName": "PromoteT2_oeTreat", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T17:06:24.185Z" - }, - "updatedAt": { - "$date": "2026-06-15T17:06:44.903Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", - "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_prod-n6GVQ2Atnw", - "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_prod", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-05-05T19:22:26.291Z" - }, - "updatedAt": { - "$date": "2026-05-05T19:22:57.162Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", - "onlineEvaluationConfigId": "TEMPAGENTTJAB_MyOnlineEval_staging-vPcBdKCji0", - "onlineEvaluationConfigName": "TEMPAGENTTJAB_MyOnlineEval_staging", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-05-05T19:22:33.950Z" - }, - "updatedAt": { - "$date": "2026-05-05T19:22:57.441Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abdemo_evalconfig-ntDb6Q5PLh", - "onlineEvaluationConfigId": "abdemo_evalconfig-ntDb6Q5PLh", - "onlineEvaluationConfigName": "abdemo_evalconfig", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-05-27T22:10:42.270Z" - }, - "updatedAt": { - "$date": "2026-05-27T22:10:51.516Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestdemo_eval_config-Kp7uUyBqZB", - "onlineEvaluationConfigId": "abtestdemo_eval_config-Kp7uUyBqZB", - "onlineEvaluationConfigName": "abtestdemo_eval_config", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-05-26T19:13:52.461Z" - }, - "updatedAt": { - "$date": "2026-05-26T19:14:02.417Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_control_eval-seHjGZ7TeH", - "onlineEvaluationConfigId": "abtestval_cs_control_eval-seHjGZ7TeH", - "onlineEvaluationConfigName": "abtestval_cs_control_eval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-24T22:58:24.637Z" - }, - "updatedAt": { - "$date": "2026-06-24T22:58:38.138Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/abtestval_cs_treatment_eval-AdSjkL4CHu", - "onlineEvaluationConfigId": "abtestval_cs_treatment_eval-AdSjkL4CHu", - "onlineEvaluationConfigName": "abtestval_cs_treatment_eval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-24T22:58:24.648Z" - }, - "updatedAt": { - "$date": "2026-06-24T22:58:38.393Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ac570c_myonlineeval-6IRBzU3dkS", - "onlineEvaluationConfigId": "ac570c_myonlineeval-6IRBzU3dkS", - "onlineEvaluationConfigName": "ac570c_myonlineeval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-07-06T18:21:13.033Z" - }, - "updatedAt": { - "$date": "2026-07-06T18:21:28.312Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_abtesteval-53zYJX8x4X", - "onlineEvaluationConfigId": "bugbashagent_abtesteval-53zYJX8x4X", - "onlineEvaluationConfigName": "bugbashagent_abtesteval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T18:19:55.627Z" - }, - "updatedAt": { - "$date": "2026-06-15T18:20:08.431Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", - "onlineEvaluationConfigId": "bugbashagent_lambda_ctrl_eval-EY3GWNCDgH", - "onlineEvaluationConfigName": "bugbashagent_lambda_ctrl_eval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T19:41:54.132Z" - }, - "updatedAt": { - "$date": "2026-06-15T19:42:10.231Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_lambda_trtm_eval-IlN5S34Bc5", - "onlineEvaluationConfigId": "bugbashagent_lambda_trtm_eval-IlN5S34Bc5", - "onlineEvaluationConfigName": "bugbashagent_lambda_trtm_eval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-06-15T19:41:54.111Z" - }, - "updatedAt": { - "$date": "2026-06-15T19:42:10.335Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEval1-QErY9BEWT5", - "onlineEvaluationConfigId": "demoEval_onlineEval1-QErY9BEWT5", - "onlineEvaluationConfigName": "demoEval_onlineEval1", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-07-16T17:26:21.325Z" - }, - "updatedAt": { - "$date": "2026-07-16T17:26:36.933Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", - "onlineEvaluationConfigId": "demoEval_onlineEvalBoto3_20260716180853-tbD9Ob7VUX", - "onlineEvaluationConfigName": "demoEval_onlineEvalBoto3_20260716180853", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-07-16T18:08:54.221Z" - }, - "updatedAt": { - "$date": "2026-07-16T18:08:54.477Z" - }, - "description": "boto3-created: onlineEvalBoto3" - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsights1-QE6As2GTTB", - "onlineEvaluationConfigId": "demoEval_onlineInsights1-QE6As2GTTB", - "onlineEvaluationConfigName": "demoEval_onlineInsights1", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-07-16T17:26:21.625Z" - }, - "updatedAt": { - "$date": "2026-07-16T17:26:37.218Z" - }, - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "clusteringConfig": { - "frequencies": [ - "DAILY" - ] - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", - "onlineEvaluationConfigId": "demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", - "onlineEvaluationConfigName": "demoEval_onlineInsightsBoto3_20260716180854", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-07-16T18:08:54.762Z" - }, - "updatedAt": { - "$date": "2026-07-16T18:08:54.954Z" - }, - "description": "boto3-created: onlineInsightsBoto3", - "insights": [ - { - "insightId": "Builtin.Insight.UserIntent" - } - ], - "clusteringConfig": { - "frequencies": [ - "DAILY" - ] - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/deploytest_myeval-JMv0ihHd30", - "onlineEvaluationConfigId": "deploytest_myeval-JMv0ihHd30", - "onlineEvaluationConfigName": "deploytest_myeval", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-04-21T22:47:43.896Z" - }, - "updatedAt": { - "$date": "2026-04-21T22:47:52.237Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/oiVerify1_prodInsights-rPYFuF2WsE", - "onlineEvaluationConfigId": "oiVerify1_prodInsights-rPYFuF2WsE", - "onlineEvaluationConfigName": "oiVerify1_prodInsights", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2026-08-21T16:49:04.244Z" - }, - "updatedAt": { - "$date": "2026-08-21T16:49:04.485Z" - }, - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ] - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", - "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", - "onlineEvaluationConfigName": "temp", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": { - "$date": "2025-12-02T21:09:17.495Z" - }, - "updatedAt": { - "$date": "2025-12-02T21:09:17.831Z" - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/tjHarnessDerivTest-8rZzgXEHiM", - "onlineEvaluationConfigId": "tjHarnessDerivTest-8rZzgXEHiM", - "onlineEvaluationConfigName": "tjHarnessDerivTest", - "status": "ACTIVE", - "executionStatus": "DISABLED", - "createdAt": { - "$date": "2026-07-27T17:56:04.618Z" - }, - "updatedAt": { - "$date": "2026-07-27T17:56:04.805Z" - } - } - ] -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.3ab70a72f531453e.json b/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.3ab70a72f531453e.json deleted file mode 100644 index 7330cc2531..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.3ab70a72f531453e.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "updatedAt": { - "$date": "2026-08-25T19:22:19.799Z" - }, - "status": "UPDATING", - "executionStatus": "DISABLED" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.a3122d8485a8784.json b/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.a3122d8485a8784.json deleted file mode 100644 index c5cc2dc5f6..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.a3122d8485a8784.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "updatedAt": { - "$date": "2026-08-25T19:22:14.430Z" - }, - "status": "UPDATING", - "executionStatus": "ENABLED" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.db059fecaf31eb68.json b/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.db059fecaf31eb68.json deleted file mode 100644 index 368136dd33..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/UpdateOnlineEvaluationConfigCommand.db059fecaf31eb68.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "updatedAt": { - "$date": "2026-08-25T19:22:25.112Z" - }, - "status": "ACTIVE", - "executionStatus": "DISABLED" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/create.golden.json b/src/handlers/eval/online-insight/__fixtures__/create.golden.json deleted file mode 100644 index 76c549b5be..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/create.golden.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "createdAt": "2026-08-25T19:22:09.493Z", - "status": "CREATING", - "executionStatus": "DISABLED", - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-NPzirZ4e60" - } - } -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/delete.golden.json b/src/handlers/eval/online-insight/__fixtures__/delete.golden.json deleted file mode 100644 index 78913e4575..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/delete.golden.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "status": "DELETING" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/get.golden.json b/src/handlers/eval/online-insight/__fixtures__/get.golden.json deleted file mode 100644 index d5d6754f28..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/get.golden.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigName": "agentcore_cli_online_insight_fixture", - "rule": { - "samplingConfig": { - "samplingPercentage": 25 - }, - "sessionConfig": { - "sessionTimeoutMinutes": 30 - } - }, - "dataSourceConfig": { - "cloudWatchLogs": { - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/testAgent_Agent-wm9hYBD93Y-DEFAULT" - ], - "serviceNames": [ - "testAgent_Agent.DEFAULT" - ] - } - }, - "status": "ACTIVE", - "executionStatus": "DISABLED", - "createdAt": "2026-08-25T19:22:09.493Z", - "updatedAt": "2026-08-25T19:22:25.112Z", - "evaluators": [], - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_insight_fixture-NPzirZ4e60" - } - }, - "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/list.golden.json b/src/handlers/eval/online-insight/__fixtures__/list.golden.json deleted file mode 100644 index 306254aa92..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/list.golden.json +++ /dev/null @@ -1,57 +0,0 @@ -{ - "onlineEvaluationConfigs": [ - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsights1-QE6As2GTTB", - "onlineEvaluationConfigId": "demoEval_onlineInsights1-QE6As2GTTB", - "onlineEvaluationConfigName": "demoEval_onlineInsights1", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": "2026-07-16T17:26:21.625Z", - "updatedAt": "2026-07-16T17:26:37.218Z", - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ], - "clusteringConfig": { - "frequencies": [ - "DAILY" - ] - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", - "onlineEvaluationConfigId": "demoEval_onlineInsightsBoto3_20260716180854-aTLOAI48Jr", - "onlineEvaluationConfigName": "demoEval_onlineInsightsBoto3_20260716180854", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": "2026-07-16T18:08:54.762Z", - "updatedAt": "2026-07-16T18:08:54.954Z", - "description": "boto3-created: onlineInsightsBoto3", - "insights": [ - { - "insightId": "Builtin.Insight.UserIntent" - } - ], - "clusteringConfig": { - "frequencies": [ - "DAILY" - ] - } - }, - { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/oiVerify1_prodInsights-rPYFuF2WsE", - "onlineEvaluationConfigId": "oiVerify1_prodInsights-rPYFuF2WsE", - "onlineEvaluationConfigName": "oiVerify1_prodInsights", - "status": "ACTIVE", - "executionStatus": "ENABLED", - "createdAt": "2026-08-21T16:49:04.244Z", - "updatedAt": "2026-08-21T16:49:04.485Z", - "insights": [ - { - "insightId": "Builtin.Insight.FailureAnalysis" - } - ] - } - ] -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/pause.golden.json b/src/handlers/eval/online-insight/__fixtures__/pause.golden.json deleted file mode 100644 index 3aa0b50004..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/pause.golden.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "updatedAt": "2026-08-25T19:22:19.799Z", - "status": "UPDATING", - "executionStatus": "DISABLED" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/resume.golden.json b/src/handlers/eval/online-insight/__fixtures__/resume.golden.json deleted file mode 100644 index 75c2b6ad31..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/resume.golden.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "updatedAt": "2026-08-25T19:22:14.430Z", - "status": "UPDATING", - "executionStatus": "ENABLED" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/__fixtures__/update.golden.json b/src/handlers/eval/online-insight/__fixtures__/update.golden.json deleted file mode 100644 index eeb9bdbdcd..0000000000 --- a/src/handlers/eval/online-insight/__fixtures__/update.golden.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_insight_fixture-NPzirZ4e60", - "onlineEvaluationConfigId": "agentcore_cli_online_insight_fixture-NPzirZ4e60", - "updatedAt": "2026-08-25T19:22:25.112Z", - "status": "ACTIVE", - "executionStatus": "DISABLED" -} \ No newline at end of file diff --git a/src/handlers/eval/online-insight/create/index.tsx b/src/handlers/eval/online-insight/create/index.tsx deleted file mode 100644 index f3cbf4f3ed..0000000000 --- a/src/handlers/eval/online-insight/create/index.tsx +++ /dev/null @@ -1,115 +0,0 @@ -import z from "zod"; -import type { DataSourceConfig, Filter } from "@aws-sdk/client-bedrock-agentcore-control"; -import { createHandler, flag } from "../../../../router"; -import { InputValidationError } from "../../../../errors"; -import { JsonRendererKey } from "../../../../tui"; -import { SourceResolver, type AppIO } from "../../../../io"; -import type { Core } from "../../../types"; -import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; - -const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight."; -const ARN_PREFIX = "arn:"; - -export const createCreateOnlineInsightHandler = (core: Core, io: AppIO) => - createHandler({ - name: "create", - description: "create an online insight config", - flags: [ - flag("name", "the name of the online insight config", z.string().min(1)), - flag("role-arn", "IAM role the online insight assumes", z.string().min(1)), - flag("agent", "harness ID or Runtime ID whose traffic to sample", z.string().optional()), - flag( - "endpoint", - "the agent endpoint qualifier to scope monitoring to (default DEFAULT)", - z.string().optional(), - ), - flag( - "data-source-config", - "the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin), as an alternative to --agent", - z.string().optional(), - ), - flag( - "insight", - "insight ID(s) to apply: Builtin.Insight.* identifiers or full ARNs", - z.array(z.string()).min(1), - ), - flag( - "clustering-frequency", - "insight clustering cadence(s): DAILY, WEEKLY, MONTHLY", - z.array(z.enum(["DAILY", "WEEKLY", "MONTHLY"])).optional(), - ), - flag( - "sampling-rate", - "percentage of sessions to sample (0.01-100)", - z.number().min(0.01).max(100), - ), - flag( - "session-timeout-minutes", - "minutes of inactivity before a session is considered complete (1-1440, default 15)", - z.number().int().min(1).max(1440).optional(), - ), - flag( - "filters", - "trace filters (JSON Filter[]; inline, file://, or - for stdin)", - z.string().optional(), - ), - flag( - "enable-on-create", - "whether to enable evaluation immediately (default true; pass false to create it paused)", - z.enum(["true", "false"]).optional(), - ), - flag( - "description", - "a description of the config's monitoring purpose", - z.string().optional(), - ), - ], - handle: async (ctx, flags) => { - for (const id of flags["insight"]) { - if (!id.startsWith(BUILTIN_INSIGHT_PREFIX) && !id.startsWith(ARN_PREFIX)) - throw new InputValidationError( - `invalid insight "${id}": must be a ${BUILTIN_INSIGHT_PREFIX}* identifier or a full ARN`, - ); - } - - assertMutuallyExclusiveFlags(flags, ["agent", "data-source-config"], { exactlyOne: true }); - const hasAgent = flags["agent"] !== undefined; - const hasDataSource = flags["data-source-config"] !== undefined; - if (hasDataSource && flags["endpoint"]) - throw new InputValidationError("'--endpoint' can only be used with '--agent'"); - - const source = new SourceResolver({ stdin: io.stdin }); - const frequencies = flags["clustering-frequency"]; - const common = { - name: flags["name"], - description: flags["description"], - samplingRate: flags["sampling-rate"], - sessionTimeoutMinutes: flags["session-timeout-minutes"], - filters: parseJsonFlag( - "filters", - await source.resolveText("filters", flags["filters"]), - ), - insightIds: flags["insight"], - clusteringConfig: frequencies ? { frequencies } : undefined, - evaluationExecutionRoleArn: flags["role-arn"], - enableOnCreate: - flags["enable-on-create"] === undefined - ? undefined - : flags["enable-on-create"] === "true", - }; - - const response = await core.eval.createOnlineInsight( - hasAgent - ? { ...common, agent: flags["agent"]!, endpoint: flags["endpoint"] } - : { - ...common, - dataSourceConfig: parseJsonFlag( - "data-source-config", - await source.resolveText("data-source-config", flags["data-source-config"]), - )!, - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); diff --git a/src/handlers/eval/online-insight/delete/index.tsx b/src/handlers/eval/online-insight/delete/index.tsx deleted file mode 100644 index 21784e4908..0000000000 --- a/src/handlers/eval/online-insight/delete/index.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; -import type { Core } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -export const createDeleteOnlineInsightHandler = (core: Core) => - createHandler({ - name: "delete", - description: "delete an online insight config by ID", - flags: [flag("id", "the ID of the online insight config to delete", z.string().min(1))], - handle: async (ctx, flags) => { - ctx - .require(JsonRendererKey) - .renderJson(await core.eval.deleteOnlineInsight(flags["id"], coreOptsFromCtx(ctx))); - }, - }); diff --git a/src/handlers/eval/online-insight/get/index.tsx b/src/handlers/eval/online-insight/get/index.tsx deleted file mode 100644 index d0385f47de..0000000000 --- a/src/handlers/eval/online-insight/get/index.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; -import type { Core } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -export const createGetOnlineInsightHandler = (core: Core) => - createHandler({ - name: "get", - description: "get an online insight config by ID", - flags: [flag("id", "the ID of the online insight config", z.string().min(1))], - handle: async (ctx, flags) => { - ctx - .require(JsonRendererKey) - .renderJson(await core.eval.getOnlineInsight(flags["id"], coreOptsFromCtx(ctx))); - }, - }); diff --git a/src/handlers/eval/online-insight/get/screen.tsx b/src/handlers/eval/online-insight/get/screen.tsx deleted file mode 100644 index db119b2a67..0000000000 --- a/src/handlers/eval/online-insight/get/screen.tsx +++ /dev/null @@ -1,78 +0,0 @@ -import { useQuery } from "@tanstack/react-query"; -import { useNavigate, useParams } from "react-router"; -import { JsonDetail } from "../../../../components/JsonDetail"; -import { ResourceDetailScreen } from "../../../../components/ResourceDetailScreen"; -import type { ScreenProps } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -function useOnlineInsightDetail({ ctx, core }: ScreenProps, configId: string | undefined) { - const opts = coreOptsFromCtx(ctx); - return useQuery({ - queryKey: ["online-insight", opts.region, configId], - queryFn: () => core.eval.getOnlineInsight(configId!, opts), - enabled: configId !== undefined, - }); -} - -export function OnlineInsightGetScreen(props: ScreenProps) { - const navigate = useNavigate(); - const { configId } = useParams(); - const detail = useOnlineInsightDetail(props, configId); - const config = detail.data; - const samplingPercentage = config?.rule?.samplingConfig?.samplingPercentage; - const insightCount = config?.insights?.length ?? 0; - const frequencies = config?.clusteringConfig?.frequencies ?? []; - - return ( - 0 ? insightCount.toString() : "-", - clustering: frequencies.length > 0 ? frequencies.join(", ") : "-", - ...(config?.failureReason ? { failureReason: config.failureReason } : {}), - role: config?.evaluationExecutionRoleArn ?? "-", - arn: config?.onlineEvaluationConfigArn ?? "", - }} - actions={ - configId && config - ? [ - { - name: "detail", - description: "show the full JSON (insights, clustering, filters, data source)", - onSelect: () => - navigate( - `/agentcore/eval/online-insight/get/${encodeURIComponent(configId)}/json`, - ), - }, - ] - : [] - } - loadingLabel="loading online insight config…" - onRetry={() => void detail.refetch()} - selectLabel="open" - /> - ); -} - -export function OnlineInsightGetJsonScreen(props: ScreenProps) { - const { configId } = useParams(); - const detail = useOnlineInsightDetail(props, configId); - - return ( - void detail.refetch()} - /> - ); -} diff --git a/src/handlers/eval/online-insight/index.tsx b/src/handlers/eval/online-insight/index.tsx deleted file mode 100644 index b54329b493..0000000000 --- a/src/handlers/eval/online-insight/index.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import { Router } from "../../../router"; -import { renderTui } from "../../../tui"; -import { withTuiOnEmptyFlagsAndArgs } from "../../../middleware"; -import type { AppIO } from "../../../io"; -import type { Core } from "../../types"; -import { createCreateOnlineInsightHandler } from "./create"; -import { createGetOnlineInsightHandler } from "./get"; -import { createListOnlineInsightHandler } from "./list"; -import { createUpdateOnlineInsightHandler } from "./update"; -import { createPauseOnlineInsightHandler } from "./pause"; -import { createResumeOnlineInsightHandler } from "./resume"; -import { createDeleteOnlineInsightHandler } from "./delete"; - -export function createOnlineInsightHandler(core: Core, io: AppIO): Router { - return new Router("online-insight", "manage AgentCore online insight configs") - .use(withTuiOnEmptyFlagsAndArgs(core, io)) - .default(renderTui(core, io)) - .supportedTuiCommands("get", "list") - .handler(createCreateOnlineInsightHandler(core, io)) - .handler(createGetOnlineInsightHandler(core)) - .handler(createListOnlineInsightHandler(core)) - .handler(createUpdateOnlineInsightHandler(core, io)) - .handler(createPauseOnlineInsightHandler(core)) - .handler(createResumeOnlineInsightHandler(core)) - .handler(createDeleteOnlineInsightHandler(core)); -} - -export { OnlineInsightScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/online-insight/list/index.tsx b/src/handlers/eval/online-insight/list/index.tsx deleted file mode 100644 index 284fd21d82..0000000000 --- a/src/handlers/eval/online-insight/list/index.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; -import type { Core } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -export const createListOnlineInsightHandler = (core: Core) => - createHandler({ - name: "list", - description: "list online insight configs", - flags: [ - flag("next-token", "pagination token returned by a previous request", z.string().optional()), - flag("max-results", "maximum number of items to return", z.number().optional()), - ], - handle: async (ctx, flags) => { - const response = await core.eval.listOnlineInsights( - flags["next-token"], - flags["max-results"], - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); diff --git a/src/handlers/eval/online-insight/list/screen.tsx b/src/handlers/eval/online-insight/list/screen.tsx deleted file mode 100644 index fcd5a3d03d..0000000000 --- a/src/handlers/eval/online-insight/list/screen.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { useNavigate } from "react-router"; -import { OnlineInsightPicker } from "../../../../components/OnlineInsightPicker"; -import type { ScreenProps } from "../../../types"; - -export function OnlineInsightListScreen(props: ScreenProps) { - const navigate = useNavigate(); - - return ( - - navigate(`/agentcore/eval/online-insight/get/${encodeURIComponent(configId)}`) - } - /> - ); -} diff --git a/src/handlers/eval/online-insight/online-insight.screen.test.tsx b/src/handlers/eval/online-insight/online-insight.screen.test.tsx deleted file mode 100644 index 2ecbc9d2f9..0000000000 --- a/src/handlers/eval/online-insight/online-insight.screen.test.tsx +++ /dev/null @@ -1,195 +0,0 @@ -import { afterEach, describe, expect, test } from "bun:test"; -import type { - GetOnlineEvaluationConfigResponse, - OnlineEvaluationConfigSummary, -} from "@aws-sdk/client-bedrock-agentcore-control"; -import { - cleanupScreens, - renderScreen, - TestCoreClient, - waitFor, - waitForText, - menuEntries, -} from "../../../testing"; - -afterEach(cleanupScreens); - -const evalEndpointUrl = "https://eval.test"; - -function configSummary( - overrides: Partial = {}, -): OnlineEvaluationConfigSummary { - return { - onlineEvaluationConfigArn: - "arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/oic-1", - onlineEvaluationConfigId: "oic-1", - onlineEvaluationConfigName: "prod_failure_insights", - status: "ACTIVE", - executionStatus: "ENABLED", - insights: [{ insightId: "ins-failure" }], - createdAt: new Date("2026-07-19T01:02:03.000Z"), - updatedAt: new Date("2026-07-20T12:34:56.000Z"), - ...overrides, - }; -} - -function getConfigResponse( - overrides: Partial = {}, -): GetOnlineEvaluationConfigResponse { - return { - onlineEvaluationConfigArn: - "arn:aws:bedrock-agentcore:us-east-1:123456789012:online-evaluation-config/oic-1", - onlineEvaluationConfigId: "oic-1", - onlineEvaluationConfigName: "prod_failure_insights", - status: "ACTIVE", - executionStatus: "ENABLED", - rule: { samplingConfig: { samplingPercentage: 5 } }, - dataSourceConfig: { - cloudWatchLogs: { logGroupNames: ["/aws/bedrock-agentcore/runtime/x"], serviceNames: [] }, - }, - insights: [{ insightId: "ins-failure" }, { insightId: "ins-intent" }], - clusteringConfig: { frequencies: ["DAILY", "WEEKLY"] }, - evaluationExecutionRoleArn: "arn:aws:iam::123456789012:role/online-insight-role", - createdAt: new Date("2026-07-19T01:02:03.000Z"), - updatedAt: new Date("2026-07-20T12:34:56.000Z"), - ...overrides, - }; -} - -function coreWithConfigs(configs: OnlineEvaluationConfigSummary[]): TestCoreClient { - const core = new TestCoreClient(); - core.eval.setOnlineEvalListResponse({ onlineEvaluationConfigs: configs }); - return core; -} - -describe("online-insight menu", () => { - test("lists the read-only commands, then the rest as command line only", async () => { - const screen = renderScreen("/agentcore/eval/online-insight"); - - await waitForText(screen.lastFrame, "get an online insight config by ID"); - expect(menuEntries(screen.lastFrame()!)).toEqual({ - screens: ["get", "list"], - cliOnly: [], - }); - }); -}); - -describe("online-insight picker", () => { - test("renders name, execution status, and update time", async () => { - const core = coreWithConfigs([ - configSummary({ - onlineEvaluationConfigName: "staging_intent_insights", - executionStatus: "DISABLED", - updatedAt: new Date("2026-07-21T02:03:04.000Z"), - }), - ]); - const screen = renderScreen("/agentcore/eval/online-insight/list", { core }); - - await waitForText(screen.lastFrame, "staging_intent_insights"); - const frame = screen.lastFrame()!; - expect(frame).toContain("DISABLED"); - expect(frame).toContain("2026-07-21 02:03"); - }); - - test("calls listOnlineInsights with exact Core options", async () => { - const core = coreWithConfigs([configSummary()]); - renderScreen("/agentcore/eval/online-insight/list", { core, endpointUrl: evalEndpointUrl }); - - await waitFor(() => core.eval.calls.some((call) => call.method === "listOnlineInsights")); - expect(core.eval.calls.filter((call) => call.method === "listOnlineInsights")).toEqual([ - { - method: "listOnlineInsights", - args: [ - undefined, - expect.any(Number), - { region: "us-east-1", endpointUrl: evalEndpointUrl }, - ], - }, - ]); - }); - - test("bare online-insight get redirects to the picker", async () => { - const core = coreWithConfigs([ - configSummary({ - onlineEvaluationConfigId: "redirected-oic", - onlineEvaluationConfigName: "redirected_insight", - }), - ]); - const screen = renderScreen("/agentcore/eval/online-insight/get", { core }); - - await waitForText(screen.lastFrame, "redirected_insight"); - expect(core.eval.calls[0]?.method).toBe("listOnlineInsights"); - }); - - test("selection opens the matching config detail", async () => { - const core = coreWithConfigs([configSummary({ onlineEvaluationConfigId: "oic-1" })]); - core.eval.setOnlineEvalGetResponse(getConfigResponse({ onlineEvaluationConfigId: "oic-1" })); - const screen = renderScreen("/agentcore/eval/online-insight/list", { core }); - - await waitForText(screen.lastFrame, "prod_failure_insights"); - await screen.press("return"); - await waitForText(screen.lastFrame, "agentcore → eval → online-insight → get → oic-1"); - await waitFor(() => - core.eval.calls.some( - (call) => call.method === "getOnlineInsight" && call.args[0] === "oic-1", - ), - ); - }); - - test("shows the empty state", async () => { - const empty = renderScreen("/agentcore/eval/online-insight/list"); - await waitForText(empty.lastFrame, "No online insight configs found in this Region."); - }); -}); - -describe("online-insight detail", () => { - test("renders sampling, execution status, insight count, and clustering frequencies", async () => { - const core = new TestCoreClient(); - core.eval.setOnlineEvalGetResponse(getConfigResponse()); - const screen = renderScreen("/agentcore/eval/online-insight/get/oic-1", { - core, - endpointUrl: evalEndpointUrl, - }); - - await waitForText(screen.lastFrame, "show the full JSON"); - const frame = screen.lastFrame()!; - expect(frame).toContain("prod_failure_insights"); - expect(frame).toMatch(/sampling\s+5%/); - expect(frame).toMatch(/execution\s+ENABLED/); - expect(frame).toMatch(/insights\s+2/); - expect(frame).toContain("DAILY, WEEKLY"); - expect(frame).toMatch(/arn\s+arn:aws:bedrock-agentcore:/); - expect(frame.replace(/\s+/g, "")).toContain(getConfigResponse().onlineEvaluationConfigArn!); - expect(frame.replace(/\s+/g, "")).toContain(getConfigResponse().evaluationExecutionRoleArn!); - expect(frame).not.toContain("evaluators"); - expect(core.eval.calls.find((call) => call.method === "getOnlineInsight")).toEqual({ - method: "getOnlineInsight", - args: ["oic-1", { region: "us-east-1", endpointUrl: evalEndpointUrl }], - }); - }); - - test("opens the complete config JSON", async () => { - const core = new TestCoreClient(); - core.eval.setOnlineEvalGetResponse(getConfigResponse()); - const screen = renderScreen("/agentcore/eval/online-insight/get/oic-1", { core }); - - await waitForText(screen.lastFrame, "show the full JSON"); - await screen.press("return"); - await waitForText(screen.lastFrame, "agentcore → eval → online-insight → get → oic-1 → json"); - expect(screen.lastFrame()).toContain('"clusteringConfig"'); - }); - - test("retries a failed detail query", async () => { - const core = new TestCoreClient(); - core.eval.setError(new Error("insight unavailable")); - const screen = renderScreen("/agentcore/eval/online-insight/get/oic-1", { core }); - - await waitForText(screen.lastFrame, "insight unavailable"); - expect(screen.lastFrame()).toContain("[r] retry"); - - core.eval.setError(undefined); - core.eval.setOnlineEvalGetResponse(getConfigResponse()); - await screen.write("r"); - await waitForText(screen.lastFrame, "show the full JSON"); - }); -}); diff --git a/src/handlers/eval/online-insight/online-insight.test.tsx b/src/handlers/eval/online-insight/online-insight.test.tsx deleted file mode 100644 index 6c798ccead..0000000000 --- a/src/handlers/eval/online-insight/online-insight.test.tsx +++ /dev/null @@ -1,392 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { join } from "node:path"; -import { CoreClient } from "../../../core"; -import { - createSilentLogger, - expectError, - fixtureFactories, - matchGolden, - settle, - TestGlobalConfigAccessor, - testIO, -} from "../../../testing"; -import { createRootHandler } from "../../index"; -import { InputValidationError } from "../../../errors"; - -const REGION = "us-west-2"; -const FIXTURES = join(import.meta.dir, "__fixtures__"); - -// Record with RECORD=1 bun test src/handlers/eval/online-insight/online-insight.test.tsx -// The RECORD run creates one online insight config against a real runtime, -// exercises list/get/resume/pause against it, then deletes it, so a recording -// leaves no residue. The id the service assigns is captured from the recorded -// create response, which keeps the dependent fixtures (keyed by request input) -// stable on replay. -const CONFIG_NAME = "agentcore_cli_online_insight_fixture"; - -// online-insight requires an insight id (no evaluators); a builtin needs no setup. -const FIXTURE_INSIGHT_ID = "Builtin.Insight.FailureAnalysis"; - -// The runtime whose traffic the recorded config samples. `--agent` resolves it to -// the CloudWatch log group / service name pair, so the id must exist in the -// fixture account. It is only referenced, never invoked. -const FIXTURE_AGENT_ID = "testAgent_Agent-wm9hYBD93Y"; - -// online-insight has no auto-provisioning, so --role-arn is required. This role -// pre-exists in the fixture account, scoped to online-evaluation-config/*. -const FIXTURE_ROLE_ARN = "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339"; - -// Well-formed but absent id: it passes the id pattern so the request reaches the -// service and comes back not-found, rather than failing local validation first. -const MISSING_CONFIG_ID = "missing-online-0000000000"; - -// A plain online-EVAL config (no insights) that pre-exists in the fixture account. -// The insight id-guards GET-verify it, see no insights, and reject before touching -// it — so pointing get/delete at it is a safe negative test that mutates nothing. -const EVAL_ONLY_CONFIG_ID = "ABVfyLatest_ProdEval-2vqlCb2UiG"; - -function createFixtureCore(): CoreClient { - const { - createControlClient, - createDataClient, - createIamClient, - createLogsClient, - createXrayClient, - createApplicationSignalsClient, - } = fixtureFactories(FIXTURES); - return new CoreClient({ - createControlClient, - createDataClient, - createIamClient, - createLogsClient, - createXrayClient, - createApplicationSignalsClient, - logger: createSilentLogger(), - }); -} - -// run drives the real router (parsing → middleware → handler → CoreClient) against -// the fixture-backed SDK clients and returns captured stdout. -async function run(args: string[]): Promise { - const io = testIO(); - const root = createRootHandler(createFixtureCore(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - - await root.route(["node", "agentcore", ...args, "--region", REGION]); - return io.stdout(); -} - -// The id assigned by CreateOnlineEvaluationConfig, shared by the tests below. -let configId: string; - -describe("eval online-insight command hierarchy", () => { - test("registers the eval → online-insight command tree", () => { - const root = createRootHandler(createFixtureCore(), { - io: testIO().io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - const onlineInsight = root - .children() - .find((c) => c.name() === "eval") - ?.children() - .find((c) => c.name() === "online-insight"); - - expect(onlineInsight?.children().map((c) => c.name())).toEqual([ - "create", - "get", - "list", - "update", - "pause", - "resume", - "delete", - ]); - }); -}); - -describe("online-insight CRUDL", () => { - test("creates an online insight config from an agent", async () => { - const stdout = await run([ - "eval", - "online-insight", - "create", - "--name", - CONFIG_NAME, - "--agent", - FIXTURE_AGENT_ID, - "--role-arn", - FIXTURE_ROLE_ARN, - "--insight", - FIXTURE_INSIGHT_ID, - "--sampling-rate", - "10", - "--session-timeout-minutes", - "30", - "--enable-on-create", - "false", - ]); - - matchGolden(FIXTURES, "create.golden.json", stdout); - configId = JSON.parse(stdout).onlineEvaluationConfigId; - expect(configId).toBeString(); - }); - - test("lists online insight configs", async () => { - // --json forces the headless path; a bare `list` (no flags) would otherwise - // open the TUI under the empty-invocation middleware. - const stdout = await run(["eval", "online-insight", "list", "--json"]); - - matchGolden(FIXTURES, "list.golden.json", stdout); - const configs = JSON.parse(stdout).onlineEvaluationConfigs; - expect(configs).toBeArray(); - // list is insight-only now: the client filters the online-eval list down to - // configs that carry insights, so every returned row must have a non-empty one. - expect(configs.length).toBeGreaterThan(0); - for (const c of configs) expect(c.insights?.length ?? 0).toBeGreaterThan(0); - }); - - // resume and pause are asserted in one test because the service rejects an - // update while the previous one is still settling (ConflictException, state - // UPDATING). Recording therefore waits for the config to leave UPDATING between - // the two calls; on replay the fixtures are served instantly and the wait is a - // no-op, so the test stays fast and deterministic. - test("resumes then pauses the config, toggling execution status", async () => { - const resumeStdout = await run(["eval", "online-insight", "resume", "--id", configId]); - matchGolden(FIXTURES, "resume.golden.json", resumeStdout); - expect(JSON.parse(resumeStdout).executionStatus).toBe("ENABLED"); - - await settle(); - - const pauseStdout = await run(["eval", "online-insight", "pause", "--id", configId]); - matchGolden(FIXTURES, "pause.golden.json", pauseStdout); - expect(JSON.parse(pauseStdout).executionStatus).toBe("DISABLED"); - // Extended timeout so the settle() wait fits while recording; it is a no-op on replay. - }, 60_000); - - // update runs after resume/pause and folds get in on purpose. Every id-scoped op - // (get, and the GET-verify guard in update/pause/resume/delete) issues - // GetOnlineEvaluationConfig with the same id, so they all share one input-keyed - // fixture. Recording the update+get last — settling before the update so pause has - // left UPDATING, and again before the get so the update has — makes the fixture's - // final state (sampling 25, settled) the one get.golden captures, so replay, which - // serves that final fixture to every GET, matches get.golden. update merges over - // the current config because UpdateOnlineEvaluationConfig replaces the whole `rule`; - // asserting the unset fields survive proves the merge. - test("updates only the sampling rate, preserving the timeout, insight, and role", async () => { - await settle(); - - const stdout = await run([ - "eval", - "online-insight", - "update", - "--id", - configId, - "--sampling-rate", - "25", - ]); - matchGolden(FIXTURES, "update.golden.json", stdout); - - await settle(); - - const getStdout = await run(["eval", "online-insight", "get", "--id", configId]); - matchGolden(FIXTURES, "get.golden.json", getStdout); - - const after = JSON.parse(getStdout); - expect(after.onlineEvaluationConfigName).toBe(CONFIG_NAME); - expect(after.rule.samplingConfig.samplingPercentage).toBe(25); - // --session-timeout-minutes was never passed to update, so the value set at - // create must survive the whole-`rule` replacement. - expect(after.rule.sessionConfig.sessionTimeoutMinutes).toBe(30); - // online-insight carries an insight list, never evaluators; the sampling-only - // update must leave both untouched. - expect(after.insights.map((i: { insightId: string }) => i.insightId)).toContain( - FIXTURE_INSIGHT_ID, - ); - expect(after.evaluators ?? []).toEqual([]); - // --role-arn is used verbatim; online-insight never provisions a role. - expect(after.evaluationExecutionRoleArn).toBe(FIXTURE_ROLE_ARN); - }, 60_000); - - test("deletes the online insight config", async () => { - const stdout = await run(["eval", "online-insight", "delete", "--id", configId]); - matchGolden(FIXTURES, "delete.golden.json", stdout); - }); - - // Guard (Nico's fix): the id-scoped ops model online-insight as a distinct - // resource, so they GET-verify and reject a plain online-EVAL config instead of - // silently operating on it. delete rejects at the guard, before any delete call, - // so aiming it at a real eval config mutates nothing. - test("rejects a plain online-eval config that has no insights", async () => { - await expect( - run(["eval", "online-insight", "get", "--id", EVAL_ONLY_CONFIG_ID, "--json"]), - ).rejects.toThrow(/is not an online-insight config/); - - await expect( - run(["eval", "online-insight", "delete", "--id", EVAL_ONLY_CONFIG_ID, "--json"]), - ).rejects.toThrow(/is not an online-insight config/); - }); - - test("propagates ResourceNotFoundException from get", async () => { - await expect( - run(["eval", "online-insight", "get", "--id", MISSING_CONFIG_ID]), - ).rejects.toMatchObject({ name: "ResourceNotFoundException" }); - }); -}); - -// Flag parsing never reaches the SDK, so these need no fixtures. -describe("flag validation", () => { - test("create requires --role-arn", async () => { - await expectError( - run([ - "eval", - "online-insight", - "create", - "--name", - CONFIG_NAME, - "--agent", - FIXTURE_AGENT_ID, - "--insight", - FIXTURE_INSIGHT_ID, - "--sampling-rate", - "10", - ]), - "required option '--role-arn' not specified", - InputValidationError, - ); - }); - - test("create requires --insight", async () => { - await expectError( - run([ - "eval", - "online-insight", - "create", - "--name", - CONFIG_NAME, - "--agent", - FIXTURE_AGENT_ID, - "--role-arn", - FIXTURE_ROLE_ARN, - "--sampling-rate", - "10", - ]), - "required option '--insight' not specified", - InputValidationError, - ); - }); - - test("create rejects an insight id that is neither a builtin nor an ARN", async () => { - await expect( - run([ - "eval", - "online-insight", - "create", - "--name", - CONFIG_NAME, - "--agent", - FIXTURE_AGENT_ID, - "--role-arn", - FIXTURE_ROLE_ARN, - "--insight", - "nope", - "--sampling-rate", - "10", - ]), - ).rejects.toThrow(/invalid insight "nope"/); - }); - - test("create requires exactly one of --agent or --data-source-config", async () => { - await expect( - run([ - "eval", - "online-insight", - "create", - "--name", - CONFIG_NAME, - "--role-arn", - FIXTURE_ROLE_ARN, - "--insight", - FIXTURE_INSIGHT_ID, - "--sampling-rate", - "10", - ]), - ).rejects.toThrow(/exactly one of --agent, --data-source-config/); - }); - - test("create rejects both --agent and --data-source-config", async () => { - await expect( - run([ - "eval", - "online-insight", - "create", - "--name", - CONFIG_NAME, - "--agent", - FIXTURE_AGENT_ID, - "--data-source-config", - '{"cloudWatchLogs":{"logGroupNames":["/custom"],"serviceNames":["svc"]}}', - "--role-arn", - FIXTURE_ROLE_ARN, - "--insight", - FIXTURE_INSIGHT_ID, - "--sampling-rate", - "10", - ]), - ).rejects.toThrow(/exactly one of --agent, --data-source-config/); - }); - - test("create rejects --endpoint without --agent", async () => { - await expect( - run([ - "eval", - "online-insight", - "create", - "--name", - CONFIG_NAME, - "--data-source-config", - '{"cloudWatchLogs":{"logGroupNames":["/custom"],"serviceNames":["svc"]}}', - "--endpoint", - "prod", - "--role-arn", - FIXTURE_ROLE_ARN, - "--insight", - FIXTURE_INSIGHT_ID, - "--sampling-rate", - "10", - ]), - ).rejects.toThrow(/'--endpoint' can only be used with '--agent'/); - }); - - test("create rejects malformed --data-source-config JSON", async () => { - await expect( - run([ - "eval", - "online-insight", - "create", - "--name", - CONFIG_NAME, - "--data-source-config", - "not-json", - "--role-arn", - FIXTURE_ROLE_ARN, - "--insight", - FIXTURE_INSIGHT_ID, - "--sampling-rate", - "10", - ]), - ).rejects.toThrow(/Invalid JSON for option '--data-source-config'/); - }); - - // --json forces the headless path so the required-flag error surfaces; without - // it a bare invocation opens the TUI under the empty-invocation middleware. - test.each(["get", "pause", "resume", "delete"])("%s requires --id", async (command) => { - await expectError( - run(["eval", "online-insight", command, "--json"]), - /required option '--id' not specified/, - InputValidationError, - ); - }); -}); diff --git a/src/handlers/eval/online-insight/pause/index.tsx b/src/handlers/eval/online-insight/pause/index.tsx deleted file mode 100644 index f950b9d958..0000000000 --- a/src/handlers/eval/online-insight/pause/index.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; -import type { Core } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -export const createPauseOnlineInsightHandler = (core: Core) => - createHandler({ - name: "pause", - description: "pause an online insight config", - flags: [flag("id", "the ID of the online insight config to pause", z.string().min(1))], - handle: async (ctx, flags) => { - ctx - .require(JsonRendererKey) - .renderJson( - await core.eval.setOnlineInsightExecutionStatus( - flags["id"], - "DISABLED", - coreOptsFromCtx(ctx), - ), - ); - }, - }); diff --git a/src/handlers/eval/online-insight/resume/index.tsx b/src/handlers/eval/online-insight/resume/index.tsx deleted file mode 100644 index 3e1d2b8d73..0000000000 --- a/src/handlers/eval/online-insight/resume/index.tsx +++ /dev/null @@ -1,23 +0,0 @@ -import z from "zod"; -import { createHandler, flag } from "../../../../router"; -import { JsonRendererKey } from "../../../../tui"; -import type { Core } from "../../../types"; -import { coreOptsFromCtx } from "../../../utils"; - -export const createResumeOnlineInsightHandler = (core: Core) => - createHandler({ - name: "resume", - description: "resume a paused online insight config", - flags: [flag("id", "the ID of the online insight config to resume", z.string().min(1))], - handle: async (ctx, flags) => { - ctx - .require(JsonRendererKey) - .renderJson( - await core.eval.setOnlineInsightExecutionStatus( - flags["id"], - "ENABLED", - coreOptsFromCtx(ctx), - ), - ); - }, - }); diff --git a/src/handlers/eval/online-insight/screen.tsx b/src/handlers/eval/online-insight/screen.tsx deleted file mode 100644 index 68277b04e5..0000000000 --- a/src/handlers/eval/online-insight/screen.tsx +++ /dev/null @@ -1,6 +0,0 @@ -import { RouterScreen } from "../../../components/RouterScreen"; -import type { ScreenProps } from "../../types"; - -export function OnlineInsightScreen(props: ScreenProps) { - return ; -} diff --git a/src/handlers/eval/online-insight/update/index.tsx b/src/handlers/eval/online-insight/update/index.tsx deleted file mode 100644 index ddd55f6d7b..0000000000 --- a/src/handlers/eval/online-insight/update/index.tsx +++ /dev/null @@ -1,106 +0,0 @@ -import z from "zod"; -import type { DataSourceConfig, Filter } from "@aws-sdk/client-bedrock-agentcore-control"; -import { createHandler, flag } from "../../../../router"; -import { InputValidationError } from "../../../../errors"; -import { JsonRendererKey } from "../../../../tui"; -import { SourceResolver, type AppIO } from "../../../../io"; -import type { Core } from "../../../types"; -import { assertMutuallyExclusiveFlags, coreOptsFromCtx, parseJsonFlag } from "../../../utils"; - -const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight."; -const ARN_PREFIX = "arn:"; - -export const createUpdateOnlineInsightHandler = (core: Core, io: AppIO) => - createHandler({ - name: "update", - description: "update an online insight config", - flags: [ - flag("id", "the ID of the online insight config to update", z.string().min(1)), - flag( - "sampling-rate", - "percentage of sessions to sample (0.01-100)", - z.number().min(0.01).max(100).optional(), - ), - flag( - "session-timeout-minutes", - "minutes of inactivity before a session is considered complete (1-1440)", - z.number().int().min(1).max(1440).optional(), - ), - flag( - "filters", - "trace filters (JSON Filter[]; inline, file://, or - for stdin)", - z.string().optional(), - ), - flag( - "insight", - "insight ID(s) to apply (replaces the existing list): Builtin.Insight.* or ARN", - z.array(z.string()).optional(), - ), - flag( - "clustering-frequency", - "insight clustering cadence(s) (replaces the existing set): DAILY, WEEKLY, MONTHLY", - z.array(z.enum(["DAILY", "WEEKLY", "MONTHLY"])).optional(), - ), - flag("agent", "repoint at a different Runtime ID", z.string().optional()), - flag( - "endpoint", - "re-scope monitoring to a different agent endpoint qualifier", - z.string().optional(), - ), - flag( - "clear-endpoint", - "reset the endpoint scope to the default qualifier (pass true)", - z.enum(["true", "false"]).optional(), - ), - flag( - "data-source-config", - "replace the traces to evaluate (JSON DataSourceConfig; inline, file://, or - for stdin)", - z.string().optional(), - ), - flag("role-arn", "replace the IAM role the online insight assumes", z.string().optional()), - ], - handle: async (ctx, flags) => { - if (flags["endpoint"] && flags["clear-endpoint"] === "true") - throw new InputValidationError( - "'--endpoint' and '--clear-endpoint' are mutually exclusive", - ); - assertMutuallyExclusiveFlags(flags, ["agent", "data-source-config"]); - if (flags["data-source-config"] && (flags["endpoint"] || flags["clear-endpoint"] === "true")) - throw new InputValidationError( - "'--endpoint' cannot be combined with '--data-source-config'", - ); - - for (const id of flags["insight"] ?? []) { - if (!id.startsWith(BUILTIN_INSIGHT_PREFIX) && !id.startsWith(ARN_PREFIX)) - throw new InputValidationError( - `invalid insight "${id}": must be a ${BUILTIN_INSIGHT_PREFIX}* identifier or a full ARN`, - ); - } - - const source = new SourceResolver({ stdin: io.stdin }); - const frequencies = flags["clustering-frequency"]; - const response = await core.eval.updateOnlineInsight( - flags["id"], - { - samplingRate: flags["sampling-rate"], - sessionTimeoutMinutes: flags["session-timeout-minutes"], - filters: parseJsonFlag( - "filters", - await source.resolveText("filters", flags["filters"]), - ), - insightIds: flags["insight"], - clusteringConfig: frequencies ? { frequencies } : undefined, - agent: flags["agent"], - endpoint: flags["endpoint"], - clearEndpoint: flags["clear-endpoint"] === "true", - dataSourceConfig: parseJsonFlag( - "data-source-config", - await source.resolveText("data-source-config", flags["data-source-config"]), - ), - evaluationExecutionRoleArn: flags["role-arn"], - }, - coreOptsFromCtx(ctx), - ); - ctx.require(JsonRendererKey).renderJson(response); - }, - }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 0b362d5269..d84fc8e506 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -171,42 +171,6 @@ export type CreateOnlineEvalInput = { | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } ); -export type CreateOnlineInsightInput = { - name: string; - description?: string; - samplingRate: number; - sessionTimeoutMinutes?: number; - filters?: Rule["filters"]; - insightIds: string[]; - clusteringConfig?: { frequencies: ("DAILY" | "WEEKLY" | "MONTHLY")[] }; - evaluationExecutionRoleArn: string; - enableOnCreate?: boolean; -} & ( - | { agent: string; endpoint?: string; dataSourceConfig?: undefined } - | { agent?: undefined; endpoint?: undefined; dataSourceConfig: DataSourceConfig } -); - -export type UpdateOnlineInsightInput = { - samplingRate?: number; - sessionTimeoutMinutes?: number; - filters?: Rule["filters"]; - insightIds?: string[]; - clusteringConfig?: { frequencies: ("DAILY" | "WEEKLY" | "MONTHLY")[] }; - agent?: string; - endpoint?: string; - clearEndpoint?: boolean; - dataSourceConfig?: DataSourceConfig; - evaluationExecutionRoleArn?: string; -}; - -// Online insight configs are the same OnlineEvaluationConfig resource with insights -// instead of evaluators, so the responses share the SDK shapes under insight names. -export type CreateOnlineInsightResponse = CreateOnlineEvaluationConfigResponse; -export type GetOnlineInsightResponse = GetOnlineEvaluationConfigResponse; -export type UpdateOnlineInsightResponse = UpdateOnlineEvaluationConfigResponse; -export type ListOnlineInsightsResponse = ListOnlineEvaluationConfigsResponse; -export type DeleteOnlineInsightResponse = DeleteOnlineEvaluationConfigResponse; - // UpdateOnlineEvalInput carries the fields a caller may change on an online // evaluation config. Undefined fields are left untouched by Core (merged over // the current config, since UpdateOnlineEvaluationConfig replaces the whole @@ -293,17 +257,6 @@ export type StartBatchEvaluationInput = { outputConfig?: OutputConfig; }; -// Batch insights use the same service job API as batch evaluations, but remain -// a distinct Core operation so each command keeps its own required fields. -export type StartBatchInsightsInput = { - name: string; - description?: string; - insightIds?: string[]; - evaluatorIds?: string[]; - source: SessionSourceValue; - kmsKeyArn?: string; -}; - // InvokeDatasetInput is the runtime-level shape for replaying a dataset: invoke each // example against the runtime, one client-generated session per example. Runtime fields // only — no evaluator/name/kms (those belong to the grader the handler composes on top, @@ -434,18 +387,12 @@ export interface CoreEvalClient { options: CoreOptions, opts?: { includeResults?: boolean }, ): Promise; - getBatchInsights(id: string, options: CoreOptions): Promise; listBatchEvaluations( nextToken: string | undefined, maxResults: number | undefined, options: CoreOptions, ): Promise; stopBatchEvaluation(id: string, options: CoreOptions): Promise; - listBatchInsights( - nextToken: string | undefined, - maxResults: number | undefined, - options: CoreOptions, - ): Promise; getABTest(id: string, options: CoreOptions): Promise; listABTests( @@ -474,10 +421,6 @@ export interface CoreEvalClient { input: StartBatchEvaluationInput, options: CoreOptions, ): Promise; - startBatchInsights( - input: StartBatchInsightsInput, - options: CoreOptions, - ): Promise; // getTracesForAgent resolves the agent to its runtime log group and reads the // target sessions' traces client-side (CloudWatch Logs Insights), grouped by @@ -525,28 +468,6 @@ export interface CoreEvalClient { options: CoreOptions, ): Promise; - createOnlineInsight( - input: CreateOnlineInsightInput, - options: CoreOptions, - ): Promise; - updateOnlineInsight( - id: string, - update: UpdateOnlineInsightInput, - options: CoreOptions, - ): Promise; - getOnlineInsight(id: string, options: CoreOptions): Promise; - listOnlineInsights( - nextToken: string | undefined, - maxResults: number | undefined, - options: CoreOptions, - ): Promise; - setOnlineInsightExecutionStatus( - id: string, - executionStatus: "ENABLED" | "DISABLED", - options: CoreOptions, - ): Promise; - deleteOnlineInsight(id: string, options: CoreOptions): Promise; - createConfigurationBundle( input: CreateConfigurationBundleInput, options: CoreOptions, diff --git a/src/handlers/project/add/add.screen.test.tsx b/src/handlers/project/add/add.screen.test.tsx index e69524db3b..0823d3309f 100644 --- a/src/handlers/project/add/add.screen.test.tsx +++ b/src/handlers/project/add/add.screen.test.tsx @@ -42,7 +42,6 @@ const ADD_COMMAND_KINDS: Record = { "gateway-target": "gateway-target", "gateway-connector": "gateway-connector", "online-eval": "online-eval", - "online-insight": "online-insight", harness: "harness", "config-bundle": "config-bundle", "policy-engine": "policy-engine", diff --git a/src/handlers/project/add/index.ts b/src/handlers/project/add/index.ts index 15eb2a758a..ad61155239 100644 --- a/src/handlers/project/add/index.ts +++ b/src/handlers/project/add/index.ts @@ -8,7 +8,6 @@ import { createAddHarnessHandler } from "./harness"; import { createAddMemoryHandler } from "./memory"; import { createAddRuntimeHandler } from "./runtime"; import { createAddOnlineEvalHandler } from "./online-eval"; -import { createAddOnlineInsightHandler } from "./online-insight"; import { createAddEvaluatorHandler } from "./evaluator"; import { createAddGatewayHandler } from "./gateway"; import { createAddGatewayTargetHandler } from "./gateway-target"; @@ -33,7 +32,6 @@ export function createAddProjectResourceHandler( "gateway-target", "gateway-connector", "online-eval", - "online-insight", "harness", "config-bundle", "policy-engine", @@ -67,7 +65,6 @@ export function createAddProjectResourceHandler( projectAdd.handler(createAddMemoryHandler(config)); projectAdd.handler(createAddEvaluatorHandler(config, core)); projectAdd.handler(createAddOnlineEvalHandler(config)); - projectAdd.handler(createAddOnlineInsightHandler(config)); projectAdd.handler(createAddConfigBundleHandler(config)); projectAdd.handler(createAddPaymentManagerHandler(config)); projectAdd.handler(createAddPaymentConnectorHandler(config)); diff --git a/src/handlers/project/add/online-insight/index.test.ts b/src/handlers/project/add/online-insight/index.test.ts deleted file mode 100644 index bdfa6d1e09..0000000000 --- a/src/handlers/project/add/online-insight/index.test.ts +++ /dev/null @@ -1,281 +0,0 @@ -import { afterEach, describe, expect, test } from "bun:test"; -import { join } from "node:path"; -import { createRootHandler } from "../../../index"; -import { - createSilentLogger, - expectError, - initProject, - TestCoreClient, - TestGlobalConfigAccessor, - testIO, -} from "../../../../testing"; -import { DeserializationError, InputValidationError } from "../../../../errors"; - -const cleanups: Array<() => Promise> = []; -afterEach(() => Promise.all(cleanups.splice(0).map((cleanup) => cleanup()))); - -async function run(args: string[], opts?: { core?: TestCoreClient }) { - const io = testIO(); - const core = opts?.core ?? new TestCoreClient(); - const root = createRootHandler(core, { - io: io.io, - globalConfigAccessor: new TestGlobalConfigAccessor(), - logger: createSilentLogger(), - }); - await root.route(["node", "agentcore", ...args]); - return { io, core }; -} - -const INSIGHT = "Builtin.Insight.FailureAnalysis"; - -describe("project add online-insight", () => { - test.each<[string, string[], Record]>([ - [ - "minimal — agent source", - ["--name", "x", "--agent", "agent", "--insight", INSIGHT, "--sampling-rate", "50"], - { agent: "agent", insights: [INSIGHT], samplingRate: 50 }, - ], - [ - "custom log-group source", - [ - "--name", - "x", - "--log-group-name", - "/aws/foo", - "--insight", - INSIGHT, - "--sampling-rate", - "50", - ], - { logGroupNames: ["/aws/foo"], insights: [INSIGHT], samplingRate: 50 }, - ], - [ - "agent source with endpoint", - [ - "--name", - "x", - "--agent", - "agent", - "--endpoint", - "PROD", - "--insight", - INSIGHT, - "--sampling-rate", - "10", - ], - { agent: "agent", endpoint: "PROD" }, - ], - [ - "clustering frequencies", - [ - "--name", - "x", - "--agent", - "agent", - "--insight", - INSIGHT, - "--sampling-rate", - "10", - "--clustering-frequency", - "DAILY", - "WEEKLY", - ], - { clusteringConfig: { frequencies: ["DAILY", "WEEKLY"] } }, - ], - [ - "service-name filter on a custom source", - [ - "--name", - "x", - "--log-group-name", - "/aws/foo", - "--service-name", - "svc", - "--insight", - INSIGHT, - "--sampling-rate", - "25", - ], - { logGroupNames: ["/aws/foo"], serviceNames: ["svc"] }, - ], - [ - "description, enable-on-create false, tags", - [ - "--name", - "x", - "--agent", - "agent", - "--insight", - INSIGHT, - "--sampling-rate", - "5", - "--description", - "monitor prod", - "--enable-on-create", - "false", - "--tags", - "team=ml", - "--tags", - "env=prod", - ], - { - description: "monitor prod", - enableOnCreate: false, - tags: { team: "ml", env: "prod" }, - }, - ], - ])("%s", async (_label, flags, expected) => { - const { projectRoot, cleanup } = await initProject({ - flags: ["--template", "agent-python-minimal"], - }); - cleanups.push(cleanup); - await run(["add", "online-insight", ...flags]); - - const agentcoreJson = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); - const config = agentcoreJson.onlineEvalConfigs.find((c: { name: string }) => c.name === "x"); - expect(config).toMatchObject(expected); - }); - - test("rejects a duplicate online-insight name", async () => { - const { cleanup } = await initProject({ flags: ["--template", "agent-python-minimal"] }); - cleanups.push(cleanup); - const flags = [ - "--name", - "x", - "--agent", - "agent", - "--insight", - INSIGHT, - "--sampling-rate", - "50", - ]; - await run(["add", "online-insight", ...flags]); - await expect(run(["add", "online-insight", ...flags])).rejects.toBeInstanceOf( - InputValidationError, - ); - }); - - test("rejects a name whose deployed form exceeds the service limit", async () => { - const { cleanup } = await initProject({ - name: "TestProject", - flags: ["--template", "agent-python-minimal"], - }); - cleanups.push(cleanup); - - await expectError( - run([ - "add", - "online-insight", - "--name", - `a${"x".repeat(28)}`, - "--agent", - "agent", - "--insight", - INSIGHT, - "--sampling-rate", - "50", - ]), - "The maximum is 48", - InputValidationError, - ); - }); - - test("rejects when the existing spec is invalid", async () => { - const { projectRoot, cleanup } = await initProject({ - flags: ["--template", "agent-python-minimal"], - }); - cleanups.push(cleanup); - - const specPath = join(projectRoot, "agentcore", "agentcore.json"); - const spec = await Bun.file(specPath).json(); - spec.unknownField = "bad"; - await Bun.write(specPath, JSON.stringify(spec)); - - await expect( - run([ - "add", - "online-insight", - "--name", - "x", - "--agent", - "agent", - "--insight", - INSIGHT, - "--sampling-rate", - "5", - ]), - ).rejects.toBeInstanceOf(DeserializationError); - }); - - test.each<[string, string[], string?]>([ - [ - "missing --name", - ["--agent", "a", "--insight", INSIGHT, "--sampling-rate", "10"], - "required option '--name' not specified", - ], - [ - "missing --sampling-rate", - ["--name", "x", "--agent", "a", "--insight", INSIGHT], - "required option '--sampling-rate' not specified", - ], - [ - "no --insight", - ["--name", "x", "--agent", "a", "--sampling-rate", "10"], - "required option '--insight' not specified", - ], - [ - "invalid insight id", - ["--name", "x", "--agent", "a", "--insight", "NotAnInsight", "--sampling-rate", "10"], - ], - [ - "--agent and --log-group-name are mutually exclusive", - [ - "--name", - "x", - "--agent", - "a", - "--log-group-name", - "/x", - "--insight", - INSIGHT, - "--sampling-rate", - "10", - ], - ], - [ - "--endpoint without --agent", - [ - "--name", - "x", - "--log-group-name", - "/x", - "--endpoint", - "PROD", - "--insight", - INSIGHT, - "--sampling-rate", - "10", - ], - ], - [ - "--service-name without --log-group-name", - [ - "--name", - "x", - "--agent", - "a", - "--service-name", - "svc", - "--insight", - INSIGHT, - "--sampling-rate", - "10", - ], - ], - ])("%s", async (...[_label, flags, requiredMessage]) => { - const { cleanup } = await initProject({ flags: ["--template", "agent-python-minimal"] }); - cleanups.push(cleanup); - const promise = run(["add", "online-insight", ...flags]); - await expectError(promise, requiredMessage ?? /./, InputValidationError); - }); -}); diff --git a/src/handlers/project/add/online-insight/index.ts b/src/handlers/project/add/online-insight/index.ts deleted file mode 100644 index 99542baf08..0000000000 --- a/src/handlers/project/add/online-insight/index.ts +++ /dev/null @@ -1,167 +0,0 @@ -import z from "zod"; -import { createHandler, flag, ProjectKey } from "../../../../router"; -import { InputValidationError } from "../../../../errors"; -import { OnlineEvalConfigSchema } from "../../../../projectSchemas/online-eval-config"; -import { parseTags } from "../../../utils"; -import type { AwsDeploymentTarget } from "../../../../projectSchemas/aws-targets"; -import type { AddResourceInput, Project } from "../../types"; -import type { AddProjectResourceConfig } from "../types"; -import { addProjectResource, requireDeployedNameFits, addDescription } from "../shared"; - -export const BUILTIN_INSIGHT_PREFIX = "Builtin.Insight."; -const ARN_PREFIX = "arn:"; -export const ONLINE_INSIGHT_DEPLOYED_NAME_MAX_LENGTH = 48; - -export interface OnlineInsightInput { - name: string; - agent?: string; - endpoint?: string; - logGroupNames?: string[]; - serviceNames?: string[]; - insights: string[]; - clusteringFrequencies?: Array<"DAILY" | "WEEKLY" | "MONTHLY">; - samplingRate: number; - description?: string; - enableOnCreate?: boolean; - tags?: Record; -} - -export function validateInsightIds(insights: readonly string[]): void { - for (const id of insights) { - if (!id.startsWith(BUILTIN_INSIGHT_PREFIX) && !id.startsWith(ARN_PREFIX)) { - throw new InputValidationError( - `invalid insight "${id}": must be a ${BUILTIN_INSIGHT_PREFIX}* identifier or a full ARN`, - ); - } - } -} - -export function toAddOnlineInsightInput( - project: Project, - targets: readonly AwsDeploymentTarget[], - input: OnlineInsightInput, -): AddResourceInput { - requireDeployedNameFits( - "Online insight config", - project.name, - input.name, - "_", - ONLINE_INSIGHT_DEPLOYED_NAME_MAX_LENGTH, - targets, - ); - validateInsightIds(input.insights); - - const parsed = OnlineEvalConfigSchema.safeParse({ - name: input.name, - agent: input.agent, - endpoint: input.endpoint, - logGroupNames: input.logGroupNames, - serviceNames: input.serviceNames, - insights: input.insights, - clusteringConfig: input.clusteringFrequencies - ? { frequencies: input.clusteringFrequencies } - : undefined, - samplingRate: input.samplingRate, - description: input.description, - enableOnCreate: input.enableOnCreate, - tags: input.tags, - }); - if (!parsed.success) throw new InputValidationError(z.prettifyError(parsed.error)); - - return { - resourceType: "online-insight", - resourceConfig: parsed.data, - }; -} - -export const createAddOnlineInsightHandler = (config: AddProjectResourceConfig) => - createHandler({ - name: "online-insight", - description: addDescription( - "online-insight", - "add an online insight config to the current project", - ), - flags: [ - flag("name", "the name of the online insight config", z.string().min(1)), - flag( - "agent", - "Runtime name whose traffic to sample (mutually exclusive with --log-group-name)", - z.string().optional(), - ), - flag( - "endpoint", - "the agent endpoint qualifier to scope monitoring to (requires --agent)", - z.string().optional(), - ), - flag( - "log-group-name", - "CloudWatch log group name(s) for custom data sources (1-5; mutually exclusive with --agent)", - z.array(z.string()).optional(), - ), - flag( - "service-name", - "service name(s) to filter traces for custom data sources (requires --log-group-name)", - z.array(z.string()).optional(), - ), - flag( - "insight", - "insight ID(s) to apply: Builtin.Insight.* identifiers or full ARNs", - z.array(z.string()).min(1), - ), - flag( - "clustering-frequency", - "insight clustering cadence(s): DAILY, WEEKLY, MONTHLY", - z.array(z.enum(["DAILY", "WEEKLY", "MONTHLY"])).optional(), - ), - flag( - "sampling-rate", - "percentage of sessions to sample (0.01-100)", - z.number().min(0.01).max(100), - ), - flag( - "description", - "a description of the config's monitoring purpose", - z.string().optional(), - ), - flag( - "enable-on-create", - "enable insights immediately after deploy (default true; pass false to add it paused)", - z.enum(["true", "false"]).optional(), - ), - flag( - "tags", - "tags as repeated key=value entries or a JSON object", - z.array(z.string()).optional(), - ), - ], - handle: async (ctx, flags) => { - const project = ctx.require(ProjectKey); - const input = toAddOnlineInsightInput( - project, - await config.projectManager.listTargets(project), - { - name: flags["name"], - agent: flags["agent"], - endpoint: flags["endpoint"], - logGroupNames: flags["log-group-name"], - serviceNames: flags["service-name"], - insights: flags["insight"], - clusteringFrequencies: flags["clustering-frequency"], - samplingRate: flags["sampling-rate"], - description: flags["description"], - enableOnCreate: - flags["enable-on-create"] === undefined - ? undefined - : flags["enable-on-create"] === "true", - tags: parseTags(flags["tags"]), - }, - ); - await addProjectResource( - ctx, - config, - project, - input, - `added online-insight config '${flags["name"]}' to '${project.name}'`, - ); - }, - }); diff --git a/src/handlers/project/add/online-insight/online-insight.screen.test.tsx b/src/handlers/project/add/online-insight/online-insight.screen.test.tsx deleted file mode 100644 index 300cf2bf9d..0000000000 --- a/src/handlers/project/add/online-insight/online-insight.screen.test.tsx +++ /dev/null @@ -1,276 +0,0 @@ -import { afterEach, describe, expect, test } from "bun:test"; -import { writeFile } from "node:fs/promises"; -import { join } from "node:path"; -import { - cleanupScreens, - flatFrame, - renderScreen, - waitForFlatText, - waitForText, - type RenderScreenResult, -} from "../../../../testing"; -import { createGatewayProjectTestHarness } from "../gateway-test-support"; - -const { cleanup, inProject, projectSpec, writeProjectSpec } = createGatewayProjectTestHarness( - "add-online-insight-wizard", -); - -afterEach(cleanup); -afterEach(cleanupScreens); - -async function chooseFailureAnalysis(screen: RenderScreenResult): Promise { - await waitForText(screen.lastFrame, "which insights should run?"); - await screen.write(" "); - await screen.press("return"); -} - -async function finishDefaults(screen: RenderScreenResult, samplingRate = "10"): Promise { - await waitForText(screen.lastFrame, "how should insight sessions be sampled and clustered?"); - await screen.press("return"); - await screen.write(samplingRate); - await screen.press("return"); - await screen.press("return"); -} - -describe("project add online-insight wizard", () => { - test("adds a Runtime-backed config with the wizard's advanced inputs", async () => { - const projectRoot = await inProject(); - const screen = renderScreen("/agentcore/add/online-insight"); - - await waitForText(screen.lastFrame, "what should this online insight config be called?"); - await screen.write("production_insights"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "where should sessions be sampled from?"); - expect(screen.lastFrame()).toContain("❯ ● project Runtime"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "which Runtime should be monitored?"); - expect(screen.lastFrame()).toContain("❯ ● agent"); - await screen.press("return"); - - await chooseFailureAnalysis(screen); - - await waitForText(screen.lastFrame, "how should insight sessions be sampled and clustered?"); - await screen.write(" "); - await screen.press("return"); - await screen.write("12.5"); - await screen.press("return"); - await screen.write("Monitor production sessions"); - await screen.press("return"); - - await waitForText( - screen.lastFrame, - "this online insight config will be added to agentcore.json", - ); - const review = flatFrame(screen.lastFrame); - expect(review).toContain("source agent:DEFAULT"); - expect(review).toContain("insights Builtin.Insight.FailureAnalysis"); - expect(review).toContain("clustering DAILY"); - expect(review).toContain("sampling 12.5%"); - await screen.press("return"); - - await waitForText( - screen.lastFrame, - "added online-insight config 'production_insights' to 'TestProject'", - ); - expect((await projectSpec(projectRoot)).onlineEvalConfigs).toContainEqual({ - name: "production_insights", - agent: "agent", - insights: ["Builtin.Insight.FailureAnalysis"], - clusteringConfig: { frequencies: ["DAILY"] }, - samplingRate: 12.5, - description: "Monitor production sessions", - }); - screen.unmount(); - }); - - test("adds a custom-log config with services, a custom insight, and clustering", async () => { - const projectRoot = await inProject(); - const screen = renderScreen("/agentcore/add/online-insight"); - - await waitForText(screen.lastFrame, "what should this online insight config be called?"); - await screen.write("custom_logs"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "where should sessions be sampled from?"); - await screen.press("down"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "which CloudWatch log groups contain the sessions?"); - await screen.write("/aws/example/one, /aws/example/two"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "limit traces to particular service names?"); - await screen.write("checkout, inventory"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "which insights should run?"); - await screen.press("down"); - await screen.write(" "); - await screen.press("down"); - await screen.press("down"); - await screen.write(" "); - expect(screen.lastFrame()).toContain("Custom insight ARN"); - await screen.press("return"); - await screen.write("arn:aws:bedrock-agentcore:us-east-1:111122223333:insight/custom"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "how should insight sessions be sampled and clustered?"); - await screen.write(" "); - await screen.press("down"); - await screen.write(" "); - await screen.press("return"); - await screen.write("0.5"); - await screen.press("return"); - await screen.press("return"); - - await waitForText( - screen.lastFrame, - "this online insight config will be added to agentcore.json", - ); - await screen.press("return"); - await waitForText(screen.lastFrame, "added online-insight config 'custom_logs'"); - - expect((await projectSpec(projectRoot)).onlineEvalConfigs).toContainEqual({ - name: "custom_logs", - logGroupNames: ["/aws/example/one", "/aws/example/two"], - serviceNames: ["checkout", "inventory"], - insights: [ - "Builtin.Insight.UserIntent", - "arn:aws:bedrock-agentcore:us-east-1:111122223333:insight/custom", - ], - clusteringConfig: { frequencies: ["DAILY", "WEEKLY"] }, - samplingRate: 0.5, - }); - screen.unmount(); - }); - - test("distinguishes a named DEFAULT endpoint from the Runtime default", async () => { - const projectRoot = await inProject(); - const spec = await projectSpec(projectRoot); - spec.runtimes[0].endpoints = { DEFAULT: { version: 1 } }; - await writeProjectSpec(projectRoot, spec); - const screen = renderScreen("/agentcore/add/online-insight"); - - await waitForText(screen.lastFrame, "what should this online insight config be called?"); - await screen.write("endpoint_insights"); - await screen.press("return"); - await waitForText(screen.lastFrame, "where should sessions be sampled from?"); - await screen.press("return"); - await waitForText(screen.lastFrame, "which Runtime should be monitored?"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "which Runtime endpoint should be monitored?"); - expect(screen.lastFrame()).toContain("NO ENDPOINT (default)"); - expect(screen.lastFrame()).toContain("DEFAULT"); - await screen.press("down"); - await screen.press("return"); - - await chooseFailureAnalysis(screen); - await finishDefaults(screen); - await waitForText( - screen.lastFrame, - "this online insight config will be added to agentcore.json", - ); - expect(flatFrame(screen.lastFrame)).toContain("source agent:DEFAULT"); - await screen.press("return"); - await waitForText(screen.lastFrame, "added online-insight config 'endpoint_insights'"); - - expect((await projectSpec(projectRoot)).onlineEvalConfigs[0]).toMatchObject({ - agent: "agent", - endpoint: "DEFAULT", - }); - screen.unmount(); - }); - - test("requires at least one built-in or custom insight", async () => { - await inProject(); - const screen = renderScreen("/agentcore/add/online-insight"); - - await waitForText(screen.lastFrame, "what should this online insight config be called?"); - await screen.write("required_insight"); - await screen.press("return"); - await waitForText(screen.lastFrame, "where should sessions be sampled from?"); - await screen.press("return"); - await waitForText(screen.lastFrame, "which Runtime should be monitored?"); - await screen.press("return"); - await waitForText(screen.lastFrame, "which insights should run?"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "At least one insight is required"); - expect(screen.lastFrame()).toContain("which insights should run?"); - screen.unmount(); - }); - - test("requires an ARN when the custom insight option is selected", async () => { - await inProject(); - const screen = renderScreen("/agentcore/add/online-insight"); - - await waitForText(screen.lastFrame, "what should this online insight config be called?"); - await screen.write("custom_insight"); - await screen.press("return"); - await waitForText(screen.lastFrame, "where should sessions be sampled from?"); - await screen.press("return"); - await waitForText(screen.lastFrame, "which Runtime should be monitored?"); - await screen.press("return"); - await waitForText(screen.lastFrame, "which insights should run?"); - await screen.press("down"); - await screen.press("down"); - await screen.press("down"); - await screen.write(" "); - await screen.press("return"); - await screen.press("return"); - - await waitForText(screen.lastFrame, "At least one custom insight ARN is required"); - expect(screen.lastFrame()).toContain("which insights should run?"); - expect(screen.lastFrame()).toContain("Custom insight ARN"); - screen.unmount(); - }); - - test("validates decimal sampling rates against the service range", async () => { - await inProject(); - const screen = renderScreen("/agentcore/add/online-insight"); - - await waitForText(screen.lastFrame, "what should this online insight config be called?"); - await screen.write("sampled_insights"); - await screen.press("return"); - await waitForText(screen.lastFrame, "where should sessions be sampled from?"); - await screen.press("return"); - await waitForText(screen.lastFrame, "which Runtime should be monitored?"); - await screen.press("return"); - await chooseFailureAnalysis(screen); - await waitForText(screen.lastFrame, "how should insight sessions be sampled and clustered?"); - await screen.press("return"); - await screen.write("0.001"); - await screen.press("return"); - - await waitForText(screen.lastFrame, ">=0.01"); - expect(screen.lastFrame()).toContain("how should insight sessions be sampled and clustered?"); - screen.unmount(); - }); - - test("validates the deployed name against the longest project target", async () => { - const projectRoot = await inProject(); - await writeFile( - join(projectRoot, "agentcore", "aws-targets.json"), - JSON.stringify([ - { - name: "production", - account: "111122223333", - region: "us-east-1", - }, - ]), - ); - const screen = renderScreen("/agentcore/add/online-insight"); - - await waitForText(screen.lastFrame, "what should this online insight config be called?"); - await screen.write(`a${"x".repeat(25)}`); - await screen.press("return"); - - await waitForFlatText(screen.lastFrame, "is 49 characters. The maximum is 48."); - expect(flatFrame(screen.lastFrame)).toContain("TestProject_production_"); - expect(screen.lastFrame()).toContain("what should this online insight config be called?"); - screen.unmount(); - }); -}); diff --git a/src/handlers/project/add/online-insight/screen.tsx b/src/handlers/project/add/online-insight/screen.tsx deleted file mode 100644 index f7c0e82be2..0000000000 --- a/src/handlers/project/add/online-insight/screen.tsx +++ /dev/null @@ -1,568 +0,0 @@ -import { useMemo, useState } from "react"; -import { useQueryClient } from "@tanstack/react-query"; -import { Box, Text, useInput } from "ink"; -import { useNavigate } from "react-router"; -import z from "zod"; -import { FormCheckboxMultiSelect } from "../../../../components/FormCheckboxMultiSelect"; -import { FormTextInput } from "../../../../components/FormTextInput"; -import { darkTheme } from "../../../../components/ui/_core.js"; -import { - Step, - Summary, - TextField, - Wizard, - useKeyHints, - useWizard, - type Choice, -} from "../../../../components/wizard"; -import type { AwsDeploymentTarget } from "../../../../projectSchemas/aws-targets"; -import { OnlineEvalConfigNameSchema } from "../../../../projectSchemas/online-eval-config"; -import { ProjectKey } from "../../../../router"; -import type { ScreenProps } from "../../../types"; -import { LoadingFrame, ProjectGate, projectQueryKey, useProjectTargets } from "../../ProjectGate"; -import type { Project } from "../../types"; -import { requireDeployedNameFits } from "../shared"; -import { - initialTrafficSourceValues, - splitCommaList, - toTrafficSourceInput, - trafficSourceSteps, - trafficSourceSummary, - type TrafficSourceFormValues, -} from "../traffic-source-fields"; -import { - ONLINE_INSIGHT_DEPLOYED_NAME_MAX_LENGTH, - toAddOnlineInsightInput, - type OnlineInsightInput, -} from "./index"; -import { RegionKey } from "../../../keys"; - -const theme = darkTheme; -const BREADCRUMB = ["agentcore", "add", "online-insight"]; -const DESCRIPTION = "add an online insight config to the current project"; -const ADD_MENU = "/agentcore/add"; - -type ClusteringFrequency = "DAILY" | "WEEKLY" | "MONTHLY"; -const CUSTOM_INSIGHT = "CUSTOM"; - -const BUILTIN_INSIGHT_CHOICES: Choice[] = [ - { - value: "Builtin.Insight.FailureAnalysis", - label: "FailureAnalysis", - description: "identify and explain failed sessions", - }, - { - value: "Builtin.Insight.UserIntent", - label: "UserIntent", - description: "group sessions by what users are trying to accomplish", - }, - { - value: "Builtin.Insight.ExecutionSummary", - label: "ExecutionSummary", - description: "summarize how sessions execute", - }, -]; - -const INSIGHT_CHOICES: Choice[] = [ - ...BUILTIN_INSIGHT_CHOICES, - { - value: CUSTOM_INSIGHT, - label: "Custom insight ARN", - description: "use an insight defined outside the built-in set", - }, -]; - -const CLUSTERING_CHOICES: Choice[] = [ - { value: "DAILY", label: "DAILY" }, - { value: "WEEKLY", label: "WEEKLY" }, - { value: "MONTHLY", label: "MONTHLY" }, -]; - -interface OnlineInsightFormValues extends TrafficSourceFormValues { - name: string; - builtinInsights: string[]; - includeCustomInsight: boolean; - customInsights: string; - clusteringFrequencies: ClusteringFrequency[]; - samplingRate: string; - description: string; -} - -function insightsOf(values: OnlineInsightFormValues): string[] { - return [ - ...values.builtinInsights, - ...(values.includeCustomInsight ? splitCommaList(values.customInsights) : []), - ]; -} - -export function toOnlineInsightInput(values: OnlineInsightFormValues): OnlineInsightInput { - return { - name: values.name, - ...toTrafficSourceInput(values), - insights: insightsOf(values), - clusteringFrequencies: - values.clusteringFrequencies.length === 0 ? undefined : values.clusteringFrequencies, - samplingRate: Number(values.samplingRate), - description: values.description === "" ? undefined : values.description, - }; -} - -function summaryOf(values: OnlineInsightFormValues): Record { - return { - config: values.name, - ...trafficSourceSummary(values), - insights: insightsOf(values).join(", "), - clustering: - values.clusteringFrequencies.length === 0 - ? "(none)" - : values.clusteringFrequencies.join(", "), - sampling: `${values.samplingRate}%`, - description: values.description === "" ? "(none)" : values.description, - }; -} - -export function AddOnlineInsightScreen({ ctx, core }: ScreenProps) { - const navigate = useNavigate(); - return ( - navigate(ADD_MENU)} - > - {(project) => ( - - )} - - ); -} - -function AddOnlineInsightLoader({ - project, - core, - region, -}: { - project: Project; - core: ScreenProps["core"]; - region: string | undefined; -}) { - const navigate = useNavigate(); - const targets = useProjectTargets(core, project); - - if (targets.data !== undefined) { - return ( - - ); - } - - return ( - navigate(ADD_MENU)} - /> - ); -} - -function AddOnlineInsightWizard({ - project, - targets, - core, - region, -}: { - project: Project; - targets: readonly AwsDeploymentTarget[]; - core: ScreenProps["core"]; - /** The command\'s resolved region, for the China gate of a project without targets. */ - region: string | undefined; -}) { - const navigate = useNavigate(); - const queryClient = useQueryClient(); - const runtimes = project.spec.runtimes; - const initialRuntime = runtimes[0]?.name ?? ""; - const [values, setValues] = useState({ - ...initialTrafficSourceValues(initialRuntime), - name: "", - builtinInsights: [], - includeCustomInsight: false, - customInsights: "", - clusteringFrequencies: [], - samplingRate: "", - description: "", - }); - const set = (update: Partial) => - setValues((current) => ({ ...current, ...update })); - - const nameSchema = useMemo( - () => - OnlineEvalConfigNameSchema.superRefine((name, ctx) => { - try { - requireDeployedNameFits( - "Online insight config", - project.name, - name, - "_", - ONLINE_INSIGHT_DEPLOYED_NAME_MAX_LENGTH, - targets, - ); - } catch (error) { - ctx.addIssue({ - code: "custom", - message: error instanceof Error ? error.message : String(error), - }); - } - }), - [project.name, targets], - ); - - return ( - navigate(ADD_MENU)} - onSubmit={async function* () { - const updated = yield* core.projectManager.addResource( - project, - toAddOnlineInsightInput(project, targets, toOnlineInsightInput(values)), - { region }, - ); - queryClient.setQueryData(projectQueryKey(), updated); - return updated; - }} - runningLabel={`adding online insight config ${values.name}…`} - successLabel={`added online-insight config '${values.name}' to '${project.name}'`} - successNextSteps={["agentcore deploy"]} - onDone={() => navigate(ADD_MENU)} - doneLabel="go back" - > - - set({ name })} - required - schema={nameSchema} - live - /> - - - {trafficSourceSteps({ values, runtimes, onChange: set })} - - - set(update)} - /> - - - - set(update)} - /> - - - - - - - ); -} - -type InsightSelectionUpdate = Partial< - Pick ->; - -function InsightSelectionField({ - builtinInsights, - includeCustomInsight, - customInsights, - onChange, -}: { - builtinInsights: string[]; - includeCustomInsight: boolean; - customInsights: string; - onChange: (update: InsightSelectionUpdate) => void; -}) { - const { advance, back } = useWizard(); - const [cursor, setCursor] = useState(0); - const [customFocused, setCustomFocused] = useState(false); - const [error, setError] = useState(); - - useKeyHints([ - { key: "↑↓", label: "navigate" }, - { key: "space", label: "toggle" }, - { key: "enter", label: "continue" }, - ]); - - useInput((input, key) => { - if (customFocused) { - if (key.escape || key.upArrow) { - setCustomFocused(false); - setError(undefined); - return; - } - if (!key.return) return; - - const customArns = splitCommaList(customInsights); - if (customArns.length === 0) { - setError("At least one custom insight ARN is required"); - return; - } - const invalidArn = customArns.find((arn) => !arn.startsWith("arn:")); - if (invalidArn !== undefined) { - setError(`invalid insight "${invalidArn}": must be a full ARN`); - return; - } - - setError(undefined); - advance(); - return; - } - - if (key.escape) { - back(); - return; - } - if (key.upArrow) { - setCursor((current) => Math.max(0, current - 1)); - return; - } - if (key.downArrow) { - setCursor((current) => Math.min(INSIGHT_CHOICES.length - 1, current + 1)); - return; - } - if (input === " ") { - const selected = INSIGHT_CHOICES[cursor]!.value; - if (selected === CUSTOM_INSIGHT) { - onChange({ includeCustomInsight: !includeCustomInsight }); - } else { - const toggled = builtinInsights.includes(selected) - ? builtinInsights.filter((insight) => insight !== selected) - : [...builtinInsights, selected]; - onChange({ - builtinInsights: BUILTIN_INSIGHT_CHOICES.filter((choice) => - toggled.includes(choice.value), - ).map((choice) => choice.value), - }); - } - setError(undefined); - return; - } - if (!key.return) return; - - if (builtinInsights.length === 0 && !includeCustomInsight) { - setError("At least one insight is required"); - return; - } - if (includeCustomInsight) { - setCustomFocused(true); - setError(undefined); - return; - } - - setError(undefined); - advance(); - }); - - return ( - - ({ - label: choice.label, - description: choice.description ?? "", - checked: - choice.value === CUSTOM_INSIGHT - ? includeCustomInsight - : builtinInsights.includes(choice.value), - }))} - cursorIndex={customFocused ? -1 : cursor} - /> - {includeCustomInsight && ( - { - onChange({ customInsights: value }); - setError(undefined); - }} - focused={customFocused} - /> - )} - {error !== undefined && {error}} - - ); -} - -type InsightSettingsUpdate = Partial< - Pick ->; - -const SamplingRateSchema = z.number().min(0.01).max(100); -const DescriptionSchema = z.string().max(200); - -function firstIssue(schema: z.ZodType, value: unknown): string | undefined { - const result = schema.safeParse(value); - if (result.success) return undefined; - const issue = result.error.issues[0]; - if (!issue) return "invalid value"; - const path = issue.path.join("."); - return path === "" ? issue.message : `${path}: ${issue.message}`; -} - -function samplingRateIssue(value: string): string | undefined { - if (value.trim() === "") return "Sampling rate is required"; - if (!/^(?:\d+(?:\.\d*)?|\.\d+)$/.test(value)) return "Sampling rate must be a number"; - return firstIssue(SamplingRateSchema, Number(value)); -} - -function InsightSettingsField({ - clusteringFrequencies, - samplingRate, - description, - onChange, -}: { - clusteringFrequencies: ClusteringFrequency[]; - samplingRate: string; - description: string; - onChange: (update: InsightSettingsUpdate) => void; -}) { - const { advance, back } = useWizard(); - const [focusedField, setFocusedField] = useState(0); - const [clusteringCursor, setClusteringCursor] = useState(0); - const [error, setError] = useState(); - - useKeyHints([ - { key: "↑↓", label: "navigate" }, - { key: "space", label: "toggle cadence" }, - { key: "enter", label: "continue" }, - ]); - - useInput((input, key) => { - if (key.escape) { - back(); - return; - } - - if (focusedField === 0) { - if (key.upArrow) { - setClusteringCursor((current) => Math.max(0, current - 1)); - return; - } - if (key.downArrow) { - setClusteringCursor((current) => Math.min(CLUSTERING_CHOICES.length - 1, current + 1)); - return; - } - if (input === " ") { - const cadence = CLUSTERING_CHOICES[clusteringCursor]!.value; - const toggled = clusteringFrequencies.includes(cadence) - ? clusteringFrequencies.filter((frequency) => frequency !== cadence) - : [...clusteringFrequencies, cadence]; - onChange({ - clusteringFrequencies: CLUSTERING_CHOICES.filter((choice) => - toggled.includes(choice.value), - ).map((choice) => choice.value), - }); - return; - } - if (key.return) { - setFocusedField(1); - setError(undefined); - } - return; - } - - if (key.upArrow) { - setFocusedField(focusedField - 1); - setError(undefined); - return; - } - if (key.downArrow && focusedField === 1) { - setFocusedField(2); - setError(undefined); - return; - } - if (!key.return) return; - - const samplingIssue = samplingRateIssue(samplingRate); - if (samplingIssue !== undefined) { - setFocusedField(1); - setError(samplingIssue); - return; - } - if (focusedField === 1) { - setFocusedField(2); - setError(undefined); - return; - } - - const descriptionIssue = - description === "" ? undefined : firstIssue(DescriptionSchema, description); - if (descriptionIssue !== undefined) { - setError(descriptionIssue); - return; - } - - setError(undefined); - advance(); - }); - - return ( - - ({ - label: choice.label, - description: choice.description ?? "", - checked: clusteringFrequencies.includes(choice.value), - }))} - cursorIndex={focusedField === 0 ? clusteringCursor : -1} - /> - { - onChange({ samplingRate: value }); - setError(undefined); - }} - focused={focusedField === 1} - /> - { - onChange({ description: value }); - setError(undefined); - }} - focused={focusedField === 2} - /> - {error !== undefined && {error}} - - ); -} diff --git a/src/handlers/project/remove/index.test.ts b/src/handlers/project/remove/index.test.ts index c09beedec1..2c02f5980f 100644 --- a/src/handlers/project/remove/index.test.ts +++ b/src/handlers/project/remove/index.test.ts @@ -121,26 +121,6 @@ describe("project remove", () => { specKey: "onlineEvalConfigs", expectedRemaining: [], }, - { - label: "online-insight", - commands: [ - [ - "add", - "online-insight", - "--name", - "failures", - "--agent", - "agent", - "--insight", - "Builtin.Insight.FailureAnalysis", - "--sampling-rate", - "5", - ], - ["remove", "online-insight", "--name", "failures"], - ], - specKey: "onlineEvalConfigs", - expectedRemaining: [], - }, { label: "memory", commands: [ diff --git a/src/handlers/project/remove/index.ts b/src/handlers/project/remove/index.ts index cbd26a8de0..2826479d75 100644 --- a/src/handlers/project/remove/index.ts +++ b/src/handlers/project/remove/index.ts @@ -52,7 +52,6 @@ export const createRemoveProjectHandler = (config: RemoveProjectResourceConfig) "credential", "config-bundle", "online-eval", - "online-insight", "memory", "evaluator", "gateway", diff --git a/src/handlers/project/remove/remove.screen.test.tsx b/src/handlers/project/remove/remove.screen.test.tsx index fe314c0799..a6c37d4f12 100644 --- a/src/handlers/project/remove/remove.screen.test.tsx +++ b/src/handlers/project/remove/remove.screen.test.tsx @@ -107,15 +107,6 @@ describe("project remove screen", () => { evaluators: ["Builtin.Helpfulness"], }, }, - { - resourceType: "online-insight", - resourceConfig: { - name: "trends", - samplingRate: 10, - agent: RUNTIME, - insights: ["Builtin.Coherence"], - }, - }, { resourceType: "evaluator", resourceConfig: { @@ -177,7 +168,6 @@ describe("project remove screen", () => { "credential", "config-bundle", "online-eval", - "online-insight", "evaluator", "gateway", "gateway-target", diff --git a/src/handlers/project/remove/screen.tsx b/src/handlers/project/remove/screen.tsx index 600f8713d1..dbfbb7d2b1 100644 --- a/src/handlers/project/remove/screen.tsx +++ b/src/handlers/project/remove/screen.tsx @@ -26,7 +26,6 @@ type RootResourceType = type RemovableResourceType = | RootResourceType | "online-eval" - | "online-insight" | "gateway-target" | "gateway-connector" | "policy" @@ -58,21 +57,10 @@ const RESOURCE_PICKER_CONFIGS: RemovableResourcePickerConfig[] = [ rootResourceTypePickerConfig("credential", (spec) => spec.credentials), rootResourceTypePickerConfig("config-bundle", (spec) => spec.configBundles), rootResourceTypePickerConfig("evaluator", (spec) => spec.evaluators), - // online-eval and online-insight share the onlineEvalConfigs collection; an - // insight config is the one with a non-empty `insights` array. { resourceType: "online-eval", listResources: (spec) => - spec.onlineEvalConfigs - .filter((config) => (config.insights?.length ?? 0) === 0) - .map(({ name }) => ({ resourceType: "online-eval", name })), - }, - { - resourceType: "online-insight", - listResources: (spec) => - spec.onlineEvalConfigs - .filter((config) => (config.insights?.length ?? 0) > 0) - .map(({ name }) => ({ resourceType: "online-insight", name })), + spec.onlineEvalConfigs.map(({ name }) => ({ resourceType: "online-eval", name })), }, rootResourceTypePickerConfig("gateway", (spec) => spec.agentCoreGateways), rootResourceTypePickerConfig("policy-engine", (spec) => spec.policyEngines), diff --git a/src/handlers/project/types.ts b/src/handlers/project/types.ts index c4c082ecc6..e0424d0780 100644 --- a/src/handlers/project/types.ts +++ b/src/handlers/project/types.ts @@ -379,10 +379,6 @@ export type AddResourceInput = resourceType: "online-eval"; resourceConfig: z.input; } - | { - resourceType: "online-insight"; - resourceConfig: z.input; - } | { resourceType: "memory"; resourceConfig: z.input; @@ -481,7 +477,6 @@ export type RemoveResourceInput = | "credential" | "config-bundle" | "online-eval" - | "online-insight" | "memory" | "evaluator" | "gateway" diff --git a/src/handlers/root.test.tsx b/src/handlers/root.test.tsx index 3547b01806..400e9f9b8f 100644 --- a/src/handlers/root.test.tsx +++ b/src/handlers/root.test.tsx @@ -75,7 +75,6 @@ describe("createRootHandler", () => { "memory", "evaluator", "online-eval", - "online-insight", "config-bundle", "payment-manager", "payment-connector", diff --git a/src/projectSchemas/online-eval-config.test.ts b/src/projectSchemas/online-eval-config.test.ts index e7161f626a..5d8027fdad 100644 --- a/src/projectSchemas/online-eval-config.test.ts +++ b/src/projectSchemas/online-eval-config.test.ts @@ -30,29 +30,12 @@ describe("online evaluation custom validation", () => { .success, ).toBe(false); }); - it("requires exactly one analysis mode and binds clustering to insights", () => { + it("requires at least one evaluator", () => { const source = { ...base, agent: "agent" }; - expect( - OnlineEvalConfigSchema.safeParse({ ...source, evaluators: [], insights: [] }).success, - ).toBe(false); - expect(OnlineEvalConfigSchema.safeParse({ ...source, insights: ["Latency"] }).success).toBe( + expect(OnlineEvalConfigSchema.safeParse({ ...source, evaluators: [] }).success).toBe(false); + expect(OnlineEvalConfigSchema.safeParse({ ...source, evaluators: undefined }).success).toBe( false, ); - expect( - OnlineEvalConfigSchema.safeParse({ - ...source, - evaluators: undefined, - clusteringConfig: { frequencies: ["DAILY"] }, - }).success, - ).toBe(false); - expect( - OnlineEvalConfigSchema.safeParse({ - ...source, - evaluators: undefined, - insights: ["Latency"], - clusteringConfig: { frequencies: ["DAILY"] }, - }).success, - ).toBe(true); }); it("limits online evaluations to ten evaluators", () => { const source = { ...base, agent: "agent" }; diff --git a/src/projectSchemas/online-eval-config.ts b/src/projectSchemas/online-eval-config.ts index b1f8be0484..7a1772cfb2 100644 --- a/src/projectSchemas/online-eval-config.ts +++ b/src/projectSchemas/online-eval-config.ts @@ -10,13 +10,6 @@ export const OnlineEvalConfigNameSchema = z /^[a-zA-Z][a-zA-Z0-9_]{0,47}$/, "Must begin with a letter and contain only alphanumeric characters and underscores (max 48 chars)", ); -export const ClusteringConfigSchema = z.object({ - frequencies: z - .array(z.enum(["DAILY", "WEEKLY", "MONTHLY"])) - .min(1) - .max(3), -}); -export type ClusteringConfig = z.infer; export const OnlineEvalEvaluatorsSchema = z .array(z.string().min(1)) .max(ONLINE_EVAL_MAX_EVALUATORS, "At most 10 evaluators may be selected"); @@ -27,9 +20,7 @@ export const OnlineEvalConfigSchema = z endpoint: z.string().min(1).optional(), logGroupNames: z.array(z.string().min(1)).min(1).max(5).optional(), serviceNames: z.array(z.string().min(1)).min(1).optional(), - evaluators: OnlineEvalEvaluatorsSchema.optional(), - insights: z.array(z.string().min(1)).optional(), - clusteringConfig: ClusteringConfigSchema.optional(), + evaluators: OnlineEvalEvaluatorsSchema.min(1, "At least one evaluator is required"), samplingRate: z.number().min(0.01).max(100), description: z.string().max(200).optional(), enableOnCreate: z.boolean().optional(), @@ -46,23 +37,5 @@ export const OnlineEvalConfigSchema = z }) .refine((data) => !data.serviceNames || data.logGroupNames, { message: '"serviceNames" requires "logGroupNames"', - }) - .refine( - (data) => { - const hasEvaluators = data.evaluators != null && data.evaluators.length > 0; - const hasInsights = data.insights != null && data.insights.length > 0; - return hasEvaluators || hasInsights; - }, - { message: "At least one of evaluators or insights must be provided" }, - ) - .refine( - (data) => - !(data.evaluators && data.evaluators.length > 0 && data.insights && data.insights.length > 0), - { - message: "Cannot have both evaluators and insights (preview constraint)", - }, - ) - .refine((data) => !data.clusteringConfig || (data.insights && data.insights.length > 0), { - message: "clusteringConfig requires insights to be set", }); export type OnlineEvalConfig = z.infer; diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 153406ecee..6cb2832f27 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -183,8 +183,6 @@ import type { CreateTargetBasedABTestInput, CreateDatasetInput, CreateOnlineEvalInput, - CreateOnlineInsightInput, - UpdateOnlineInsightInput, DatasetUpdateResult, DatasetUpdateProgressEvent, EvaluateInput, @@ -195,7 +193,6 @@ import type { GetTracesInput, LlmAsAJudgeUpdate, SessionTrace, - StartBatchInsightsInput, StartBatchEvaluationInput, StartRecommendationInput, UpdateConfigurationBundleInput, @@ -1686,7 +1683,6 @@ export class TestEvalClient implements CoreEvalClient { // simulates a CloudWatch read failure surfaced as `resultsError`. private batchEvalGetResponse: GetBatchEvaluationResponse = DEFAULT_GET_BATCH_EVAL_RESPONSE; private batchEvalListResponses = new Map(); - private batchInsightsListResponses = new Map(); private abTestGetResponse: GetABTestResponse = DEFAULT_GET_ABTEST_RESPONSE; private abTestListResponses = new Map(); private abTestUpdateResponse: UpdateABTestResponse = DEFAULT_UPDATE_ABTEST_RESPONSE; @@ -1899,16 +1895,6 @@ export class TestEvalClient implements CoreEvalClient { return this; } - // setBatchInsightsListResponse sets what listBatchInsights resolves to (when - // not erroring). Pass `forNextToken` to serve a later logical Insights page. - setBatchInsightsListResponse( - response: ListBatchEvaluationsResponse, - forNextToken?: string, - ): this { - this.batchInsightsListResponses.set(forNextToken, response); - return this; - } - // setBatchEvalResults sets the per-session results getBatchEvaluation merges in // (when results are requested, the job is terminal, and it has a CloudWatch // output config). @@ -2084,17 +2070,6 @@ export class TestEvalClient implements CoreEvalClient { return { detail }; } - async getBatchInsights(id: string, options: CoreOptions): Promise { - this.calls.push({ method: "getBatchInsights", args: [id, options] }); - if (this.error) throw this.error; - - const detail: BatchEvaluationDetail = { ...this.batchEvalGetResponse }; - if (!detail.insights?.length) { - throw new InputValidationError(`batch evaluation "${id}" is not a batch insights run`); - } - return detail; - } - async listBatchEvaluations( nextToken: string | undefined, maxResults: number | undefined, @@ -2118,20 +2093,6 @@ export class TestEvalClient implements CoreEvalClient { return this.stopBatchEvalResponse; } - async listBatchInsights( - nextToken: string | undefined, - maxResults: number | undefined, - options: CoreOptions, - ): Promise { - this.calls.push({ method: "listBatchInsights", args: [nextToken, maxResults, options] }); - if (this.error) throw this.error; - const response = - this.batchInsightsListResponses.get(nextToken) ?? - this.batchInsightsListResponses.get(undefined) ?? - DEFAULT_LIST_BATCH_EVALS_RESPONSE; - return { ...response, batchEvaluations: response.batchEvaluations ?? [] }; - } - async getABTest(id: string, options: CoreOptions): Promise { this.calls.push({ method: "getABTest", args: [id, options] }); if (this.error) throw this.error; @@ -2195,15 +2156,6 @@ export class TestEvalClient implements CoreEvalClient { return this.startBatchEvalResponse; } - async startBatchInsights( - input: StartBatchInsightsInput, - options: CoreOptions, - ): Promise { - this.calls.push({ method: "startBatchInsights", args: [input, options] }); - if (this.error) throw this.error; - return this.startBatchEvalResponse; - } - // setGetTracesResponse sets what getTracesForAgent resolves to (when not // erroring). setGetTracesResponse(traces: SessionTrace[]): this { @@ -2308,66 +2260,6 @@ export class TestEvalClient implements CoreEvalClient { return this.onlineEvalDeleteResponse; } - async createOnlineInsight( - input: CreateOnlineInsightInput, - options: CoreOptions, - ): Promise { - this.calls.push({ method: "createOnlineInsight", args: [input, options] }); - if (this.error) throw this.error; - return this.onlineEvalCreateResponse; - } - - async updateOnlineInsight( - id: string, - update: UpdateOnlineInsightInput, - options: CoreOptions, - ): Promise { - this.calls.push({ method: "updateOnlineInsight", args: [id, update, options] }); - if (this.error) throw this.error; - return this.onlineEvalUpdateResponse; - } - - async getOnlineInsight( - id: string, - options: CoreOptions, - ): Promise { - this.calls.push({ method: "getOnlineInsight", args: [id, options] }); - if (this.error) throw this.error; - return this.onlineEvalGetResponse; - } - - async listOnlineInsights( - nextToken: string | undefined, - maxResults: number | undefined, - options: CoreOptions, - ): Promise { - this.calls.push({ method: "listOnlineInsights", args: [nextToken, maxResults, options] }); - if (this.error) throw this.error; - return this.onlineEvalListResponses.get(nextToken) ?? { onlineEvaluationConfigs: [] }; - } - - async setOnlineInsightExecutionStatus( - id: string, - executionStatus: "ENABLED" | "DISABLED", - options: CoreOptions, - ): Promise { - this.calls.push({ - method: "setOnlineInsightExecutionStatus", - args: [id, executionStatus, options], - }); - if (this.error) throw this.error; - return this.onlineEvalUpdateResponse; - } - - async deleteOnlineInsight( - id: string, - options: CoreOptions, - ): Promise { - this.calls.push({ method: "deleteOnlineInsight", args: [id, options] }); - if (this.error) throw this.error; - return this.onlineEvalDeleteResponse; - } - async createConfigurationBundle( input: CreateConfigurationBundleInput, options: CoreOptions, From 4a2a4aa396f4b48c2fef12f65a12815a9fcc689f Mon Sep 17 00:00:00 2001 From: jariy17 Date: Mon, 12 Oct 2026 17:41:49 +0000 Subject: [PATCH 2/2] fix(add): require --evaluators for online-eval Evaluators became mandatory once insights were removed; mark the flag required so a missing value fails at parse time with a clear message. --- src/handlers/project/add/online-eval/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/handlers/project/add/online-eval/index.ts b/src/handlers/project/add/online-eval/index.ts index 3fd681aca3..ecbc447265 100644 --- a/src/handlers/project/add/online-eval/index.ts +++ b/src/handlers/project/add/online-eval/index.ts @@ -18,7 +18,7 @@ export interface OnlineEvalInput { endpoint?: string; logGroupNames?: string[]; serviceNames?: string[]; - evaluators?: string[]; + evaluators: string[]; samplingRate: number; description?: string; enableOnCreate?: boolean; @@ -53,7 +53,7 @@ export function toAddOnlineEvalInput( ONLINE_EVAL_DEPLOYED_NAME_MAX_LENGTH, targets, ); - validateEvaluatorReferences(project, input.evaluators ?? []); + validateEvaluatorReferences(project, input.evaluators); const parsed = OnlineEvalConfigSchema.safeParse({ name: input.name, @@ -107,7 +107,7 @@ export const createAddOnlineEvalHandler = (config: AddProjectResourceConfig) => flag( "evaluators", "evaluator name(s), Builtin.* IDs, or ARNs to apply", - z.array(z.string()).optional(), + z.array(z.string()).min(1), ), flag( "sampling-rate",