diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index dfbeafd63..c19a340d9 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -62,6 +62,40 @@ When the same model id is offered by multiple providers, use the Both provider name and provider id are accepted as the prefix. +### Targeting a specific AI Hub agent + +GoodData has no admin-settable "default agent": when a conversation doesn't +name one, the platform picks whichever agent was last used or last edited in +that workspace. If your org has several AI Hub agents configured (e.g. one +scoped to visualization only, another with every skill enabled), evaluating +without `--agent-id` can silently exercise the wrong one — a +`metric_skill`/`alert_skill` item run against a visualization-only agent will +never pass, no matter how well-formed the question is. + +```bash +export GD_EVAL_AGENT_ID='eval-all-skills' + +gd-eval run \ + --host https://your.gooddata.cloud \ + --workspace ecommerce_demo \ + --dataset ./my-dataset \ + --model gpt-5.2 \ + --runs 1 \ + --json results.json +``` + +Or pass it explicitly instead of via the env var: + +```bash +gd-eval run \ + --host https://your.gooddata.cloud \ + --workspace ecommerce_demo \ + --dataset ./my-dataset \ + --agent-id eval-all-skills \ + --model gpt-5.2 \ + --runs 1 +``` + ### All flags #### Connection @@ -72,6 +106,7 @@ Both provider name and provider id are accepted as the prefix. | `--token TOKEN` | `GOODDATA_TOKEN` | API token. Pass via flag or env var. | | `--profile NAME` | — | Profile name in `~/.gooddata/profiles.yaml` (same file as the `gdc` CLI). | | `--workspace ID` | — | **Required.** Workspace id to evaluate against. | +| `--agent-id ID` | `GD_EVAL_AGENT_ID` | AI Hub agent every conversation should target. GoodData has no admin-settable default agent — without this, each conversation falls back to whichever agent the platform's last-used/last-edited heuristic resolves, which may not have every skill under test enabled. | #### Dataset source (pick one) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 31104660e..d9971a5b4 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -80,6 +80,7 @@ def _dispatch_agentic( langfuse: Any, run_ts: str, model_version_override: str | None, + agent_id: str | None = None, ) -> None: """Call the appropriate evaluate_agentic_* function for the item's test_kind.""" kind = item.test_kind @@ -100,6 +101,7 @@ def _dispatch_agentic( question=item.question, expected_outputs=_parse_visualization_expected(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_metric_skill": @@ -110,6 +112,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, (dict, list)) else {}, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_alert_skill": @@ -120,6 +123,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, dict) else {}, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_search": @@ -133,6 +137,7 @@ def _dispatch_agentic( question=item.question, expected_tool_call=expected_args, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_general_question": @@ -143,6 +148,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_guardrail": @@ -153,6 +159,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_conversation": @@ -162,6 +169,7 @@ def _dispatch_agentic( token=token, workspace_id=workspace_id, fixture=ConversationFixture.model_validate(fixture_data), + agent_id=agent_id, **lf_kw, ) else: @@ -180,6 +188,7 @@ def run_agentic_items( run_ts: str, on_item_start: Any = None, on_item_done: Any = None, + agent_id: str | None = None, ) -> EvalReport: """Run agentic items through evaluate_agentic_* and return an EvalReport.""" langfuse = make_langfuse_client() if use_langfuse else None @@ -202,7 +211,7 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version) + _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, agent_id) item_report.pass_at_k = True item_report.runs = k except AssertionError as exc: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 0303270be..be7a2b9e1 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -2,6 +2,7 @@ """`gd-eval` command-line entry point.""" import argparse +import os import sys import threading from datetime import datetime, timezone @@ -109,6 +110,16 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).", ) + run.add_argument( + "--agent-id", + dest="agent_id", + help=( + "AI Hub agent id every conversation should target (or set GD_EVAL_AGENT_ID). " + "GoodData has no admin-settable default agent -- without this, each conversation " + "falls back to whichever agent the platform's last-used/last-edited heuristic " + "resolves, which may not have every skill under test enabled." + ), + ) models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.") models_cmd.add_argument("--host", help="GoodData host URL.") models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).") @@ -333,6 +344,7 @@ def on_langfuse_item_done( run_ts=run_ts, on_item_start=on_item_start, on_item_done=on_item_done, + agent_id=config.agent_id, ) # --- non-agentic items (single-turn, use Evaluator) --- @@ -342,6 +354,7 @@ def on_langfuse_item_done( token=config.token, workspace_id=config.workspace_id, preserve_failed=config.preserve_failed, + agent_id=config.agent_id, ), SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id), ) @@ -433,6 +446,7 @@ def main(argv: list[str] | None = None) -> int: quiet=args.quiet, kind=args.kind, preserve_failed=args.preserve_failed, + agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"), ) return _run(config) except ( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 1a7d2a188..085e45098 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -342,11 +342,12 @@ def run_agentic_alert_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticAlertSummary: """Run the alert-skill agentic evaluation K times and return a summary.""" expected = _normalize_expected_output(expected_output) run_results: list[AlertRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) sdk = GoodDataSdk.create(host, token) def _run_once(conv_id: str) -> AlertRunResult: @@ -456,6 +457,7 @@ def evaluate_agentic_alert_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "alert_skill", @@ -481,6 +483,7 @@ def evaluate_agentic_alert_skill( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 6b79b3279..da7e0fc91 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -278,6 +278,7 @@ def run_agentic_conversation( fixture: ConversationFixture, max_clarification_turns: int = 20, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> ConversationResult: """Run a multi-turn, multi-skill conversation evaluation (no K-runs). @@ -285,7 +286,7 @@ def run_agentic_conversation( trigger up to *max_clarification_turns* additional rounds of simulated-user replies before the agent produces the expected output. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) sdk = GoodDataSdk.create(host, token) turn_results: list[TurnResult] = [] turn_outputs: dict[str, dict] = {} @@ -397,6 +398,7 @@ def evaluate_agentic_conversation( fixture: ConversationFixture, max_clarification_turns: int = 20, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "conversation", @@ -420,6 +422,7 @@ def evaluate_agentic_conversation( fixture=fixture, max_clarification_turns=max_clarification_turns, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index 653a77956..014855ea2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -71,10 +71,11 @@ def run_agentic_general_question( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticGeneralQuestionSummary: """Run the general-question agentic evaluation K times and return a summary.""" run_results: list[GeneralQuestionResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o") try: @@ -147,6 +148,7 @@ def evaluate_agentic_general_question( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "general_question", @@ -171,6 +173,7 @@ def evaluate_agentic_general_question( expected_output=expected_output, k=k, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index cb61da24a..16afeb2e6 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -68,10 +68,11 @@ def run_agentic_guardrail( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticGuardrailSummary: """Run the guardrail agentic evaluation K times and return a summary.""" run_results: list[GuardrailResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o") try: @@ -144,6 +145,7 @@ def evaluate_agentic_guardrail( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "guardrail", @@ -168,6 +170,7 @@ def evaluate_agentic_guardrail( expected_output=expected_output, k=k, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index a2758f6a8..d25088af8 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -232,6 +232,7 @@ def run_agentic_metric_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticMetricSummary: """Run the metric-skill agentic evaluation K times and return a summary. @@ -240,7 +241,7 @@ def run_agentic_metric_skill( """ expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] run_results: list[MetricRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) sdk = GoodDataSdk.create(host, token) try: @@ -294,6 +295,7 @@ def evaluate_agentic_metric_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "metric_skill", @@ -319,6 +321,7 @@ def evaluate_agentic_metric_skill( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index cdbce48ab..9a92eb079 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -67,11 +67,12 @@ def run_agentic_search_tool( expected_tool_call: dict, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticSearchSummary: """Run the search-tool agentic evaluation K times (single-turn each).""" run_results: list[SearchResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: @@ -138,6 +139,7 @@ def evaluate_agentic_search_tool( expected_tool_call: dict, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "search", @@ -162,6 +164,7 @@ def evaluate_agentic_search_tool( expected_tool_call=expected_tool_call, k=k, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 12914157a..6abf33fa4 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -203,6 +203,7 @@ def run_agentic_visualization( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, ) -> AgenticRunSummary: """Run K independent conversations and return evaluation results. @@ -211,7 +212,7 @@ def run_agentic_visualization( fresh conversations. Caller-supplied conversations are not deleted; all conversations created by this function are deleted on completion. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id) + client = ChatClient(host=host, token=token, workspace_id=workspace_id, agent_id=agent_id) run_results: list[RunResult] = [] try: @@ -258,6 +259,7 @@ def evaluate_agentic_visualization( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "visualization", @@ -285,6 +287,7 @@ def evaluate_agentic_visualization( k=k, max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 091436b27..df54ddda6 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -242,16 +242,25 @@ class ChatClient: """Single-turn AI chat client over the GoodData AI conversation endpoints.""" def __init__( - self, host: str, token: str, workspace_id: str, *, timeout: float = 300.0, preserve_failed: bool = False + self, + host: str, + token: str, + workspace_id: str, + *, + timeout: float = 300.0, + preserve_failed: bool = False, + agent_id: str | None = None, ): self._base = f"{host.rstrip('/')}/api/v1/ai/workspaces/{workspace_id}/chat/conversations" self._auth = {"Authorization": f"Bearer {token}"} self._client = httpx.Client(timeout=timeout) self._preserve_failed = preserve_failed + self._agent_id = agent_id def create_conversation(self) -> str: def _do() -> str: - resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"}) + body = {"agentId": self._agent_id} if self._agent_id else {} + resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body) resp.raise_for_status() body = resp.json() if "conversationId" not in body: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 277785176..800fa8f27 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -20,3 +20,4 @@ class RunConfig: quiet: bool = False kind: str = "visualization" preserve_failed: bool = False + agent_id: str | None = None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py new file mode 100644 index 000000000..3cbff3137 --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -0,0 +1,93 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +from unittest.mock import patch + +import pytest +from gooddata_eval.cli.agentic_runner import _dispatch_agentic +from gooddata_eval.core.models import DatasetItem + + +def test_dispatch_agentic_passes_agent_id_through_to_alert_skill(): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind="agentic_alert_skill", + question="Alert me when spend exceeds 100", + expected_output={"Operator": "GREATER_THAN", "Threshold": 100}, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + agent_id="agent-1", + ) + assert mock_eval.call_args.kwargs["agent_id"] == "agent-1" + + +def test_dispatch_agentic_omits_agent_id_by_default(): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind="agentic_metric_skill", + question="Create a metric for total spend", + expected_output={"maql": "SELECT {metric/spend}"}, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_metric_skill") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + ) + assert mock_eval.call_args.kwargs["agent_id"] is None + + +_MIN_VIZ = {"id": "v1", "type": "table", "query": {"fields": {}, "filter_by": {}}, "metrics": [], "view_by": []} +_MIN_CONVERSATION_FIXTURE = { + "id": "c1", + "expected_skills": ["visualization"], + "turns": [{"turn_id": "t1", "message": "hi", "expected_skill": "visualization"}], +} + + +@pytest.mark.parametrize( + ("kind", "expected_output", "target"), + [ + ("vis_agentic", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_visualization", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_search", {"tool_call": {"function_arguments": {}}}, "evaluate_agentic_search_tool"), + ("agentic_general_question", "What is X?", "evaluate_agentic_general_question"), + ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), + ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ], +) +def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_output, target): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind=kind, + question="q", + expected_output=expected_output, + ) + with patch(f"gooddata_eval.cli.agentic_runner.{target}") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + agent_id="agent-1", + ) + assert mock_eval.call_args.kwargs["agent_id"] == "agent-1" diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 28aecb3c4..0f7735956 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -26,6 +26,16 @@ def test_build_run_config_requires_a_source(): cli_main.parse_args(["run", "--host", "h", "--workspace", "w"]) +def test_parse_args_agent_id_flag(): + args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d", "--agent-id", "agent-1"]) + assert args.agent_id == "agent-1" + + +def test_parse_args_agent_id_defaults_to_none(): + args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d"]) + assert args.agent_id is None + + def test_cli_run_end_to_end(monkeypatch, tmp_path, fixtures_dir): # Stub connection + model activation + chat backend so no network is needed. monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) @@ -93,6 +103,106 @@ def _fake_run( assert orjson.loads(out.read_bytes())["runs"]["Test Provider/gpt-5.2"]["summary"]["passed"] == 1 +def _stub_run_for_agent_id_test(monkeypatch, seen_chat_client_kwargs): + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="prov", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel( + provider_id="prov", model_id=requested or "gpt-5.2", switched=False, provider_name="Test Provider" + ) + + def restore(self, original): ... + def close(self): ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + + def _fake_run(items, backend, *, runs, model, workspace_id, **kw): + return EvalReport(model=model, workspace_id=workspace_id, items=[]) + + monkeypatch.setattr(cli_main, "run_items", _fake_run) + + def _spy_chat_client(**kwargs): + seen_chat_client_kwargs.update(kwargs) + return object() + + monkeypatch.setattr(cli_main, "ChatClient", _spy_chat_client) + + +def test_cli_run_passes_agent_id_flag_to_chat_client(monkeypatch, tmp_path, fixtures_dir): + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--agent-id", + "agent-1", + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] == "agent-1" + + +def test_cli_run_falls_back_to_agent_id_env_var(monkeypatch, tmp_path, fixtures_dir): + monkeypatch.setenv("GD_EVAL_AGENT_ID", "agent-from-env") + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] == "agent-from-env" + + +def test_cli_run_agent_id_omitted_when_unset(monkeypatch, tmp_path, fixtures_dir): + monkeypatch.delenv("GD_EVAL_AGENT_ID", raising=False) + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] is None + + def test_cli_operational_error_exits_nonzero(monkeypatch, fixtures_dir): def _boom(host, token, profile): raise ConnectionError_("Missing token.") @@ -535,8 +645,6 @@ def close(self): ... monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) - original_chat_client = cli_main.ChatClient - def _capture_chat_client(**kwargs): captured_kwargs.update(kwargs) return object() diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index c5590e428..8da0154e4 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -145,8 +145,8 @@ def test_parse_sse_lines_non_retryable_status_is_chat_error_not_transient(): assert ei.value.status_code == 400 -def _client_with_handler(handler): - client = ChatClient(host="https://example.invalid", token="t", workspace_id="w") +def _client_with_handler(handler, **client_kwargs): + client = ChatClient(host="https://example.invalid", token="t", workspace_id="w", **client_kwargs) client._client = httpx.Client(transport=httpx.MockTransport(handler)) return client @@ -238,6 +238,32 @@ def handler(request): assert sleeps == [] +def test_create_conversation_omits_agent_id_by_default(): + # No agent_id given -> unchanged, existing behavior: GoodData's own + # last-used/last-edited default-agent resolution still applies. + seen = {} + + def handler(request): + seen["body"] = request.content + return httpx.Response(200, json={"conversationId": "abc"}) + + client = _client_with_handler(handler) + client.create_conversation() + assert seen["body"] in (b"", b"{}") + + +def test_create_conversation_sends_agent_id_when_given(): + seen = {} + + def handler(request): + seen["body"] = json.loads(request.content) + return httpx.Response(200, json={"conversationId": "abc"}) + + client = _client_with_handler(handler, agent_id="agent-123") + client.create_conversation() + assert seen["body"] == {"agentId": "agent-123"} + + def test_int_env_uses_default_when_unset(monkeypatch): monkeypatch.delenv("GD_TEST_INT", raising=False) assert sse_mod._int_env("GD_TEST_INT", 5) == 5