feat(gooddata-eval): support requesting a per-message reasoning effort

packages/gooddata-eval/README.md

@@ -92,6 +92,7 @@ Both provider name and provider id are accepted as the prefix.
 |---|---|---|
 | `--runs K` | `2` | Independent runs per item (pass@K). An item passes if any run passes. |
 | `--concurrency K` | `1` | Number of items evaluated concurrently. `1` = sequential (default). Increase to load-test the agent under simultaneous requests. Progress output interleaves when K > 1. |
+| `--reasoning-effort {LOW,MEDIUM,HIGH}` | — | Requested LLM reasoning effort, sent with every message this run makes (or set `GD_EVAL_REASONING_EFFORT`). See [Requesting a reasoning effort](#requesting-a-reasoning-effort) below. |
 
 #### Output
 
@@ -106,6 +107,25 @@ Both provider name and provider id are accepted as the prefix.
 |---|---|
 | `--langfuse` | Log scores and traces to Langfuse after each item. Requires `--langfuse-dataset`. Creates one named experiment run per model (`gd-eval-{timestamp}-{model}`). Requires `LANGFUSE_PUBLIC_KEY`, `LANGFUSE_SECRET_KEY`, `LANGFUSE_HOST`. |
 
+### Requesting a reasoning effort
+
+GoodData Cloud has an experimental per-message `reasoningEffort` option (`LOW`, `MEDIUM`, or `HIGH`) that hints how much the LLM should reason before answering. `gd-eval` can send it on every message it makes:
+
+```bash
+# One-off, via flag
+gd-eval run --workspace my-ws --dataset ./data --reasoning-effort HIGH
+
+# Session-wide, via env var
+export GD_EVAL_REASONING_EFFORT=LOW
+gd-eval run --workspace my-ws --dataset ./data
+```
+
+Things to know before using it:
+
+- **Not persisted server-side.** The setting applies only to the messages this run sends — it is not saved as a conversation or workspace default. Every `gd-eval` message in the run carries the value; nothing else on the server is affected.
+- **Feature-flag gated.** GoodData Cloud must have the reasoning-effort feature enabled for the org; when it isn't, the value is ignored and the platform falls back to `MEDIUM` regardless of what was requested.
+- **A hint, not a hard budget.** Providers with adaptive-thinking models (e.g. Anthropic, Bedrock) treat the value as a hint rather than an exact token allocation, so actual reasoning depth can still vary by model.
+
 ### JSON report shape
 
 The JSON report always uses the nested multi-model shape:

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -80,6 +80,7 @@ def _dispatch_agentic(
     langfuse: Any,
     run_ts: str,
     model_version_override: str | None,
+    reasoning_effort: str | None = None,
 ) -> None:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind."""
     kind = item.test_kind
@@ -100,6 +101,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_outputs=_parse_visualization_expected(eo),
             k=k,
+            reasoning_effort=reasoning_effort,
             **lf_kw,
         )
     elif kind == "agentic_metric_skill":
@@ -110,6 +112,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, (dict, list)) else {},
             k=k,
+            reasoning_effort=reasoning_effort,
             **lf_kw,
         )
     elif kind == "agentic_alert_skill":
@@ -120,6 +123,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
+            reasoning_effort=reasoning_effort,
             **lf_kw,
         )
     elif kind == "agentic_search":
@@ -133,6 +137,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_tool_call=expected_args,
             k=k,
+            reasoning_effort=reasoning_effort,
             **lf_kw,
         )
     elif kind == "agentic_general_question":
@@ -143,6 +148,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, str) else str(eo),
             k=k,
+            reasoning_effort=reasoning_effort,
             **lf_kw,
         )
     elif kind == "agentic_guardrail":
@@ -153,6 +159,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, str) else str(eo),
             k=k,
+            reasoning_effort=reasoning_effort,
             **lf_kw,
         )
     elif kind == "agentic_conversation":
@@ -162,6 +169,7 @@ def _dispatch_agentic(
             token=token,
             workspace_id=workspace_id,
             fixture=ConversationFixture.model_validate(fixture_data),
+            reasoning_effort=reasoning_effort,
             **lf_kw,
         )
     else:
@@ -180,6 +188,7 @@ def run_agentic_items(
     run_ts: str,
     on_item_start: Any = None,
     on_item_done: Any = None,
+    reasoning_effort: str | None = None,
 ) -> EvalReport:
     """Run agentic items through evaluate_agentic_* and return an EvalReport."""
     langfuse = make_langfuse_client() if use_langfuse else None
@@ -202,7 +211,7 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version)
+            _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort)
             item_report.pass_at_k = True
             item_report.runs = k
         except AssertionError as exc:

packages/gooddata-eval/src/gooddata_eval/cli/main.py

@@ -2,6 +2,7 @@
 """`gd-eval` command-line entry point."""
 
 import argparse
+import os
 import sys
 import threading
 from datetime import datetime, timezone
@@ -37,14 +38,15 @@ class _RoutingBackend:
     else uses the conversational chat endpoint.
     """
 
-    def __init__(self, chat: ChatClient, summary: SummaryClient):
+    def __init__(self, chat: ChatClient, summary: SummaryClient, *, reasoning_effort: str | None = None):
         self._chat = chat
         self._summary = summary
+        self._reasoning_effort = reasoning_effort
 
     def ask(self, item: DatasetItem) -> ChatResult:
         if item.test_kind == _SUMMARY_TEST_KIND:
             return self._summary.ask(item)
-        return self._chat.ask(item)
+        return self._chat.ask(item, reasoning_effort=self._reasoning_effort)
 
     def close(self) -> None:
         for backend in (self._chat, self._summary):
@@ -109,6 +111,17 @@ def _build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).",
     )
+    run.add_argument(
+        "--reasoning-effort",
+        dest="reasoning_effort",
+        choices=["LOW", "MEDIUM", "HIGH"],
+        default=None,
+        help=(
+            "Requested LLM reasoning effort for this run's messages (or set GD_EVAL_REASONING_EFFORT). "
+            "Experimental GoodData feature, gated behind an org-level flag — when disabled, the value is "
+            "ignored and MEDIUM is used. Not persisted server-side: applies only to messages this run sends."
+        ),
+    )
     models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.")
     models_cmd.add_argument("--host", help="GoodData host URL.")
     models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).")
@@ -333,6 +346,7 @@ def on_langfuse_item_done(
                     run_ts=run_ts,
                     on_item_start=on_item_start,
                     on_item_done=on_item_done,
+                    reasoning_effort=config.reasoning_effort,
                 )
 
             # --- non-agentic items (single-turn, use Evaluator) ---
@@ -344,6 +358,7 @@ def on_langfuse_item_done(
                     preserve_failed=config.preserve_failed,
                 ),
                 SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id),
+                reasoning_effort=config.reasoning_effort,
             )
             try:
                 single_report = run_items(
@@ -410,11 +425,23 @@ def on_langfuse_item_done(
     return _EXIT_OK
 
 
+_VALID_REASONING_EFFORTS = frozenset({"LOW", "MEDIUM", "HIGH"})
+
+
 def main(argv: list[str] | None = None) -> int:
     args = parse_args(argv if argv is not None else sys.argv[1:])
     if hasattr(args, "concurrency") and args.concurrency < 1:
         print("error: --concurrency must be >= 1.", file=sys.stderr)
         return _EXIT_OPERATIONAL_ERROR
+    reasoning_effort = None
+    if hasattr(args, "reasoning_effort"):
+        reasoning_effort = args.reasoning_effort or os.environ.get("GD_EVAL_REASONING_EFFORT")
+        if reasoning_effort is not None and reasoning_effort not in _VALID_REASONING_EFFORTS:
+            print(
+                f"error: reasoning effort must be one of {sorted(_VALID_REASONING_EFFORTS)}, got {reasoning_effort!r}.",
+                file=sys.stderr,
+            )
+            return _EXIT_OPERATIONAL_ERROR
     try:
         host, token = resolve_connection(host=args.host, token=args.token, profile=args.profile)
         if args.command == "models":
@@ -433,6 +460,7 @@ def main(argv: list[str] | None = None) -> int:
             quiet=args.quiet,
             kind=args.kind,
             preserve_failed=args.preserve_failed,
+            reasoning_effort=reasoning_effort,
         )
         return _run(config)
     except (

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -342,6 +342,7 @@ def run_agentic_alert_skill(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    reasoning_effort: str | None = None,
 ) -> AgenticAlertSummary:
     """Run the alert-skill agentic evaluation K times and return a summary."""
     expected = _normalize_expected_output(expected_output)
@@ -361,7 +362,7 @@ def _run_once(conv_id: str) -> AlertRunResult:
             current_question = question
 
             for _iteration in range(max_iterations):
-                chat_result = client.send_message(conv_id, current_question)
+                chat_result = client.send_message(conv_id, current_question, reasoning_effort=reasoning_effort)
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id
@@ -462,6 +463,7 @@ def evaluate_agentic_alert_skill(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: str | None = None,
 ) -> None:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -481,6 +483,7 @@ def evaluate_agentic_alert_skill(
         k=k,
         max_iterations=max_iterations,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -278,6 +278,7 @@ def run_agentic_conversation(
     fixture: ConversationFixture,
     max_clarification_turns: int = 20,
     initial_conversation_id: str | None = None,
+    reasoning_effort: str | None = None,
 ) -> ConversationResult:
     """Run a multi-turn, multi-skill conversation evaluation (no K-runs).
 
@@ -315,7 +316,7 @@ def run_agentic_conversation(
             final_result: ChatResult | None = None
 
             for _iter in range(max_clarification_turns + 1):
-                chat_result = client.send_message(conversation_id, current_message)
+                chat_result = client.send_message(conversation_id, current_message, reasoning_effort=reasoning_effort)
                 final_result = chat_result
                 all_tool_calls.extend(chat_result.tool_call_events or [])
 
@@ -403,6 +404,7 @@ def evaluate_agentic_conversation(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: str | None = None,
 ) -> None:
     """Run conversation evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -420,6 +422,7 @@ def evaluate_agentic_conversation(
         fixture=fixture,
         max_clarification_turns=max_clarification_turns,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -71,6 +71,7 @@ def run_agentic_general_question(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: str | None = None,
 ) -> AgenticGeneralQuestionSummary:
     """Run the general-question agentic evaluation K times and return a summary."""
     run_results: list[GeneralQuestionResult] = []
@@ -80,7 +81,7 @@ def run_agentic_general_question(
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
-            chat_result = client.send_message(conv_id_0, question)
+            chat_result = client.send_message(conv_id_0, question, reasoning_effort=reasoning_effort)
             actual_output = (chat_result.text_response or "").strip()
             passed, reasoning = judge.score(
                 input=question, expected_output=expected_output, actual_output=actual_output
@@ -102,7 +103,7 @@ def run_agentic_general_question(
         for _ in range(1, k):
             conv_id = client.create_conversation()
             try:
-                chat_result = client.send_message(conv_id, question)
+                chat_result = client.send_message(conv_id, question, reasoning_effort=reasoning_effort)
                 actual_output = (chat_result.text_response or "").strip()
                 passed, reasoning = judge.score(
                     input=question, expected_output=expected_output, actual_output=actual_output
@@ -153,6 +154,7 @@ def evaluate_agentic_general_question(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: str | None = None,
 ) -> None:
     """Run general-question evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -171,6 +173,7 @@ def evaluate_agentic_general_question(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id:

packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py

@@ -68,6 +68,7 @@ def run_agentic_guardrail(
     expected_output: str,
     k: int = _DEFAULT_K,
     initial_conversation_id: str | None = None,
+    reasoning_effort: str | None = None,
 ) -> AgenticGuardrailSummary:
     """Run the guardrail agentic evaluation K times and return a summary."""
     run_results: list[GuardrailResult] = []
@@ -77,7 +78,7 @@ def run_agentic_guardrail(
     try:
         conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation()
         try:
-            chat_result = client.send_message(conv_id_0, question)
+            chat_result = client.send_message(conv_id_0, question, reasoning_effort=reasoning_effort)
             actual_output = (chat_result.text_response or "").strip()
             passed, reasoning = judge.score(
                 input=question, expected_output=expected_output, actual_output=actual_output
@@ -99,7 +100,7 @@ def run_agentic_guardrail(
         for _ in range(1, k):
             conv_id = client.create_conversation()
             try:
-                chat_result = client.send_message(conv_id, question)
+                chat_result = client.send_message(conv_id, question, reasoning_effort=reasoning_effort)
                 actual_output = (chat_result.text_response or "").strip()
                 passed, reasoning = judge.score(
                     input=question, expected_output=expected_output, actual_output=actual_output
@@ -150,6 +151,7 @@ def evaluate_agentic_guardrail(
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
+    reasoning_effort: str | None = None,
 ) -> None:
     """Run guardrail evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -168,6 +170,7 @@ def evaluate_agentic_guardrail(
         expected_output=expected_output,
         k=k,
         initial_conversation_id=initial_conversation_id,
+        reasoning_effort=reasoning_effort,
     )
 
     if langfuse is not None and dataset_item_id: