feat(suite): expose provider sampling options

README.md

@@ -49,6 +49,8 @@ Write a suite (`suite.eval.yaml`):
 ```yaml
 name: my-agent
 provider: mock          # works with no API key
+temperature: 0          # deterministic by default; case values override
+maxTokens: 512          # optional output cap; case values override
 threshold: 0.9          # mean score required to pass
 cases:
   - id: greeting
@@ -93,6 +95,8 @@ A suite is YAML or JSON with this shape:
 | `name` | suite | Suite name shown in reports. Required. |
 | `provider` | suite / case | Provider to call (`mock`, `openai`, `anthropic`, `groq`, `openrouter`). |
 | `model` | suite / case | Model id. Case overrides suite. |
+| `temperature` | suite / case | Non-negative sampling temperature. Case overrides suite; default `0`. Provider-specific upper limits still apply. |
+| `maxTokens` | suite / case | Positive integer output-token cap. Case overrides suite; omitted by default. |
 | `threshold` | suite | Mean score in `[0,1]` required for the run to pass. |
 | `cases[].id` | case | Unique id. Required. |
 | `cases[].input.prompt` | case | A single-string prompt. |

examples/summarizer.eval.json

@@ -3,6 +3,8 @@
   "description": "A small JSON-format suite showing that summaries stay on-topic and within budget.",
   "provider": "mock",
   "model": "mock",
+  "temperature": 0,
+  "maxTokens": 256,
   "threshold": 0.85,
   "cases": [
     {
@@ -29,6 +31,7 @@
     {
       "id": "close-to-reference",
       "description": "The summary should be semantically close to the gold summary.",
+      "maxTokens": 128,
       "input": {
         "prompt": "Summarize.\necho: The quarterly report shows revenue growth driven by enterprise customers."
       },

examples/support-agent.eval.yaml

@@ -5,6 +5,8 @@ description: >
   it passes with zero API keys and no network access.
 provider: mock
 model: mock
+temperature: 0
+maxTokens: 512
 threshold: 0.9
 
 cases:
@@ -79,6 +81,8 @@ cases:
 
   - id: tone-rubric
     description: The reply must be polite, actionable, and reference the ticket.
+    temperature: 0.4
+    maxTokens: 128
     input:
       prompt: |
         Write a closing message for ticket 8842.

src/runner.ts

@@ -57,12 +57,14 @@ async function mapPool<T, R>(
 }
 
 /** Turn a case into a provider request. */
-function toRequest(c: EvalCase, model: string): ProviderRequest {
+function toRequest(c: EvalCase, suite: EvalSuite, model: string): ProviderRequest {
   return {
     model,
     prompt: c.input.prompt,
     messages: c.input.messages,
-    temperature: 0,
+    // Deterministic regression runs remain the default when neither level opts in.
+    temperature: c.temperature ?? suite.temperature ?? 0,
+    maxTokens: c.maxTokens ?? suite.maxTokens,
   };
 }
 
@@ -89,7 +91,7 @@ export async function runCase(
   const model = c.model ?? suite.model ?? options.defaultModel ?? "mock";
   const provider: Provider = providers.get(providerName);
 
-  const response = await provider.complete(toRequest(c, model));
+  const response = await provider.complete(toRequest(c, suite, model));
 
   const scoreResults: ScoreResult[] = [];
   for (const spec of c.scorers) {

src/suite.ts

@@ -47,6 +47,7 @@ export function validateSuite(data: unknown): EvalSuite {
       "suite.threshold must be a number in [0, 1]",
     );
   }
+  validateSamplingOptions(d, "suite");
 
   const ids = new Set<string>();
   const cases = (d.cases as unknown[]).map((c, i) => validateCase(c, i, ids));
@@ -56,6 +57,8 @@ export function validateSuite(data: unknown): EvalSuite {
     description: typeof d.description === "string" ? d.description : undefined,
     model: typeof d.model === "string" ? d.model : undefined,
     provider: typeof d.provider === "string" ? d.provider : undefined,
+    temperature: typeof d.temperature === "number" ? d.temperature : undefined,
+    maxTokens: typeof d.maxTokens === "number" ? d.maxTokens : undefined,
     threshold: typeof d.threshold === "number" ? d.threshold : undefined,
     vars: isVars(d.vars) ? d.vars : undefined,
     cases,
@@ -69,6 +72,25 @@ function isVars(v: unknown): v is Record<string, string | number | boolean> {
   );
 }
 
+function validateSamplingOptions(data: Record<string, unknown>, label: string): void {
+  if (data.temperature !== undefined) {
+    assert(
+      typeof data.temperature === "number" &&
+        Number.isFinite(data.temperature) &&
+        data.temperature >= 0,
+      `${label}.temperature must be a non-negative finite number`,
+    );
+  }
+  if (data.maxTokens !== undefined) {
+    assert(
+      typeof data.maxTokens === "number" &&
+        Number.isInteger(data.maxTokens) &&
+        data.maxTokens > 0,
+      `${label}.maxTokens must be a positive integer`,
+    );
+  }
+}
+
 function validateCase(data: unknown, index: number, ids: Set<string>): EvalCase {
   assert(data && typeof data === "object", `cases[${index}] must be an object`);
   const c = data as Record<string, unknown>;
@@ -86,6 +108,7 @@ function validateCase(data: unknown, index: number, ids: Set<string>): EvalCase
   assert(Array.isArray(c.scorers), `case "${c.id}" requires a scorers array`);
   assert((c.scorers as unknown[]).length > 0, `case "${c.id}" needs at least one scorer`);
   const scorers = (c.scorers as unknown[]).map((s, i) => validateScorer(s, c.id as string, i));
+  validateSamplingOptions(c, `case "${c.id}"`);
 
   return {
     id: c.id,
@@ -96,6 +119,8 @@ function validateCase(data: unknown, index: number, ids: Set<string>): EvalCase
     },
     model: typeof c.model === "string" ? c.model : undefined,
     provider: typeof c.provider === "string" ? c.provider : undefined,
+    temperature: typeof c.temperature === "number" ? c.temperature : undefined,
+    maxTokens: typeof c.maxTokens === "number" ? c.maxTokens : undefined,
     expected: typeof c.expected === "string" ? c.expected : undefined,
     scorers,
     tags: Array.isArray(c.tags) ? (c.tags as string[]).map(String) : undefined,

src/types.ts

@@ -122,6 +122,10 @@ export interface EvalCase {
   model?: string;
   /** Optional per-case provider override. */
   provider?: string;
+  /** Optional per-case sampling temperature override. */
+  temperature?: number;
+  /** Optional per-case output token cap override. */
+  maxTokens?: number;
   /** Optional expected value, shared by many scorers. */
   expected?: string;
   /** One or more scorers applied to the output. */
@@ -142,6 +146,10 @@ export interface EvalSuite {
   model?: string;
   /** Default provider applied to cases that do not override it. */
   provider?: string;
+  /** Default sampling temperature applied to cases that do not override it. */
+  temperature?: number;
+  /** Default output token cap applied to cases that do not override it. */
+  maxTokens?: number;
   /** Global pass threshold in [0, 1] for the aggregate score. */
   threshold?: number;
   /** Suite-level template variables applied to every case. */

tests/runner.test.ts

@@ -1,6 +1,7 @@
 import { describe, it, expect } from "vitest";
-import type { EvalSuite } from "../src/types.js";
+import type { EvalSuite, ProviderRequest } from "../src/types.js";
 import { runSuite, runCase, aggregateScore } from "../src/runner.js";
+import { ProviderRegistry } from "../src/providers/registry.js";
 
 const suite: EvalSuite = {
   name: "unit",
@@ -89,4 +90,39 @@ describe("runCase", () => {
     expect(res.provider).toBe("mock");
     expect(res.passed).toBe(true);
   });
+
+  it("resolves case, suite, and default sampling options", async () => {
+    const requests: ProviderRequest[] = [];
+    const providers = new ProviderRegistry().register("capture", () => ({
+      name: "capture",
+      async complete(request) {
+        requests.push(request);
+        return { output: "y", latencyMs: 0, model: request.model };
+      },
+    }));
+    const samplingSuite: EvalSuite = {
+      name: "sampling",
+      provider: "capture",
+      model: "capture-model",
+      temperature: 0.2,
+      maxTokens: 128,
+      cases: [],
+    };
+    const baseCase = {
+      id: "x",
+      input: { prompt: "exactly: y" },
+      expected: "y",
+      scorers: [{ type: "exact-match" }],
+    };
+
+    await runCase({ ...baseCase, temperature: 0.7, maxTokens: 64 }, samplingSuite, { providers });
+    await runCase(baseCase, samplingSuite, { providers });
+    await runCase(baseCase, { ...samplingSuite, temperature: undefined, maxTokens: undefined }, { providers });
+
+    expect(requests.map(({ temperature, maxTokens }) => ({ temperature, maxTokens }))).toEqual([
+      { temperature: 0.7, maxTokens: 64 },
+      { temperature: 0.2, maxTokens: 128 },
+      { temperature: 0, maxTokens: undefined },
+    ]);
+  });
 });

tests/suite.test.ts

@@ -28,6 +28,28 @@ cases:
     expect(suite.cases[0]!.id).toBe("a");
   });
 
+  it("parses suite and case sampling options", () => {
+    const suite = parseSuite(`
+name: sampling
+temperature: 0.2
+maxTokens: 512
+cases:
+  - id: creative
+    temperature: 0.7
+    maxTokens: 64
+    input:
+      prompt: write something
+    scorers:
+      - type: regex
+        pattern: .+
+`, "sampling.yaml");
+
+    expect(suite.temperature).toBe(0.2);
+    expect(suite.maxTokens).toBe(512);
+    expect(suite.cases[0]!.temperature).toBe(0.7);
+    expect(suite.cases[0]!.maxTokens).toBe(64);
+  });
+
   it("rejects a suite without a name", () => {
     expect(() => validateSuite({ cases: [] })).toThrow(SuiteValidationError);
   });
@@ -59,4 +81,38 @@ cases:
       }),
     ).toThrow(/threshold/);
   });
+
+  it.each([
+    ["temperature", -0.1],
+    ["temperature", Number.POSITIVE_INFINITY],
+    ["maxTokens", 0],
+    ["maxTokens", 1.5],
+  ])("rejects invalid suite %s", (field, value) => {
+    expect(() =>
+      validateSuite({
+        name: "d",
+        [field]: value,
+        cases: [{ id: "x", input: { prompt: "a" }, scorers: [{ type: "regex" }] }],
+      }),
+    ).toThrow(new RegExp(`suite\\.${field}`));
+  });
+
+  it.each([
+    ["temperature", Number.NaN],
+    ["maxTokens", -1],
+  ])("rejects invalid case %s", (field, value) => {
+    expect(() =>
+      validateSuite({
+        name: "d",
+        cases: [
+          {
+            id: "x",
+            [field]: value,
+            input: { prompt: "a" },
+            scorers: [{ type: "regex" }],
+          },
+        ],
+      }),
+    ).toThrow(new RegExp(`case "x"\\.${field}`));
+  });
 });