From d391fd83fb4167d8f4c50ecfefc6bd72f814a3a9 Mon Sep 17 00:00:00 2001 From: Victor <70475442+vsolano9@users.noreply.github.com> Date: Sun, 23 Aug 2026 16:35:18 +0200 Subject: [PATCH] feat(suite): expose provider sampling options Add validated suite- and case-level temperature and maxTokens fields, resolve them with case-over-suite precedence, and retain temperature zero as the deliberate deterministic default. Document the public format, update both examples, and cover parsing, validation, precedence, and provider request forwarding offline. Closes AgentPostmortem/Evalgate#17 --- README.md | 4 +++ examples/summarizer.eval.json | 3 ++ examples/support-agent.eval.yaml | 4 +++ src/runner.ts | 8 +++-- src/suite.ts | 25 ++++++++++++++ src/types.ts | 8 +++++ tests/runner.test.ts | 38 +++++++++++++++++++++- tests/suite.test.ts | 56 ++++++++++++++++++++++++++++++++ 8 files changed, 142 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6507b1a..883a5f1 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,8 @@ Write a suite (`suite.eval.yaml`): ```yaml name: my-agent provider: mock # works with no API key +temperature: 0 # deterministic by default; case values override +maxTokens: 512 # optional output cap; case values override threshold: 0.9 # mean score required to pass cases: - id: greeting @@ -93,6 +95,8 @@ A suite is YAML or JSON with this shape: | `name` | suite | Suite name shown in reports. Required. | | `provider` | suite / case | Provider to call (`mock`, `openai`, `anthropic`, `groq`, `openrouter`). | | `model` | suite / case | Model id. Case overrides suite. | +| `temperature` | suite / case | Non-negative sampling temperature. Case overrides suite; default `0`. Provider-specific upper limits still apply. | +| `maxTokens` | suite / case | Positive integer output-token cap. Case overrides suite; omitted by default. | | `threshold` | suite | Mean score in `[0,1]` required for the run to pass. | | `cases[].id` | case | Unique id. Required. | | `cases[].input.prompt` | case | A single-string prompt. | diff --git a/examples/summarizer.eval.json b/examples/summarizer.eval.json index 279e54c..998c6e9 100644 --- a/examples/summarizer.eval.json +++ b/examples/summarizer.eval.json @@ -3,6 +3,8 @@ "description": "A small JSON-format suite showing that summaries stay on-topic and within budget.", "provider": "mock", "model": "mock", + "temperature": 0, + "maxTokens": 256, "threshold": 0.85, "cases": [ { @@ -29,6 +31,7 @@ { "id": "close-to-reference", "description": "The summary should be semantically close to the gold summary.", + "maxTokens": 128, "input": { "prompt": "Summarize.\necho: The quarterly report shows revenue growth driven by enterprise customers." }, diff --git a/examples/support-agent.eval.yaml b/examples/support-agent.eval.yaml index 1ae34ca..14eb928 100644 --- a/examples/support-agent.eval.yaml +++ b/examples/support-agent.eval.yaml @@ -5,6 +5,8 @@ description: > it passes with zero API keys and no network access. provider: mock model: mock +temperature: 0 +maxTokens: 512 threshold: 0.9 cases: @@ -79,6 +81,8 @@ cases: - id: tone-rubric description: The reply must be polite, actionable, and reference the ticket. + temperature: 0.4 + maxTokens: 128 input: prompt: | Write a closing message for ticket 8842. diff --git a/src/runner.ts b/src/runner.ts index ba9b916..3ec8834 100644 --- a/src/runner.ts +++ b/src/runner.ts @@ -57,12 +57,14 @@ async function mapPool( } /** Turn a case into a provider request. */ -function toRequest(c: EvalCase, model: string): ProviderRequest { +function toRequest(c: EvalCase, suite: EvalSuite, model: string): ProviderRequest { return { model, prompt: c.input.prompt, messages: c.input.messages, - temperature: 0, + // Deterministic regression runs remain the default when neither level opts in. + temperature: c.temperature ?? suite.temperature ?? 0, + maxTokens: c.maxTokens ?? suite.maxTokens, }; } @@ -89,7 +91,7 @@ export async function runCase( const model = c.model ?? suite.model ?? options.defaultModel ?? "mock"; const provider: Provider = providers.get(providerName); - const response = await provider.complete(toRequest(c, model)); + const response = await provider.complete(toRequest(c, suite, model)); const scoreResults: ScoreResult[] = []; for (const spec of c.scorers) { diff --git a/src/suite.ts b/src/suite.ts index 7e9e04a..214c139 100644 --- a/src/suite.ts +++ b/src/suite.ts @@ -47,6 +47,7 @@ export function validateSuite(data: unknown): EvalSuite { "suite.threshold must be a number in [0, 1]", ); } + validateSamplingOptions(d, "suite"); const ids = new Set(); const cases = (d.cases as unknown[]).map((c, i) => validateCase(c, i, ids)); @@ -56,6 +57,8 @@ export function validateSuite(data: unknown): EvalSuite { description: typeof d.description === "string" ? d.description : undefined, model: typeof d.model === "string" ? d.model : undefined, provider: typeof d.provider === "string" ? d.provider : undefined, + temperature: typeof d.temperature === "number" ? d.temperature : undefined, + maxTokens: typeof d.maxTokens === "number" ? d.maxTokens : undefined, threshold: typeof d.threshold === "number" ? d.threshold : undefined, vars: isVars(d.vars) ? d.vars : undefined, cases, @@ -69,6 +72,25 @@ function isVars(v: unknown): v is Record { ); } +function validateSamplingOptions(data: Record, label: string): void { + if (data.temperature !== undefined) { + assert( + typeof data.temperature === "number" && + Number.isFinite(data.temperature) && + data.temperature >= 0, + `${label}.temperature must be a non-negative finite number`, + ); + } + if (data.maxTokens !== undefined) { + assert( + typeof data.maxTokens === "number" && + Number.isInteger(data.maxTokens) && + data.maxTokens > 0, + `${label}.maxTokens must be a positive integer`, + ); + } +} + function validateCase(data: unknown, index: number, ids: Set): EvalCase { assert(data && typeof data === "object", `cases[${index}] must be an object`); const c = data as Record; @@ -86,6 +108,7 @@ function validateCase(data: unknown, index: number, ids: Set): EvalCase assert(Array.isArray(c.scorers), `case "${c.id}" requires a scorers array`); assert((c.scorers as unknown[]).length > 0, `case "${c.id}" needs at least one scorer`); const scorers = (c.scorers as unknown[]).map((s, i) => validateScorer(s, c.id as string, i)); + validateSamplingOptions(c, `case "${c.id}"`); return { id: c.id, @@ -96,6 +119,8 @@ function validateCase(data: unknown, index: number, ids: Set): EvalCase }, model: typeof c.model === "string" ? c.model : undefined, provider: typeof c.provider === "string" ? c.provider : undefined, + temperature: typeof c.temperature === "number" ? c.temperature : undefined, + maxTokens: typeof c.maxTokens === "number" ? c.maxTokens : undefined, expected: typeof c.expected === "string" ? c.expected : undefined, scorers, tags: Array.isArray(c.tags) ? (c.tags as string[]).map(String) : undefined, diff --git a/src/types.ts b/src/types.ts index 8e8ce6e..eeffeb9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -122,6 +122,10 @@ export interface EvalCase { model?: string; /** Optional per-case provider override. */ provider?: string; + /** Optional per-case sampling temperature override. */ + temperature?: number; + /** Optional per-case output token cap override. */ + maxTokens?: number; /** Optional expected value, shared by many scorers. */ expected?: string; /** One or more scorers applied to the output. */ @@ -142,6 +146,10 @@ export interface EvalSuite { model?: string; /** Default provider applied to cases that do not override it. */ provider?: string; + /** Default sampling temperature applied to cases that do not override it. */ + temperature?: number; + /** Default output token cap applied to cases that do not override it. */ + maxTokens?: number; /** Global pass threshold in [0, 1] for the aggregate score. */ threshold?: number; /** Suite-level template variables applied to every case. */ diff --git a/tests/runner.test.ts b/tests/runner.test.ts index 85cd1f2..2bfaf46 100644 --- a/tests/runner.test.ts +++ b/tests/runner.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from "vitest"; -import type { EvalSuite } from "../src/types.js"; +import type { EvalSuite, ProviderRequest } from "../src/types.js"; import { runSuite, runCase, aggregateScore } from "../src/runner.js"; +import { ProviderRegistry } from "../src/providers/registry.js"; const suite: EvalSuite = { name: "unit", @@ -89,4 +90,39 @@ describe("runCase", () => { expect(res.provider).toBe("mock"); expect(res.passed).toBe(true); }); + + it("resolves case, suite, and default sampling options", async () => { + const requests: ProviderRequest[] = []; + const providers = new ProviderRegistry().register("capture", () => ({ + name: "capture", + async complete(request) { + requests.push(request); + return { output: "y", latencyMs: 0, model: request.model }; + }, + })); + const samplingSuite: EvalSuite = { + name: "sampling", + provider: "capture", + model: "capture-model", + temperature: 0.2, + maxTokens: 128, + cases: [], + }; + const baseCase = { + id: "x", + input: { prompt: "exactly: y" }, + expected: "y", + scorers: [{ type: "exact-match" }], + }; + + await runCase({ ...baseCase, temperature: 0.7, maxTokens: 64 }, samplingSuite, { providers }); + await runCase(baseCase, samplingSuite, { providers }); + await runCase(baseCase, { ...samplingSuite, temperature: undefined, maxTokens: undefined }, { providers }); + + expect(requests.map(({ temperature, maxTokens }) => ({ temperature, maxTokens }))).toEqual([ + { temperature: 0.7, maxTokens: 64 }, + { temperature: 0.2, maxTokens: 128 }, + { temperature: 0, maxTokens: undefined }, + ]); + }); }); diff --git a/tests/suite.test.ts b/tests/suite.test.ts index 37e3c1d..70dcc91 100644 --- a/tests/suite.test.ts +++ b/tests/suite.test.ts @@ -28,6 +28,28 @@ cases: expect(suite.cases[0]!.id).toBe("a"); }); + it("parses suite and case sampling options", () => { + const suite = parseSuite(` +name: sampling +temperature: 0.2 +maxTokens: 512 +cases: + - id: creative + temperature: 0.7 + maxTokens: 64 + input: + prompt: write something + scorers: + - type: regex + pattern: .+ +`, "sampling.yaml"); + + expect(suite.temperature).toBe(0.2); + expect(suite.maxTokens).toBe(512); + expect(suite.cases[0]!.temperature).toBe(0.7); + expect(suite.cases[0]!.maxTokens).toBe(64); + }); + it("rejects a suite without a name", () => { expect(() => validateSuite({ cases: [] })).toThrow(SuiteValidationError); }); @@ -59,4 +81,38 @@ cases: }), ).toThrow(/threshold/); }); + + it.each([ + ["temperature", -0.1], + ["temperature", Number.POSITIVE_INFINITY], + ["maxTokens", 0], + ["maxTokens", 1.5], + ])("rejects invalid suite %s", (field, value) => { + expect(() => + validateSuite({ + name: "d", + [field]: value, + cases: [{ id: "x", input: { prompt: "a" }, scorers: [{ type: "regex" }] }], + }), + ).toThrow(new RegExp(`suite\\.${field}`)); + }); + + it.each([ + ["temperature", Number.NaN], + ["maxTokens", -1], + ])("rejects invalid case %s", (field, value) => { + expect(() => + validateSuite({ + name: "d", + cases: [ + { + id: "x", + [field]: value, + input: { prompt: "a" }, + scorers: [{ type: "regex" }], + }, + ], + }), + ).toThrow(new RegExp(`case "x"\\.${field}`)); + }); });