From e82bf13f07076da7c67e7407d3d503cab2bf63d6 Mon Sep 17 00:00:00 2001 From: Luca Belli <129434630+sator-labs@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:12:51 -0700 Subject: [PATCH 1/5] refactor: move run_for_user_models into the generation package `vera generate` imported `run_for_user_models` from the legacy root `generate.py`, so the new CLI depended on a deprecated script. Move it and its `_legacy_model_config` helper into `generate_conversations/main.py`, where it calls `run_generation` directly instead of going through the legacy `main` wrapper, and export it from the package. Tests that patched `generate.main` for the `vera` path now patch `generate_conversations.main.run_generation`; the keyword arguments are the same, since `main` only forwarded them. Co-Authored-By: Claude Opus 5.5 --- generate.py | 80 ---------------------------- generate_conversations/__init__.py | 4 +- generate_conversations/main.py | 79 +++++++++++++++++++++++++++ tests/integration/test_cli_parity.py | 9 ++-- tests/unit/test_vera_cli.py | 14 ++--- vera_cli/generate.py | 4 +- 6 files changed, 96 insertions(+), 94 deletions(-) diff --git a/generate.py b/generate.py index 1a224337..596bf7f2 100644 --- a/generate.py +++ b/generate.py @@ -10,7 +10,6 @@ from generate_conversations import run_generation from llm_clients.llm_interface import DEFAULT_START_PROMPT -from utils.config_schema import GenerationConfig, ModelSpec from utils.debug import set_debug from utils.rubric_manifest import ( load_manifest_persona_context_template, @@ -63,85 +62,6 @@ async def main( ) -async def run_for_user_models( - generation: GenerationConfig, - *, - max_personas: Optional[int], - into: Optional[str] = None, -) -> List[str]: - """Run one generation per user-side model, sequentially. - - STOPGAP. This exists so the CLI hands the generation domain a resolved - `GenerationConfig` and lets the domain decide how a multi-model request - expands into runs, instead of the CLI looping over models and flattening - each one into the legacy dict shape itself. - - Deliberately thin: sequential, with no shared run identity across models. - Sequential is not incidental — `max_concurrent` is applied *within* one - generation, so running models concurrently would silently multiply the - caller's concurrency cap against the same provider. - - Placed beside `main` so it moves into the permanent `generate/` package in - the same atomic change (see docs/architecture.md). The real fix is for the - generation domain to accept these types natively, at which point - `_legacy_model_config` below and this wrapper both disappear. - - Returns the run folder each user model wrote to, in the order the models - were given. `main` has always returned its output folder and this wrapper - used to drop it; `vera pipeline` needs it, because chaining generation into - judging means knowing where the transcripts landed without re-deriving it - from the filesystem or scraping it out of a log. `vera generate` ignores - the return. - """ - run_folders: List[str] = [] - for user in generation.user: - _, run_folder = await main( - persona_model_config=_legacy_model_config(user), - # Only the agent side may carry `name`; see `_legacy_model_config`. - agent_model_config=_legacy_model_config(generation.chatbot) - | {"name": generation.chatbot.name}, - persona_files=list(generation.personas), - persona_extra_run_params=dict(user.extra_params), - agent_extra_run_params=dict(generation.chatbot.extra_params), - max_turns=generation.turns, - runs_per_prompt=user.repeats, - persona_names=None, - verbose=True, - # `into` names an existing run folder to continue; without it, - # `output` is a parent to mint a new run under. - output_folder=into or generation.output, - run_id=None, - max_concurrent=generation.max_concurrent, - max_total_words=generation.max_total_words, - max_personas=max_personas, - persona_speaks_first=generation.persona_speaks_first, - session_types=generation.sessions, - resume=into is not None, - persona_context_template_path=generation.persona_context_template, - ) - run_folders.append(run_folder) - return run_folders - - -def _legacy_model_config(model: ModelSpec) -> Dict[str, Any]: - """Flatten a `ModelSpec` into the dict shape `main` still expects. - - STOPGAP translation shim, deleted once the generation domain takes - `ModelSpec` directly. `repeats` is dropped because it is passed separately - as `runs_per_prompt`. - - Callers add `name` for the agent side only. That asymmetry is not this - function's to decide, and it is not about output naming — run folders are - built from `model` (`utils/naming.py:model_token_for_run_folder`). `name` is - the provider's display name, defaulted to `"Provider"` in - `generate_conversations/runner.py`. It is legal on the agent config only - because the runner filters reserved keys out of that one before splatting - it into `LLMFactory.create_llm`, while splatting the persona config raw — - so a `name` key there would collide. Fix belongs in the runner. - """ - return {**model.extra_params, "model": model.name} - - async def resolve_persona_inputs(manifest: str) -> tuple[List[str], str]: """Resolve generation inputs for legacy callers of ``generate.py``.""" persona_files = await load_manifest_personas(manifest) diff --git a/generate_conversations/__init__.py b/generate_conversations/__init__.py index 73b7f5f1..13032b88 100644 --- a/generate_conversations/__init__.py +++ b/generate_conversations/__init__.py @@ -1,6 +1,6 @@ """Generate Conversations Package - LLM Conversation Simulation""" -from .main import run_generation +from .main import run_for_user_models, run_generation from .runner import ConversationRunner -__all__ = ["ConversationRunner", "run_generation"] +__all__ = ["ConversationRunner", "run_for_user_models", "run_generation"] diff --git a/generate_conversations/main.py b/generate_conversations/main.py index 5561e40e..66c4fa4b 100644 --- a/generate_conversations/main.py +++ b/generate_conversations/main.py @@ -12,6 +12,7 @@ from datetime import datetime from typing import Any, Dict, List, Optional +from utils.config_schema import GenerationConfig, ModelSpec from utils.naming import ( build_generation_run_folder_name, model_token_for_run_folder, @@ -147,3 +148,81 @@ async def run_generation( print(message) return results, output_folder + + +async def run_for_user_models( + generation: GenerationConfig, + *, + max_personas: Optional[int], + into: Optional[str] = None, +) -> List[str]: + """Run one generation per user-side model, sequentially. + + STOPGAP. This exists so the CLI hands the generation domain a resolved + `GenerationConfig` and lets the domain decide how a multi-model request + expands into runs, instead of the CLI looping over models and flattening + each one into the legacy dict shape itself. + + Deliberately thin: sequential, with no shared run identity across models. + Sequential is not incidental — `max_concurrent` is applied *within* one + generation, so running models concurrently would silently multiply the + caller's concurrency cap against the same provider. + + The real fix is for the generation domain to accept these types natively, + at which point `_legacy_model_config` below and this wrapper both + disappear. + + Returns the run folder each user model wrote to, in the order the models + were given. `run_generation` has always returned its output folder and + this wrapper used to drop it; `vera pipeline` needs it, because chaining + generation into judging means knowing where the transcripts landed + without re-deriving it from the filesystem or scraping it out of a log. + `vera generate` ignores the return. + """ + run_folders: List[str] = [] + for user in generation.user: + _, run_folder = await run_generation( + persona_model_config=_legacy_model_config(user), + # Only the agent side may carry `name`; see `_legacy_model_config`. + agent_model_config=_legacy_model_config(generation.chatbot) + | {"name": generation.chatbot.name}, + persona_files=list(generation.personas), + persona_extra_run_params=dict(user.extra_params), + agent_extra_run_params=dict(generation.chatbot.extra_params), + max_turns=generation.turns, + runs_per_prompt=user.repeats, + persona_names=None, + verbose=True, + # `into` names an existing run folder to continue; without it, + # `output` is a parent to mint a new run under. + output_folder=into or generation.output, + run_id=None, + max_concurrent=generation.max_concurrent, + max_total_words=generation.max_total_words, + max_personas=max_personas, + persona_speaks_first=generation.persona_speaks_first, + session_types=generation.sessions, + resume=into is not None, + persona_context_template_path=generation.persona_context_template, + ) + run_folders.append(run_folder) + return run_folders + + +def _legacy_model_config(model: ModelSpec) -> Dict[str, Any]: + """Flatten a `ModelSpec` into the dict shape `run_generation` expects. + + STOPGAP translation shim, deleted once the generation domain takes + `ModelSpec` directly. `repeats` is dropped because it is passed separately + as `runs_per_prompt`. + + Callers add `name` for the agent side only. That asymmetry is not this + function's to decide, and it is not about output naming — run folders are + built from `model` (`utils/naming.py:model_token_for_run_folder`). `name` is + the provider's display name, defaulted to `"Provider"` in + `generate_conversations/runner.py`. It is legal on the agent config only + because the runner filters reserved keys out of that one before splatting + it into `LLMFactory.create_llm`, while splatting the persona config raw — + so a `name` key there would collide. Fix belongs in the runner. + """ + return {**model.extra_params, "model": model.name} diff --git a/tests/integration/test_cli_parity.py b/tests/integration/test_cli_parity.py index 33848a3a..a847f412 100644 --- a/tests/integration/test_cli_parity.py +++ b/tests/integration/test_cli_parity.py @@ -72,14 +72,15 @@ def calls(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[dict[str, Any]]]: """Stub the domain boundary and collect the kwargs each CLI sends it. - Both patches are needed: `generate` bound `run_generation` at import time, - while a fresh `runpy` exec of `generate.py` re-imports it from the package. + Both patches are needed: `run_for_user_models` (the `vera` path) calls the + `run_generation` bound in `generate_conversations.main`, while a fresh + `runpy` exec of `generate.py` re-imports it from the package. `chdir` matters because the legacy CLI resolves paths against the working directory, so its manifest argument only resolves from the repository root. """ - import generate import generate_conversations + import generate_conversations.main recorded: list[dict[str, Any]] = [] @@ -89,7 +90,7 @@ async def record(**kwargs: Any) -> tuple[list[dict[str, Any]], str]: monkeypatch.chdir(REPO_ROOT) with ( - patch.object(generate, "run_generation", record), + patch.object(generate_conversations.main, "run_generation", record), patch.object(generate_conversations, "run_generation", record), ): yield recorded diff --git a/tests/unit/test_vera_cli.py b/tests/unit/test_vera_cli.py index 3316f603..23be43a9 100644 --- a/tests/unit/test_vera_cli.py +++ b/tests/unit/test_vera_cli.py @@ -8,7 +8,7 @@ import pytest -import generate as generation_domain +import generate_conversations.main as generation_domain import vera from utils.config_schema import InvocationConfig, ModelSpec from vera_cli import ( @@ -222,7 +222,7 @@ def test_cli_defaults_resolve_before_print(capsys: pytest.CaptureFixture) -> Non def test_generation_delegates_every_value_to_generate_main() -> None: with patch.object( - generation_domain, "main", new_callable=AsyncMock + generation_domain, "run_generation", new_callable=AsyncMock ) as generate_main: generate_main.return_value = ([], "output/generated") result = vera.main( @@ -273,7 +273,7 @@ def test_generation_delegates_every_value_to_generate_main() -> None: def test_each_user_model_gets_one_generate_main_call(tmp_path: Path) -> None: manifest = _write_target(tmp_path, persona_count=2) with patch.object( - generation_domain, "main", new_callable=AsyncMock + generation_domain, "run_generation", new_callable=AsyncMock ) as generate_main: generate_main.return_value = ([], "output/generated") vera.main( @@ -452,7 +452,9 @@ def test_incomplete_target_fails_before_dispatch(tmp_path: Path) -> None: manifest.write_text(json.dumps({"personas": ["personas.tsv"]}), encoding="utf-8") with ( - patch.object(generation_domain, "main", new_callable=AsyncMock) as runner, + patch.object( + generation_domain, "run_generation", new_callable=AsyncMock + ) as runner, pytest.raises(SystemExit) as error, ): vera.main( @@ -534,7 +536,7 @@ def test_into_continues_an_existing_run_folder(tmp_path: Path) -> None: run_folder = tmp_path / "p_user__a_bot__t5__r1" run_folder.mkdir() with patch.object( - generation_domain, "main", new_callable=AsyncMock + generation_domain, "run_generation", new_callable=AsyncMock ) as generate_main: generate_main.return_value = ([], "output/generated") result = vera.main( @@ -610,7 +612,7 @@ def test_into_is_invocation_only_so_it_may_accompany_config(tmp_path: Path) -> N run_folder.mkdir() config = _write_config(tmp_path, _generation_config()) with patch.object( - generation_domain, "main", new_callable=AsyncMock + generation_domain, "run_generation", new_callable=AsyncMock ) as generate_main: generate_main.return_value = ([], "output/generated") result = vera.main( diff --git a/vera_cli/generate.py b/vera_cli/generate.py index d34e375c..41b1a89c 100644 --- a/vera_cli/generate.py +++ b/vera_cli/generate.py @@ -25,7 +25,7 @@ from pathlib import Path from typing import Any -from generate import run_for_user_models +from generate_conversations import run_for_user_models from utils.config_schema import GenerationConfig, InvocationConfig, ModelSpec, RunConfig from utils.debug import set_debug from utils.utils import parse_key_value_list @@ -342,7 +342,7 @@ async def _execute(run_configs: list[RunConfig]) -> list[str]: Expanding a run's user models into individual generations belongs to the domain, not the CLI, so this passes the resolved `GenerationConfig` straight - through to `generate.run_for_user_models` — a stopgap wrapper, see its + through to `generate_conversations.run_for_user_models` — a stopgap wrapper, see its docstring. Targets stay sequential here: `--target all` runs share chatbot, user models, turns, and repeats, and run folder names carry only second-granularity timestamps, so concurrent starts would collide. From 62671840c92d81add5bd58c482e9ebe681e794a3 Mon Sep 17 00:00:00 2001 From: Luca Belli <129434630+sator-labs@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:13:17 -0700 Subject: [PATCH 2/5] chore: move the legacy scripts into legacy/ (rename only) Pure renames, so git history follows each file. The tree does not run between this commit and the next, which updates every reference; skip it when bisecting. Co-Authored-By: Claude Opus 5.5 --- generate.py => legacy/generate.py | 0 judge.py => legacy/judge.py | 0 run_pipeline.py => legacy/run_pipeline.py | 0 {scripts => legacy}/run_recommended_vera_pipeline.sh | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename generate.py => legacy/generate.py (100%) rename judge.py => legacy/judge.py (100%) rename run_pipeline.py => legacy/run_pipeline.py (100%) rename {scripts => legacy}/run_recommended_vera_pipeline.sh (100%) diff --git a/generate.py b/legacy/generate.py similarity index 100% rename from generate.py rename to legacy/generate.py diff --git a/judge.py b/legacy/judge.py similarity index 100% rename from judge.py rename to legacy/judge.py diff --git a/run_pipeline.py b/legacy/run_pipeline.py similarity index 100% rename from run_pipeline.py rename to legacy/run_pipeline.py diff --git a/scripts/run_recommended_vera_pipeline.sh b/legacy/run_recommended_vera_pipeline.sh similarity index 100% rename from scripts/run_recommended_vera_pipeline.sh rename to legacy/run_recommended_vera_pipeline.sh From f39bdeb007598a151bbed03ecea6dd7e97359447 Mon Sep 17 00:00:00 2001 From: Luca Belli <129434630+sator-labs@users.noreply.github.com> Date: Thu, 1 Oct 2026 16:17:19 -0700 Subject: [PATCH 3/5] chore: run the legacy scripts from legacy/ and update every reference The legacy entry points now run as modules from the repository root: `uv run python -m legacy.generate`, `-m legacy.judge`, `-m legacy.score`, and `-m legacy.run_pipeline`. - Move `judge/score.py`'s CLI (`main`) to `legacy/score.py`; the scoring domain (`run_scoring`) stays in `judge/score.py`. - Add `legacy/README.md`: deprecated, may be removed at any time. - `legacy/run_pipeline.py` imports `legacy.generate` and loads its sibling `judge.py` by path relative to itself, not to the working directory. - Update tests, shell scripts, Dockerfile `CMD` (now `vera.py --help`), CODEOWNERS, the Claude slash commands and permissions, and the docs. With `generate.py` out of the root, `generate_conversations/` can be renamed to `generate/` without shadowing. Co-Authored-By: Claude Opus 5.5 --- .claude/commands/run-generator.md | 2 +- .claude/commands/run-judge.md | 2 +- .claude/commands/verify.md | 2 +- .claude/settings.json | 4 +- .github/CODEOWNERS | 2 +- AGENTS.md | 2 +- Dockerfile | 2 +- README.md | 2 +- docs/architecture.md | 16 +- docs/evaluating.md | 4 +- docs/legacy-scripts.md | 40 ++--- judge/score.py | 65 +------- legacy/README.md | 5 + legacy/__init__.py | 1 + legacy/run_pipeline.py | 12 +- legacy/run_recommended_vera_pipeline.sh | 6 +- legacy/score.py | 68 ++++++++ llm_clients/config.py | 2 +- scripts/pool_vera_scores.py | 2 +- scripts/run_combinations.sh | 2 +- scripts/score_publication_evaluations.sh | 2 +- tests/integration/test_cli_parity.py | 5 +- .../test_judge_against_clinician_ratings.py | 4 +- tests/integration/test_pipeline.py | 150 +++++++++--------- tests/integration/test_score_parity.py | 6 +- tests/integration/test_scoring.py | 22 ++- .../test_generate_cli.py | 2 +- tests/unit/judge/test_judge_cli.py | 4 +- tests/unit/test_vera_pipeline.py | 2 +- vera_cli/pipeline.py | 2 +- 30 files changed, 234 insertions(+), 206 deletions(-) create mode 100644 legacy/README.md create mode 100644 legacy/__init__.py create mode 100644 legacy/score.py diff --git a/.claude/commands/run-generator.md b/.claude/commands/run-generator.md index c96b9972..138f92ff 100644 --- a/.claude/commands/run-generator.md +++ b/.claude/commands/run-generator.md @@ -1,4 +1,4 @@ -Help the user run the conversation generator (generate.py): +Help the user run the conversation generator (`uv run python vera.py generate`; flags in docs/cli.md): 1. Ask the user for parameters: - Model to use (default: claude-3-7-sonnet) diff --git a/.claude/commands/run-judge.md b/.claude/commands/run-judge.md index 95ba68b2..450c1b38 100644 --- a/.claude/commands/run-judge.md +++ b/.claude/commands/run-judge.md @@ -1,4 +1,4 @@ -Help the user run the conversation evaluator (judge.py): +Help the user run the conversation evaluator (`uv run python vera.py judge`; flags in docs/cli.md): 1. Ask the user for parameters: - Input directory (default: output/conversations) diff --git a/.claude/commands/verify.md b/.claude/commands/verify.md index 998863bc..f29c8cca 100644 --- a/.claude/commands/verify.md +++ b/.claude/commands/verify.md @@ -1,6 +1,6 @@ Verify that a code change works correctly without hitting live APIs. -IMPORTANT: Never run `generate.py`, `judge.py`, or `run_pipeline.py` directly — these make live LLM API calls. Verification must use the test suite only. +IMPORTANT: Never run the `vera` commands or the scripts in `legacy/` directly — these make live LLM API calls. Verification must use the test suite only. Steps: 1. Identify what changed (git diff --stat) to understand the scope diff --git a/.claude/settings.json b/.claude/settings.json index 45c78a2f..9e09a161 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -10,8 +10,8 @@ "Bash(.venv/bin/ruff check --fix .)", "Bash(.venv/bin/pyright:*)", "Bash(.venv/bin/pytest:*)", - "Bash(python3 generate.py:*)", - "Bash(python3 judge.py:*)", + "Bash(python3 -m legacy.generate:*)", + "Bash(python3 -m legacy.judge:*)", "Bash(python3 -m pytest:*)", "Bash(pytest:*)", "Bash(git status)", diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8815575b..accdb1fe 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,7 +4,7 @@ /llm_clients/ @SpringCare/vera-mh-maintainers /judge/ @SpringCare/vera-mh-maintainers /generate/ @SpringCare/vera-mh-maintainers -/run_pipeline.py @SpringCare/vera-mh-maintainers +/legacy/ @SpringCare/vera-mh-maintainers /openspec/ @SpringCare/vera-mh-maintainers /pyproject.toml @SpringCare/vera-mh-maintainers diff --git a/AGENTS.md b/AGENTS.md index 1540c0b8..c3576cf2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -134,7 +134,7 @@ uv run pyright pre-commit run --all-files ``` -`generate.py`, `judge.py`, `run_pipeline.py`, `judge/score.py`, and `scripts/run_recommended_vera_pipeline.sh` are deprecated and will be removed; don't build on them. Their flags and `vera` equivalents are in [docs/legacy-scripts.md](docs/legacy-scripts.md). +The scripts in [`legacy/`](legacy/) (`generate.py`, `judge.py`, `score.py`, `run_pipeline.py`, `run_recommended_vera_pipeline.sh`) are deprecated and may be removed at any time; don't build on them, and never import them from `vera_cli/` or the domain packages. Their flags and `vera` equivalents are in [docs/legacy-scripts.md](docs/legacy-scripts.md). Use dated model IDs (e.g. `claude-sonnet-4-5-20250929`) as in README; shorthand aliases may not resolve. diff --git a/Dockerfile b/Dockerfile index fd4c91ab..264b4c35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,4 +11,4 @@ RUN uv sync --frozen COPY . . -CMD ["python", "generate.py"] \ No newline at end of file +CMD ["python", "vera.py", "--help"] \ No newline at end of file diff --git a/README.md b/README.md index ca0cce54..9290f51a 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ pre-commit install # optional: format and lint on commit ## Legacy scripts -The pre-2.0 entry points—`generate.py`, `judge.py`, `run_pipeline.py`, `judge/score.py`, and `scripts/run_recommended_vera_pipeline.sh`—still work, but they are **deprecated and will be removed soon**. Their flags differ from the `vera` CLI (for example `-c` means `--max-concurrent` in `generate.py`). Use `vera` for new work; if you still depend on a script, see [docs/legacy-scripts.md](docs/legacy-scripts.md) for its options and the `vera` equivalent. +The pre-2.0 entry points live in [`legacy/`](legacy/) and run as modules, for example `uv run python -m legacy.generate`. They still work, but they are **deprecated and may be removed at any time**. Their flags differ from the `vera` CLI (for example `-c` means `--max-concurrent` in `legacy/generate.py`). Use `vera` for new work; if you still depend on a script, see [docs/legacy-scripts.md](docs/legacy-scripts.md) for its options and the `vera` equivalent. ## License diff --git a/docs/architecture.md b/docs/architecture.md index f97bb14e..79ed2969 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -123,14 +123,12 @@ namespaces, input config files, or target manifests. They define no CLI behavior defaults. Pooling likewise delegates to the function owned by the scoring domain rather than to a script entry point. -Legacy root scripts may remain temporarily while their replacement feature is -migrated. During that transition, `vera_cli` may import the reusable function -from a root script, but never its argument parser. For generation, the temporary -flow is `vera_cli.generate` → `generate.run_for_user_models` → `generate.main`. -Removing `generate.py` and moving those functions plus the existing -`generate_conversations/` code into the permanent `generate/` package is one -atomic later change, so a root `generate.py` module and a top-level `generate/` -package never coexist. +Legacy scripts live in `legacy/` until they are removed, and nothing outside +`legacy/` and its tests imports them. For generation, the flow is +`vera_cli.generate` → `generate_conversations.run_for_user_models` → +`generate_conversations.run_generation`. With `generate.py` out of the root, +renaming `generate_conversations/` to the permanent `generate/` package no +longer collides with a root `generate` module. `run_for_user_models` and its `_legacy_model_config` helper are explicit stopgaps. They put the expansion of a run's user models, and the flattening of @@ -141,7 +139,7 @@ models identically. | Subcommand | Delegates to | Purpose | |------------|--------------|---------| -| `vera generate` | generation application function (temporarily `generate.run_for_user_models`) | Simulate conversations → `/c_/u_*/conversations/` | +| `vera generate` | generation application function (temporarily `generate_conversations.run_for_user_models`) | Simulate conversations → `/c_/u_*/conversations/` | | `vera judge` | `judge.runner` | Evaluate transcripts → `/c_/u_*/evaluations/j_*` | | `vera score` | `score.score` | Aggregate `results.csv` → scores and visualizations | | `vera pool` | `score.pool` | Concatenate multiple evaluation folders into one pooled result → `/c_/pooled/u_+_*` | diff --git a/docs/evaluating.md b/docs/evaluating.md index 4ae2346d..3efbe068 100644 --- a/docs/evaluating.md +++ b/docs/evaluating.md @@ -242,8 +242,8 @@ Add configuration to [config.py](../llm_clients/config.py) if your LLM requires ### 5. Use the new LLM in your simulations ```bash -python3 generate.py -u your-model-name -p your-model-name -t 5 -r 1 -python3 judge.py -f conversations/{YOUR_FOLDER} -j your-model-name +uv run python vera.py generate -c your-model-name -u your-model-name -t 5 --target SI +uv run python vera.py judge --conversations output/{YOUR_RUN_FOLDER} -j your-model-name --target SI ``` ## Prompt caching (by provider) diff --git a/docs/legacy-scripts.md b/docs/legacy-scripts.md index 63de80b7..74f6c668 100644 --- a/docs/legacy-scripts.md +++ b/docs/legacy-scripts.md @@ -1,23 +1,25 @@ # Legacy scripts (deprecated) -> **These scripts will be removed soon.** Use the [`vera` CLI](cli.md) for new work. This page exists so workflows that still depend on a script keep running while they migrate. +> **These scripts may be removed at any time.** Use the [`vera` CLI](cli.md) for new work. This page exists so workflows that still depend on a script keep running while they migrate. + +The scripts live in [`legacy/`](../legacy/). Run the Python ones from the repository root as modules (`uv run python -m legacy.