From b9f40f49c88a2af59a3444175359eaf9aa07b559 Mon Sep 17 00:00:00 2001 From: Jason Date: Sun, 9 Aug 2026 14:28:48 -0500 Subject: [PATCH] feat(cli): stop accepting related_to in bulk (#508) First slice of #508, and the one that needs no prompt change. `--accept structure` was applying, unreviewed, exactly the suggestions where the model had declined to claim anything. `related_to` is not a wrong answer -- the rubric defines it as an answer, and an honest one beats a guessed `part_of` that asserts something false about how the knowledge fits together. What sets it apart is narrower and uncontested: applying it adds no claim to the graph beyond the untyped link that was already there. That makes it the cheapest place to spend a human glance and, at a measured 67% of accepted edges, also the largest. An accepted Structure now applies every specific type without asking and routes `related_to` to the operator. On a non-TTY run there is no channel to ask on, so it is counted as skipped rather than prompted -- reaching `typer.prompt` with no terminal would kill the walk mid-run, the same failure the Identity non-TTY guard exists to prevent. An unattended run therefore writes the confident suggestions and leaves the rest queued. The exemption is scoped to the ITEM, not the stage: the same run still applies every specific suggestion in bulk. The numeric-confidence route stays out of scope. `EdgeSuggestion` and `TierSuggestion` carry no confidence, adding it is a prompt change, and no eval under `evals/` scores either suggester -- so there is nothing to A/B against. Building that harness is its own piece of work and #508 stays open for it. Spec, docs and CHANGELOG updated. Suite 4048 -> 4050. --- CHANGELOG.md | 15 ++++ docs/cli.md | 2 +- openspec/specs/curate-command/spec.md | 32 +++++++++ src/openkos/cli/curate.py | 30 +++++++- src/openkos/resolution/edge_typing.py | 14 ++++ tests/unit/cli/test_curate.py | 98 +++++++++++++++++++++++++++ 6 files changed, 188 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79521b9..de4e02f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -120,6 +120,21 @@ and commit history follows [Conventional Commits](https://www.conventionalcommit mid-session. Confidence-threshold auto-acceptance stays out: the two stages that cause the prompt volume expose no confidence at all — only adjudication does, and that is the destructive one (#385). +- **An accepted Structure stage still asks about `related_to`**: bulk + acceptance was applying, unreviewed, exactly the suggestions where the + model had declined to claim anything. `related_to` is not a wrong answer — + the rubric defines it as an answer, and an honest one beats a guessed + `part_of` that asserts something false about how the knowledge fits + together. What sets it apart is narrower: applying it adds no claim to the + graph beyond the untyped link that was already there, which makes it the + cheapest place to spend a human glance and, at a measured 67% of accepted + edges, also the largest. So `--accept structure` now applies every + specific type without asking and routes this one to the operator. On a + pipe, where there is nothing to prompt on, it is counted as skipped rather + than applied — an unattended run writes the confident suggestions and + leaves the rest queued. This is the first slice of #508, and the one that + needed no prompt change: the numeric-confidence route stays blocked on + there being no eval harness for these suggesters at all (#508). ### Changed diff --git a/docs/cli.md b/docs/cli.md index 7921fa6..2247d59 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -412,7 +412,7 @@ Identity reuses `adjudicate --apply`'s exact merge walk rather than reimplementi | `--auto` | Accept every stage's cost gate without prompting (model spend only — per-item write prompts are never auto-accepted). | | `--include-confidential` | Include confidential concepts, forwarded to every stage's underlying call. Excluded by default when the LLM backend is **not** verifiably on this machine. See [Sensitivity and the local backend](#sensitivity-and-the-local-backend). | | `--include-deprecated` | Include deprecated and superseded concepts, forwarded the same way. Excluded by default. | -| `--accept STAGES` | Comma-separated, case-insensitive list of stages whose per-item write prompts are answered yes without asking: `structure`, `metadata`. **`identity` is refused (exit 2)** — a merge deletes the absorbed concept, so no flag may apply one unreviewed; an unknown name is refused the same way, and both refusals run before the workspace gate so a typo reports as itself. Naming a stage IS per-item write consent for it, so an accepted stage also passes the non-TTY write refusal: `curate --auto --accept structure` writes on a pipe, matching `suggest-relations --auto`. | +| `--accept STAGES` | Comma-separated, case-insensitive list of stages whose per-item write prompts are answered yes without asking: `structure`, `metadata`. **`identity` is refused (exit 2)** — a merge deletes the absorbed concept, so no flag may apply one unreviewed; an unknown name is refused the same way, and both refusals run before the workspace gate so a typo reports as itself. Naming a stage IS per-item write consent for it, so an accepted stage also passes the non-TTY write refusal: `curate --auto --accept structure` writes on a pipe, matching `suggest-relations --auto`. **One exemption**: an accepted Structure still prompts for a `related_to` suggestion, since that is the type the suggester uses when the documents do not support a specific claim — applying it adds no claim beyond the untyped link that already existed. On a pipe, where there is nothing to prompt on, it is counted as skipped instead. | `review: false` in `openkos.yaml` now reaches `curate` (#385): with no `--accept` flag it accepts every acceptable stage — `structure` and `metadata` — and **never** Identity, since a value set for the standalone verbs cannot become retroactive authorization to delete a concept. An explicit `--accept` overrides it and names the exact set rather than widening it, so an operator running with `review: false` can still re-review one stage without editing the config file. diff --git a/openspec/specs/curate-command/spec.md b/openspec/specs/curate-command/spec.md index b897268..369fb82 100644 --- a/openspec/specs/curate-command/spec.md +++ b/openspec/specs/curate-command/spec.md @@ -257,6 +257,38 @@ Identity MUST remain subject to that refusal on every path. - WHEN `curate --accept strcture` runs - THEN the exit code is 2 and stderr names the offending value +### Requirement: Bulk Acceptance Excludes The Least-Specific Relation Type + +An accepted Structure stage MUST still route a `related_to` suggestion to +the operator. Every other suggestable type asserts a specific relationship; +`related_to` is the answer the prompt designates as correct when the +documents do not support one, so applying it adds no claim beyond the +untyped link that already existed. It is therefore the cheapest place to +spend a human glance, and at a measured 67% of accepted edges also the +largest. + +On a TTY the exempted item falls back to the per-item prompt. On a non-TTY +run there is no channel to ask on, so it MUST be counted as skipped rather +than prompted — reaching the prompt with no terminal would kill the walk +mid-run. + +This exemption is scoped to the item, not the stage: the same run still +applies every specific suggestion without asking. + +#### Scenario: A specific type applies while `related_to` is prompted + +- GIVEN a Structure queue with one specific suggestion and one `related_to` +- WHEN `curate --accept structure` runs on a TTY +- THEN the specific suggestion is written with no prompt +- AND the `related_to` suggestion is prompted per item + +#### Scenario: On a pipe the exempted item is skipped, not prompted + +- GIVEN the same queue +- WHEN `curate --auto --accept structure` runs with stdout piped +- THEN the specific suggestion is written, the `related_to` suggestion is + counted as skipped, and no prompt is printed + ### Requirement: `review: false` Accepts Only The Non-Destructive Stages When `--accept` is absent, `review: false` in `openkos.yaml` MUST accept diff --git a/src/openkos/cli/curate.py b/src/openkos/cli/curate.py index 7541ac9..44301cd 100644 --- a/src/openkos/cli/curate.py +++ b/src/openkos/cli/curate.py @@ -81,6 +81,7 @@ plan_candidates, ) from openkos.resolution.edge_typing import ( + LEAST_SPECIFIC_RELATION_TYPE, EdgeSuggestion, candidate_edges, candidate_truncation_notice, @@ -333,15 +334,33 @@ def _accepts(ctx: CurateContext, stage_name: str) -> bool: return stage_name in ctx.accepted_stages -def _confirm_item(ctx: CurateContext, stage_name: str, prompt_text: str) -> bool: +def _confirm_item( + ctx: CurateContext, + stage_name: str, + prompt_text: str, + *, + acceptable_in_bulk: bool = True, +) -> bool: """`_confirm`, unless this stage was accepted in bulk for the run, in which case the answer is yes and no prompt is printed (issue #385). + `acceptable_in_bulk=False` exempts ONE item from that acceptance + (issue #508): the stage is still accepted, but this particular + suggestion asserts nothing specific, so it is worth the operator's + glance even in a run that opted out of the rest. On a TTY it falls + back to the prompt; on a pipe there is no channel to ask on, so it is + SKIPPED rather than prompted -- reaching `typer.prompt` with no + terminal would kill the walk mid-run, which is the same failure the + Identity non-TTY guard exists to prevent. + Identity calls `_confirm` DIRECTLY rather than routing through here: that keeps the merge walk structurally incapable of being skipped, so a future edit to the acceptance rules cannot reach it by accident.""" if _accepts(ctx, stage_name): - return True + if acceptable_in_bulk: + return True + if not sys.stdin.isatty(): + return False return _confirm(prompt_text) @@ -757,6 +776,13 @@ def _structure_run(ctx: CurateContext, probe: StageProbe) -> StageOutcome: "Structure", f"Relate {edge.source_id} -> {edge.target_id} " f"[{suggestion.suggested_type}]? [y/N]", + # #508: `--accept structure` applies every specific type in + # bulk, but the least-specific one asserts nothing beyond the + # untyped link that already existed, so it still reaches a + # human. See `edge_typing.LEAST_SPECIFIC_RELATION_TYPE`. + acceptable_in_bulk=( + suggestion.suggested_type != LEAST_SPECIFIC_RELATION_TYPE + ), ): skipped += 1 declined.append( diff --git a/src/openkos/resolution/edge_typing.py b/src/openkos/resolution/edge_typing.py index 2d7016c..0e47dce 100644 --- a/src/openkos/resolution/edge_typing.py +++ b/src/openkos/resolution/edge_typing.py @@ -155,6 +155,20 @@ `related_to` is defined as an ANSWER, not as a shrug. The aim of this rubric is NOT to drive its share down -- see `_SYSTEM_PROMPT`.""" +LEAST_SPECIFIC_RELATION_TYPE = "related_to" +"""The one suggestable type that asserts no specific relationship (#508). + +NOT a synonym for "wrong", and not a walk-back of the rubric above: it is +the answer `_SYSTEM_PROMPT` calls CORRECT when the documents do not support +a specific claim, and an honest one is better than a guessed `part_of`. What +sets it apart is narrower and uncontested -- accepting it adds no claim to +the graph beyond the untyped link that was already there. + +That is why `curate --accept structure` applies every other type in bulk but +still routes this one to the operator: it is the cheapest place to spend a +human glance, and at a measured 67% of accepted edges it is also where the +volume is.""" + _RUBRIC_LINES = "\n".join( f"- {name}: {_RELATION_RUBRIC[name]}." for name in sorted(SUGGESTABLE_RELATION_TYPES) diff --git a/tests/unit/cli/test_curate.py b/tests/unit/cli/test_curate.py index 409cc0f..65a769a 100644 --- a/tests/unit/cli/test_curate.py +++ b/tests/unit/cli/test_curate.py @@ -3510,3 +3510,101 @@ def test_review_false_never_lifts_the_non_tty_write_refusal_for_identity( "Identity: non-interactive write consent unavailable -- run " "`openkos adjudicate --apply-same --confirm-count ` instead." ) in _lines(result.stdout) + + +def _mixed_structure_queue(monkeypatch: pytest.MonkeyPatch) -> None: + """A COMPLETE 2-edge Structure queue: one SPECIFIC suggestion and one + `related_to`, the type the prompt designates as correct when the + documents do not support a specific claim (#508).""" + from openkos.graph.base import Edge + from openkos.resolution.edge_typing import EdgeSuggestion, EdgeSuggestionBatch + + specific = Edge(source_id="concepts/a", target_id="concepts/b") + vague = Edge(source_id="concepts/a", target_id="concepts/c") + monkeypatch.setattr( + "openkos.cli.curate.find_candidates_report", + lambda *a, **k: CandidateGroupReport(), + ) + monkeypatch.setattr( + "openkos.cli.curate.candidate_edges", lambda *a, **k: [specific, vague] + ) + monkeypatch.setattr( + "openkos.cli.curate.suggest_edge_types", + lambda *a, **k: EdgeSuggestionBatch( + results=[ + EdgeSuggestion( + edge=specific, suggested_type="references", rationale="names it" + ), + EdgeSuggestion( + edge=vague, suggested_type="related_to", rationale="cannot say how" + ), + ] + ), + ) + monkeypatch.setattr("openkos.cli.curate._concept_type_names", lambda *a, **k: []) + monkeypatch.setattr( + "openkos.cli.curate._contradiction_plan", lambda *a, **k: _empty_plan() + ) + + +def test_accept_structure_still_prompts_for_a_related_to_suggestion( + tmp_path: Path, + tmp_path_factory: pytest.TempPathFactory, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """`--accept structure` applies the SPECIFIC suggestion silently but + still prompts for `related_to` (#508). + + `related_to` is the answer the prompt calls correct when the documents + do not support a specific claim, so it is the one type whose bulk + acceptance adds no specific claim to the graph -- and, measured on a + real bundle, 67% of accepted edges. Accepting those unreviewed is + exactly the low-value material #385 warned about.""" + _init_apply_workspace(tmp_path, tmp_path_factory, monkeypatch) + _write_doc(tmp_path / "bundle" / "concepts" / "a.md", title="Concept A") + _write_doc(tmp_path / "bundle" / "concepts" / "b.md", title="Concept B") + _write_doc(tmp_path / "bundle" / "concepts" / "c.md", title="Concept C") + _reindexed_workspace(tmp_path, monkeypatch) + _mixed_structure_queue(monkeypatch) + _simulate_tty(monkeypatch) + + # The cost gate's "y", then a decline for the `related_to` prompt that + # must still be asked. + result = runner.invoke(app, ["curate", "--accept", "structure"], input="y\nn\n") + + assert result.exit_code == 0 + assert "Relate concepts/a -> concepts/b" not in result.stdout + assert "Relate concepts/a -> concepts/c [related_to]? [y/N]" in result.stdout + assert "Structure: applied 1, skipped 1." in _lines(result.stdout) + assert " declined: concepts/a -> concepts/c [related_to]" in result.stdout + + +def test_accept_structure_on_a_pipe_skips_related_to_instead_of_prompting( + tmp_path: Path, + tmp_path_factory: pytest.TempPathFactory, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """On a non-TTY run there is no channel to ask on, so an unacceptable + item is SKIPPED rather than prompted (#508). + + Without this the run would reach `typer.prompt` with no terminal and + die mid-walk -- the same failure the Identity non-TTY guard exists to + prevent. Unattended acceptance therefore applies exactly the specific + suggestions and leaves the rest queued for a human.""" + _init_apply_workspace(tmp_path, tmp_path_factory, monkeypatch) + _write_doc(tmp_path / "bundle" / "concepts" / "a.md", title="Concept A") + _write_doc(tmp_path / "bundle" / "concepts" / "b.md", title="Concept B") + _write_doc(tmp_path / "bundle" / "concepts" / "c.md", title="Concept C") + _reindexed_workspace(tmp_path, monkeypatch) + _mixed_structure_queue(monkeypatch) + + result = runner.invoke(app, ["curate", "--auto", "--accept", "structure"]) + + assert result.exit_code == 0 + assert "Relate concepts/a" not in result.stdout + assert "Structure: applied 1, skipped 1." in _lines(result.stdout) + source_text = (tmp_path / "bundle" / "concepts" / "a.md").read_text( + encoding="utf-8" + ) + assert "concepts/b" in source_text + assert "concepts/c" not in source_text