From 639ba3327b7ce702fb48532fa387d50ae4fdce0b Mon Sep 17 00:00:00 2001 From: Zhibo Lin <147509942+LE0-Lin@users.noreply.github.com> Date: Mon, 7 Sep 2026 00:07:22 +0800 Subject: [PATCH] Fix prohibited command table false positive --- CHANGELOG.md | 9 +++++++++ README.md | 10 ++++++---- benchmarks/README.md | 15 +++++++++------ benchmarks/adversarial-v1-report.md | 8 ++++---- benchmarks/adversarial_cases.json | 14 ++++++++++++++ benchmarks/corpus.json | 30 +++++++++++++++++++++++++++++ docs/limitations.md | 2 +- src/agent_config_score/scanner.py | 29 +++++++++++++++++++++++++--- tests/test_scanner.py | 27 ++++++++++++++++++++++++++ 9 files changed, 126 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95f0df5..e1093b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +### Fixed + +- Do not report a dangerous command repeated as the left-hand example in a nearby Markdown replacement table when preceding text explicitly prohibits that same command. + +### Changed + +- Expanded Adversarial Benchmark v1 from 122 to 134 deterministic contracts with positive and negative Markdown-table cases across six instruction formats. +- Expanded the pinned real-repository smoke benchmark from three to five projects with one reviewed negative control and one active infrastructure-danger finding. + ## v0.21.0 ### Added diff --git a/README.md b/README.md index 1e06c0f..5d0b228 100644 --- a/README.md +++ b/README.md @@ -251,8 +251,8 @@ formats and separates current guarantees from open research challenges: | Tier | Result | Meaning | |---|---:|---| -| Deterministic contract | 122/122 exact matches | Current rule behavior stayed reproducible | -| Clean negative controls | 50/50 passed | No unexpected finding in controlled clean cases | +| Deterministic contract | 134/134 exact matches | Current rule behavior stayed reproducible | +| Clean negative controls | 56/56 passed | No unexpected finding in controlled clean cases | | Open challenge set | 4/8 detected | Four danger surfaces are covered; four semantic cases remain unsolved | The contract suite reports 100% precision and recall only for its closed labeled @@ -275,15 +275,17 @@ sanitized before/after example, then submit the ## Real-repository benchmark -The v0.17.0 scanner was replayed against pinned commits from three public AI coding projects; source code was scanned but never executed. +The scanner is replayed against pinned commits from five public projects with coding-agent instructions; source code is scanned but never executed. | Repository | Commit | Files found | Result | Manually reviewed signal | |---|---|---:|---:|---| | [`openai/codex`](https://github.com/openai/codex) | `d58d0e5` | 2 | B 82 | 3 references to absent `.rs` files; 1 context-size warning | | [`anomalyco/opencode`](https://github.com/anomalyco/opencode) | `9f69463` | 18 | A 94 | 1 context-size warning | | [`browser-use/browser-use`](https://github.com/browser-use/browser-use) | `d379a32` | 2 | B 88 | 1 context-size warning | +| [`Reaparr/Reaparr`](https://github.com/Reaparr/Reaparr) | `d9926d6` | 1 | A 100 | Prohibited `rm -rf` replacement-table example remains clean | +| [`olup/origan`](https://github.com/olup/origan) | `95ac789` | 1 | B 88 | 1 active aggressive Docker cleanup error | -All six findings matched their rule definitions in manual review. This small corpus is a reproducible smoke benchmark, not a quality leaderboard or a claim of broad statistical accuracy. The pinned inputs, reviewed expectations, limitations, and one-command runner live in [`benchmarks/`](https://github.com/LE0-Lin/AgentConfigScore/blob/v0/benchmarks/README.md). +All seven findings matched their rule definitions in manual review. This small corpus is a reproducible smoke benchmark, not a quality leaderboard or a claim of broad statistical accuracy. The pinned inputs, reviewed expectations, limitations, and one-command runner live in [`benchmarks/`](https://github.com/LE0-Lin/AgentConfigScore/blob/v0/benchmarks/README.md). ## Manual GitHub Actions setup diff --git a/benchmarks/README.md b/benchmarks/README.md index 6824bfe..0be012a 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -9,16 +9,16 @@ Neither benchmark is a claim that A 100 means semantic prompt quality. ## Adversarial mutation Benchmark v1 -Benchmark v1 contains 122 deterministic contract cases and 8 explicitly labeled +Benchmark v1 contains 134 deterministic contract cases and 8 explicitly labeled open challenges. The contracts cover positive detections and clean negative controls across `AGENTS.md`, `CLAUDE.md`, `GEMINI.md`, legacy and modern Cursor rules, and GitHub Copilot instructions. The current committed result is: -- 122/122 exact contract matches; -- 73 expected rule detections with no extra or missing rule IDs; -- 50/50 clean negative controls; +- 134/134 exact contract matches; +- 79 expected rule detections with no extra or missing rule IDs; +- 56/56 clean negative controls; - 4/8 challenge cases detected after adding four narrow danger-surface rules. The 100% contract precision and recall describe this closed, maintained test @@ -40,7 +40,8 @@ behavior. ## Real-repository smoke benchmark -This benchmark scans pinned commits from three public AI coding projects. It is +This benchmark scans pinned commits from five public projects with coding-agent +instructions. It is designed to make scanner behavior reproducible and to catch noisy path heuristics before release. It is not a ranking of the projects or a claim that three repositories represent every instruction style. @@ -53,8 +54,10 @@ cannot judge, read the [score contract and known limitations](../docs/limitation | `openai/codex` | `d58d0e5` | 2 | B 82 | 3 absent `.rs` path occurrences; 1 context-size warning | | `anomalyco/opencode` | `9f69463` | 18 | A 94 | 1 context-size warning | | `browser-use/browser-use` | `d379a32` | 2 | B 88 | 1 context-size warning | +| `Reaparr/Reaparr` | `d9926d6` | 1 | A 100 | Prohibited `rm -rf` replacement-table example remains clean | +| `olup/origan` | `95ac789` | 1 | B 88 | 1 active aggressive Docker cleanup error | -All six findings in the recorded run were manually checked against their rule +All seven findings in the recorded run were manually checked against their rule definitions. In particular, the reviewed output contains no `dead-path` finding for API symbols, package imports, documentation URLs, code-fence examples, platform paths, or generic filename conventions. diff --git a/benchmarks/adversarial-v1-report.md b/benchmarks/adversarial-v1-report.md index a9e75b7..cae23ef 100644 --- a/benchmarks/adversarial-v1-report.md +++ b/benchmarks/adversarial-v1-report.md @@ -1,11 +1,11 @@ # AgentConfigScore Benchmark v1 -Deterministic contract cases: **122/122 exact matches**. +Deterministic contract cases: **134/134 exact matches**. - Precision: **100.0%** - Recall: **100.0%** - F1: **100.0%** -- Clean negative controls: **50/50 passed** +- Clean negative controls: **56/56 passed** | Category | Exact matches | Accuracy | |---|---:|---:| @@ -13,8 +13,8 @@ Deterministic contract cases: **122/122 exact matches**. | `contradiction` | 6/6 | 100.0% | | `coordination` | 1/1 | 100.0% | | `coverage` | 7/7 | 100.0% | -| `danger` | 42/42 | 100.0% | -| `negation` | 36/36 | 100.0% | +| `danger` | 48/48 | 100.0% | +| `negation` | 42/42 | 100.0% | | `path` | 12/12 | 100.0% | | `regression` | 5/5 | 100.0% | | `secret` | 6/6 | 100.0% | diff --git a/benchmarks/adversarial_cases.json b/benchmarks/adversarial_cases.json index ec7aa0d..b2af104 100644 --- a/benchmarks/adversarial_cases.json +++ b/benchmarks/adversarial_cases.json @@ -45,6 +45,20 @@ "files": {"{instruction}": "Do not run rm -rf build.\n"}, "expected_codes": [] }, + { + "id": "prohibited-rm-rf-replacement-table", + "category": "negation", + "note": "A prohibited command repeated in a nearby replacement table is a negative control.", + "files": {"{instruction}": "## Safety\n\nNever use `rm -rf` under any circumstances.\n\nUse `trash` instead:\n\n| Instead of | Use |\n|---|---|\n| `rm -rf ` | `trash ` |\n"}, + "expected_codes": [] + }, + { + "id": "active-rm-rf-command-table", + "category": "danger", + "note": "An active dangerous command in an ordinary Markdown table must still be reported.", + "files": {"{instruction}": "| Situation | Command |\n|---|---|\n| Clean build outputs | `rm -rf build` |\n"}, + "expected_codes": ["rm-rf"] + }, { "id": "double-negative-rm-rf", "category": "danger", diff --git a/benchmarks/corpus.json b/benchmarks/corpus.json index 6650d8d..1074f85 100644 --- a/benchmarks/corpus.json +++ b/benchmarks/corpus.json @@ -70,6 +70,36 @@ ] }, "review_note": "URLs, platform-specific paths, generic filenames, and placeholder test paths were reviewed and are not reported as repository dead paths." + }, + { + "name": "Reaparr/Reaparr", + "url": "https://github.com/Reaparr/Reaparr.git", + "commit": "d9926d649ae6dc9d35b4c0faf183aa1d4d7b137b", + "expected": { + "files": 1, + "score": 100, + "grade": "A", + "findings": [] + }, + "review_note": "The safety section explicitly prohibits rm -rf and repeats it only in the left-hand side of a nearby replacement table; that example is not an active dangerous instruction." + }, + { + "name": "olup/origan", + "url": "https://github.com/olup/origan.git", + "commit": "95ac7898544aea4f817c911152c196c54f56c4af", + "expected": { + "files": 1, + "score": 88, + "grade": "B", + "findings": [ + { + "code": "docker-system-prune", + "file": "AGENTS.md", + "line": 31 + } + ] + }, + "review_note": "The AGENTS.md disk-cleanup section actively instructs docker system prune -af --volumes, matching the narrow all-plus-force danger rule." } ] } diff --git a/docs/limitations.md b/docs/limitations.md index 132562b..4643713 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -44,7 +44,7 @@ judgment requires a separately evaluated model-assisted mode and a labeled corpus, not a stronger marketing claim for the deterministic score. The offline [Adversarial Benchmark v1](../benchmarks/adversarial-v1-report.md) -keeps both sides visible: 122/122 maintained deterministic contracts currently +keeps both sides visible: 134/134 maintained deterministic contracts currently match, while 4/8 labeled challenges are detected. The four remaining misses are semantic cases rather than literal command patterns. The contract figure is a regression guarantee for a closed fixture suite, not a diff --git a/src/agent_config_score/scanner.py b/src/agent_config_score/scanner.py index e15eb3e..3f0e02d 100644 --- a/src/agent_config_score/scanner.py +++ b/src/agent_config_score/scanner.py @@ -158,7 +158,30 @@ def _line(text: str, index: int) -> int: return text.count("\n", 0, index) + 1 -def _dangerous_command_is_prohibited(text: str, index: int) -> bool: +def _prohibited_markdown_table_example(text: str, index: int, command: str) -> bool: + line_start = text.rfind("\n", 0, index) + 1 + line_end = text.find("\n", index) + if line_end == -1: + line_end = len(text) + line = text[line_start:line_end].strip() + if not line.startswith("|") or line.count("|") < 3: + return False + + normalized_command = re.sub(r"\s+", " ", command.lower()).strip() + preceding_lines = text[:line_start].splitlines()[-8:] + for candidate in reversed(preceding_lines): + cleaned = re.sub(r"[`*_>#]", " ", candidate) + cleaned = re.sub(r"\s+", " ", cleaned.lower()).strip() + negation = DANGER_NEGATION.search(cleaned) + if not negation or normalized_command not in cleaned: + continue + after_negation = cleaned[negation.end():] + if DANGER_NEGATION_EXCEPTION.search(after_negation) is None: + return True + return False + + +def _dangerous_command_is_prohibited(text: str, index: int, command: str) -> bool: """Ignore dangerous commands that are explicitly prohibited in the same clause.""" line_start = text.rfind("\n", 0, index) + 1 line_end = text.find("\n", index) @@ -173,7 +196,7 @@ def _dangerous_command_is_prohibited(text: str, index: int) -> bool: return False negations = list(DANGER_NEGATION.finditer(clause_prefix)) if not negations: - return False + return _prohibited_markdown_table_example(text, index, command) after_negation = clause_prefix[negations[-1].end():] + clause_suffix return DANGER_NEGATION_EXCEPTION.search(after_negation) is None @@ -378,7 +401,7 @@ def analyze(root: Path, *, suppressions: tuple[Suppression, ...] = ()) -> Report for match in pattern_rule.pattern.finditer(text): if ( pattern_rule.rule.category == "danger" - and _dangerous_command_is_prohibited(text, match.start()) + and _dangerous_command_is_prohibited(text, match.start(), match.group(0)) ): continue findings.append(_finding(pattern_rule.rule.code, rel, _line(text, match.start()))) diff --git a/tests/test_scanner.py b/tests/test_scanner.py index 1c9332e..e849330 100644 --- a/tests/test_scanner.py +++ b/tests/test_scanner.py @@ -143,6 +143,33 @@ def test_extended_danger_surface_prohibitions_remain_clean(self): } self.assertFalse(any(f.code in danger_codes for f in report.findings)) + def test_prohibited_command_repeated_in_replacement_table_remains_clean(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "AGENTS.md").write_text( + "## Safety\n\n" + "> **NEVER** use `rm`, `rmdir`, or `rm -rf` under any circumstances.\n\n" + "Use `trash` instead:\n\n" + "| Instead of | Use |\n" + "|---|---|\n" + "| `rm -rf ` | `trash ` |\n", + encoding="utf-8", + ) + report = analyze(root) + self.assertFalse(any(f.code == "rm-rf" for f in report.findings)) + + def test_active_command_in_markdown_table_is_still_reported(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "AGENTS.md").write_text( + "| Situation | Command |\n" + "|---|---|\n" + "| Clean build outputs | `rm -rf build` |\n", + encoding="utf-8", + ) + report = analyze(root) + self.assertTrue(any(f.code == "rm-rf" for f in report.findings)) + def test_double_negative_does_not_hide_dangerous_command(self): with tempfile.TemporaryDirectory() as d: root = Path(d)