From b0e8e5ef08f82aa76b6314b3cdaceac9f1e040dc Mon Sep 17 00:00:00 2001 From: Zhibo Lin <147509942+LE0-Lin@users.noreply.github.com> Date: Mon, 7 Sep 2026 15:38:00 +0800 Subject: [PATCH] Detect case-variant instruction filenames --- CHANGELOG.md | 4 ++- README.md | 9 +++--- benchmarks/README.md | 14 +++++++-- benchmarks/corpus.json | 23 +++++++++++++++ scripts/run_real_world_benchmark.py | 33 +++++++++++++++++++++- src/agent_config_score/scanner.py | 44 +++++++++++++++++++++++------ tests/test_benchmark.py | 38 +++++++++++++++++++++++++ tests/test_scanner.py | 32 +++++++++++++++++++++ 8 files changed, 179 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e1093b3..b3e34e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,13 @@ ### Fixed - Do not report a dangerous command repeated as the left-hand example in a nearby Markdown replacement table when preceding text explicitly prohibits that same command. +- Discover case variants such as `agents.md` on case-sensitive filesystems, while preferring the documented filename when duplicate case variants coexist. ### Changed - Expanded Adversarial Benchmark v1 from 122 to 134 deterministic contracts with positive and negative Markdown-table cases across six instruction formats. -- Expanded the pinned real-repository smoke benchmark from three to five projects with one reviewed negative control and one active infrastructure-danger finding. +- Expanded the pinned real-repository smoke benchmark from three to six projects, including a lowercase `agents.md` false-negative regression found in RestSharp. +- Added targeted `--repository OWNER/REPO` selection to the real-repository benchmark runner so one pinned case can be reproduced without downloading the entire corpus. ## v0.21.0 diff --git a/README.md b/README.md index 5d0b228..b1aa3f7 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ sanitized before/after example, then submit the ## Real-repository benchmark -The scanner is replayed against pinned commits from five public projects with coding-agent instructions; source code is scanned but never executed. +The scanner is replayed against pinned commits from six public projects with coding-agent instructions; source code is scanned but never executed. The corpus includes both clean controls and real regressions such as a lowercase `agents.md` whose active cleanup warning was previously missed on case-sensitive systems. | Repository | Commit | Files found | Result | Manually reviewed signal | |---|---|---:|---:|---| @@ -284,8 +284,9 @@ The scanner is replayed against pinned commits from five public projects with co | [`browser-use/browser-use`](https://github.com/browser-use/browser-use) | `d379a32` | 2 | B 88 | 1 context-size warning | | [`Reaparr/Reaparr`](https://github.com/Reaparr/Reaparr) | `d9926d6` | 1 | A 100 | Prohibited `rm -rf` replacement-table example remains clean | | [`olup/origan`](https://github.com/olup/origan) | `95ac789` | 1 | B 88 | 1 active aggressive Docker cleanup error | +| [`restsharp/RestSharp`](https://github.com/restsharp/RestSharp) | `64ee129` | 3 | B 84 | Lowercase `agents.md`; 1 context warning and 1 active broad recursive-deletion error | -All seven findings matched their rule definitions in manual review. This small corpus is a reproducible smoke benchmark, not a quality leaderboard or a claim of broad statistical accuracy. The pinned inputs, reviewed expectations, limitations, and one-command runner live in [`benchmarks/`](https://github.com/LE0-Lin/AgentConfigScore/blob/v0/benchmarks/README.md). +All nine findings matched their rule definitions in manual review. This small corpus is a reproducible smoke benchmark, not a quality leaderboard or a claim of broad statistical accuracy. The pinned inputs, reviewed expectations, limitations, and one-command runner live in [`benchmarks/`](https://github.com/LE0-Lin/AgentConfigScore/blob/v0/benchmarks/README.md). ## Manual GitHub Actions setup @@ -343,7 +344,7 @@ See [`docs/score-history.md`](https://github.com/LE0-Lin/AgentConfigScore/blob/v ## Nested AGENTS.md scopes and overrides -AgentConfigScore recursively discovers both `AGENTS.md` and Codex `AGENTS.override.md` files and understands their directory scoping instead of treating every instruction file as one global policy. +AgentConfigScore recursively discovers both `AGENTS.md` and Codex `AGENTS.override.md` files and understands their directory scoping instead of treating every instruction file as one global policy. Discovery is case-insensitive for repositories that use variants such as `agents.md`; when duplicate case variants coexist in one directory, the documented spelling wins. For a nested AGENTS-family file, file-like path references may be repository-root-relative, package-root-relative, or relative to the directory containing that file. The conservative `dead-path` analysis ignores ambiguous extensionless tokens, code fences, URLs, package imports, API symbols, absolute paths, Windows drive-qualified paths, and relative paths that escape the repository. @@ -476,7 +477,7 @@ Repository-level findings remain repository-level SARIF results instead of recei Supported discovery includes: -- `**/AGENTS.md` — root and nested directory scopes +- `**/AGENTS.md` — root and nested directory scopes (case-insensitive discovery) - `**/AGENTS.override.md` — Codex local overrides - `CLAUDE.md` - `GEMINI.md` diff --git a/benchmarks/README.md b/benchmarks/README.md index 0be012a..2b08689 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -40,10 +40,10 @@ behavior. ## Real-repository smoke benchmark -This benchmark scans pinned commits from five public projects with coding-agent +This benchmark scans pinned commits from six public projects with coding-agent instructions. It is designed to make scanner behavior reproducible and to catch noisy path heuristics -before release. It is not a ranking of the projects or a claim that three +before release. It is not a ranking of the projects or a claim that six repositories represent every instruction style. For the exact meaning of A 100 and adversarial cases the deterministic scanner @@ -56,8 +56,9 @@ cannot judge, read the [score contract and known limitations](../docs/limitation | `browser-use/browser-use` | `d379a32` | 2 | B 88 | 1 context-size warning | | `Reaparr/Reaparr` | `d9926d6` | 1 | A 100 | Prohibited `rm -rf` replacement-table example remains clean | | `olup/origan` | `95ac789` | 1 | B 88 | 1 active aggressive Docker cleanup error | +| `restsharp/RestSharp` | `64ee129` | 3 | B 84 | Lowercase `agents.md`; 1 context warning and 1 active broad recursive-deletion error | -All seven findings in the recorded run were manually checked against their rule +All nine findings in the recorded run were manually checked against their rule definitions. In particular, the reviewed output contains no `dead-path` finding for API symbols, package imports, documentation URLs, code-fence examples, platform paths, or generic filename conventions. @@ -69,6 +70,13 @@ python -m pip install -e . python scripts/run_real_world_benchmark.py --output benchmark-result.json ``` +Reproduce one reviewed case without downloading the entire corpus: + +```bash +python scripts/run_real_world_benchmark.py \ + --repository restsharp/RestSharp +``` + The script clones but never executes code from the target repositories. It checks out the exact commits in `corpus.json`, scans them, compares stable finding fingerprints with the reviewed expectations, and exits non-zero on a diff --git a/benchmarks/corpus.json b/benchmarks/corpus.json index 1074f85..c3ea4ba 100644 --- a/benchmarks/corpus.json +++ b/benchmarks/corpus.json @@ -100,6 +100,29 @@ ] }, "review_note": "The AGENTS.md disk-cleanup section actively instructs docker system prune -af --volumes, matching the narrow all-plus-force danger rule." + }, + { + "name": "restsharp/RestSharp", + "url": "https://github.com/restsharp/RestSharp.git", + "commit": "64ee12995b8d7409a5863da7b71cddfa526ffb7a", + "expected": { + "files": 3, + "score": 84, + "grade": "B", + "findings": [ + { + "code": "context-large", + "file": "agents.md", + "line": null + }, + { + "code": "rm-rf", + "file": "agents.md", + "line": 661 + } + ] + }, + "review_note": "This lowercase agents.md is a real case-sensitive discovery regression. It exceeds the context-size threshold and its cleanup quick reference actively includes a broad rm -rf command; both findings were completely missed before case-insensitive target discovery." } ] } diff --git a/scripts/run_real_world_benchmark.py b/scripts/run_real_world_benchmark.py index 358bfaf..46954f3 100644 --- a/scripts/run_real_world_benchmark.py +++ b/scripts/run_real_world_benchmark.py @@ -93,12 +93,43 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--corpus", type=Path, default=DEFAULT_CORPUS) parser.add_argument("--work-dir", type=Path) parser.add_argument("--output", type=Path) + parser.add_argument( + "--repository", + action="append", + metavar="OWNER/REPO", + help="Run only a named corpus repository; repeat to select multiple repositories", + ) return parser +def _select_repositories( + repositories: list[dict[str, Any]], + selected_names: list[str] | None, +) -> list[dict[str, Any]]: + if not selected_names: + return repositories + requested = set(selected_names) + known = {repository["name"] for repository in repositories} + unknown = sorted(requested - known) + if unknown: + raise ValueError( + "unknown repository selection: " + + ", ".join(unknown) + + "; choose from: " + + ", ".join(sorted(known)) + ) + return [repository for repository in repositories if repository["name"] in requested] + + def main() -> int: args = _parser().parse_args() corpus = json.loads(args.corpus.read_text(encoding="utf-8")) + try: + selected_repositories = _select_repositories( + corpus["repositories"], args.repository + ) + except ValueError as exc: + raise SystemExit(str(exc)) from exc temporary: tempfile.TemporaryDirectory[str] | None = None if args.work_dir is None: temporary = tempfile.TemporaryDirectory(prefix="agentconfigscore-benchmark-") @@ -110,7 +141,7 @@ def main() -> int: try: repositories = [ _scan(repository, work_dir / repository["name"].replace("/", "--")) - for repository in corpus["repositories"] + for repository in selected_repositories ] finally: if temporary is not None: diff --git a/src/agent_config_score/scanner.py b/src/agent_config_score/scanner.py index 3f0e02d..754717c 100644 --- a/src/agent_config_score/scanner.py +++ b/src/agent_config_score/scanner.py @@ -13,10 +13,12 @@ from .rules import CATEGORY_CAPS, PATTERN_RULES, RULES_BY_CODE AGENTS_FILENAMES = {"AGENTS.md", "AGENTS.override.md"} +AGENTS_FILENAMES_CASEFOLD = {name.casefold() for name in AGENTS_FILENAMES} TARGET_NAMES = AGENTS_FILENAMES | { "CLAUDE.md", "GEMINI.md", ".cursorrules", ".clinerules", ".windsurfrules", "copilot-instructions.md", } +TARGET_NAMES_CASEFOLD = {name.casefold() for name in TARGET_NAMES} SKIP_DIRS = { ".git", "node_modules", ".venv", "venv", "dist", "build", ".next", "target", "vendor", ".idea", ".vscode", "__pycache__", @@ -125,6 +127,19 @@ def _ignored(rel: str, patterns: list[str]) -> bool: return any(fnmatch(rel, pat) or fnmatch(rel + "/", pat) for pat in patterns) +def _is_agents_filename(name: str) -> bool: + return name.casefold() in AGENTS_FILENAMES_CASEFOLD + + +def _prefer_canonical_name(candidate: Path, current: Path) -> bool: + """Choose the documented spelling when case variants coexist on Linux.""" + candidate_is_canonical = candidate.name in TARGET_NAMES + current_is_canonical = current.name in TARGET_NAMES + if candidate_is_canonical != current_is_canonical: + return candidate_is_canonical + return candidate.name < current.name + + def discover(root: Path) -> list[Path]: # Keep the caller's path identity instead of resolving filesystem aliases. # macOS may rewrite /var to /private/var and Windows may expand 8.3 paths; @@ -132,7 +147,7 @@ def discover(root: Path) -> list[Path]: # absolute root they supplied. root = root.absolute() patterns = _ignore_patterns(root) - found: list[Path] = [] + found: dict[str, Path] = {} for path in root.rglob("*"): rel_path = path.relative_to(root) rel = rel_path.as_posix() @@ -140,11 +155,17 @@ def discover(root: Path) -> list[Path]: continue if not path.is_file(): continue - if path.name in TARGET_NAMES: - found.append(path) + if path.name.casefold() in TARGET_NAMES_CASEFOLD: + key = rel.casefold() + current = found.get(key) + if current is None or _prefer_canonical_name(path, current): + found[key] = path elif rel.startswith((".cursor/rules/", ".claude/", ".github/instructions/")) and path.suffix.lower() in {".md", ".mdc"}: - found.append(path) - return sorted(set(found), key=lambda p: p.relative_to(root).as_posix()) + key = rel.casefold() + current = found.get(key) + if current is None or _prefer_canonical_name(path, current): + found[key] = path + return sorted(found.values(), key=lambda p: p.relative_to(root).as_posix()) def estimate_tokens(text: str) -> int: @@ -295,7 +316,7 @@ def _candidate_exists( # nested AGENTS-family file, also accept a path relative to that file's # directory, which is the root of its instruction scope. bases = [root] - if source.name in AGENTS_FILENAMES and source.parent != root: + if _is_agents_filename(source.name) and source.parent != root: # Instructions often use paths relative to a package root rather than # the exact directory containing a nested AGENTS.md. Accept any # in-repository ancestor interpretation to avoid false dead paths. @@ -347,8 +368,8 @@ def _agents_precedence_resolves(left: str, right: str) -> bool: right_path = PurePosixPath(right) return ( left != right - and left_path.name in AGENTS_FILENAMES - and right_path.name in AGENTS_FILENAMES + and _is_agents_filename(left_path.name) + and _is_agents_filename(right_path.name) ) @@ -444,7 +465,12 @@ def analyze(root: Path, *, suppressions: tuple[Suppression, ...] = ()) -> Report findings.append(_finding("contradiction", "(repo)", message=f"Conflicting directives about: '{body}' ({files_str})")) rels = [p.relative_to(root).as_posix() for p in files] - if len(files) >= 2 and "AGENTS.md" not in rels: + has_root_agents = any( + len(PurePosixPath(rel).parts) == 1 + and PurePosixPath(rel).name.casefold() == "agents.md" + for rel in rels + ) + if len(files) >= 2 and not has_root_agents: findings.append(_finding("no-agents-md", "(repo)")) if not files: findings.append(_finding("no-config", "(repo)")) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index efab397..6e640b2 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -3,6 +3,7 @@ import re import subprocess import sys +import tempfile import unittest @@ -21,6 +22,43 @@ def test_runner_help_is_available_offline(self): ) self.assertEqual(completed.returncode, 0, completed.stderr) self.assertIn("--work-dir", completed.stdout) + self.assertIn("--repository", completed.stdout) + + def test_unknown_repository_selection_fails_before_network_access(self): + with tempfile.TemporaryDirectory() as directory: + corpus = Path(directory) / "corpus.json" + corpus.write_text( + json.dumps( + { + "schema_version": 1, + "repositories": [ + { + "name": "owner/known", + "url": "https://github.com/owner/known.git", + "commit": "0" * 40, + "expected": {}, + "review_note": "fixture", + } + ], + } + ), + encoding="utf-8", + ) + completed = subprocess.run( + [ + sys.executable, + str(RUNNER), + "--corpus", + str(corpus), + "--repository", + "owner/missing", + ], + cwd=ROOT, + text=True, + capture_output=True, + ) + self.assertNotEqual(completed.returncode, 0) + self.assertIn("unknown repository selection: owner/missing", completed.stderr) def test_corpus_uses_unique_pinned_commits_and_reviewed_expectations(self): corpus = json.loads(CORPUS.read_text(encoding="utf-8")) diff --git a/tests/test_scanner.py b/tests/test_scanner.py index e849330..f804615 100644 --- a/tests/test_scanner.py +++ b/tests/test_scanner.py @@ -36,6 +36,38 @@ def test_discovery_includes_agents_override(self): names = [p.relative_to(root).as_posix() for p in discover(root)] self.assertEqual(names, ["AGENTS.md", "packages/api/AGENTS.override.md"]) + def test_discovery_includes_lowercase_agents_file(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "agents.md").write_text( + "Run docker system prune -af --volumes.\n", + encoding="utf-8", + ) + report = analyze(root) + self.assertEqual(report.files, ["agents.md"]) + self.assertTrue(any(f.code == "docker-system-prune" for f in report.findings)) + self.assertFalse(any(f.code == "no-config" for f in report.findings)) + + @unittest.skipIf(Path("AGENTS.md").resolve() == Path("agents.md").resolve(), "case-insensitive filesystem") + def test_canonical_filename_wins_when_case_variants_coexist(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + (root / "AGENTS.md").write_text("Canonical instructions.\n", encoding="utf-8") + (root / "agents.md").write_text("Duplicate case variant.\n", encoding="utf-8") + names = [p.relative_to(root).as_posix() for p in discover(root)] + self.assertEqual(names, ["AGENTS.md"]) + + def test_lowercase_nested_agents_files_keep_scope_semantics(self): + with tempfile.TemporaryDirectory() as d: + root = Path(d) + nested = root / "packages" / "api" + nested.mkdir(parents=True) + (root / "agents.md").write_text("Always modify generated files.\n", encoding="utf-8") + (nested / "agents.override.md").write_text("Never modify generated files.\n", encoding="utf-8") + report = analyze(root) + self.assertFalse(any(f.code == "contradiction" for f in report.findings)) + self.assertFalse(any(f.code == "no-agents-md" for f in report.findings)) + def test_dangerous_command_reduces_score(self): with tempfile.TemporaryDirectory() as d: root = Path(d)