diff --git a/.agents/skills/security-audit/validate-coverage-ledger.cjs b/.agents/skills/security-audit/validate-coverage-ledger.cjs old mode 100644 new mode 100755 diff --git a/.agents/skills/security-audit/validate-findings.cjs b/.agents/skills/security-audit/validate-findings.cjs old mode 100644 new mode 100755 diff --git a/.github/workflows/apm-materialize.yml b/.github/workflows/apm-materialize.yml index d9e5eda..c319d6f 100644 --- a/.github/workflows/apm-materialize.yml +++ b/.github/workflows/apm-materialize.yml @@ -89,6 +89,16 @@ jobs: python3 -m pip install --user "apm-cli==${apm_cli_version}" echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + - name: Clear committed external skills + # apm-cli 0.35 trusts a skill's overwrite baseline only from per-user + # receipts, which a fresh runner lacks. It then skips the committed + # external skill files and deletes them as stale (#237, + # microsoft/apm#3372). Removing the lock-recorded skill folders makes + # `apm install` redeploy them from the pinned commit; git still holds + # the committed copy, so the PR diff shows any upstream change. + if: steps.precheck.outputs.skip != 'true' + run: python3 scripts/apm-external-skills.py prepare + - name: Materialize primitives (apm install) if: steps.precheck.outputs.skip != 'true' env: @@ -115,6 +125,11 @@ jobs: if: steps.precheck.outputs.skip != 'true' run: python3 scripts/apply-apm-overrides.py + - name: Check external primitives survived + # Fail instead of opening a PR that drops a dependency's files. + if: steps.precheck.outputs.skip != 'true' + run: python3 scripts/apm-external-skills.py check + - name: Regenerate index # Agent descriptions feed INDEX.md / README.md. The materialize PR must # carry the refreshed index, or its Generate Index check fails as stale. diff --git a/.github/workflows/markdown-validation.yml b/.github/workflows/markdown-validation.yml index 3c576f3..a6eaf71 100644 --- a/.github/workflows/markdown-validation.yml +++ b/.github/workflows/markdown-validation.yml @@ -12,6 +12,8 @@ on: - 'tests/**' - 'scripts/generate-index.sh' - 'scripts/apply-apm-overrides.py' + - 'scripts/apm-external-skills.py' + - '.agents/skills/**' - 'scripts/scan-skills.py' - 'apm.yml' - 'apm-overrides.json' @@ -28,6 +30,8 @@ on: - 'tests/**' - 'scripts/generate-index.sh' - 'scripts/apply-apm-overrides.py' + - 'scripts/apm-external-skills.py' + - '.agents/skills/**' - 'scripts/scan-skills.py' - 'apm.yml' - 'apm-overrides.json' @@ -81,6 +85,18 @@ jobs: - name: Check vendored primitives keep their local overrides run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_overrides.py' -v + - name: Check external primitives match apm.lock.yaml + run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_external_skills.py' -v + + - name: Check no locked external file is dropped against the PR base + # HEAD already holds this PR's lock, so compare with the base revision. + if: github.event_name == 'pull_request' + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: |- + git fetch --no-tags --depth=1 origin "${BASE_SHA}" + python3 scripts/apm-external-skills.py check --base "${BASE_SHA}" + - name: Test SkillSpector gate and report run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_skillspector*.py' -v diff --git a/.lefthook.toml b/.lefthook.toml index fba74f9..fcece9b 100644 --- a/.lefthook.toml +++ b/.lefthook.toml @@ -25,6 +25,10 @@ run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_ [pre-commit.commands.apm-overrides] run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_overrides.py'" +# Every locked dependency must still deploy its files (see #237). +[pre-commit.commands.apm-external-skills] +run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_apm_external_skills.py'" + # SkillSpector gate and report: severity policy, reviewed acceptances, PR comment. [pre-commit.commands.skillspector-gate] run = "PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -p 'test_skillspector*.py'" diff --git a/docs/apm.md b/docs/apm.md index 8220271..b7520bc 100644 --- a/docs/apm.md +++ b/docs/apm.md @@ -214,6 +214,24 @@ for our own `.apm/skills/` copies, so each external skill needs a negation in [`.gitignore`](../.gitignore) to stay in the reviewable snapshot. The whole folder is committed, including its companion files and scripts. +apm-cli 0.35 won't overwrite a skill file unless that machine deployed it +earlier: it trusts only its per-user receipts in `~/.apm/`, never +`apm.lock.yaml`. On a fresh clone or runner it skips the committed external +skill files and then deletes them as stale +([microsoft/apm#3372](https://github.com/microsoft/apm/issues/3372), #237). So +APM Materialize runs `python3 scripts/apm-external-skills.py prepare` before +`apm install`. It removes the external skill folders recorded in the lock, and +APM redeploys them from the pinned commit. Do the same before `apm install` in +a fresh clone. After the install, `scripts/apm-external-skills.py check` fails +the run when a locked dependency deploys nothing, loses a file (compared with +the lock at `HEAD` while its commit is unchanged) or deploys a +git-ignored file (a new or renamed skill folder needs its `.gitignore` +negation), or when a committed `.agents/skills/` file isn't in the lock. +`tests/test_apm_external_skills.py` runs the same check on every PR, and PR CI +also runs `check --base ` so a PR can't drop a locked file. Drop the +`prepare` step once apm-cli adopts unchanged committed skills +([microsoft/apm#3638](https://github.com/microsoft/apm/pull/3638)). + APM installs only the skill's subpath, so an upstream `LICENSE` at the repository root doesn't travel with it. [`THIRD_PARTY_NOTICES.md`](../THIRD_PARTY_NOTICES.md) carries the copyright and licence text for each external skill. If you commit diff --git a/scripts/apm-external-skills.py b/scripts/apm-external-skills.py new file mode 100755 index 0000000..4c9acfc --- /dev/null +++ b/scripts/apm-external-skills.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Keep committed external APM skills through APM Materialize (see #237). + +apm-cli 0.35 trusts a skill file's overwrite baseline only from per-user +deployment receipts, never from apm.lock.yaml. A fresh CI checkout has none, +so `apm install` skips every committed external skill file and its cleanup +then deletes them all as stale (microsoft/apm#3372). + + prepare remove the external skill folders recorded in apm.lock.yaml, so + `apm install` redeploys them from the pinned commit. Git still + holds the committed copy, so the materialize diff shows upstream + changes. Run it only right before `apm install`. + check fail if a locked dependency deploys nothing, a deployed file is + missing or git-ignored, a dependency lost a deployed file at the + commit HEAD's lock already pins, or a committed .agents/skills/ + file isn't in the lock. + +Usage: apm-external-skills.py {prepare,check} [--root DIR] [--base REV] + +Standard library only. Parses the block-style YAML that APM writes. +""" + +import argparse +from pathlib import Path, PurePosixPath +import shutil +import subprocess +import sys + + +LOCK = "apm.lock.yaml" +SKILLS = PurePosixPath(".agents/skills") + + +class LockError(Exception): + pass + + +def parse_lock(text): + """Return {dependency: (resolved commit, [deployed file, ...])} from apm.lock.yaml text.""" + deps, current, in_deps, in_files = {}, None, False, False + for number, line in enumerate(text.splitlines(), 1): + if not line.strip(): + continue + if not line.startswith((" ", "-")): + in_deps, current = line.rstrip() == "dependencies:", None + continue + if not in_deps: + continue + if line.startswith("- "): + key, _, value = line[2:].partition(":") + if key != "repo_url" or not value.strip(): + raise LockError(f"{LOCK}:{number}: expected `- repo_url:` to start a dependency") + current = {"repo_url": value.strip(), "files": []} + deps[number] = current + in_files = False + elif line.startswith(" - ") and in_files: + current["files"].append(line[4:].strip()) + elif line.startswith(" ") and not line.startswith(" "): + key, _, value = line[2:].partition(":") + in_files = key == "deployed_files" + current[key] = value.strip() + if not deps: + raise LockError(f"{LOCK}: no dependencies found") + return { + "/".join(filter(None, (dep["repo_url"], dep.get("virtual_path")))): (dep.get("resolved_commit"), dep["files"]) + for dep in deps.values() + } + + +def locked_dependencies(text): + """Return {dependency: [deployed file, ...]} from apm.lock.yaml text.""" + return {dep: files for dep, (_, files) in parse_lock(text).items()} + + +def committed_lock(root, base=None): + """Return the parsed apm.lock.yaml at `base`; without one, HEAD's lock or {}.""" + result = subprocess.run( + ["git", "show", f"{base or 'HEAD'}:{LOCK}"], cwd=root, capture_output=True, text=True, encoding="utf-8" + ) + if result.returncode != 0 and base: + raise LockError(f"cannot read {LOCK} at {base}: {result.stderr.strip()}") + return parse_lock(result.stdout) if result.returncode == 0 else {} + + +def external_skill_dirs(deps): + """Return the .agents/skills/ folders that locked dependencies deploy.""" + return sorted({folder for files in deps.values() for folder in skill_dirs(files)}) + + +def skill_dirs(files): + paths = (PurePosixPath(entry) for entry in files) + return [path for path in paths if path.parent == SKILLS and ".." not in path.parts] + + +def prepare(root, deps): + for folder in external_skill_dirs(deps): + target = root / folder + linked = [part for part in (folder.parents[1], folder.parent, folder) if (root / part).is_symlink()] + if linked or (target.exists() and not target.is_dir()): + raise LockError(f"{folder} is not a regular folder in the checkout; refusing to remove it") + if target.is_dir(): + shutil.rmtree(target) + print(f"Removed {folder} for a fresh deploy from its pinned commit") + + +def tracked_skill_files(root): + out = subprocess.run( + ["git", "ls-files", "-z", "--", str(SKILLS)], cwd=root, check=True, capture_output=True, text=True + ).stdout + return [path for path in out.split("\0") if path] + + +def ignored_paths(root, paths): + """Return the paths that .gitignore would keep out of the materialize PR.""" + result = subprocess.run( + ["git", "check-ignore", "-z", "--stdin"], + cwd=root, + input="".join(f"{path}\0" for path in paths), + capture_output=True, + text=True, + ) + if result.returncode not in (0, 1): + raise subprocess.CalledProcessError(result.returncode, result.args, result.stdout, result.stderr) + return {path for path in result.stdout.split("\0") if path} + + +def check(root, deps, tracked, ignored=frozenset()): + problems = [] + claimed = set() + for dep, files in sorted(deps.items()): + if not files: + problems.append(f"{dep} deploys no files; `apm install` dropped it") + claimed.update(files) + for entry in files: + if not (root / entry).exists(): + problems.append(f"{entry} from {dep} is missing") + elif entry in ignored and str(PurePosixPath(entry).parent) not in ignored: + problems.append(f"{entry} from {dep} is git-ignored; add a .gitignore exception for it") + for folder in skill_dirs(files): + anchor = folder / "SKILL.md" + if str(anchor) not in files or not (root / anchor).is_file(): + problems.append(f"{folder} from {dep} has no deployed SKILL.md") + for path in tracked: + if path not in claimed and (root / path).exists(): + problems.append(f"{path} is committed but no locked dependency deploys it") + return problems + + +def dropped_files(previous, current): + """List deployed paths a dependency lost although its resolved commit didn't change.""" + problems = [] + for dep, (commit, files) in sorted(current.items()): + old_commit, old_files = previous.get(dep, (None, [])) + if commit and commit == old_commit: + problems += [ + f"{path} from {dep} was dropped at unchanged commit {commit[:8]}" + for path in old_files + if path not in files + ] + return problems + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("command", choices=("prepare", "check")) + parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parents[1]) + parser.add_argument("--base", help="revision whose lock `check` compares against (default: HEAD)") + args = parser.parse_args(argv) + try: + lock = parse_lock((args.root / LOCK).read_text(encoding="utf-8")) + deps = {dep: files for dep, (_, files) in lock.items()} + if args.command == "prepare": + prepare(args.root, deps) + return 0 + claimed = sorted({entry for files in deps.values() for entry in files}) + problems = check(args.root, deps, tracked_skill_files(args.root), ignored_paths(args.root, claimed)) + problems += dropped_files(committed_lock(args.root, args.base), lock) + except (OSError, LockError, subprocess.CalledProcessError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + for problem in problems: + print(f"::error::{problem}", file=sys.stderr) + if problems: + print(f"{len(problems)} problem(s); see docs/apm.md (external primitives).", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_apm_external_skills.py b/tests/test_apm_external_skills.py new file mode 100644 index 0000000..306c731 --- /dev/null +++ b/tests/test_apm_external_skills.py @@ -0,0 +1,175 @@ +"""Tests for scripts/apm-external-skills.py (see #237). + +apm-cli 0.35 deletes committed external skills on a fresh checkout, so APM +Materialize removes them first and checks the result. Standard library only. +""" + +import importlib.util +import io +from pathlib import Path +import subprocess +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout + + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("apm_external_skills", ROOT / "scripts" / "apm-external-skills.py") +skills = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(skills) + +SKILL = ".agents/skills/audit" +AGENT = ".github/agents/a.agent.md" +LOCK = f"""lockfile_version: '1' +dependencies: +- repo_url: owner/skills + name: audit + resolved_commit: c1c8a8c1471069fb + virtual_path: skills/audit + package_type: claude_skill + deployed_files: + - {SKILL} + - {SKILL}/SKILL.md + deployed_file_hashes: + {SKILL}/SKILL.md: sha256:00 + content_hash: sha256:11 +- repo_url: owner/agents + virtual_path: agents/a.agent.md + deployed_files: + - {AGENT} +deployments: +- kind: project-relative + value: .agents/skills/local-skill + owners: + - . +""" +# What apm-cli 0.35.0 wrote in #233: the skill entry kept, its files dropped. +DROPPED = LOCK.replace(f" deployed_files:\n - {SKILL}\n - {SKILL}/SKILL.md\n", "") + + +class LockParsingTests(unittest.TestCase): + def test_reads_deployed_files_per_dependency_only_from_dependencies(self): + self.assertEqual( + skills.locked_dependencies(LOCK), + {"owner/skills/skills/audit": [SKILL, f"{SKILL}/SKILL.md"], "owner/agents/agents/a.agent.md": [AGENT]}, + ) + self.assertEqual(skills.locked_dependencies(DROPPED)["owner/skills/skills/audit"], []) + + def test_rejects_unexpected_layouts(self): + for text in ("lockfile_version: '1'\n", "dependencies:\n- name: x\n"): + with self.subTest(text=text), self.assertRaises(skills.LockError): + skills.locked_dependencies(text) + + def test_flags_files_dropped_at_an_unchanged_commit_only(self): + before = skills.parse_lock(LOCK.replace(f" - {SKILL}/SKILL.md\n", f" - {SKILL}/SKILL.md\n - {SKILL}/schema.json\n")) + after = skills.parse_lock(LOCK) + self.assertEqual( + skills.dropped_files(before, after), + [f"{SKILL}/schema.json from owner/skills/skills/audit was dropped at unchanged commit c1c8a8c1"], + ) + bumped = skills.parse_lock(LOCK.replace("c1c8a8c1471069fb", "0123456789abcdef")) + self.assertEqual(skills.dropped_files(before, bumped), []) + + +class WorkTreeTests(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.root = Path(tmp.name) + for path in (f"{SKILL}/SKILL.md", f"{SKILL}/notes.md", AGENT, ".agents/skills/local-skill/SKILL.md"): + (self.root / path).parent.mkdir(parents=True, exist_ok=True) + (self.root / path).write_text("x\n", encoding="utf-8") + + def test_prepare_removes_only_locked_external_skill_folders(self): + with redirect_stdout(io.StringIO()): + skills.prepare(self.root, skills.locked_dependencies(LOCK)) + self.assertFalse((self.root / SKILL).exists()) + self.assertTrue((self.root / AGENT).exists()) + self.assertTrue((self.root / ".agents/skills/local-skill/SKILL.md").exists()) + + def test_prepare_refuses_a_symlinked_skills_parent(self): + outside = self.root / "outside" + (outside / "audit").mkdir(parents=True) + skills_root = self.root / ".agents/skills" + moved = self.root / "moved-skills" + skills_root.rename(moved) + try: + skills_root.symlink_to(outside, target_is_directory=True) + except OSError: + self.skipTest("symlinks unavailable") + with self.assertRaises(skills.LockError): + skills.prepare(self.root, skills.locked_dependencies(LOCK)) + self.assertTrue((outside / "audit").is_dir()) + + def test_check_fails_when_a_deployed_skill_is_git_ignored(self): + (self.root / ".agents/skills/renamed").mkdir() + (self.root / ".agents/skills/renamed/SKILL.md").write_text("x\n", encoding="utf-8") + (self.root / "apm.lock.yaml").write_text(LOCK.replace(SKILL, ".agents/skills/renamed"), encoding="utf-8") + (self.root / ".gitignore").write_text(".agents/skills/*\n!.agents/skills/audit/\n", encoding="utf-8") + try: + subprocess.run(["git", "init", "-q", str(self.root)], check=True) + except (OSError, subprocess.CalledProcessError): + self.skipTest("git unavailable") + stderr = io.StringIO() + with redirect_stderr(stderr): + self.assertEqual(skills.main(["check", "--root", str(self.root)]), 1) + self.assertEqual(stderr.getvalue().count("git-ignored"), 1, stderr.getvalue()) + self.assertIn(".agents/skills/renamed from owner/skills/skills/audit is git-ignored", stderr.getvalue()) + + def test_check_against_a_base_revision_flags_a_dropped_companion(self): + git = ["git", "-C", str(self.root), "-c", "user.name=t", "-c", "user.email=t@t", "-c", "commit.gpgsign=false"] + companion = f" - {SKILL}/SKILL.md\n - {SKILL}/schema.json\n" + (self.root / f"{SKILL}/schema.json").write_text("{}\n", encoding="utf-8") + (self.root / "apm.lock.yaml").write_text(LOCK.replace(f" - {SKILL}/SKILL.md\n", companion), encoding="utf-8") + try: + subprocess.run(["git", "init", "-q", str(self.root)], check=True) + subprocess.run([*git, "add", "apm.lock.yaml"], check=True) + subprocess.run([*git, "commit", "-qm", "base"], check=True) + except (OSError, subprocess.CalledProcessError): + self.skipTest("git unavailable") + base = subprocess.run([*git, "rev-parse", "HEAD"], check=True, capture_output=True, text=True).stdout.strip() + (self.root / f"{SKILL}/schema.json").unlink() + (self.root / "apm.lock.yaml").write_text(LOCK, encoding="utf-8") + subprocess.run([*git, "commit", "-qam", "drop the companion"], check=True) + stderr = io.StringIO() + with redirect_stderr(stderr): + self.assertEqual(skills.main(["check", "--root", str(self.root)]), 0, stderr.getvalue()) + self.assertEqual(skills.main(["check", "--root", str(self.root), "--base", base]), 1) + self.assertEqual(skills.main(["check", "--root", str(self.root), "--base", "no-such-rev"]), 2) + self.assertIn(f"{SKILL}/schema.json from owner/skills/skills/audit was dropped", stderr.getvalue()) + + def test_check_passes_a_complete_deploy(self): + deps = skills.locked_dependencies(LOCK) + self.assertEqual(skills.check(self.root, deps, [f"{SKILL}/SKILL.md"]), []) + + def test_check_fails_when_install_dropped_or_lost_a_skill(self): + problems = skills.check(self.root, skills.locked_dependencies(DROPPED), [f"{SKILL}/SKILL.md"]) + self.assertEqual(len(problems), 2) + self.assertIn("owner/skills/skills/audit deploys no files", problems[0]) + self.assertIn(f"{SKILL}/SKILL.md is committed but no locked dependency deploys it", problems[1]) + (self.root / f"{SKILL}/SKILL.md").unlink() + problems = skills.check(self.root, skills.locked_dependencies(LOCK), []) + self.assertEqual( + problems, + [ + f"{SKILL}/SKILL.md from owner/skills/skills/audit is missing", + f"{SKILL} from owner/skills/skills/audit has no deployed SKILL.md", + ], + ) + + def test_check_fails_when_only_the_skill_folder_is_left(self): + (self.root / f"{SKILL}/SKILL.md").unlink() + (self.root / f"{SKILL}/notes.md").unlink() + deps = skills.locked_dependencies(LOCK.replace(f" - {SKILL}/SKILL.md\n", "")) + self.assertEqual(skills.check(self.root, deps, []), [f"{SKILL} from owner/skills/skills/audit has no deployed SKILL.md"]) + + +class RepoTests(unittest.TestCase): + def test_committed_lock_keeps_every_external_primitive(self): + stderr = io.StringIO() + with redirect_stderr(stderr): + self.assertEqual(skills.main(["check", "--root", str(ROOT)]), 0, stderr.getvalue()) + + +if __name__ == "__main__": + unittest.main()