diff --git a/.mise.toml b/.mise.toml index 05571f78..768d8eb3 100644 --- a/.mise.toml +++ b/.mise.toml @@ -32,4 +32,5 @@ run = [ "bash ./tests/open-pr-labels.sh", "bash ./tests/config-sync-ai-toolkit-instructions.sh", "python3 ./tests/workflow-job-conditions.py", + "python3 ./tests/ai-commits-badge.py", ] diff --git a/README.md b/README.md index cda87b69..097345cf 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ repositories. | ---------------------------- | ----------------------------------------------------------------------------------------- | | Reusable workflows | [`.github/workflows/`](.github/workflows/) | | Composite actions | [`actions/`](actions/) | +| AI commits badge | [`actions/ai-commits-badge/`](actions/ai-commits-badge/README.md) | | Workflow templates | [`workflow-templates/`](workflow-templates/) | | Config sync (files) | [`config-sync/files/`](config-sync/files/) | | Config sync (templates) | [`config-sync/templates/`](config-sync/templates/) | diff --git a/actions/ai-commits-badge/README.md b/actions/ai-commits-badge/README.md new file mode 100644 index 00000000..41cd195f --- /dev/null +++ b/actions/ai-commits-badge/README.md @@ -0,0 +1,217 @@ +# `ai-commits-badge` composite action + +Generate an AI commits badge and README card from Git history. No git-ai, +third-party actions, Python packages, network calls or publishing are needed. +Run on Linux with Bash, Git and Python 3.8+ already available. + +## What it measures + +Each non-merge commit is classified once, in this order: + +1. **AI agent:** its author matches a bundled AI identity. +2. **Bot automation:** its author contains `[bot]`, unless recognized above. + Excluded from the denominator, including bot commits with AI trailers. +3. **Co-authored with AI:** a `Co-authored-by` trailer matches an AI identity. +4. **No recorded AI attribution:** neither author nor trailers match. + +The percentage is `(agent + coauthored) / (agent + coauthored + unattributed)`. +This counts commits, not lines, effort or the percentage of code written by AI. +Missing attribution is **not proof of human authorship**. Squashing, lost +trailers and unrecognized tools affect the result. Tool counts use the first +matching identity, not every co-author. Defaults recognize Copilot, Claude, +Cursor, Devin, Jules and Aider; the exact rules live in +[`ai_commits_badge.py`](ai_commits_badge.py). + +The card legend shortens the final category to **No recorded AI** to fit its +existing 480px layout. Tooltips and accessible hover text retain the full +**No recorded AI attribution** wording. The corresponding machine-readable +key and action output are `unattributed`. + +Long repository names are clipped to the card's 200px header region without +stretching the text. The full name remains in the tooltip and accessible text. + +## Inputs + +| Input | Default | Meaning | +| ---------- | -------------------------- | --------------------------------------------------------------- | +| `ref` | `HEAD` | Commit, branch or tag reachable in the local checkout. | +| `since` | empty | Optional Git date filter, e.g. `90 days ago`. | +| `name` | `${{ github.repository }}` | Name displayed on the card and in tooltips. | +| `label` | `AI commits` | Compact badge and Shields endpoint label. | +| `out` | `badges` | Output directory relative to the caller's working directory. | +| `themes` | `"false"` | Set `"true"` to also write light/dark cards. Other values fail. | +| `repo-dir` | `.` | Checkout to measure; the action never changes it. | + +Identity rules are centralized and cannot be overridden per caller, so repos +using the same action revision use comparable classification rules. Add new +AI identities to the bundled list through a PR in this repository. The same +case-insensitive rules match complete recognized names or email addresses for +authors and individual trailers. Email patterns include the expected domain; +partial names and addresses do not establish AI attribution. + +## Outputs and files + +| Step output | Meaning | +| --------------------- | --------------------------------------------------------- | +| `ai-percent` | One-decimal percentage, or `n/a` when no commits qualify. | +| `commits` | Denominator, excluding merge and automation commits. | +| `ai-commits` | Agent-authored plus AI-co-authored commits. | +| `agent` | AI-authored commits. | +| `coauthored` | AI-co-authored commits. | +| `unattributed` | No recorded AI attribution. | +| `excluded-automation` | Excluded non-merge bot commits. | + +The badge displays a whole-number percentage calculated directly from the +commit counts; `ai-percent` and stats JSON retain one decimal. Display values +are not rounded again from the statistics. Empty history produces a grey +`n/a` badge. + +| File in `out` | Purpose | +| ------------------------------------------------------- | ----------------------------------------------------------------- | +| `ai-commits.svg` | Compact badge with a proportional strip and hover breakdown. | +| `ai-commits.json` | Shields.io endpoint JSON (`schemaVersion: 1`). | +| `ai-commits-stats.json` | Counts, tool breakdown, ref, date filter and UTC generation time. | +| `ai-commits-card.svg` | Card following `prefers-color-scheme`. | +| `ai-commits-card-light.svg`, `ai-commits-card-dark.svg` | Additional cards when `themes: "true"`. | + +The action writes files only. Callers own uploads, branch updates, permissions, +retention and publication. Counts and tool names become visible wherever you +publish these files. Publishing results from private or work repositories is a +separate, explicitly approved decision for each repository owner. + +## Complete history is required + +Use `actions/checkout` with `fetch-depth: 0`. The default shallow checkout fails +with an actionable message before writing files or step outputs; `since` does +not bypass this guard. The action never fetches history automatically. +Fetch any requested non-default ref in the caller as well. + +## Example: add badges to a MkDocs site + +In the docs workflow, add a daily refresh and generate **after** `mkdocs build` +(which cleans `site`) and **before** uploading the Pages artifact. Keep your +existing pinned MkDocs setup/build steps and Pages deployment configuration. + +```yaml +--- +name: Docs +on: + push: + branches: [main] + workflow_dispatch: + schedule: + - cron: "17 4 * * *" +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-24.04 + steps: + # renovate: datasource=github-tags depName=actions/checkout + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + # Keep the site's existing pinned toolchain setup here. + - run: mkdocs build + - uses: DevSecNinja/.github/actions/ai-commits-badge@ + id: ai + with: + out: site/badges + themes: "true" + - run: printf 'AI commits %s/%s (%s%%)\n' "$AI" "$TOTAL" "$PERCENT" + env: + AI: ${{ steps.ai.outputs.ai-commits }} + TOTAL: ${{ steps.ai.outputs.commits }} + PERCENT: ${{ steps.ai.outputs.ai-percent }} + # renovate: datasource=github-tags depName=actions/upload-pages-artifact + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: site +``` + +The snippet assumes MkDocs is already installed by your existing docs setup. +Keep the existing Pages deploy job after `build`, with `pages: write` and +`id-token: write`. The daily trigger regenerates rolling `since` windows even +when the source has no new commits. + +In a README, reference the published site, for example: + +```markdown +[![AI commits](https://example.org/badges/ai-commits.svg)](https://example.org/badges/ai-commits-stats.json) +``` + +Or use Shields with the publicly accessible, URL-encoded JSON endpoint: +`https://img.shields.io/endpoint?url=https%3A%2F%2Fexample.org%2Fbadges%2Fai-commits.json`. + +## Example: standalone badge-only Pages site + +For a repo without documentation, publish just the generated files to Pages. +Configure its Pages source as **GitHub Actions** first. This avoids committing +generated artifacts or needing a separate `badges` branch. + +```yaml +--- +name: Badges +on: + push: + branches: [main] + workflow_dispatch: + schedule: + - cron: "17 4 * * *" +permissions: + contents: read +concurrency: + group: badges-pages + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-24.04 + permissions: + contents: read + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + # renovate: datasource=github-tags depName=actions/checkout + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - uses: DevSecNinja/.github/actions/ai-commits-badge@ + with: + out: public/badges + themes: "true" + # renovate: datasource=github-tags depName=actions/upload-pages-artifact + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: public + # renovate: datasource=github-tags depName=actions/deploy-pages + - uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 + id: deploy +``` + +For a project site, the badge URL is +`https://OWNER.github.io/REPO/badges/ai-commits.svg`. +Do not use this workflow alongside another Pages site in the same repo: it +would replace that deployment. Use the docs-site pattern instead. + +Both examples intentionally leave the new action's ref as +`` until it is released. Replace it with the released full +40-character SHA and a version comment before use; let Renovate track the +release as with other centralized actions. + +## Local checks + +```sh +python3 tests/ai-commits-badge.py +python3 actions/ai-commits-badge/ai_commits_badge.py \ + --repo-dir . --ref HEAD --out /tmp/ai-badges --themes +``` + +The tests build isolated synthetic Git history and execute the composite +action's actual Bash step, including its environment wiring and outputs. diff --git a/actions/ai-commits-badge/action.yml b/actions/ai-commits-badge/action.yml new file mode 100644 index 00000000..3376e8ca --- /dev/null +++ b/actions/ai-commits-badge/action.yml @@ -0,0 +1,82 @@ +--- +name: AI commits badge +description: Generate README badges from commit authors and recorded AI co-authorship, excluding merges and automation bots. +inputs: + ref: + description: "Commit, branch or tag to measure." + default: HEAD + since: + description: "Optional git date filter, for example 90 days ago." + default: "" + name: + description: "Repository name shown in the card and tooltip." + default: ${{ github.repository }} + label: + description: "Compact badge and Shields endpoint label." + default: AI commits + out: + description: "Output directory, relative to the caller's working directory." + default: badges + themes: + description: "Also generate separate light and dark cards (true or false)." + default: "false" + repo-dir: + description: "Repository to measure. Must have complete history." + default: "." +outputs: + ai-percent: + description: "AI share of considered commits, rounded to one decimal; n/a for an empty denominator." + value: ${{ steps.generate.outputs.ai-percent }} + commits: + description: "Non-merge commits excluding automation bots." + value: ${{ steps.generate.outputs.commits }} + ai-commits: + description: "Agent-authored plus AI-co-authored commits." + value: ${{ steps.generate.outputs.ai-commits }} + agent: + description: "Commits authored by a recognized AI identity." + value: ${{ steps.generate.outputs.agent }} + coauthored: + description: "Commits with a recognized AI Co-authored-by trailer." + value: ${{ steps.generate.outputs.coauthored }} + unattributed: + description: "Commits with no recorded AI attribution." + value: ${{ steps.generate.outputs.unattributed }} + excluded-automation: + description: "Excluded non-merge automation-bot commits." + value: ${{ steps.generate.outputs.excluded-automation }} +runs: + using: composite + steps: + - name: Generate AI commits badges + id: generate + shell: bash + env: + ACTION_PATH: ${{ github.action_path }} + BADGE_REF: ${{ inputs.ref }} + BADGE_SINCE: ${{ inputs.since }} + BADGE_NAME: ${{ inputs.name }} + BADGE_LABEL: ${{ inputs.label }} + BADGE_OUT: ${{ inputs.out }} + BADGE_THEMES: ${{ inputs.themes }} + BADGE_REPO_DIR: ${{ inputs.repo-dir }} + run: |- + set -euo pipefail + args=( + "--repo-dir=${BADGE_REPO_DIR}" + "--ref=${BADGE_REF}" + "--since=${BADGE_SINCE}" + "--name=${BADGE_NAME}" + "--label=${BADGE_LABEL}" + "--out=${BADGE_OUT}" + "--github-output=${GITHUB_OUTPUT}" + ) + case "${BADGE_THEMES}" in + true) args+=(--themes) ;; + false) ;; + *) + echo "::error::themes must be true or false." >&2 + exit 1 + ;; + esac + python3 "${ACTION_PATH}/ai_commits_badge.py" "${args[@]}" diff --git a/actions/ai-commits-badge/ai_commits_badge.py b/actions/ai-commits-badge/ai_commits_badge.py new file mode 100644 index 00000000..451e616b --- /dev/null +++ b/actions/ai-commits-badge/ai_commits_badge.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Render an "AI commits" README badge from commit authors and Co-authored-by trailers. + +Needs only git + Python 3.8+. Merge commits and bot automation are excluded. +Outputs in --out: ai-commits.svg, ai-commits.json (Shields endpoint), +ai-commits-stats.json and ai-commits-card.svg (auto light/dark). +""" +import argparse +import json +import re +import subprocess +from collections import Counter +from datetime import datetime, timezone +from html import escape +from pathlib import Path + +# Complete name/email patterns; first matching tool wins for authors and trailers. +AI_IDENTITIES = [ + (r"copilot-swe-agent(?:\[bot\])?", r"198982749\+Copilot@users\.noreply\.github\.com", "Copilot coding agent"), + (None, r"223556219\+Copilot@users\.noreply\.github\.com", "Copilot CLI"), + (None, r"175728472\+Copilot@users\.noreply\.github\.com", "Copilot"), + (r"anthropic-code-agent(?:\[bot\])?", r"242468646\+Claude@users\.noreply\.github\.com", "Claude agent"), + (None, r"noreply@anthropic\.com", "Claude Code"), + (None, r"cursoragent@cursor\.com", "Cursor"), + (r"devin-ai-integration\[bot\]", None, "Devin"), + (r"google-labs-jules\[bot\]", None, "Jules"), + (None, r"noreply@aider\.chat", "Aider"), +] +NO_ATTRIBUTION = "No recorded AI attribution" +SEGMENTS = [ + ("agent", "AI agent", "#8250df"), + ("coauthored", "co-authored with AI", "#c297ff"), + ("unattributed", NO_ATTRIBUTION, "#0969da"), +] +PURPLE, GREY = "#8250df", "#8c959f" +SPARKLE = "M5 0 6.2 3.8 10 5 6.2 6.2 5 10 3.8 6.2 0 5 3.8 3.8Z" +VERDANA = {" ": 3.9, "%": 11.9, "/": 4.7, "I": 4.6, "A": 7.5, "i": 3.1, "m": 10.7, "t": 4.3, + "s": 5.7, "c": 5.7, "o": 6.7, "n": 7.0, "a": 6.6} + + +def text_width(s): + return sum(VERDANA.get(ch, 7.0 if ch.isdigit() else 6.6) for ch in s) + + +def ai_tool(identity): + match = re.fullmatch(r"\s*(.*?)\s*<([^<>]+)>\s*", identity) + if not match: + return None + name, email = (part.strip() for part in match.groups()) + for name_pattern, email_pattern, tool in AI_IDENTITIES: + if (name_pattern and re.fullmatch(name_pattern, name, re.IGNORECASE)) or ( + email_pattern and re.fullmatch(email_pattern, email, re.IGNORECASE) + ): + return tool + return None + + +def git(repo_dir, *args): + return subprocess.run( + ["git", "-C", str(repo_dir), *args], check=True, + capture_output=True, text=True, encoding="utf-8", + ).stdout + + +def collect(repo_dir, ref, since): + if git(repo_dir, "rev-parse", "--is-shallow-repository").strip() == "true": + raise ValueError( + "Shallow history cannot be measured accurately. " + "Use actions/checkout with fetch-depth: 0, or git fetch --unshallow." + ) + revision = git(repo_dir, "rev-parse", "--verify", "--end-of-options", ref + "^{commit}").strip() + fmt = "%an <%ae>%x1f%(trailers:key=Co-authored-by,valueonly,separator=%x1e)%x00" + cmd = ["log", "--encoding=UTF-8", "--no-show-signature", "--no-merges", f"--format={fmt}"] + if since: + cmd.append(f"--since={since}") + out = git(repo_dir, *cmd, revision, "--") + counts, tools = Counter(), Counter() + for record in filter(None, (r.strip("\n") for r in out.split("\0"))): + author, _, trailers = record.partition("\x1f") + if tool := ai_tool(author): + counts["agent"] += 1 + tools[tool] += 1 + elif "[bot]" in author: + counts["automation"] += 1 + elif co := next(filter(None, map(ai_tool, trailers.split("\x1e"))), None): + counts["coauthored"] += 1 + tools[co] += 1 + else: + counts["unattributed"] += 1 + considered = counts["agent"] + counts["coauthored"] + counts["unattributed"] + ai = counts["agent"] + counts["coauthored"] + return { + "source": "commit-authors-and-trailers", + "ref": ref, + "since": since, + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "commits": considered, + "ai_commits": ai, + "ai_percent": round(100 * ai / considered, 1) if considered else None, + "agent": counts["agent"], + "coauthored": counts["coauthored"], + "unattributed": counts["unattributed"], + "excluded_automation": counts["automation"], + "tools": dict(tools.most_common()), + } + + +def message(st): + return "n/a" if not st["commits"] else f"{100 * st['ai_commits'] / st['commits']:.0f}%" + + +def tooltip(st, name): + if not st["commits"]: + return f"{name}: no commits to measure" + window = f" since {st['since']}" if st["since"] else "" + by_tool = ", ".join(f"{t} {n}" for t, n in st["tools"].items()) or "none" + return (f"{name}: {st['ai_commits']:,} of {st['commits']:,} commits{window} ({message(st)}) involved AI: " + f"{st['agent']:,} by an AI agent, {st['coauthored']:,} co-authored with AI, " + f"{st['unattributed']:,} {NO_ATTRIBUTION}. " + f"By tool: {by_tool}. Excludes merges and {st['excluded_automation']:,} bot-automation commits.") + + +def badge_svg(st, name, label="AI commits"): + msg = message(st) + lw = round(text_width(label) + 10 + 14) + rw = round(text_width(msg) + 10) + w = lw + rw + title = escape(tooltip(st, name)) + strip, x = [], 0.0 + for key, _, colour in SEGMENTS: + seg = w * st[key] / st["commits"] if st["commits"] else 0 + if seg > 0: + strip.append(f'') + x += seg + + def txt(cx, t): + tl = text_width(t) * 10 + return (f'{escape(t)}' + f'{escape(t)}') + + return ( + f'' + f"{title}" + '' + '' + f'' + f'' + f'' + f'{"".join(strip)}' + f'' + '' + f"{txt(14 + (lw - 14) / 2, label)}{txt(lw + rw / 2, msg)}" + ) + + +CARD_THEMES = { + "light": ".bg{fill:#fff;stroke:#d0d7de}.t{fill:#1f2328}.m{fill:#59636e}", + "dark": ".bg{fill:#0d1117;stroke:#3d444d}.t{fill:#f0f6fc}.m{fill:#9198a1}", +} +CARD_LEGEND = {"agent": "AI agent", "coauthored": "Co-authored", "unattributed": NO_ATTRIBUTION} + + +def fits(s, font_px, max_w): + # Shorten lines instead of squeezing them with textLength. + return len(s) * font_px * 0.55 <= max_w + + +def legend_label(key, share): + for label in (CARD_LEGEND[key], "No recorded AI"): + if fits(f"{label} {share}", 12, 129): + return label + return "No AI record" + + +def tools_line(tools, max_w=432): + items = list(tools.items()) + for k in range(len(items), 0, -1): + more = len(items) - k + line = "Tools: " + " · ".join(f"{t} {c:,}" for t, c in items[:k]) + (f" · +{more} more" if more else "") + if fits(line, 12, max_w): + return line + return f"Tools: {len(items)} detected" if items else "Tools: none detected" + + +def card_svg(st, name, theme="auto"): + css = CARD_THEMES["dark" if theme == "dark" else "light"] + if theme == "auto": + css += "@media (prefers-color-scheme:dark){" + CARD_THEMES["dark"] + "}" + css += 'text{font-family:-apple-system,BlinkMacSystemFont,"Segoe UI","Noto Sans",Helvetica,Arial,sans-serif}' + n = st["commits"] + window = f" since {st['since']}" if st["since"] else "" + sub = f"of {n:,} commits{window} involved AI" if n else "no commits to measure" + shown_name = name if fits(name, 12, 200) else name.rsplit("/", 1)[-1] + bar, legend, x = [], [], 24.0 + for i, (key, _, colour) in enumerate(SEGMENTS): + w = 432 * st[key] / n if n else 0 + if w > 0: + bar.append(f'') + x += w + share = f"{100 * st[key] / n:.0f}%" if n else "0%" + lx = 24 + i * 144 + legend.append(f'' + f'{legend_label(key, share)} ' + f'{share}') + title = escape(tooltip(st, name)) + return ( + f'{title}' + '' + f'' + 'Who wrote the commits?' + '' + f'' + f'{escape(shown_name)}' + f'' + f'{escape(message(st))}{escape(sub)}' + '' + f'' + f'{"".join(bar)}{"".join(legend)}' + f'{escape(tools_line(st["tools"]))}' + 'Commit authors + Co-authored-by trailers · bots excluded' + f'{st["generated_at"][:10]}' + "" + ) + + +def endpoint_json(st, label="AI commits"): + logo = f'' + return {"schemaVersion": 1, "label": label, "message": message(st), + "color": PURPLE.lstrip("#") if st["commits"] else "lightgrey", "logoSvg": logo} + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--repo-dir", default=".") + ap.add_argument("--ref", default="HEAD") + ap.add_argument("--since", help='only count recent commits, e.g. "90 days ago"') + ap.add_argument("--name", default="this repository", help="owner/repo shown in the tooltip") + ap.add_argument("--label", default="AI commits") + ap.add_argument("--out", default="badges") + ap.add_argument("--themes", action="store_true", + help="also write ai-commits-card-light.svg / -dark.svg for tags") + ap.add_argument("--github-output", help="append numeric step outputs to this file") + args = ap.parse_args() + try: + st = collect(args.repo_dir, args.ref, args.since) + except (ValueError, subprocess.CalledProcessError) as error: + detail = error.stderr.strip() if isinstance(error, subprocess.CalledProcessError) else str(error) + ap.exit(1, f"AI commits badge: {detail}\n") + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + files = { + "ai-commits.svg": badge_svg(st, args.name, args.label), + "ai-commits.json": json.dumps(endpoint_json(st, args.label), indent=2), + "ai-commits-stats.json": json.dumps(st, indent=2), + "ai-commits-card.svg": card_svg(st, args.name), + } + if args.themes: + for theme in ("light", "dark"): + files[f"ai-commits-card-{theme}.svg"] = card_svg(st, args.name, theme) + for filename, content in files.items(): + (out / filename).write_bytes((content + "\n").encode("utf-8")) + if args.github_output: + with Path(args.github_output).open("a", encoding="utf-8", newline="\n") as output: + for key in ("ai_percent", "commits", "ai_commits", "agent", "coauthored", "unattributed", "excluded_automation"): + value = "n/a" if st[key] is None else st[key] + output.write(f"{key.replace('_', '-')}={value}\n") + print(json.dumps(st, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/docs/architecture.md b/docs/architecture.md index bf69034e..af7b82b6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -684,6 +684,16 @@ OIDC subject, or pre-existing job state — things a reusable workflow Pin the `uses:` ref to a full commit SHA with a version comment, just like reusable workflows. +### AI commits badge (`actions/ai-commits-badge/`) + +Generates compact badges, Shields endpoint JSON, statistics and theme-aware +README cards from complete Git history. Measures recognized AI authors and +co-author trailers, excludes merges and automation bots, and exposes the +percentage and counts as step outputs. Publishing stays with the caller. + +See [`actions/ai-commits-badge/README.md`](../actions/ai-commits-badge/README.md) +for attribution caveats, inputs and docs-site or standalone Pages examples. + ### Open PR (`actions/open-pr/`) Opens (or force-updates) a pull request from the current working tree. diff --git a/tests/ai-commits-badge.py b/tests/ai-commits-badge.py new file mode 100644 index 00000000..e35926d4 --- /dev/null +++ b/tests/ai-commits-badge.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Offline contract tests using synthetic history and the actual composite step.""" +import importlib.util +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from xml.etree import ElementTree as ET + +sys.dont_write_bytecode = True + +ROOT = Path(__file__).resolve().parent.parent +ACTION = ROOT / "actions" / "ai-commits-badge" +SPEC = importlib.util.spec_from_file_location("badge", ACTION / "ai_commits_badge.py") +badge = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(badge) +NS = {"svg": "http://www.w3.org/2000/svg"} + + +class BadgeContract(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="ai-badge-") + self.addCleanup(self.temp.cleanup) + self.work = Path(self.temp.name) + self.repo = self.work / "repo" + self.env = dict(os.environ, GIT_CONFIG_NOSYSTEM="1", GIT_CONFIG_GLOBAL=os.devnull) + for key in list(self.env): + if key.startswith(("GIT_CONFIG_KEY_", "GIT_CONFIG_VALUE_")) or key == "GIT_CONFIG_COUNT": + del self.env[key] + self.git("init", "--quiet", str(self.repo), cwd=self.work) + self.tree = self.git("mktree", input="").stdout.strip() + self.tip = None + self.commit("Human", "human@example.org", "Initial") + self.commit("copilot-swe-agent[bot]", "agent@example.org", "Agent") + self.commit( + "Human", "human@example.org", + "Pair\n\nCo-authored-by: Someone \n" + "Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>", + ) + self.commit( + "renovate[bot]", "bot@example.org", + "Automation\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>", + ) + side = self.commit("Human", "human@example.org", "Side") + self.commit("Human", "human@example.org", "Merge", parents=[side, self.initial]) + + def git(self, *args, cwd=None, input=None, env=None): + return subprocess.run( + ["git", *args], cwd=cwd or self.repo, env=env or self.env, + input=input, capture_output=True, text=True, encoding="utf-8", check=True, + ) + + def commit(self, name, email, message, parents=None): + env = dict( + self.env, GIT_AUTHOR_NAME=name, GIT_AUTHOR_EMAIL=email, + GIT_COMMITTER_NAME=name, GIT_COMMITTER_EMAIL=email, + GIT_AUTHOR_DATE="2020-01-01T12:00:00+00:00", + GIT_COMMITTER_DATE="2020-01-01T12:00:00+00:00", + ) + args = ["commit-tree", self.tree] + for parent in parents if parents is not None else ([self.tip] if self.tip else []): + args += ["-p", parent] + self.tip = self.git(*args, input=message + "\n", env=env).stdout.strip() + if not hasattr(self, "initial"): + self.initial = self.tip + self.git("update-ref", "refs/heads/main", self.tip) + self.git("symbolic-ref", "HEAD", "refs/heads/main") + return self.tip + + def run_action(self, **overrides): + text = (ACTION / "action.yml").read_text(encoding="utf-8") + inputs = { + "ref": "HEAD", "since": "", "name": "owner/repo", "label": "AI commits", + "out": str(self.work / "out"), "themes": "false", "repo-dir": str(self.repo), + } + inputs.update(overrides) + env = dict(self.env, ACTION_PATH=str(ACTION), GITHUB_OUTPUT=str(self.work / "outputs")) + mappings = re.findall(r"^ ([A-Z_]+): \$\{\{ inputs\.([\w-]+) \}\}$", text, re.M) + self.assertEqual({name for _, name in mappings}, set(inputs)) + for variable, name in mappings: + env[variable] = inputs[name] + step = re.split(r" run: \|[-+]?\n", text, maxsplit=1)[1] + script = "\n".join(line[8:] for line in step.splitlines()) + "\n" + # Windows's python3 app alias is not necessarily the interpreter running this test. + interpreter = shlex.quote(sys.executable.replace("\\", "/")) + script = f"python3() {{ {interpreter} \"$@\"; }}\n" + script + return subprocess.run( + [shutil.which("bash"), "--noprofile", "--norc", "-c", script], + env=env, cwd=self.work, text=True, capture_output=True, encoding="utf-8", + ) + + def stats(self): + return json.loads((self.work / "out" / "ai-commits-stats.json").read_text(encoding="utf-8")) + + def test_classification_files_and_step_outputs(self): + result = self.run_action(themes="true", name='owner/ & "name"') + self.assertEqual(result.returncode, 0, result.stderr) + st = self.stats() + expected = { + "commits": 4, "ai_commits": 2, "ai_percent": 50.0, + "agent": 1, "coauthored": 1, "unattributed": 2, "excluded_automation": 1, + } + for key, value in expected.items(): + self.assertEqual(st[key], value, key) + self.assertEqual(st["tools"], {"Copilot coding agent": 1, "Copilot CLI": 1}) + out = self.work / "out" + self.assertEqual({p.name for p in out.iterdir()}, { + "ai-commits.svg", "ai-commits.json", "ai-commits-stats.json", + "ai-commits-card.svg", "ai-commits-card-light.svg", "ai-commits-card-dark.svg", + }) + endpoint = json.loads((out / "ai-commits.json").read_text(encoding="utf-8")) + self.assertEqual(endpoint["schemaVersion"], 1) + self.assertEqual(endpoint["message"], "50%") + for path in out.glob("*.svg"): + svg = ET.parse(path).getroot() + self.assertIn("No recorded AI attribution", svg.get("aria-label")) + self.assertIn('owner/ & "name"', svg.find("svg:title", NS).text) + self.assertNotIn("human only", path.read_text(encoding="utf-8").lower()) + if "card" in path.name: + self.assertIsNone(svg.find(".//*[@textLength]")) + self.assertIsNone(svg.find(".//*[@lengthAdjust]")) + labels = svg.findall("svg:text[@y='121']", NS) + self.assertEqual(len(labels), 3) + for label in labels: + self.assertTrue(badge.fits("".join(label.itertext()), 12, 129)) + actual = dict(line.split("=", 1) for line in (self.work / "outputs").read_text().splitlines()) + self.assertEqual(actual, {key.replace("_", "-"): str(value) for key, value in expected.items()}) + text = (ACTION / "action.yml").read_text() + for key in actual: + self.assertIn(f"value: ${{{{ steps.generate.outputs.{key} }}}}", text) + + def test_shallow_guard(self): + shallow = self.work / "shallow" + self.git("clone", "--quiet", "--depth=1", self.repo.as_uri(), str(shallow), cwd=self.work) + result = self.run_action(**{"repo-dir": str(shallow), "since": "2019-01-01"}) + self.assertNotEqual(result.returncode, 0) + self.assertIn("fetch-depth: 0", result.stderr) + self.assertFalse((self.work / "out").exists()) + self.assertFalse((self.work / "outputs").exists()) + + def test_empty_denominator_and_default_files(self): + result = self.run_action(since="2100-01-01", label="Custom") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIsNone(self.stats()["ai_percent"]) + endpoint = json.loads((self.work / "out" / "ai-commits.json").read_text()) + self.assertEqual((endpoint["message"], endpoint["label"]), ("n/a", "Custom")) + self.assertEqual(len(list((self.work / "out").iterdir())), 4) + self.assertIn("ai-percent=n/a\n", (self.work / "outputs").read_text()) + + def test_central_identities_recognize_agent_bots_and_unknown_authors(self): + self.commit("google-labs-jules[bot]", "jules@example.org", "Jules") + self.commit("Unknown assistant", "unknown@example.org", "Unrecognized") + result = self.run_action() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.stats()["agent"], 2) + self.assertEqual(self.stats()["unattributed"], 3) + self.assertEqual(self.stats()["ai_percent"], 50.0) + self.assertEqual(self.stats()["excluded_automation"], 1) + self.assertEqual(self.stats()["tools"]["Jules"], 1) + + def test_invalid_inputs_fail_before_writing(self): + for overrides in ( + {"themes": "yes"}, + {"ref": "--all"}, {"ref": "missing"}, + ): + with self.subTest(overrides=overrides): + result = self.run_action(**overrides) + self.assertNotEqual(result.returncode, 0) + self.assertTrue(result.stderr) + self.assertFalse((self.work / "out").exists()) + self.assertFalse((self.work / "outputs").exists()) + + def test_ref_limits_measured_history(self): + result = self.run_action(ref=self.initial) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.stats()["commits"], 1) + self.assertEqual(self.stats()["ai_percent"], 0.0) + + def test_partial_identities_do_not_count_as_ai(self): + self.commit("Human", "noreply@anthropic.com.example.org", "Not Claude") + self.commit("Human", "alice+223556219+Copilot@example.org", "Not Copilot") + self.commit("not-copilot-swe-agent", "human@example.org", "Not the agent") + self.commit( + "Human", "human@example.org", + "Unrecognized trailers\n\n" + "Co-authored-by: Human \n" + "Co-authored-by: Human ", + ) + result = self.run_action() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.stats()["agent"], 1) + self.assertEqual(self.stats()["coauthored"], 1) + self.assertEqual(self.stats()["unattributed"], 6) + self.assertEqual(self.stats()["ai_percent"], 25.0) + + def test_display_percentage_avoids_double_rounding(self): + self.tip = None + self.commit("copilot-swe-agent[bot]", "agent@example.org", "Agent") + for i in range(21): + self.commit("Human", "human@example.org", f"Unattributed {i}") + result = self.run_action(themes="true") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.stats()["ai_percent"], 4.5) + out = self.work / "out" + endpoint = json.loads((out / "ai-commits.json").read_text(encoding="utf-8")) + self.assertEqual(endpoint["message"], "5%") + compact = ET.parse(out / "ai-commits.svg") + self.assertIn("5%", [text.text for text in compact.findall(".//svg:text", NS)]) + for path in out.glob("ai-commits-card*.svg"): + card = ET.parse(path) + self.assertEqual(card.find("svg:text[@y='74']/svg:tspan", NS).text, "5%") + self.assertEqual(card.find("svg:text[@y='121']/svg:tspan", NS).text, "5%") + self.assertIn("ai-percent=4.5\n", (self.work / "outputs").read_text()) + + def test_git_log_encoding_does_not_change_output_decoding(self): + self.commit("Jos\u00e9", "human@example.org", "Non-ASCII author") + self.git("config", "i18n.logOutputEncoding", "ISO-8859-1") + result = self.run_action() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.stats()["unattributed"], 3) + self.assertEqual(self.stats()["commits"], 5) + + def test_long_repository_name_stays_in_header(self): + name = "owner/" + "W" * 100 + result = self.run_action(themes="true", name=name) + self.assertEqual(result.returncode, 0, result.stderr) + for path in (self.work / "out").glob("ai-commits-card*.svg"): + card = ET.parse(path) + header = card.find("svg:text[@x='456'][@y='34']", NS) + clip = header.get("clip-path") + self.assertIsNotNone(clip) + rect = card.find(f"svg:clipPath[@id='{clip[5:-1]}']/svg:rect", NS) + self.assertIsNotNone(rect) + self.assertLessEqual(float(rect.get("width")), 200) + self.assertEqual(float(rect.get("x")) + float(rect.get("width")), float(header.get("x"))) + self.assertIn(name, card.find("svg:title", NS).text) + self.assertIn(name, card.getroot().get("aria-label")) + self.assertIsNone(card.find(".//*[@textLength]")) + self.assertIsNone(card.find(".//*[@lengthAdjust]")) + + +if __name__ == "__main__": + unittest.main(verbosity=2)