From 3ef974d4df3cd972e903430006e6fc53546fe31e Mon Sep 17 00:00:00 2001 From: Jean-Paul van Ravensberg <14926452+DevSecNinja@users.noreply.github.com> Date: Mon, 16 Nov 2026 11:31:16 +0100 Subject: [PATCH] chore(hooks): install the latest stable Python from .python-version with mise Make .python-version the single source of the Python the hooks and CI use: the latest stable minor that has been out for 90 days (3.14; 3.15 is only 38 days old). mise installs it from the file (idiomatic version file) for contributors and Windows CI. scripts/py.sh parses its major.minor, prefers mise's Python, rejects malformed values and points to `mise install`. The Ubuntu test job sets it up via python-version-file. test_py_launcher.py fails if py.sh, mise or CI stop reading the file. Renovate looks the version up on endoflife.date, which dates each minor by its x.y.0 release (the Docker tag the pyenv manager uses is re-pushed with every patch), and auto-merges a newer minor 90 days after release in its own PR. Closes #246 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/markdown-validation.yml | 12 +++ .github/workflows/windows-hooks.yml | 18 ++-- .lefthook.toml | 5 +- .mise.toml | 5 + .python-version | 1 + CONTRIBUTING.md | 35 +++++-- renovate.json5 | 22 ++++ scripts/py.sh | 75 ++++++++++--- tests/test_py_launcher.py | 122 +++++++++++++++++++--- 9 files changed, 244 insertions(+), 51 deletions(-) create mode 100644 .python-version diff --git a/.github/workflows/markdown-validation.yml b/.github/workflows/markdown-validation.yml index 688dd9d..d9595a9 100644 --- a/.github/workflows/markdown-validation.yml +++ b/.github/workflows/markdown-validation.yml @@ -12,6 +12,9 @@ on: - 'tests/**' - 'scripts/generate-index.sh' - 'scripts/py.sh' + - '.python-version' + - '.mise.toml' + - '.github/workflows/windows-hooks.yml' - 'scripts/apply-apm-overrides.py' - 'scripts/apm-external-skills.py' - '.agents/skills/**' @@ -31,6 +34,9 @@ on: - 'tests/**' - 'scripts/generate-index.sh' - 'scripts/py.sh' + - '.python-version' + - '.mise.toml' + - '.github/workflows/windows-hooks.yml' - 'scripts/apply-apm-overrides.py' - 'scripts/apm-external-skills.py' - '.agents/skills/**' @@ -71,6 +77,12 @@ jobs: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Test with the Python the hooks use (#246), not ubuntu-latest's python3. + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version-file: .python-version + - name: Run validation script run: |- bash tests/validate-prompts.sh diff --git a/.github/workflows/windows-hooks.yml b/.github/workflows/windows-hooks.yml index 6366e8b..3abe267 100644 --- a/.github/workflows/windows-hooks.yml +++ b/.github/workflows/windows-hooks.yml @@ -12,6 +12,7 @@ on: paths: - '.lefthook.toml' - '.mise.toml' + - '.python-version' - 'scripts/**' - 'tests/**' - '.apm/skills/*/scripts/**' @@ -20,6 +21,7 @@ on: paths: - '.lefthook.toml' - '.mise.toml' + - '.python-version' - 'scripts/**' - 'tests/**' - '.apm/skills/*/scripts/**' @@ -44,21 +46,15 @@ jobs: with: persist-credentials: false - # The floor scripts/py.sh accepts; tests/test_py_launcher.py keeps them equal. - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: "3.12" - - - name: Install lefthook + # Python comes from .python-version via .mise.toml, as for contributors, and + # scripts/py.sh picks mise's Python first (#246); test_py_launcher.py fails + # if this step stops installing it. + - name: Install lefthook and Python uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 with: # renovate: datasource=github-releases depName=jdx/mise version: "2026.10.19" - install_args: lefthook + install_args: lefthook python - name: Run pre-commit hooks - env: - # Make `py -3` pick the Python set up above, not a newer preinstalled one. - PY_PYTHON3: "3.12" run: '& (mise which lefthook) run pre-commit --all-files' diff --git a/.lefthook.toml b/.lefthook.toml index 2b98d70..b691c54 100644 --- a/.lefthook.toml +++ b/.lefthook.toml @@ -10,8 +10,9 @@ parallel = false [pre-commit.commands.validate-prompts] run = "bash tests/validate-prompts.sh" -# Python hooks go through scripts/py.sh, which finds Python >= 3.12 on Linux, -# macOS and native Windows (skipping the Microsoft Store python3 stub; #229). +# Python hooks go through scripts/py.sh, which runs the .python-version Python +# that mise installs, or another new enough one, on Linux, macOS and native +# Windows (skipping the Microsoft Store python3 stub; #229, #246). [pre-commit.commands.py-launcher] run = "PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_py_launcher.py'" diff --git a/.mise.toml b/.mise.toml index 9055a10..64f9ff9 100644 --- a/.mise.toml +++ b/.mise.toml @@ -14,3 +14,8 @@ trivy = "0.76.1" uv = "0.13.5" yamlfmt = "0.21.0" zizmor = "1.31.1" + +[settings] +# Install Python from .python-version, its single source (#246). The file holds +# major.minor, so mise installs that minor's latest patch. +idiomatic_version_file_enable_tools = ["python"] diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b46c38..01b69fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,10 +101,11 @@ for commands and clean-context behavior scenarios. This repo uses [lefthook](https://lefthook.dev) to run the same quality gates as CI before each commit. Inside the **Dev Container / Codespaces** the hooks are installed for you by `.devcontainer/post-create.sh`. For a local setup, install -the pinned lefthook (see [`.mise.toml`](/.mise.toml)) and wire up the hooks: +the pinned lefthook and the repo's Python (see [`.mise.toml`](/.mise.toml)) and +wire up the hooks: ```bash -mise install # installs the pinned lefthook +mise install # installs the pinned lefthook and Python mise exec -- lefthook install # wires up the git hooks ``` @@ -112,7 +113,8 @@ What runs on **pre-commit**: - `bash tests/validate-prompts.sh` — validates every primitive's frontmatter. - `PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_py_launcher.py'` - — checks the Python launcher the other hooks use (see below). + — checks the Python launcher the other hooks use and that it, mise and CI + take the Python version from `.python-version` (see below). - `PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_renovate_prs.py'` — exercises the Renovate helper with offline GitHub fixtures. - `PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_release_please_config.py'` @@ -133,17 +135,32 @@ What runs on **pre-commit**: **staged** primitives and stages only that result (unstaged README edits and untracked primitives stay out of the commit), so the index never drifts. -The Python hooks run through [`scripts/py.sh`](/scripts/py.sh), which uses the -first of `python3`, `py -3` (the Windows launcher) and `python` that is -**Python 3.12 or newer**, the version CI tests. It skips the Microsoft Store -`python3` stub and fails with a clear message when no suitable Python is found. +The Python hooks run on the version in [`.python-version`](/.python-version) +(major.minor, e.g. `3.14`), which `mise install` installs (with that minor's +latest patch). [`scripts/py.sh`](/scripts/py.sh) uses mise's Python, or else +the first of `python3`, `py -3` (the Windows launcher) and `python` that is at +least that version. It skips the Microsoft Store `python3` stub and, when no +suitable Python is found, fails with a clear message that says to run +`mise install`. + +`.python-version` is the only place the version lives: mise installs it +(`idiomatic_version_file_enable_tools` in `.mise.toml`), `py.sh` reads it, the +**Windows Hooks** workflow installs it with mise, the other CI jobs set it up +with `actions/setup-python`'s `python-version-file`, and +`tests/test_py_launcher.py` fails if any of them stops doing so. It holds the +**latest stable Python minor that has been out for 90 days**. Renovate dates +each minor by its x.y.0 release on [endoflife.date](https://endoflife.date/python) +and opens its own PR for a newer minor once it is 90 days old. That PR +auto-merges when CI, including the Windows hooks on the new Python, passes. +After it lands, run `mise install` to get the new Python. ### Windows The hooks run natively on Windows: lefthook runs them with Git for Windows' `sh`, so you don't need WSL. You need [Git for Windows](https://gitforwindows.org/) -and Python 3.12+ (`winget install Python.Python.3.12`); the `py` launcher finds -it even when `python3` is the Store stub. Clone with LF line endings +and [mise](https://mise.jdx.dev/) (`winget install jdx.mise`); `mise install` +downloads a prebuilt Python, and `py.sh` finds it even when `python3` is the +Store stub. Clone with LF line endings (`git config --global core.autocrlf false` or `input`) because the shell hooks break on CRLF. The **Windows Hooks** workflow runs the same suite on `windows-latest` for changes to hooks, scripts and tests. diff --git a/renovate.json5 b/renovate.json5 index 7a95aa7..c100a8c 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -56,5 +56,27 @@ commitMessageSuffix: "", addLabels: ["external-primitives", "merge: manual"], }, + { + // .python-version is the Python that mise installs for the hooks and CI + // sets up: the latest stable minor once it has been out for 90 days, so + // its first patches and tooling support have landed. The file holds + // major.minor; mise installs that minor's latest patch, so patches need + // no PR. The pyenv manager looks Python up as the `python` Docker image, + // whose `3.x` tag is re-pushed with every patch and image rebuild, so its + // timestamp never ages 90 days. endoflife.date dates each minor by its + // x.y.0 release instead; extractVersion turns its latest patch (3.14.10) + // into the minor (3.14). Its own PR keeps the 90-day bump separate from + // the weekly non-major group; the shared autoMerge preset merges a minor + // bump once CI, including the Windows hooks on the new Python, passes + // (a major, Python 4, stays manual like every major). + description: "Track the latest stable Python minor in .python-version, adopted 90 days after its release and auto-merged", + matchManagers: ["pyenv"], + matchFileNames: [".python-version"], + overrideDatasource: "endoflife-date", + overridePackageName: "python", + extractVersion: "^(?\\d+\\.\\d+)", + minimumReleaseAge: "90 days", + groupName: null, + }, ], } diff --git a/scripts/py.sh b/scripts/py.sh index 3f92c4e..20c3d66 100755 --- a/scripts/py.sh +++ b/scripts/py.sh @@ -1,27 +1,75 @@ #!/bin/sh -# Run Python portably for the git hooks (#229): use the first of python3, the -# Windows `py -3` launcher and python that is at least the minimum version. -# Every candidate is probed with arguments, so the Microsoft Store stub +# Run Python portably for the git hooks (#229, #246). The version is the +# major.minor in the repo's .python-version, which mise installs (.mise.toml) +# and CI sets up. Use mise's Python for it, else the first of python3, the +# Windows `py -3` launcher and python that is at least that version. Every +# candidate is probed with arguments, so the Microsoft Store stub # (WindowsApps\python3.exe), which only prints an install hint, is skipped. -# Keep `min_minor` in sync with python-version in -# .github/workflows/windows-hooks.yml; Ubuntu CI uses ubuntu-latest's python3. +# Only shell builtins are used, so this works even when PATH has no coreutils. set -u -min_major=3 -min_minor=12 +dir="${0%/*}" +[ "${dir}" = "$0" ] && dir="${0%\\*}" # a Windows path, e.g. from Python's subprocess +[ "${dir}" = "$0" ] && dir="." +version_file="${dir}/../.python-version" +minimum="" +if [ -r "${version_file}" ]; then + # `|| [ -n ... ]` keeps a last line that has no trailing newline. + while IFS= read -r line || [ -n "${line}" ]; do + line="${line%"${line##*[![:space:]]}"}" # trailing space, CR (CRLF checkout) + line="${line#"${line%%[![:space:]]*}"}" # leading space + case "${line}" in + '' | '#'*) continue ;; + *) ;; + esac + minimum="${line}" + break + done <"${version_file}" +fi +# major.minor with an optional numeric patch (3.14 or 3.14.2); only major.minor counts. +case "${minimum}" in +*[!0-9.]* | .* | *. | *..* | *.*.*.*) bad=1 ;; +*) bad="" ;; +esac +case "${minimum}" in +*.*) ;; +*) bad=1 ;; +esac +if [ -n "${bad}" ]; then + printf '%s\n' "scripts/py.sh: ${version_file} must hold a Python version such as 3.14 or 3.14.2, not '${minimum}'." >&2 + exit 1 +fi +min_major="${minimum%%.*}" +min_minor="${minimum#*.}" +min_minor="${min_minor%%.*}" nl=' ' tried="" -for candidate in python3 "py -3" python; do - if ! path="$(command -v "${candidate%% *}" 2>/dev/null)"; then +probe='import sys; sys.stdout.write("%d.%d" % sys.version_info[:2])' + +for candidate in mise python3 "py -3" python; do + if [ "${candidate}" = mise ]; then + # `mise which` finds the installed .python-version Python whether or not + # mise is activated in this shell; it fails when that one isn't installed. + if ! command -v mise >/dev/null 2>&1; then + tried="${tried}${nl} mise: not on PATH" + continue + fi + if ! path="$(mise -C "${dir}/.." which python 2>/dev/null)" || [ -z "${path}" ]; then + tried="${tried}${nl} mise: Python ${min_major}.${min_minor} not installed" + continue + fi + version="$("${path}" -c "${probe}" 2>/dev/null)" + elif ! path="$(command -v "${candidate%% *}" 2>/dev/null)"; then tried="${tried}${nl} ${candidate}: not on PATH" continue + else + # shellcheck disable=SC2086 # split "py -3" into the command and its flag + version="$(${candidate} -c "${probe}" 2>/dev/null)" fi - # shellcheck disable=SC2086 # split "py -3" into the command and its flag - version="$(${candidate} -c 'import sys; sys.stdout.write("%d.%d" % sys.version_info[:2])' 2>/dev/null)" case "${version}" in *[!0-9.]* | *.*.*) version="" ;; *) ;; @@ -41,6 +89,7 @@ for candidate in python3 "py -3" python; do major="${version%%.*}" minor="${version#*.}" if [ "$((major * 1000 + minor))" -ge "$((min_major * 1000 + min_minor))" ]; then + [ "${candidate}" = mise ] && exec "${path}" "$@" # shellcheck disable=SC2086 # see above exec ${candidate} "$@" fi @@ -49,6 +98,6 @@ done # printf is a builtin, so this still prints when PATH has no coreutils. printf '%s\n' "scripts/py.sh: no Python >= ${min_major}.${min_minor} found. Tried:${tried}" \ - "Install Python ${min_major}.${min_minor}+ from https://www.python.org/downloads/" \ - "(on Windows: \`winget install Python.Python.${min_major}.${min_minor}\`; the hooks find it via \`py -3\`)." >&2 + "Run \`mise install\` in the repo to install Python ${min_major}.${min_minor} from .python-version" \ + "(see CONTRIBUTING.md), or install Python ${min_major}.${min_minor}+ from https://www.python.org/downloads/." >&2 exit 1 diff --git a/tests/test_py_launcher.py b/tests/test_py_launcher.py index 01c1fb2..f630a67 100644 --- a/tests/test_py_launcher.py +++ b/tests/test_py_launcher.py @@ -1,10 +1,13 @@ -"""scripts/py.sh picks the first Python that meets the minimum (#229).""" +"""scripts/py.sh runs the hooks on the Python in .python-version, the single +source that mise installs (#229, #246).""" import os from pathlib import Path import re +import shutil import subprocess import tempfile +import tomllib import unittest from posix_shell import posix_shell @@ -12,7 +15,10 @@ ROOT = Path(__file__).resolve().parents[1] LAUNCHER = ROOT / "scripts/py.sh" -WINDOWS_WORKFLOW = ROOT / ".github/workflows/windows-hooks.yml" +PYTHON_VERSION = ROOT / ".python-version" +MISE_CONFIG = ROOT / ".mise.toml" +WORKFLOWS = ROOT / ".github/workflows" +WINDOWS_WORKFLOW = WORKFLOWS / "windows-hooks.yml" STUB = '#!/bin/sh\necho "Python was not found; run without arguments to install from the Microsoft Store"\nexit 49\n' @@ -22,18 +28,55 @@ def interpreter(name, version): f'if [ "$1" = -c ]; then echo {version}; else echo "{name} $*"; fi\n') +def mise(python): + # `mise -C which python` prints the installed Python, if there is one. + found = f'echo "{python}"; exit 0' if python else "echo 'python@3.12 is not installed' >&2; exit 1" + return (f'#!/bin/sh\n[ "$1" = -C ] && [ -r "$2/.python-version" ] || exit 2\n' + f'shift 2\n[ "$*" = "which python" ] || exit 2\n{found}\n') + + +def run(launcher, path): + return subprocess.run([posix_shell("sh"), str(launcher), "-m", "unittest"], + env=dict(os.environ, PATH=path), + capture_output=True, encoding="utf-8", timeout=20, check=False) + + class PyLauncherTests(unittest.TestCase): - def run_launcher(self, **fakes): + def run_launcher(self, minimum="3.12\n", mise_python=None, **fakes): + # A copy of the launcher in a fake repo, so its .python-version is ours. + # mise_python: the version mise has installed, "" for none, None for no mise. with tempfile.TemporaryDirectory(prefix="py-launcher-") as temp: - store = Path(temp) / "WindowsApps" - store.mkdir() + repo, bin_dir, installs = Path(temp) / "repo", Path(temp) / "bin", Path(temp) / "mise" + store = bin_dir / "WindowsApps" + (repo / "scripts").mkdir(parents=True) + store.mkdir(parents=True) + installs.mkdir() + shutil.copyfile(LAUNCHER, repo / "scripts/py.sh") + if minimum is not None: + (repo / ".python-version").write_bytes(minimum.encode()) + if mise_python is not None: + python = installs / "python" + if mise_python: + python.write_text(interpreter("mise-python", mise_python), newline="\n") + python.chmod(0o755) + fakes["mise"] = mise(python.as_posix() if mise_python else "") for name, body in fakes.items(): - path = (store if body == STUB else Path(temp)) / name + path = (store if body == STUB else bin_dir) / name path.write_text(body, newline="\n") path.chmod(0o755) - return subprocess.run([posix_shell("sh"), str(LAUNCHER), "-m", "unittest"], - env=dict(os.environ, PATH=f"{store}{os.pathsep}{temp}"), - capture_output=True, encoding="utf-8", timeout=20, check=False) + return run(repo / "scripts/py.sh", f"{store}{os.pathsep}{bin_dir}") + + def test_prefers_mise_python(self): + result = self.run_launcher(mise_python="3.12", python3=interpreter("python3", "3.13")) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "mise-python -m unittest") + + def test_falls_back_when_mise_has_no_new_enough_python(self): + for mise_python in ("", "3.11"): + with self.subTest(mise_python=mise_python): + result = self.run_launcher(mise_python=mise_python, python3=interpreter("python3", "3.12")) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "python3 -m unittest") def test_skips_store_stub_and_old_python_for_first_new_enough(self): result = self.run_launcher(python3=STUB, py=interpreter("py", "3.9"), @@ -53,19 +96,66 @@ def test_fails_clearly_without_a_new_enough_python(self): self.assertEqual(result.returncode, 1) self.assertEqual(result.stdout, "") self.assertIn("no Python >= 3.12 found", result.stderr) + self.assertIn("mise: not on PATH", result.stderr) + self.assertIn("Run `mise install` in the repo to install Python 3.12", result.stderr) self.assertIn("Microsoft Store stub, runs no code", result.stderr) self.assertIn("py -3 (", result.stderr) self.assertIn("Python 3.9, too old", result.stderr) self.assertIn("python: not on PATH", result.stderr) self.assertIn("python.org", result.stderr) + result = self.run_launcher(mise_python="") + self.assertIn("mise: Python 3.12 not installed", result.stderr) + + def test_reads_major_minor_from_python_version_file(self): + for minimum in ("3.13", "3.13\n", "3.13.1\n", "3.13\r\n", "# floor\n\n 3.13 \n3.9\n"): + with self.subTest(minimum=minimum): + result = self.run_launcher(minimum, python3=interpreter("python3", "3.12"), + python=interpreter("python", "3.13")) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "python -m unittest") + + def test_rejects_a_missing_or_malformed_python_version_file(self): + for minimum in (None, "", "\n", "3", "3.", ".12", "three.12", "3.12-dev", + "3..12", "3.12.", "3.12.garbage", "3.12.4.5"): + with self.subTest(minimum=minimum): + result = self.run_launcher(minimum, python=interpreter("python", "3.13")) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, "") + self.assertIn(".python-version must hold a Python version such as 3.14 or 3.14.2", result.stderr) + + +class PythonVersionSourceTests(unittest.TestCase): + """.python-version is the only place the version lives; mise, py.sh and CI read it.""" + + def test_python_version_file_holds_major_minor(self): + self.assertRegex(PYTHON_VERSION.read_text(encoding="utf-8"), r"\A\d+\.\d+\n\Z") + + def test_launcher_enforces_the_file_version(self): + version = PYTHON_VERSION.read_text(encoding="utf-8").strip() + self.assertNotRegex(LAUNCHER.read_text(encoding="utf-8"), r"(?m)^\s*min_(major|minor)=['\"]?\d") + with tempfile.TemporaryDirectory(prefix="py-launcher-") as empty: + result = run(LAUNCHER, empty) + self.assertEqual(result.returncode, 1) + self.assertIn(f"no Python >= {version} found", result.stderr) + + def test_mise_installs_python_from_the_file(self): + config = tomllib.loads(MISE_CONFIG.read_text(encoding="utf-8")) + self.assertIn("python", config.get("settings", {}).get("idiomatic_version_file_enable_tools", [])) + self.assertNotIn("python", config.get("tools", {}), "pin Python in .python-version only") + + def test_windows_ci_installs_python_with_mise(self): + workflow = WINDOWS_WORKFLOW.read_text(encoding="utf-8") + self.assertRegex(workflow, r"(?m)^\s+install_args:.*\bpython\b") + self.assertNotRegex(workflow, r"PY_PYTHON3|actions/setup-python@") - def test_minimum_matches_windows_ci_python(self): - launcher = LAUNCHER.read_text() - minimum = "{}.{}".format(re.search(r"(?m)^min_major=(\d+)$", launcher)[1], - re.search(r"(?m)^min_minor=(\d+)$", launcher)[1]) - ci = re.search(r"python-version:\s*['\"]?([\d.]+)", WINDOWS_WORKFLOW.read_text())[1] - self.assertEqual(minimum, ci) + def test_every_setup_python_step_reads_the_file(self): + for workflow in sorted(WORKFLOWS.glob("*.y*ml")): + text = workflow.read_text(encoding="utf-8") + with self.subTest(workflow=workflow.name): + self.assertNotRegex(text, r"(?m)^\s+python-version:") + self.assertEqual(len(re.findall(r"uses:\s*actions/setup-python@", text)), + len(re.findall(r"(?m)^\s+python-version-file:\s*['\"]?\.python-version['\"]?\s*$", text))) if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file