diff --git a/.github/workflows/markdown-validation.yml b/.github/workflows/markdown-validation.yml index 688dd9d..d9595a9 100644 --- a/.github/workflows/markdown-validation.yml +++ b/.github/workflows/markdown-validation.yml @@ -12,6 +12,9 @@ on: - 'tests/**' - 'scripts/generate-index.sh' - 'scripts/py.sh' + - '.python-version' + - '.mise.toml' + - '.github/workflows/windows-hooks.yml' - 'scripts/apply-apm-overrides.py' - 'scripts/apm-external-skills.py' - '.agents/skills/**' @@ -31,6 +34,9 @@ on: - 'tests/**' - 'scripts/generate-index.sh' - 'scripts/py.sh' + - '.python-version' + - '.mise.toml' + - '.github/workflows/windows-hooks.yml' - 'scripts/apply-apm-overrides.py' - 'scripts/apm-external-skills.py' - '.agents/skills/**' @@ -71,6 +77,12 @@ jobs: - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + # Test with the Python the hooks use (#246), not ubuntu-latest's python3. + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version-file: .python-version + - name: Run validation script run: |- bash tests/validate-prompts.sh diff --git a/.github/workflows/windows-hooks.yml b/.github/workflows/windows-hooks.yml index 6366e8b..3abe267 100644 --- a/.github/workflows/windows-hooks.yml +++ b/.github/workflows/windows-hooks.yml @@ -12,6 +12,7 @@ on: paths: - '.lefthook.toml' - '.mise.toml' + - '.python-version' - 'scripts/**' - 'tests/**' - '.apm/skills/*/scripts/**' @@ -20,6 +21,7 @@ on: paths: - '.lefthook.toml' - '.mise.toml' + - '.python-version' - 'scripts/**' - 'tests/**' - '.apm/skills/*/scripts/**' @@ -44,21 +46,15 @@ jobs: with: persist-credentials: false - # The floor scripts/py.sh accepts; tests/test_py_launcher.py keeps them equal. - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: "3.12" - - - name: Install lefthook + # Python comes from .python-version via .mise.toml, as for contributors, and + # scripts/py.sh picks mise's Python first (#246); test_py_launcher.py fails + # if this step stops installing it. + - name: Install lefthook and Python uses: jdx/mise-action@c2a87611a18de5b3828c5652fe268e992400cb5c # v4.3.0 with: # renovate: datasource=github-releases depName=jdx/mise version: "2026.10.19" - install_args: lefthook + install_args: lefthook python - name: Run pre-commit hooks - env: - # Make `py -3` pick the Python set up above, not a newer preinstalled one. - PY_PYTHON3: "3.12" run: '& (mise which lefthook) run pre-commit --all-files' diff --git a/.lefthook.toml b/.lefthook.toml index 2b98d70..b691c54 100644 --- a/.lefthook.toml +++ b/.lefthook.toml @@ -10,8 +10,9 @@ parallel = false [pre-commit.commands.validate-prompts] run = "bash tests/validate-prompts.sh" -# Python hooks go through scripts/py.sh, which finds Python >= 3.12 on Linux, -# macOS and native Windows (skipping the Microsoft Store python3 stub; #229). +# Python hooks go through scripts/py.sh, which runs the .python-version Python +# that mise installs, or another new enough one, on Linux, macOS and native +# Windows (skipping the Microsoft Store python3 stub; #229, #246). [pre-commit.commands.py-launcher] run = "PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_py_launcher.py'" diff --git a/.mise.toml b/.mise.toml index 9055a10..64f9ff9 100644 --- a/.mise.toml +++ b/.mise.toml @@ -14,3 +14,8 @@ trivy = "0.76.1" uv = "0.13.5" yamlfmt = "0.21.0" zizmor = "1.31.1" + +[settings] +# Install Python from .python-version, its single source (#246). The file holds +# major.minor, so mise installs that minor's latest patch. +idiomatic_version_file_enable_tools = ["python"] diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..6324d40 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.14 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b46c38..01b69fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,10 +101,11 @@ for commands and clean-context behavior scenarios. This repo uses [lefthook](https://lefthook.dev) to run the same quality gates as CI before each commit. Inside the **Dev Container / Codespaces** the hooks are installed for you by `.devcontainer/post-create.sh`. For a local setup, install -the pinned lefthook (see [`.mise.toml`](/.mise.toml)) and wire up the hooks: +the pinned lefthook and the repo's Python (see [`.mise.toml`](/.mise.toml)) and +wire up the hooks: ```bash -mise install # installs the pinned lefthook +mise install # installs the pinned lefthook and Python mise exec -- lefthook install # wires up the git hooks ``` @@ -112,7 +113,8 @@ What runs on **pre-commit**: - `bash tests/validate-prompts.sh` — validates every primitive's frontmatter. - `PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_py_launcher.py'` - — checks the Python launcher the other hooks use (see below). + — checks the Python launcher the other hooks use and that it, mise and CI + take the Python version from `.python-version` (see below). - `PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_renovate_prs.py'` — exercises the Renovate helper with offline GitHub fixtures. - `PYTHONDONTWRITEBYTECODE=1 sh scripts/py.sh -m unittest discover -s tests -p 'test_release_please_config.py'` @@ -133,17 +135,32 @@ What runs on **pre-commit**: **staged** primitives and stages only that result (unstaged README edits and untracked primitives stay out of the commit), so the index never drifts. -The Python hooks run through [`scripts/py.sh`](/scripts/py.sh), which uses the -first of `python3`, `py -3` (the Windows launcher) and `python` that is -**Python 3.12 or newer**, the version CI tests. It skips the Microsoft Store -`python3` stub and fails with a clear message when no suitable Python is found. +The Python hooks run on the version in [`.python-version`](/.python-version) +(major.minor, e.g. `3.14`), which `mise install` installs (with that minor's +latest patch). [`scripts/py.sh`](/scripts/py.sh) uses mise's Python, or else +the first of `python3`, `py -3` (the Windows launcher) and `python` that is at +least that version. It skips the Microsoft Store `python3` stub and, when no +suitable Python is found, fails with a clear message that says to run +`mise install`. + +`.python-version` is the only place the version lives: mise installs it +(`idiomatic_version_file_enable_tools` in `.mise.toml`), `py.sh` reads it, the +**Windows Hooks** workflow installs it with mise, the other CI jobs set it up +with `actions/setup-python`'s `python-version-file`, and +`tests/test_py_launcher.py` fails if any of them stops doing so. It holds the +**latest stable Python minor that has been out for 90 days**. Renovate dates +each minor by its x.y.0 release on [endoflife.date](https://endoflife.date/python) +and opens its own PR for a newer minor once it is 90 days old. That PR +auto-merges when CI, including the Windows hooks on the new Python, passes. +After it lands, run `mise install` to get the new Python. ### Windows The hooks run natively on Windows: lefthook runs them with Git for Windows' `sh`, so you don't need WSL. You need [Git for Windows](https://gitforwindows.org/) -and Python 3.12+ (`winget install Python.Python.3.12`); the `py` launcher finds -it even when `python3` is the Store stub. Clone with LF line endings +and [mise](https://mise.jdx.dev/) (`winget install jdx.mise`); `mise install` +downloads a prebuilt Python, and `py.sh` finds it even when `python3` is the +Store stub. Clone with LF line endings (`git config --global core.autocrlf false` or `input`) because the shell hooks break on CRLF. The **Windows Hooks** workflow runs the same suite on `windows-latest` for changes to hooks, scripts and tests. diff --git a/renovate.json5 b/renovate.json5 index 7a95aa7..c100a8c 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -56,5 +56,27 @@ commitMessageSuffix: "", addLabels: ["external-primitives", "merge: manual"], }, + { + // .python-version is the Python that mise installs for the hooks and CI + // sets up: the latest stable minor once it has been out for 90 days, so + // its first patches and tooling support have landed. The file holds + // major.minor; mise installs that minor's latest patch, so patches need + // no PR. The pyenv manager looks Python up as the `python` Docker image, + // whose `3.x` tag is re-pushed with every patch and image rebuild, so its + // timestamp never ages 90 days. endoflife.date dates each minor by its + // x.y.0 release instead; extractVersion turns its latest patch (3.14.10) + // into the minor (3.14). Its own PR keeps the 90-day bump separate from + // the weekly non-major group; the shared autoMerge preset merges a minor + // bump once CI, including the Windows hooks on the new Python, passes + // (a major, Python 4, stays manual like every major). + description: "Track the latest stable Python minor in .python-version, adopted 90 days after its release and auto-merged", + matchManagers: ["pyenv"], + matchFileNames: [".python-version"], + overrideDatasource: "endoflife-date", + overridePackageName: "python", + extractVersion: "^(?\\d+\\.\\d+)", + minimumReleaseAge: "90 days", + groupName: null, + }, ], } diff --git a/scripts/py.sh b/scripts/py.sh index 3f92c4e..20c3d66 100755 --- a/scripts/py.sh +++ b/scripts/py.sh @@ -1,27 +1,75 @@ #!/bin/sh -# Run Python portably for the git hooks (#229): use the first of python3, the -# Windows `py -3` launcher and python that is at least the minimum version. -# Every candidate is probed with arguments, so the Microsoft Store stub +# Run Python portably for the git hooks (#229, #246). The version is the +# major.minor in the repo's .python-version, which mise installs (.mise.toml) +# and CI sets up. Use mise's Python for it, else the first of python3, the +# Windows `py -3` launcher and python that is at least that version. Every +# candidate is probed with arguments, so the Microsoft Store stub # (WindowsApps\python3.exe), which only prints an install hint, is skipped. -# Keep `min_minor` in sync with python-version in -# .github/workflows/windows-hooks.yml; Ubuntu CI uses ubuntu-latest's python3. +# Only shell builtins are used, so this works even when PATH has no coreutils. set -u -min_major=3 -min_minor=12 +dir="${0%/*}" +[ "${dir}" = "$0" ] && dir="${0%\\*}" # a Windows path, e.g. from Python's subprocess +[ "${dir}" = "$0" ] && dir="." +version_file="${dir}/../.python-version" +minimum="" +if [ -r "${version_file}" ]; then + # `|| [ -n ... ]` keeps a last line that has no trailing newline. + while IFS= read -r line || [ -n "${line}" ]; do + line="${line%"${line##*[![:space:]]}"}" # trailing space, CR (CRLF checkout) + line="${line#"${line%%[![:space:]]*}"}" # leading space + case "${line}" in + '' | '#'*) continue ;; + *) ;; + esac + minimum="${line}" + break + done <"${version_file}" +fi +# major.minor with an optional numeric patch (3.14 or 3.14.2); only major.minor counts. +case "${minimum}" in +*[!0-9.]* | .* | *. | *..* | *.*.*.*) bad=1 ;; +*) bad="" ;; +esac +case "${minimum}" in +*.*) ;; +*) bad=1 ;; +esac +if [ -n "${bad}" ]; then + printf '%s\n' "scripts/py.sh: ${version_file} must hold a Python version such as 3.14 or 3.14.2, not '${minimum}'." >&2 + exit 1 +fi +min_major="${minimum%%.*}" +min_minor="${minimum#*.}" +min_minor="${min_minor%%.*}" nl=' ' tried="" -for candidate in python3 "py -3" python; do - if ! path="$(command -v "${candidate%% *}" 2>/dev/null)"; then +probe='import sys; sys.stdout.write("%d.%d" % sys.version_info[:2])' + +for candidate in mise python3 "py -3" python; do + if [ "${candidate}" = mise ]; then + # `mise which` finds the installed .python-version Python whether or not + # mise is activated in this shell; it fails when that one isn't installed. + if ! command -v mise >/dev/null 2>&1; then + tried="${tried}${nl} mise: not on PATH" + continue + fi + if ! path="$(mise -C "${dir}/.." which python 2>/dev/null)" || [ -z "${path}" ]; then + tried="${tried}${nl} mise: Python ${min_major}.${min_minor} not installed" + continue + fi + version="$("${path}" -c "${probe}" 2>/dev/null)" + elif ! path="$(command -v "${candidate%% *}" 2>/dev/null)"; then tried="${tried}${nl} ${candidate}: not on PATH" continue + else + # shellcheck disable=SC2086 # split "py -3" into the command and its flag + version="$(${candidate} -c "${probe}" 2>/dev/null)" fi - # shellcheck disable=SC2086 # split "py -3" into the command and its flag - version="$(${candidate} -c 'import sys; sys.stdout.write("%d.%d" % sys.version_info[:2])' 2>/dev/null)" case "${version}" in *[!0-9.]* | *.*.*) version="" ;; *) ;; @@ -41,6 +89,7 @@ for candidate in python3 "py -3" python; do major="${version%%.*}" minor="${version#*.}" if [ "$((major * 1000 + minor))" -ge "$((min_major * 1000 + min_minor))" ]; then + [ "${candidate}" = mise ] && exec "${path}" "$@" # shellcheck disable=SC2086 # see above exec ${candidate} "$@" fi @@ -49,6 +98,6 @@ done # printf is a builtin, so this still prints when PATH has no coreutils. printf '%s\n' "scripts/py.sh: no Python >= ${min_major}.${min_minor} found. Tried:${tried}" \ - "Install Python ${min_major}.${min_minor}+ from https://www.python.org/downloads/" \ - "(on Windows: \`winget install Python.Python.${min_major}.${min_minor}\`; the hooks find it via \`py -3\`)." >&2 + "Run \`mise install\` in the repo to install Python ${min_major}.${min_minor} from .python-version" \ + "(see CONTRIBUTING.md), or install Python ${min_major}.${min_minor}+ from https://www.python.org/downloads/." >&2 exit 1 diff --git a/tests/test_py_launcher.py b/tests/test_py_launcher.py index 01c1fb2..f630a67 100644 --- a/tests/test_py_launcher.py +++ b/tests/test_py_launcher.py @@ -1,10 +1,13 @@ -"""scripts/py.sh picks the first Python that meets the minimum (#229).""" +"""scripts/py.sh runs the hooks on the Python in .python-version, the single +source that mise installs (#229, #246).""" import os from pathlib import Path import re +import shutil import subprocess import tempfile +import tomllib import unittest from posix_shell import posix_shell @@ -12,7 +15,10 @@ ROOT = Path(__file__).resolve().parents[1] LAUNCHER = ROOT / "scripts/py.sh" -WINDOWS_WORKFLOW = ROOT / ".github/workflows/windows-hooks.yml" +PYTHON_VERSION = ROOT / ".python-version" +MISE_CONFIG = ROOT / ".mise.toml" +WORKFLOWS = ROOT / ".github/workflows" +WINDOWS_WORKFLOW = WORKFLOWS / "windows-hooks.yml" STUB = '#!/bin/sh\necho "Python was not found; run without arguments to install from the Microsoft Store"\nexit 49\n' @@ -22,18 +28,55 @@ def interpreter(name, version): f'if [ "$1" = -c ]; then echo {version}; else echo "{name} $*"; fi\n') +def mise(python): + # `mise -C which python` prints the installed Python, if there is one. + found = f'echo "{python}"; exit 0' if python else "echo 'python@3.12 is not installed' >&2; exit 1" + return (f'#!/bin/sh\n[ "$1" = -C ] && [ -r "$2/.python-version" ] || exit 2\n' + f'shift 2\n[ "$*" = "which python" ] || exit 2\n{found}\n') + + +def run(launcher, path): + return subprocess.run([posix_shell("sh"), str(launcher), "-m", "unittest"], + env=dict(os.environ, PATH=path), + capture_output=True, encoding="utf-8", timeout=20, check=False) + + class PyLauncherTests(unittest.TestCase): - def run_launcher(self, **fakes): + def run_launcher(self, minimum="3.12\n", mise_python=None, **fakes): + # A copy of the launcher in a fake repo, so its .python-version is ours. + # mise_python: the version mise has installed, "" for none, None for no mise. with tempfile.TemporaryDirectory(prefix="py-launcher-") as temp: - store = Path(temp) / "WindowsApps" - store.mkdir() + repo, bin_dir, installs = Path(temp) / "repo", Path(temp) / "bin", Path(temp) / "mise" + store = bin_dir / "WindowsApps" + (repo / "scripts").mkdir(parents=True) + store.mkdir(parents=True) + installs.mkdir() + shutil.copyfile(LAUNCHER, repo / "scripts/py.sh") + if minimum is not None: + (repo / ".python-version").write_bytes(minimum.encode()) + if mise_python is not None: + python = installs / "python" + if mise_python: + python.write_text(interpreter("mise-python", mise_python), newline="\n") + python.chmod(0o755) + fakes["mise"] = mise(python.as_posix() if mise_python else "") for name, body in fakes.items(): - path = (store if body == STUB else Path(temp)) / name + path = (store if body == STUB else bin_dir) / name path.write_text(body, newline="\n") path.chmod(0o755) - return subprocess.run([posix_shell("sh"), str(LAUNCHER), "-m", "unittest"], - env=dict(os.environ, PATH=f"{store}{os.pathsep}{temp}"), - capture_output=True, encoding="utf-8", timeout=20, check=False) + return run(repo / "scripts/py.sh", f"{store}{os.pathsep}{bin_dir}") + + def test_prefers_mise_python(self): + result = self.run_launcher(mise_python="3.12", python3=interpreter("python3", "3.13")) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "mise-python -m unittest") + + def test_falls_back_when_mise_has_no_new_enough_python(self): + for mise_python in ("", "3.11"): + with self.subTest(mise_python=mise_python): + result = self.run_launcher(mise_python=mise_python, python3=interpreter("python3", "3.12")) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "python3 -m unittest") def test_skips_store_stub_and_old_python_for_first_new_enough(self): result = self.run_launcher(python3=STUB, py=interpreter("py", "3.9"), @@ -53,19 +96,66 @@ def test_fails_clearly_without_a_new_enough_python(self): self.assertEqual(result.returncode, 1) self.assertEqual(result.stdout, "") self.assertIn("no Python >= 3.12 found", result.stderr) + self.assertIn("mise: not on PATH", result.stderr) + self.assertIn("Run `mise install` in the repo to install Python 3.12", result.stderr) self.assertIn("Microsoft Store stub, runs no code", result.stderr) self.assertIn("py -3 (", result.stderr) self.assertIn("Python 3.9, too old", result.stderr) self.assertIn("python: not on PATH", result.stderr) self.assertIn("python.org", result.stderr) + result = self.run_launcher(mise_python="") + self.assertIn("mise: Python 3.12 not installed", result.stderr) + + def test_reads_major_minor_from_python_version_file(self): + for minimum in ("3.13", "3.13\n", "3.13.1\n", "3.13\r\n", "# floor\n\n 3.13 \n3.9\n"): + with self.subTest(minimum=minimum): + result = self.run_launcher(minimum, python3=interpreter("python3", "3.12"), + python=interpreter("python", "3.13")) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.strip(), "python -m unittest") + + def test_rejects_a_missing_or_malformed_python_version_file(self): + for minimum in (None, "", "\n", "3", "3.", ".12", "three.12", "3.12-dev", + "3..12", "3.12.", "3.12.garbage", "3.12.4.5"): + with self.subTest(minimum=minimum): + result = self.run_launcher(minimum, python=interpreter("python", "3.13")) + self.assertEqual(result.returncode, 1) + self.assertEqual(result.stdout, "") + self.assertIn(".python-version must hold a Python version such as 3.14 or 3.14.2", result.stderr) + + +class PythonVersionSourceTests(unittest.TestCase): + """.python-version is the only place the version lives; mise, py.sh and CI read it.""" + + def test_python_version_file_holds_major_minor(self): + self.assertRegex(PYTHON_VERSION.read_text(encoding="utf-8"), r"\A\d+\.\d+\n\Z") + + def test_launcher_enforces_the_file_version(self): + version = PYTHON_VERSION.read_text(encoding="utf-8").strip() + self.assertNotRegex(LAUNCHER.read_text(encoding="utf-8"), r"(?m)^\s*min_(major|minor)=['\"]?\d") + with tempfile.TemporaryDirectory(prefix="py-launcher-") as empty: + result = run(LAUNCHER, empty) + self.assertEqual(result.returncode, 1) + self.assertIn(f"no Python >= {version} found", result.stderr) + + def test_mise_installs_python_from_the_file(self): + config = tomllib.loads(MISE_CONFIG.read_text(encoding="utf-8")) + self.assertIn("python", config.get("settings", {}).get("idiomatic_version_file_enable_tools", [])) + self.assertNotIn("python", config.get("tools", {}), "pin Python in .python-version only") + + def test_windows_ci_installs_python_with_mise(self): + workflow = WINDOWS_WORKFLOW.read_text(encoding="utf-8") + self.assertRegex(workflow, r"(?m)^\s+install_args:.*\bpython\b") + self.assertNotRegex(workflow, r"PY_PYTHON3|actions/setup-python@") - def test_minimum_matches_windows_ci_python(self): - launcher = LAUNCHER.read_text() - minimum = "{}.{}".format(re.search(r"(?m)^min_major=(\d+)$", launcher)[1], - re.search(r"(?m)^min_minor=(\d+)$", launcher)[1]) - ci = re.search(r"python-version:\s*['\"]?([\d.]+)", WINDOWS_WORKFLOW.read_text())[1] - self.assertEqual(minimum, ci) + def test_every_setup_python_step_reads_the_file(self): + for workflow in sorted(WORKFLOWS.glob("*.y*ml")): + text = workflow.read_text(encoding="utf-8") + with self.subTest(workflow=workflow.name): + self.assertNotRegex(text, r"(?m)^\s+python-version:") + self.assertEqual(len(re.findall(r"uses:\s*actions/setup-python@", text)), + len(re.findall(r"(?m)^\s+python-version-file:\s*['\"]?\.python-version['\"]?\s*$", text))) if __name__ == "__main__": - unittest.main() + unittest.main() \ No newline at end of file