From 20e1003159599d879568408a7355acd499ba81ae Mon Sep 17 00:00:00 2001 From: "Calvin A. Allen" Date: Thu, 22 Oct 2026 18:33:02 -0400 Subject: [PATCH 1/2] fix(mirror): build full Windows Python archives instead of embeddable zips For Python versions python-build-standalone has no Windows build for, the mirror served python.org's embeddable zips. Those are a trimmed runtime for embedding in an application: a ._pth file puts them in isolated mode (a script's own directory is not on sys.path and PYTHONPATH is ignored), and they have no venv, ensurepip, headers or tkinter. The mirror now assembles a full build of the same release instead, on the existing Linux runner: - windows-amd64: python.org's per-component MSIs (core, exe, lib, dev, tcltk), each verified against its .asc signature from the Windows release signing key (pinned in keys/, fingerprint checked) and unpacked with msiextract from msitools. - windows-arm64: python.org's pythonarm64 NuGet packages, verified against the SHA-512 in nuget.org's catalog, keeping only tools/. x64 NuGet packages are not used: some (3.5.2, 3.5.3, 3.6.0) lack venv and ensurepip. Both are packed reproducibly as a .tar.gz with everything under python/, the layout of python-build-standalone's Windows archives, so dtvem installs them unchanged. Each is checked to be a full install (no ._pth, venv, ensurepip, headers, import library, and tkinter for MSI builds); an upstream build that is not is skipped rather than published, without failing the run. MirrorJob gains an optional Build step used in place of a download. The new sources run after python-build-standalone, so it still wins where it has a build. Archives go to new .tar.gz keys rather than overwriting the old .zip objects, which are cached as immutable and still referenced by released dtvem binaries. A sync-only run now also treats an assembled job as pending until its archive exists, since its metadata already exists from the old zip. mirror-sync installs msitools for the Python job, and a new mirror-tool workflow vets and tests this module, which Build & Test does not cover. --- .github/workflows/mirror-sync.yml | 5 + .github/workflows/mirror-tool.yml | 45 ++ CLAUDE.md | 5 +- scripts/mirror-binaries/go.mod | 4 + scripts/mirror-binaries/go.sum | 8 + .../keys/python-windows-signing.asc | 252 +++++++ scripts/mirror-binaries/main.go | 138 ++-- scripts/mirror-binaries/main_test.go | 129 ++++ scripts/mirror-binaries/python.go | 24 - scripts/mirror-binaries/python_winbuild.go | 671 +++++++++++++++++ .../python_winbuild_network_test.go | 137 ++++ .../mirror-binaries/python_winbuild_test.go | 690 ++++++++++++++++++ scripts/mirror-binaries/sources.go | 5 +- scripts/mirror-binaries/sources_test.go | 19 + 14 files changed, 2062 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/mirror-tool.yml create mode 100644 scripts/mirror-binaries/keys/python-windows-signing.asc create mode 100644 scripts/mirror-binaries/main_test.go create mode 100644 scripts/mirror-binaries/python_winbuild.go create mode 100644 scripts/mirror-binaries/python_winbuild_network_test.go create mode 100644 scripts/mirror-binaries/python_winbuild_test.go diff --git a/.github/workflows/mirror-sync.yml b/.github/workflows/mirror-sync.yml index 5487650..a2ab9b1 100644 --- a/.github/workflows/mirror-sync.yml +++ b/.github/workflows/mirror-sync.yml @@ -37,6 +37,11 @@ jobs: with: go-version-file: 'go.mod' + - name: Install msitools + # Full Windows Python builds are assembled from python.org's MSIs + if: matrix.runtime == 'python' + run: sudo apt-get update && sudo apt-get install -y msitools + - name: Build mirror tool run: | cd scripts/mirror-binaries diff --git a/.github/workflows/mirror-tool.yml b/.github/workflows/mirror-tool.yml new file mode 100644 index 0000000..9b9a726 --- /dev/null +++ b/.github/workflows/mirror-tool.yml @@ -0,0 +1,45 @@ +name: Mirror Tool + +# Tests for scripts/mirror-binaries, which is its own Go module and so is +# not covered by the Build & Test workflow. + +on: + pull_request: + branches: [ main ] + paths: + - 'scripts/mirror-binaries/**' + - '.github/workflows/mirror-tool.yml' + push: + branches: [ main ] + paths: + - 'scripts/mirror-binaries/**' + - '.github/workflows/mirror-tool.yml' + +permissions: + contents: read + +jobs: + test: + name: Test mirror tool + runs-on: ubuntu-latest + defaults: + run: + working-directory: scripts/mirror-binaries + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'scripts/mirror-binaries/go.mod' + cache-dependency-path: 'scripts/mirror-binaries/go.sum' + + - name: Vet + run: go vet ./... + + # Network tests (python_winbuild_network_test.go) only run when + # MIRROR_NETWORK_TESTS=1, so this needs neither network nor msitools. + - name: Test + run: go test -race ./... diff --git a/CLAUDE.md b/CLAUDE.md index 137c3cb..fc0faee 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -324,7 +324,10 @@ cd src && go test -cover ./... # With coverage | `generate-manifests-from-r2.yml` | Manual/scheduled | Generate version manifests from R2 mirror | | `deploy-manifests.yml` | Push to main (manifests/) | Deploy manifest files to R2 | | `mirror-all.yml` | Manual dispatch | Mirror all runtime binaries to R2 | -| `mirror-sync.yml` | Scheduled | Sync new versions to R2 mirror | +| `mirror-sync.yml` | Scheduled | Sync new versions to R2 mirror (installs msitools for Python) | +| `mirror-tool.yml` | PR, push to main (`scripts/mirror-binaries/`) | Vet and test the mirror tool, a separate Go module | + +**Windows Python builds in the mirror**: python-build-standalone is the first source. For versions it lacks, the mirror assembles full builds instead of using python.org's embeddable zips (`scripts/mirror-binaries/python_winbuild.go`): `windows-amd64` from python.org's per-component MSIs (verified against the pinned Windows signing key in `keys/`, unpacked with `msiextract`), `windows-arm64` from python.org's `pythonarm64` NuGet packages (verified against nuget.org's SHA-512). Both are packed as a `.tar.gz` with everything under `python/`. Builds that turn out not to be full installs are skipped, not published. `MIRROR_NETWORK_TESTS=1 go test -run Network ./...` (needs msitools) builds real archives. ### Reusable Workflows (from `dtvem/.github` repo) diff --git a/scripts/mirror-binaries/go.mod b/scripts/mirror-binaries/go.mod index 04e1475..31dc334 100644 --- a/scripts/mirror-binaries/go.mod +++ b/scripts/mirror-binaries/go.mod @@ -3,6 +3,7 @@ module github.com/CodingWithCalvin/dtvem.cli/scripts/mirror-binaries go 1.23.0 require ( + github.com/ProtonMail/go-crypto v1.5.2 github.com/aws/aws-sdk-go-v2 v1.32.6 github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/credentials v1.17.47 @@ -24,4 +25,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect github.com/aws/smithy-go v1.22.1 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/sys v0.35.0 // indirect ) diff --git a/scripts/mirror-binaries/go.sum b/scripts/mirror-binaries/go.sum index 8c425bd..6563cdb 100644 --- a/scripts/mirror-binaries/go.sum +++ b/scripts/mirror-binaries/go.sum @@ -1,3 +1,5 @@ +github.com/ProtonMail/go-crypto v1.5.2 h1:cucYnvqcY7UOXVD//mSyjeaPY0SSN3v5cDkYPxumINk= +github.com/ProtonMail/go-crypto v1.5.2/go.mod h1:/RaSu30DaKO4RY+XdV/ACcCcZkGr7AhUIduq5sjzzCo= github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= @@ -34,3 +36,9 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HS github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= diff --git a/scripts/mirror-binaries/keys/python-windows-signing.asc b/scripts/mirror-binaries/keys/python-windows-signing.asc new file mode 100644 index 0000000..f0a4a61 --- /dev/null +++ b/scripts/mirror-binaries/keys/python-windows-signing.asc @@ -0,0 +1,252 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: Hostname: +Version: Hockeypuck 2.2 + +xsFNBFUh1AUBEACdUPt6PwJVO23zGZqgtgBeA9JsO22dk3CMzrwPJdUmMd6mcRWa +vl4BoAba66fuC17GvOgGXimKI+iaw5Vt9QI3uSjUjFSfc24J8T7NB/yAr/0zEcex +raHD2dxT/JpE/iY0yWHxRlitvwGSw1Qlq3NnY8tDI1DJEJD+gBuCktvVvu1FfQTw +6bd+aEq0c4sWJHAOnKLuLH0pNFOznnynAFGPGBBsm/YwYc5BP2JVvka775LUjA+W +1h2Sgg3FAUPIm64pc4Pq6mUo6Tulw72xsWMpCL1/5atXNPXT6rJUOB8euTcNMr4l +1O6GKSsiLeLAuvq4bmhOKtLzjWzXnY1gDVoOfdgpD6o4ZHk4xiVsdVE8hCa/ylz8 +1ZwRW2gGo2jP8t3hKciR2i+Qs+6lPNZpeFIxa6Uo9ER1IBgCHHapIR/UdcOFyoS0 +MNn7Ui7DLQNM4gI/G17eG9tfvjW2dl4SgFSYWMq/OtXnPDUBGqFUWsn8adOL2PFL +B7kM5ZRTPc5SnY9hoSGa5E20rJZIXcpy1aygRz/xUjoKwNzAySSEyyIorUxZ8KaH +EEBQSsqwe04MXIENqnDozH0/cvP4JXEDSl8EkzMSCWSoavQSIYD5pQppyFQpGHqa +5CuOA25Ja+sgp2xqahtr3fEqZUknPQSoYlnJbaHnzsGSlRAVWMsklsZibQARAQAB +zUBTdGV2ZSBEb3dlciAoUHl0aG9uIFJlbGVhc2UgU2lnbmluZykgPHN0ZXZlLmRv +d2VyQG1pY3Jvc29mdC5jb20+wsF5BBMBCAAjBQJVIdQFAhsDBwsJCAcDAgEGFQgC +CQoLBBYCAwECHgECF4AACgkQ/GJGQ0hwNOUX3Q//e8D156Hu3oqT+1F0PeNs9W7l +eqUxr9C3mtmh59tKNgQYx6fe2F66Bf3v/FuofmPhavIVqF3WCXQvQ1YMyYb21wCB +vFaxevyAxkOtq+H3/wCMkGPpIgjgglYY8Lqjk41WMNqWEVHCN1MF1T3qbQ0l3CGZ +Bwi7RFJ0uWDE3ZD7mawJrnAgp33CbXr12xw9siWD+L8H1zG2pbDQQyiYurmABXOM +r/vw8ktd631yb7tqWBr7UXfn7O5gRE71E7m4SOONKSLLOk1+jJcVnsV+k8vLyORw +kE9vNR+M6nW3uRncxpiMdRImMdbCW4EgTagoebESWOfwV81Sd3XzPzL+UGGOLpTv +6lDRXQvH9XGZtHrjqN5mgeI2Pko2n3+7dFiM9BrnzuHRwEcCYN7NCzvVn+fs/321 +jjHH773H2hzW4WilpNepLJXP4OzWLzHzrpXjNOmBshj8//2vXG2l+dzojLqTby5Y +cjzuTXxqab0Z+cQBJXAMNUS4RyXIL07SLP81VXxz2iuNbvAPXFyp5iFAt/SuARrz +qT8xLwnUNmWwKs0C5mdVCAq9ecNTX7OrqBESW57qviYDjznDpHswFhC4uS3WUxJZ +6SaWT0w0NN/txr8Q+EfbEfpeUU/FJe5Bftzzz83pH0MxfjYrKwYTxW9pT7dx8N9h +YE0yLlwjEUyhAW/+s+/CwXMEEwEIACcFAlUh1AUJEPxiRkNIcDTlAhsDBwsJCAcD +AgEGFQgCCQoLBBYCAwEAAB7KD/46ODOeFoiSFZhlZuO9ZKLOn5g1RnnnX3/+9RlH +aythd9iPWR6W67TFOBLMgywOsLRtLcRJSb98njONahJ0/eA38Pinf/v30TbnyNlo +15jcS6F9dFGBcqTBemBUHMgxjJTODlc3tJ6/WRIyUPct6gSxU4VGu6gkLYUnZUj0 +1M07t1E7lBKqZqAqgtr1LgpFJIePgbDHFr8dHoSKMW8C6L87mO7IXfNu3ou9JRpn +IEWM7ZxFb+sPNKHuRCWoBmoBa/7ErEd+S0BTLS8FM3yppReGT5cOSZ/3xTI5yeP5 +WjgYQzU4vT8scnvv/YBMGYz4l/o3mUJj/ZF7pKw3OH2MpYs3rJ+VKvWPwaWxtCcS +DundXB4EInrCFkDmZn3PBvNXxlvfO5r8HWfiTNb5a65MvUkGWhb7g0Gt4kbFK0GP +2RS8664eIbsK/tafoA0uHyFGsiC6ou3pT8VV+nkxTb+NuTDDL95jxLvKpzsS7AWK +6HnfuiLyWQg5yuErMK/5dcUHGa2sYqZg7xPsJDy8Ei81+YpOPjqVXDAGnlQnx8ws +9MtINUe5aF78wOZMIiItH1ql6p2ett/eZaUKioHH8p9orhaHHpqEhfeiMl4QhPBe +qZ4lf0AwaryOnmF3GJoYEno4R4uRsqXu2Szfn1XvBY+7ZZ8vT8vS1JNd3sJfrmtT +PCT7hcLBcwQTAQgAJwUCVSHUBQkQ/GJGQ0hwNOUCGwMHCwkIBwMCAQYVCAIJCgsE +FgIDAQAAHsoQADo4M54WiJIVmGVm471kos6fmDVGeedff/71GUdrK2F32I9ZHpbr +tMU4EsyDLA6wtG0txElJv3yeM41qEnT94Dfw+Kd/+/fRNufI2WjXmNxLoX10UYFy +pMF6YFQcyDGMlM4OVze0nr9ZEjJQ9y3qBLFThUa7qCQthSdlSPTUzTu3UTuUEqpm +oCqC2vUuCkUkh4+BsMcWvx0ehIoxbwLovzuY7shd827ei70lGmcgRYztnEVv6w80 +oe5EJagGagFr/sSsR35LQFMtLwUzfKmlF4ZPlw5Jn/fFMjnJ4/laOBhDNTi9Pyxy +e+/9gEwZjPiX+jeZQmP9kXukrDc4fYylizesn5Uq9Y/BpbG0JxIO6d1cHgQiesIW +QOZmfc8G81fGW987mvwdZ+JM1vlrrky9SQZaFvuDQa3iRsUrQY/ZFLzrrh4huwr+ +1p+gDS4fIUayILqi7elPxVX6eTFNv425MMMv3mPEu8qnOxLsBYroed+6IvJZCDnK +4Sswr/l1xQcZraxipmDvE+wkPLwSLzX5ik4+OpVcMAaeVCfHzCz0y0g1R7loXvzA +5kwiIi0fWqXqnZ62395lpQqKgcfyn2iuFocemoSF96IyXhCE8F6pniV/QDBqvI6e +YXcYmhgSejhHi5Gype7ZLN+fVe8Fj7tlny9Py9LUk13ewl+ua1M8JPuFwsFcBBAB +CAAGBQJVtk8DAAoJECngR5uZyKOX2oYP/iEEkhP8h/An/Nj2COyBqg1TcEcNqO3a +vIrHAfn4RR3xRYVwDcFFBacz3gzKzK3ZqgZHu1YNQ92yBaHF2DojcH9ie3+JgTuj +i9FQLw19NlcofwtmUnNebmclOglhUDAhvCTkpoEBbN5DB5C+KrbrdWOa3qGHrJrX +iHtOlFckcPMk9UdQZMT9sIWfe6sLL1FBPXFQ9PMArx+POnn8OxLMCM2BmtSnGV44 +tDL4135LFbZBEEG8TDCO0lV6NN+OWUXsMG7oqBi+YL7wDzm4FON7JDkWpxRxcYhx +CpqoBwR1yvyyqmhVBTCnG+IhUzbNlyq4RF7BIuRDCNQBbCc43F0ak+dFotnGYV3k +XO9LheoM6q3Vr6MGtqDj4jusxg4k6mQltQAK5P0AcxrT41YfpyhHR8J76Svfdd1A +JADevNofEJKFT2FzryCg2GUQa+RH6VMkLctq/TGHsM9GDeM5Rj9TxUvRcZ3HUmYm +SakFYn9ApDqdbXUsS8px3Lo0IAE2TbYHLl/ScFMD6etDLq4DnCGYlt52cP8/+b8w +QWnUmvf0yBUiQBTQZaninSBkLopNFB/gs0UbUhW9/Be4zD8K9EFEpLbp7UNB1rrz +ao+DfAxZuOf5+azdfdHkaLa1Q12Zoxnop6ljyE7iArJRiCgKfkOst7eyQO9TCG3r +v2WOmXH7TpGuwsFcBBABCAAGBQJWUVonAAoJEMYjTPAjlc0KuW0QAJLZ13Hjtxgo +rzLfckdUK9Dr+AOEPgiUiC+OXnyxZkt9JHG5Bx7WqvtBWTYdmWjGq7QtejFI2uGt +jlHo4Od1QYWklZEK1ZP+jPxwOv6OgryIWxVv+Qvp0VgXeYH78LFr467Qt3CmXKOM +IPsmHJZA20h/thDw2TiiB6QcuOmE8/VkNfEiedVlZDVDYmakQQOjcuRhaMnehKVm +WBD3pZszXpr98zq0HVcy1duP5i3o3Cy9rSpiOp4F+F1Yl1AkVFNzB/P1fldmE0Bn +EzP+hxmh6TdC06pJ3+sxLaPavRYkU2olNpMxMYQavEH/nFasJvtZDLdBRjoKU+O1 +QVjHlFqutoEN4eTXwLOA364i4DVEZUwtXSWABKrhYQf/KAUA9Yf3fQo0MpPPIom9 +qSSJ0QQ7CthXH0AWqfYmngRWKAVk8zECcYPkblitcCJtp9KKmubqwWGbURxOUSHQ +z/GS6IKwThZH1xicZUQUA9Rmm++6qIXadEpxKxsODXt4CR/0jYSvs8x5BxESK0WQ +kfec3WJCn2do8jj80rgAVUYoBEFUDwg59IW793G5UwniWMBtpn0XZn8qkjACyiIF +3vcrydpJfEIGlTrCYzX0MyCnqBxTyJuVMN2Aeo4AxXObsxGh+1ks++AOyoRmIwu4 +bsNk7oKLB21saYSSnXcMXytVro2GgHicwsFcBBABCAAGBQJX9rDyAAoJEH2/rXbN +7KDblxEP/15FmB5ibCsng2QvDBVPv77FanYZxbOlF2ZL3+zKDRZLjAy/s4FaDKRR +37U3cgajQKBJ53WpJNtdoyvGFhfIOJEFyGvbjEaiSzOJqgpfcnKP7cCfc/RDAPiS +0l2CtGW05C3brUg2RAyoKR25npMdgtTrFpUS7PT2oReTwGULNDfpviyQZ8OmxBzk +k7+DhOblVH/Z73XrNzgEb5BkhBlETZlGhXIcVrWf0hdavSX49Cv8YLPRrfOeaLnr +3kVyyBqY4hFaCh/3a7IMihTCVa6WQw4EuBlVxvG6txkxYjALOwTz3fW4yPo4L6rb +YCLWXGLFvyECtQSKBuhXq3uHD77viLIQnJJvU/JzivfLWbqL/IS2zQ/ZltneywoS +SNa/x282yiJdJOjooHRTKyZbp9s5rQ8U/U6d7sTmEmltmIIHKTVIyNvLb+gMKAPK +JFWPdy1ozuZHWRpG5YkZ3/3icuzSWJ9CzqILLzFCQpUQBp2UxyxK99loatDCJpV2 +uN5K+qDyU+XcF18sORC7LREM7XMwndT2AZG0B1KQ+Y9z8IDVTopwp23eKXP6lbWE +qqYNoOkyYtV4idet/I4Z9pPovMKKf3cnrYjfJRke+rcGBnsfQGrxAFKVUfa+qn6a +eCgYmxIXby+qNjtN0ULtexnEc10gGM5lIuVOY2d3Qc+TGFI4FGTPwsBcBBABCAAG +BQJYsBphAAoJEEhSKohZ29goZggIALKlgyoecD5v3ulh1eoctRqtCOxkAoENEfPt +3l5x6N8Wq89yHzf10T1rVioEXOHhDi1m37DDoQmRJD0sOYQymq10xDGRYAJjyOf3 +X0pvRkZ+F7T0U4dSV3DasLIHcN26kRwv1yCYsf0QvhgT6EJZKyUNHtV9qrb9u3A1 +Zp6epC/EyT8zMZj+21GzTUrnbnug3Ak9p7+APCZS4Ahh9ZHFuD38MZ7+OwrUd6ot ++6cbb1nnQLSAGQOHSp6EP6ktrnsKzts0L+tzHurxtJgUkR01imJuSFfYpLoZa/L7 +qXNyEpEUTC/SWzRWD9y2QkM7DLzXcaReVAyJr9rix1lDQbEFIqvCwVwEEAEKAAYF +AllTh9gACgkQXLNh5VL7DRAk7Q//X8eUhwEvl/d9Sv2kBNCZFjAW3QmZp2L/sxhS +cJZXrOFzKUdmjap9Xlul1qr6/Wif7YLKbOdNUI7KziEBn+9SEd90XauoVkzU2F0J +n9ILGQfUHAIpocRTKuCwBrncaBozHQwDO3Dk33AhZ6lqTv/AVLRKHQXwigGTBJxK +4cCEZ+VwK9tKk6BrQB48Rm7pg9HF5ey5JGPRWgUnn1v0IJN5ysZ5m9ChYbqF8Vwv +Mw0txmgKgvdDKpXbF/S59Bp4TH/7Dr2DkAeNTcuzTFBaFE+siMgksZIYKZ1VkVoi +N2qQA7ZaA5LQbUom0WdrKZGefFfPt9ESA4wyL3OfxRsmWmd/5Fxrwm1VbzgPoMd1 +Dc5ExlyqnecdGzDui2bmltNqRJd9ytRq6YUGYzXp4qQkWO61CoC3mkm2M8Ex7DGb +UtXhdg0zoa08w9lXuOtHVhY7XlLWjO1Up8cp4DVxsN/wOXtyH1pcleGo4aEsgyU/ +DH57prFLGz7Egp2JhRDHnZmlonWp74G1VLfqkOqZlqTU4mPA827C8qPCx6cMsRvF +S7OEiDBswkFWBKjkUCw4rLC1tBMBCxJWtZlc+Y0LNyOryJ3h6EJmRIHO57oLen34 +5e1WOi4ROOC/wQMErFk7B3P41Lqmrwb8HGuKn3ca+Aw70hVrZ+7Q3RRFTLlOS/vv +107Fqu7CwVwEEAEKAAYFAlp9o2EACgkQw7DeDnjqz17JrRAAsKN/XBIJSzAa1Fa9 +GIWOEEvvKewveA3txkCpAZ1fRoHR6y1B6zufxVmOR+/Vl5d5MjilDWi7NzEcunBD +s0KLCBdR2oKdnWabUzsH1qokbE2dbwJ/CrHUeZwxymtlmkVz/skVz02tL5zPYO4r +TMagT07KrdVofGxpTp6+hDvtixVV5BsREzBG+2bCJDfkNKNSoVaZcikwzdJe94rN +f9RgihxQ3N0MdfyD5LcHuJdcjWy8OCygCfmxTDsCFUNnc2CXn+5DuKksppE4jBvR +KOz0s29zDEIdnJ0h35XFfEJqrVmJT/IH17TFhGiMQKmnYVkZDlsxQcNPYF/RB6Vo +PAb7U9tAIWdcGWJ5dOmwUCSGPlrbaqI1Hb6gcu59fjueXHQrmOM5WGEMdqXsshoS +X8d632p1bQ3pOKz9NGW7mUPUdal3oLvH1I0Gt8EQi/nf/IWXsUX9/SVTHoZbB5Pm +wGZ+Rsafp84vqRrrDD+gz3PiDSBHkz8Y7xBQlXJe47FXf21e+nvoEveF2VckNnCQ +Rz353LVSj/U8mNut5IZJmpTnuI9qgpAUWEBOEvGXIMUL4VzxVDxkInLdhcN+zXYv +1GZG1c7bU45h8AVoqkIwmeZuKi9r2wUtsR9M3rqWqcjE56mxtVnkYW7CrE9EhofW +dWj//WPVmP22zKGAfwa5xj3tkNHCwHMEEAEIAB0WIQRJM0HfZ47qo6KWEUnUDkBi +bqxXGwUCW+0k0wAKCRDUDkBibqxXG/flB/9vlOvtm/QGtYuBHKR1NW+lTDPv6Nkr +d+NGdRisQ5+CiDIzd2mOvHzoMNGY3dQLvwUYt4dxdUVbTlH+0+C8we6FNeNYNBDQ +X/Qt+Q5zmBO43OrfSv4nLSmqoV7bxoSY7lsp0cBD9qE0Wnhnn5xxhQJVhj8h5tUV +7+jXK2pAlPCkSi3RoHXMYFxwDwxBjBkiexMGt7gwsGcsLl0HyHkkM6ut+Z3zBOWn +0xBagBT7TjgkfRKvo1lFpW9OfZC4tkZUCnLjr7VWQola7dvepaHgMJsqHqOl87YX +e1McAdXGnwc4q5d7oG/o1f4RTg2iNWgABOUT8/sa/NNfDoVXlFtHb+2lwsBzBBAB +CAAdFiEEOOn84tqyMDqaxg0YTn3VBQ8zqOEFAlwsCVEACgkQTn3VBQ8zqOFB5AgA +lFVwqF87o7p/pqanExRT/nECrlUqHINpGVi8Fm9/UbFb3VwdoRVZkWi6UWvaOdYJ +AFsXDhtvS1fieNbwtGrE20hZkHy66m8zJp7YwQp2qyKw8bbQeyUpgk57UqgC2nGs +HStdgbvEsFiNgbtF+0bDvh4Lag7wtCcQgehXy5kpyJFBY09q6oEXd8HNXZU9+Vaf +xBuJvaT8U+8GT5VeRiYv57tB5q008qbw0o3eZklTbg5ss8VibOPq6X1Xu5+PoL3l +Xtmd4ak4guCk52KCr3GTZ1oSZaXLB8PiWLdiIG7zLQFYJ8tfqJWbzNZh4Gk/FDh7 +D2ytVcKukXVpG8B9p3Tz3MLBcwQQAQgAHRYhBEW+vuyVCr0Fzw71w1CgTQw7ZRfy +BQJc8pFGAAoJEFCgTQw7ZRfyeywP/0GH+mle8wjIVNH5haH6VWXvfeVRXzpDqObu +KNWy8r2Vb1OrfnNvZGkc0TZz1ofXo5GSEjDb2G/R9ccErtqcv2HP7WYb2kE8Gq43 +pY65U6CXnL60W4a3snNrGgXzaIA54haGGXgpE7xE3uzNshwP3m4YvAVAv4zFp/vj +aOtWyUaq0LujWZSFE2liXSF7J9yCfDGSfAYSqO8giOOJwjlLt4Y8IeW8F2wiP2jT +oFns0g16N/yvM3k79MdDaStWz2NLA5M7oYgEE61rHRkXB48HKTy6HmZ5MsgbyOW+ +e+gEgiRwWAYryNvUvK1viXJPOnGW5z9Z+ss4zwaw6yyvtsH8/APAYxBSKCDXK3k5 +agtKy8L7XkXcFR5dz4xWUMWrQPnf6L8z2MA41xjjIF6Xo748A204vsLiMJReN0EC +H/kitrbYWrFxWfuvQwQbFvj1jKOt9Y3w8gg5K40fOMbaQpMFy/MWZImMbEqnkmrc +FqGwKSI2Fx4EVG7pBRPTUl1Ts/H47nfaXvvph8255hteL5XJoSbwOy9pBvclSdbd +7ldMj012yBpMb5JstFQLsFMRj1uAc0gw93EfaVX1A6Glv4HhRNVGW1nzIVE3KYH5 +zvWX2yC9j+Olc3JjR6uBuL0+fJmS7Pv0PtcLkZcnolpj01qvX1IhY8JdG9DXlWSV +40DLHttywsBzBBABCAAdFiEEXucagSQtOm55NGslTs2ZgDD1eqIFAl08PY0ACgkQ +Ts2ZgDD1eqLA2gf+M440eyOW70xUqt3W+lFG9R31Esd3fTqYZ9XZ2Ui0qqCfv93Y +k6N2XOOhl3jB8xaegG1+DQwh9DdlVd+aLXhFmslCYEWwUNg/uIMecvjMM8/nBCd3 +ncvgDY/K6NS1MOHjkLtxelWG5Q6bcodADmxuq3jN5xaF2qoHhNza4inRHBwdtG7/ +8iHn/M6NHL0GcVAlnCYVC8TwlWssX0MFA96oOWH29ss+3gI9Sq/rSFsYfhSVMxqR +bw8hEBkCt8o9GH982yNLU6NYPoGqZAVyHZppgZQpXUhwjw4FLQE70DdEwKH7pH6T +Xzm/xoy+0rjNhMGgz8YGgHQKEFldsGsqXfHVtsLAcwQQAQgAHRYhBNZmFyqvP0un +BcHvP+z7qpo6w3+bBQJdZnjbAAoJEOz7qpo6w3+bYMAH/35M4ffLUIBaT9i/s67K +8NkLXS8VWzOVDRVRA26702+PrEk+vQ3pJoFZQQYy+PNkl6gDI8PtLMKTqn+/zI7t +/1iO0dX6Vb9xsB0BbRvxodJvjIuW9CJr+ei6EpjXolzoVDQrZ/aE5DIIu8ltmW9M +wW7DeT11VSMddSvhJjFTCJoLcdXGVbqIa6lgPq/nhsTQVN8b06uLWzjO275mft+u +qfBKzDSI0igXtYGojyKvu1zTQqUxjOvLu3/nVyR96x3a37oy6EWZGizqc6JBdLDP +qrMVvo2YWd4+2lI6PYSpg9uesCsQ7jlFNvZ6qveRG6WMBl/dn0yADhRb15ywt5/q +i27CwHMEEAEIAB0WIQRXC6+NpbbWdXnYtNo2y+0KeqAbpAUCXadaawAKCRA2y+0K +eqAbpLs8B/sFH8FM56UgiERSBQ4f+atYB6HHljHqWmsntp2L8tp9BKa7wSvbtvfI +ch+W9tMX4l/prweIUty5PCTC6R1Qxkg46dwlVEJO6eTAdq7ZYKHw5Fv6C78E4cbw +kq6X0EBnPiKgf7DfBEa/gnZvy+gG8ztPSunYZ5tsB8W8UrVRni6EANcJqjlDOqPV +duwmF8FYOOxIunK2aQiNgwEZ1mQACbc5F/vqdvk6MC00I2rDhIsCImE/S5KMHuo7 +knNKZvxg8xifoSx0iOWumpQAiSTBIaHUgaPrmKx1BfwVwLqYSyuzivp8Xy+T1aAw +N7+LADAV8l1h+rf5vw4QtKzdqGQ8SPdbwsBzBBABCAAdFiEEUOXPFISLGVtIWB06 +PF3+q8WXx/8FAl2/fvAACgkQPF3+q8WXx/8u1Qf/eX7GZBYJjfIpKfcwZVAqb4sE +paLlty127Hf0Q+fTIfqngfMBHRUw73H8JyWHG6g4OuBar6Dkf036NVL5tzWz4bbb +/YWc628f02GlX79yofQDSPt7vceo5OboGmGt2sEY6ObGAocVcLoNc1dRtOqgMjHz +kh9YH8/0hZ82FIEY1JbkdJJePTHTX9BokMM0LubVpA+L7DGrGALyrZcqpaZnG7UG +TeRSaPrq/OE/A/Icx0ckWYHyjhhmQihoE2Npp/bQaf+8iN1AOkcUjJ6PTc4+jsO6 +wOoQ0pUDFxwfgUovHO2udRWgmv/BvId2407RI3YwgYKC8uOup3eUtzl17E1QI8LB +igQQAQgANBYhBOPeodrRYC9jnme/5AUuwev5aN01BQJexiFPFhSAAAAAAA0AAHJl +bUBnbnVwZy5vcmcACgkQBS7B6/lo3TWe8xAAxOSgR4BYbOxTi2D3YmvoAQGXSCYq +5u5+sVR6LjcA6aABWFc90USwp3LkJ8BoANioA2j7R5/mAaIQpWAYaelZJ5IPoQmJ +LK7gvYnumZSxo0yMMrTQT3+1pw5fJeWiEO2YUdwtQSYvnFplnYAEFXk5udKDSnkB +HhWtahbyTDT70SrcLSNKE4OVdzBxNnZafy+W9QWiDvGTpdeg/osWW74CviX87P6j +AIxN+MAB+g+/y/gJQANvM8G8zhUFSoYswVMgR41POArfxfcjI+NL7JvKUPk4FaqX +2oBRd/zVXsUFUIOXqATjnskUk15ehYzUjl0zAJ4fL8xNg/5nyPFDdrBwpJg2Iex9 +pnY30xQILOnywG8T7YM63uD9WvJEWD7MVHjOiECLyz/jzWsD3RFlnCF3vWWPkaH2 +4u9ioxgidpsq1IBneVEVSBHT11CvsMAM77xCa+h3yUc5+9naJWdfGv0TGEGUY++w +c+HxY8a8BdbmscSs4XUIEDvnJUDe6v/WcA/GJo8bmaaTceoA032RvSw/HxhAHMEA +jU4ig1eyKbQMJixTyMpIM2qmdqi0p2njQc6sgrV60ioJYqXy18aDeUQZSLZei3Co +Y8afGdsNP6ShS4hWCk91Z4DrvZYIL8EizB9qQfvpVqZSnHieFkDGOApMkzMEAZjx +rKY7QyuzxtsezgHCwI4EEAEIADgWIQRXFpWVsN3CoDQOmQPh6tx80S+Z4wUCX1dh +ZBoUgAAAAAANAARyZW1AZ251cGcub3JnQU5OSQAKCRDh6tx80S+Z41+2CACsZCGk +gpAfqmoCDJWSGbplj6JXvO3awYJ1qByd+lo5p8bBn3fp9yle+YIyNFy9Qfk4jAxF +psxSgZrDfDHSrnBHShc5wv8bqtd1ECcZcnjxX33U3rlR+obNJFnpNX/JyID+PnKT +11tZ/iIetvOeZmkumA2iHt54yo+gZ91SUoV7bzsmYTmGFHDRFpkmyRBLc9mrtXjg +SwbI1lQwYBOvPsIXwEYn5DETP5UJIKqGrXFtbvok7y79ZGITGHLa82Jb9f0DQttz +cmeLM3BOXadbQCUzrmNil6iDKVh6s32CdPQsYkmCyW0kf22t9wX6pd+z3+rPEKFD +SJ1cSaCqxCE9homOwsFzBBABCAAdFiEEhLVxtwVtdcpLm/p2szaYZjC3UiYFAmHM +g6AACgkQszaYZjC3UiZMBA/+O10BV1HEfHny5z+V2SYgmEmNEkwWL/qdR+DJiryc +31qMcvymW7KhXH77XctWaQ5BxJ5oq2s2G7JTRLwvj5NmQF1BR85drj+387ycP71F +GUr+7Ahsq4ZoLU53MAtW2/iIjvDyosVW4CIKylKwVzc7L8as7ze6bhxPZquwMO26 +UHuY7FBEB5qNhBvWVb+QQs7nkPYnno57dX4NWb/uoBGYf2MfdAhLQYhFvpReM7lI +GnUpVv2v9vohBzJpJ+0rVCFrjnhOKP92KJrc56/NqOneH3Sy2EaDV238MhqLjvyt +0OiztAk2V0debyA57piNxJNnlaPMNeBU5+3CEEeC+Q0p3yajIHx4gW+Nm3G6hj5N +CUNn1/v634va3xtBdCdA/xSrzDw9Zv1HUjZ8Xr6U4eB+HOh96x7R0eOXgPPBRS9h +3PCqykDLMP6hgRb9OTtujNgCkc+myM+6KHl3atARV7y4pUYzaRDxZ6oM8E+MazKD +8g4dGppBbwvzn3+J1NRwNuTLo0F6UNO06zV6PPCLmx9Ynif8U70c5M15qlD2Af6f +6lilBjm63lz7Ux455raeZ8AxNKSwssycElwyc5E762b0O984sWrXB2o8y0eWU/KR +EVdoDISrElxeQjcZ2cro1BxNR0i6v85/gr7p+3adVPEtW4ZSyTYW88nCNy/07lLw +PhHCkgQQFgoAOhYhBKnDlZJc49SuizfOuCw7HjER/AkyBQJlL8vRHBSAAAAAAA0A +BnJlbUBnbnVwZy5vcmdQeXRob24ACgkQLDseMRH8CTJFXwD/dbo63+YVYZtXL3DK +2koE2R1UjCW2oTRGcll8BGSl9KsBAOwVc3LE5JleER56DFey9OeRz8L1JwXlUico +6UiLvF8GwpIEEBYKADoWIQSpw5WSXOPUros3zrgsOx4xEfwJMgUCZS/MDxwUgAAA +AAANAAZyZW1AZ251cGcub3JnUHl0aG9uAAoJECw7HjER/Aky2vcBAOnGlGPt6FCx +PyBubpCMQufpwc2xRxj+uu0jQ0lv0SYFAQCC7dTAltf0lEsokgWdAJFQA3jJaOnJ +hEEtz9/MTyZmBcKSBBAWCgA6FiEEqcOVklzj1K6LN864LDseMRH8CTIFAmUvzCMc +FIAAAAAADQAGcmVtQGdudXBnLm9yZ1B5dGhvbgAKCRAsOx4xEfwJMtqfAP4kYQFu +6lSHlm01EQNTwM35Na1GYwr6LQZ+8OysRmu+swD+MI4w6LaFU0Kw0F4ilzzIIlHz +xyq8WE+PyK+5TCsrGAjCwXkEEAEIACMWIQTXwOkHK6SvSGfdeR2iSyV8o7TD2AUC +ZWNX/AWDA8O4gAAKCRCiSyV8o7TD2DGwD/4pju58F0cq+fs4WGWSuY7XR9cuIxI9 +Oylx+4AI8IzPjmL4iBI945Pvi2R0gwbii8AoTN0vs4k363oOq8zs80pfgOf4u7gv +Mc2NrQ7Vw6vhep664u2xCXLbItoFjU25Du++9+40TuprExgCKqT+c51Mr9coxEuN +LKHakewXhWjcGjCxU637n/bQ4CyCoz7orBLKVMbCNj4udkUyGoX3dabajBC+BCmT +c3hjohr2ZM5QtDdPQDHmLgXetPVmv38aWOk+YAIPqAPYG8SVvYOZht1R2RwBr93P +B4g7lvgBOr7JgdEaf0b05d3C7njQV+I6YgP2A7/hAFLCesDcs6W7G/hSb8tFfu5H +Z81jIaQzXIQjQE4xtbSjMFq0mzqa/SFOKN+YAQyPac/nmGTXStJ9Z45CNWy6WU/P +u5PEvXoR8mI9Mt4NMn1clwLRoLsTXhxtHHiLuOj4oe+ks+k+EpXBaNI3nzhjhtaA +IpZFJBsX4A/kbOem3WEMkXTRxwnxA/vJqsIK66yJU/o0BPSfCQRkFCN197DOHeS5 +bJMzES8WcshJy9PvRtnzTf6DaCU6NH8dZsz11K/hR8r1f9GDudPN0xFK/4KeU0nx +G9mv0Ude5UrEYauVeasgPIOp2z1EOX7pa0AL+FeR4VyhVAUGh+/vLEoSaxBF9ppI +qMShYavJrVZIycJ1BBAWCgAdFiEEIJmlt63FmMoAH0rGteH1dc4zC1kFAmXisJAA +CgkQteH1dc4zC1kzLAEAqLSB2vsiUmfwCy/ABOKC4yCT8YzvrowW+D1HBz7iAHIB +AKtEpC3uewF14z7dRN1pxHWwoCFLceKXDnOTAw+WyLgAwsCaBBABCABEFiEEHGnk ++HPqa68ThNVqHbsVQ1nUiDEFAmXt0YMFgwPCZwADBQF4HIY8W14+XStbQC5dbWlj +cm9zb2Z0XC5jb20+JAAACgkQHbsVQ1nUiDGF9Af+JzQ06C1Da14WCDt+FxF2dI1R +NHAtNoWQGc0ro06+N89trzi5C50l3zZL00faBQ4OCyrPk/mLbkTYjzJ+0NnrFIGP +WGKu7HdvoWaDqFX6LenpPOHZ03mFd77jUGng9jbB/UWu9nVdOdOZcy1aQObazh3g +lJmiVR6W9ala7K5FtiIiWYH7sDeMqHb2c8uLI1vzw9fyC7/DHVSpnPjLGk2rlfhy +PLA9uJPbntUBnCtIN8sL5f542F7cJ/KSstkqoItR1nVy7AMlX7scsobzhtTN+Lmt +OPVVSPMDA5/WY7pYLiFm1RluQ5Ycvh2e/G+8qfqS4/mLciwRLoz1bz9I7vICmM7B +TQRVIdQFARAAtmnsZ9A8ovJIJ9RlWeIylEhHRyQifqzgc/r50uDZVPBjewOA462L +jH3+F6zFGEkU+q2aqSe0A0SJPF/Whj6MNYXLoibxi5D4mGkoIao9ExnXt4LXAc6o +gQpY6vFQBJU5Nr8XCefQbm0loa/oy5uK8JHLWCZ2jAossnVpzDwNeN27+B8h5+Oi +fnWhQCTun1xz5EJiyc0yoBmf46zfmU4CMUBsPvrXcLmw4J3wp35qmrHg1tNyPhd7 +VBlikMrgtrWX9IaPZ40dnrGG/WjOFYB3CKxGb0pTCj7GC4ubxo2upeWZqHLmdIVc +7Nzsfp8EcwJbTj+jZ2Zfq6F8y+jesbgh8CaxYn4hEs23aPYRq5H4/buVmZhUw3/A +AL9ZmyX6AtAQ0HktVtQe7ykP7DLsEpeLG+vPJFY363QeDsLHwOoxnZSfGziVlB4N +/KqIkixNWcFTG8GSE1zKcdJVNoW+3MB3+FtMZWUJhH0FyKg5qLaJCtC7Yo5gsddU ++QCqTn6gcZBnMX5j4LaAmW4hh1RXffwwsbfviK5uhXQCeUnbUaokieetDx4s6Kay +6t9ahTRr0r/Z3VWzvr+xATxNWZzixTdezCGOB2ycZ0vq4bKXBuN8CAyOy5X1hf7R +c1BiAVQCILHJDtz0Ak/Hax6DAa2AHnx9YlugHQf000KroLEY+GaxqYEAEQEAAcLB +XwQYAQgACQUCVSHUBQIbDAAKCRD8YkZDSHA05RtyEACdOEmGolL1xG6I+lDVdot6 +oBZqC9e021aLWqCUpWJFDp0m0aTmCfmOI1gTaFjScxhq1W0GPUoJKUZhk3tlVfdS +CtUckI+xuWKEfqJYtvUtTXpK4jDeaZBovJ3KNpJRIynbr1566zCSQJhHiCGWmE/M +5KN3gPsORbCBQXEkONSVsslf1Wm66hU6uqSWUaceD+4fl5LClbck1DPWchAP7+uL +KPEOtORyH6KRTgKl73zYo7xU1K4QMN/1aMjobPkqNvvkXnUNwO7QMz18Nx+WqPc4 +ksJgW1O1aPQ2qL/ARY5jatZ6BBd7iytfz7d6JOh0FOIlmhBqbWd7fEGrLsSA+EjB +GBwW5BnIMmxP1xhjhwrcI18y8kAK5UzdW2hbbAlc2rlsuxEc+xOYh8kGcc+mZ1j/ +aMn4gALsTbSO/0T+YJhfODNnL1dCj7oPbJGmmG6pb/o7P4azBUVC9lHOuV3XlAPj +SmJylnNsV7+PxwPlXlvKgh4S4C4ZPUc/iPetsxXR2djccOoNxVU4CqJBqYKgul/p +UphXkh7QfEKyH+42UETbVhstdBVUazJ6SeUnv9ClVDGsCEhfEZfNOnOoDzJGxDfE +SoAw7ih91vIhTyHHsK83p2HLDMLPptLzx/0AFBfo6MWGGpd2RSnMWNbvh59wiThl +DeI+Das3ln5nsAo67dMYdA== +=IpRU +-----END PGP PUBLIC KEY BLOCK----- diff --git a/scripts/mirror-binaries/main.go b/scripts/mirror-binaries/main.go index 5fbb297..2fecea3 100644 --- a/scripts/mirror-binaries/main.go +++ b/scripts/mirror-binaries/main.go @@ -42,6 +42,11 @@ type MirrorJob struct { UpstreamSHA256 string // Checksum from upstream manifest (may be empty) R2Key string MetaKey string + + // Build, when set, produces the binary to upload in place of + // downloading URL, for archives dtvem assembles from upstream files. + // URL then records where those files came from. + Build func() ([]byte, error) } // Stats tracks mirroring statistics @@ -144,7 +149,10 @@ func main() { withoutChecksum := 0 for _, job := range jobs { checksumStatus := "(will generate checksum)" - if job.UpstreamSHA256 != "" { + if job.Build != nil { + checksumStatus = "(assembled by dtvem)" + withoutChecksum++ + } else if job.UpstreamSHA256 != "" { checksumStatus = "(has upstream checksum)" withChecksum++ } else { @@ -158,18 +166,9 @@ func main() { return } - // Filter jobs if sync-only (check for .meta.json to determine if mirrored) + // Filter jobs if sync-only if *syncOnly && existingKeys != nil { - var filtered []MirrorJob - for _, job := range jobs { - if !existingKeys[job.MetaKey] { - // Not yet mirrored - filtered = append(filtered, job) - } else if builtFromSourceKeys[job.MetaKey] { - // Exists but was built from source — re-mirror from upstream - filtered = append(filtered, job) - } - } + filtered := pendingJobs(jobs, existingKeys, builtFromSourceKeys) skipped := len(jobs) - len(filtered) fmt.Printf("Skipping %d files already in R2, %d remaining\n", skipped, len(filtered)) jobs = filtered @@ -200,6 +199,27 @@ func main() { } } +// pendingJobs returns the jobs a sync-only run still has to mirror: those +// without metadata in R2, and those whose metadata says built-from-source, +// which are re-mirrored from upstream. +// +// A job dtvem assembles (Build set) is also pending until its archive +// exists. Its archive can replace an older one under the same metadata, as +// a full Windows Python .tar.gz replaces an embeddable .zip, so the +// metadata alone does not show it was mirrored. +func pendingJobs(jobs []MirrorJob, existingKeys, builtFromSourceKeys map[string]bool) []MirrorJob { + var pending []MirrorJob + for _, job := range jobs { + switch { + case !existingKeys[job.MetaKey], builtFromSourceKeys[job.MetaKey]: + pending = append(pending, job) + case job.Build != nil && !existingKeys[job.R2Key]: + pending = append(pending, job) + } + } + return pending +} + // exitOnFetchError exits non-zero when any upstream source failed to fetch. // Without this a broken source mirrors nothing while the workflow still // reports success, hiding missing versions indefinitely. @@ -331,12 +351,7 @@ func processJobs(client *s3.Client, jobs []MirrorJob) *Stats { go func() { defer wg.Done() for job := range jobChan { - if err := mirrorFile(client, job, stats); err != nil { - fmt.Fprintf(os.Stderr, "Error mirroring %s: %v\n", job.R2Key, err) - atomic.AddInt64(&stats.Failed, 1) - } else { - atomic.AddInt64(&stats.Mirrored, 1) - } + recordResult(stats, job, mirrorFile(client, job, stats)) } }() } @@ -361,8 +376,8 @@ func mirrorFile(client *s3.Client, job MirrorJob, stats *Stats) error { } err := doMirror(client, job, stats) - if err == nil { - return nil + if err == nil || errors.Is(err, errNotFullBuild) { + return err } lastErr = err } @@ -370,42 +385,77 @@ func mirrorFile(client *s3.Client, job MirrorJob, stats *Stats) error { return lastErr } -func doMirror(client *s3.Client, job MirrorJob, stats *Stats) error { - // Download file +// recordResult counts a finished job. A job whose upstream build is not a +// full install (errNotFullBuild) counts as skipped, not failed. +func recordResult(stats *Stats, job MirrorJob, err error) { + switch { + case err == nil: + atomic.AddInt64(&stats.Mirrored, 1) + case errors.Is(err, errNotFullBuild): + fmt.Fprintf(os.Stderr, "Skipping %s: %v\n", job.R2Key, err) + atomic.AddInt64(&stats.Skipped, 1) + default: + fmt.Fprintf(os.Stderr, "Error mirroring %s: %v\n", job.R2Key, err) + atomic.AddInt64(&stats.Failed, 1) + } +} + +// fetchBinary returns the binary for a job, its SHA-256 and where that +// checksum comes from: "upstream" when it matched one the upstream +// published, "dtvem" when dtvem computed it, including for archives a Build +// step assembled. +func fetchBinary(job MirrorJob, stats *Stats) (body []byte, checksum, checksumSource string, err error) { + if job.Build != nil { + body, err = job.Build() + if err != nil { + return nil, "", "", fmt.Errorf("build failed: %w", err) + } + } else { + body, err = downloadBinary(job.URL) + if err != nil { + return nil, "", "", err + } + } + atomic.AddInt64(&stats.BytesDown, int64(len(body))) + + hash := sha256.Sum256(body) + checksum = hex.EncodeToString(hash[:]) + + if job.Build == nil && job.UpstreamSHA256 != "" { + if checksum != job.UpstreamSHA256 { + return nil, "", "", fmt.Errorf("checksum mismatch: expected %s, got %s", job.UpstreamSHA256, checksum) + } + atomic.AddInt64(&stats.UpstreamChecksum, 1) + return body, checksum, "upstream", nil + } + atomic.AddInt64(&stats.GeneratedChecksum, 1) + return body, checksum, "dtvem", nil +} + +// downloadBinary downloads a binary to mirror. +func downloadBinary(url string) ([]byte, error) { httpClient := &http.Client{Timeout: 10 * time.Minute} - resp, err := httpClient.Get(job.URL) + resp, err := httpClient.Get(url) if err != nil { - return fmt.Errorf("download failed: %w", err) + return nil, fmt.Errorf("download failed: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download failed: HTTP %d", resp.StatusCode) + return nil, fmt.Errorf("download failed: HTTP %d", resp.StatusCode) } - // Read body and calculate checksum body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("read failed: %w", err) + return nil, fmt.Errorf("read failed: %w", err) } + return body, nil +} - atomic.AddInt64(&stats.BytesDown, int64(len(body))) - - // Calculate actual checksum - hash := sha256.Sum256(body) - actualChecksum := hex.EncodeToString(hash[:]) - - // Determine checksum source and verify if upstream provided one - var checksumSource string - if job.UpstreamSHA256 != "" { - if actualChecksum != job.UpstreamSHA256 { - return fmt.Errorf("checksum mismatch: expected %s, got %s", job.UpstreamSHA256, actualChecksum) - } - checksumSource = "upstream" - atomic.AddInt64(&stats.UpstreamChecksum, 1) - } else { - checksumSource = "dtvem" - atomic.AddInt64(&stats.GeneratedChecksum, 1) +func doMirror(client *s3.Client, job MirrorJob, stats *Stats) error { + body, actualChecksum, checksumSource, err := fetchBinary(job, stats) + if err != nil { + return err } // Determine content type diff --git a/scripts/mirror-binaries/main_test.go b/scripts/mirror-binaries/main_test.go new file mode 100644 index 0000000..b16369e --- /dev/null +++ b/scripts/mirror-binaries/main_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestPendingJobs(t *testing.T) { + build := func() ([]byte, error) { return nil, nil } + job := func(version string, assembled bool) MirrorJob { + j := MirrorJob{ + Version: version, + R2Key: "python/" + version + "/windows-amd64.tar.gz", + MetaKey: "python/" + version + "/windows-amd64.meta.json", + } + if assembled { + j.Build = build + } + return j + } + jobs := []MirrorJob{ + job("3.12.3", false), // archive and metadata mirrored + job("3.9.11", false), // nothing mirrored yet + job("3.8.20", false), // mirrored, but built from source + job("3.7.9", false), // metadata only: not a dtvem build, left alone + job("3.9.9", true), // dtvem build; metadata from the old embeddable .zip only + job("3.12.2", true), // dtvem build, archive and metadata mirrored + } + existing := map[string]bool{ + "python/3.12.3/windows-amd64.tar.gz": true, + "python/3.12.3/windows-amd64.meta.json": true, + "python/3.8.20/windows-amd64.tar.gz": true, + "python/3.8.20/windows-amd64.meta.json": true, + "python/3.7.9/windows-amd64.meta.json": true, + "python/3.9.9/windows-amd64.zip": true, + "python/3.9.9/windows-amd64.meta.json": true, + "python/3.12.2/windows-amd64.tar.gz": true, + "python/3.12.2/windows-amd64.meta.json": true, + } + builtFromSource := map[string]bool{"python/3.8.20/windows-amd64.meta.json": true} + + var got []string + for _, j := range pendingJobs(jobs, existing, builtFromSource) { + got = append(got, j.Version) + } + if want := []string{"3.9.11", "3.8.20", "3.9.9"}; !reflect.DeepEqual(got, want) { + t.Errorf("pendingJobs() = %v, want %v", got, want) + } +} + +func TestRecordResult(t *testing.T) { + stats := &Stats{} + job := MirrorJob{R2Key: "python/3.5.2/windows-arm64.tar.gz"} + + recordResult(stats, job, nil) + recordResult(stats, job, fmt.Errorf("build failed: %w", errNotFullBuild)) + recordResult(stats, job, errors.New("upload failed")) + + if stats.Mirrored != 1 || stats.Skipped != 1 || stats.Failed != 1 { + t.Errorf("recordResult() counted mirrored=%d skipped=%d failed=%d, want 1 each", stats.Mirrored, stats.Skipped, stats.Failed) + } +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func TestFetchBinary_Build(t *testing.T) { + archive := []byte("assembled archive") + job := MirrorJob{ + URL: "https://www.python.org/ftp/python/3.9.11/amd64/", + // A Build job's checksum is always dtvem's, even if one were set. + UpstreamSHA256: "ignored", + Build: func() ([]byte, error) { return archive, nil }, + } + + body, checksum, source, err := fetchBinary(job, &Stats{}) + if err != nil { + t.Fatalf("fetchBinary() error: %v", err) + } + if string(body) != string(archive) || checksum != sha256Hex(archive) || source != "dtvem" { + t.Errorf("fetchBinary() = (%q, %s, %s), want (%q, %s, dtvem)", body, checksum, source, archive, sha256Hex(archive)) + } + + job.Build = func() ([]byte, error) { return nil, errors.New("bad signature") } + if _, _, _, err := fetchBinary(job, &Stats{}); err == nil { + t.Error("fetchBinary() with a failing Build = nil error, want error") + } +} + +func TestFetchBinary_Download(t *testing.T) { + payload := []byte("upstream archive") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(payload) + })) + defer srv.Close() + + tests := []struct { + name string + upstream string + wantSource string + wantErr bool + }{ + {name: "matching upstream checksum", upstream: sha256Hex(payload), wantSource: "upstream"}, + {name: "no upstream checksum", wantSource: "dtvem"}, + {name: "mismatched upstream checksum", upstream: sha256Hex([]byte("other")), wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, checksum, source, err := fetchBinary(MirrorJob{URL: srv.URL, UpstreamSHA256: tt.upstream}, &Stats{}) + if tt.wantErr { + if err == nil { + t.Error("fetchBinary() = nil error, want checksum mismatch") + } + return + } + if err != nil || source != tt.wantSource || checksum != sha256Hex(payload) { + t.Errorf("fetchBinary() = (%s, %s, %v), want (%s, %s, nil)", checksum, source, err, sha256Hex(payload), tt.wantSource) + } + }) + } +} diff --git a/scripts/mirror-binaries/python.go b/scripts/mirror-binaries/python.go index 521719e..a1b0f93 100644 --- a/scripts/mirror-binaries/python.go +++ b/scripts/mirror-binaries/python.go @@ -223,27 +223,3 @@ func (s *PythonStandaloneSource) mapTripleToPlatform(triple string) string { return "" } } - -// PythonOfficialSource fetches Python versions from python.org -// This is a fallback for versions not available in python-build-standalone -type PythonOfficialSource struct{} - -func (s *PythonOfficialSource) Name() string { - return "python.org" -} - -func (s *PythonOfficialSource) FetchVersions() ([]MirrorJob, error) { - // Python.org doesn't provide prebuilt binaries for most platforms - // Only Windows installers and source tarballs are available - // For now, we rely primarily on python-build-standalone - // This source can be expanded later if needed - - // The official FTP has a complex structure: - // https://www.python.org/ftp/python/3.12.0/ - // - Python-3.12.0.tar.xz (source) - // - python-3.12.0-amd64.exe (Windows installer - not a portable archive) - // - python-3.12.0-embed-amd64.zip (Windows embeddable - limited use) - - // For now, return empty - python-build-standalone covers our needs - return []MirrorJob{}, nil -} diff --git a/scripts/mirror-binaries/python_winbuild.go b/scripts/mirror-binaries/python_winbuild.go new file mode 100644 index 0000000..b71ed72 --- /dev/null +++ b/scripts/mirror-binaries/python_winbuild.go @@ -0,0 +1,671 @@ +package main + +// Full Windows builds for Python versions python-build-standalone does not +// cover. +// +// python.org's embeddable zips, which the mirror used to fall back to, are a +// trimmed runtime meant to be shipped inside an application: a ._pth file +// puts them in isolated mode (no script directory on sys.path, PYTHONPATH +// ignored), and they have no venv, ensurepip, headers or tkinter. Instead the +// mirror assembles a full build of the same release from official files: +// +// - windows-amd64: python.org's per-component MSIs, verified against the +// Windows release signing key and unpacked with msiextract (msitools), +// so this runs on a Linux runner; +// - windows-arm64: python.org's NuGet packages, verified against +// nuget.org's SHA-512. +// +// A release with neither (3.5.2's MSIs are unsigned) keeps its embeddable +// build, and dtvem's installer disables its ._pth file instead. +// +// Either way the result is a .tar.gz with everything under python/, the +// layout of python-build-standalone's Windows archives, so dtvem installs it +// without changes. + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "crypto/sha512" + _ "embed" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/ProtonMail/go-crypto/openpgp" +) + +const ( + pythonFTPBaseURL = "https://www.python.org/ftp/python/" + nugetFlatContainer = "https://api.nuget.org/v3-flatcontainer/" + nugetRegistrationURL = "https://api.nuget.org/v3/registration5-gz-semver2/" + + // pythonWindowsSigningFingerprint is the key python.org lists for + // "Windows binaries" (Steve Dower) at + // https://www.python.org/downloads/metadata/pgp/. + pythonWindowsSigningFingerprint = "7ED10B6531D7C8E1BC296021FC624643487034E5" + + // msiDiscoveryWorkers bounds concurrent requests to python.org while + // checking which releases have signed MSIs. + msiDiscoveryWorkers = 8 +) + +// pythonWindowsSigningKey is the armored public key for +// pythonWindowsSigningFingerprint, committed so the mirror never depends on +// a keyserver. +// +//go:embed keys/python-windows-signing.asc +var pythonWindowsSigningKey []byte + +// pythonMSIComponents are the MSIs that together make a full install: +// pythonXY.dll, python.exe, the standard library (with venv and ensurepip), +// headers and import library, and tkinter with Tcl/Tk. pip.msi is left out: +// it holds no files and only runs ensurepip, which dtvem does itself. +var pythonMSIComponents = []string{"core", "exe", "lib", "dev", "tcltk"} + +// pythonNuGetPackages maps a platform to python.org's NuGet package for it. +// Only arm64 is listed: python.org has no arm64 MSIs for 3.9 and 3.10, and +// every pythonarm64 release checked is a full install. x64 is covered by +// python-build-standalone and the MSIs, and its NuGet packages are not +// reliably full (3.5.2, 3.5.3 and 3.6.0 lack venv and ensurepip). +var pythonNuGetPackages = []struct { + platform string + id string +}{ + {"windows-arm64", "pythonarm64"}, +} + +// errNotFullBuild marks an upstream build that turned out not to be a full +// install. The mirror skips such a job rather than failing: retrying cannot +// change upstream's files, and failing would fail every scheduled run. +var errNotFullBuild = errors.New("not a full Python install") + +var ( + // minWindowsPython is the first release with python.org's MSI-based + // installer and NuGet packages. + minWindowsPython = [3]int{3, 5, 0} + // firstSigstoreOnlyPython is the first release python.org signs with + // Sigstore only; earlier MSIs carry OpenPGP signatures. + firstSigstoreOnlyPython = [3]int{3, 14, 0} + + pythonFTPVersionPattern = regexp.MustCompile(`href="(\d+\.\d+\.\d+)/"`) + releaseVersionPattern = regexp.MustCompile(`^\d+\.\d+\.\d+$`) + + // archiveModTime is the modification time of every archive entry, so + // rebuilding the same release produces the same archive. + archiveModTime = time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC) +) + +// PythonMSISource assembles full windows-amd64 builds from python.org's +// per-component MSIs. +type PythonMSISource struct { + // baseURL is python.org's FTP root; tests point it at a local server. + baseURL string + // extract unpacks one MSI into a directory; nil means msiextract. + extract func(msiPath, destDir string) error +} + +func (s *PythonMSISource) Name() string { + return "python.org MSIs" +} + +func (s *PythonMSISource) ftpBase() string { + if s.baseURL != "" { + return s.baseURL + } + return pythonFTPBaseURL +} + +// FetchVersions returns a job for every release from minWindowsPython up to +// firstSigstoreOnlyPython whose amd64 folder has all of pythonMSIComponents +// with their .asc signatures. Releases without Windows binaries (source-only +// security releases) or without signatures (3.5.2) get no job. +func (s *PythonMSISource) FetchVersions() ([]MirrorJob, error) { + verifier, err := newPythonWindowsVerifier() + if err != nil { + return nil, err + } + + base := s.ftpBase() + index, err := fetchText(base) + if err != nil { + return nil, fmt.Errorf("listing %s: %w", base, err) + } + + var candidates []string + for _, v := range parsePythonFTPVersions(index) { + parts, _ := parseVersion(v) + if compareVersions(parts, minWindowsPython) >= 0 && compareVersions(parts, firstSigstoreOnlyPython) < 0 { + candidates = append(candidates, v) + } + } + + signed := make([]bool, len(candidates)) + var wg sync.WaitGroup + sem := make(chan struct{}, msiDiscoveryWorkers) + for i, v := range candidates { + wg.Add(1) + go func() { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + listing, err := fetchText(base + v + "/amd64/") + signed[i] = err == nil && hasSignedMSIs(listing) + }() + } + wg.Wait() + + var jobs []MirrorJob + for i, v := range candidates { + if !signed[i] { + continue + } + dirURL := base + v + "/amd64/" + jobs = append(jobs, MirrorJob{ + Runtime: "python", + Version: v, + Platform: "windows-amd64", + URL: dirURL, + R2Key: fmt.Sprintf("python/%s/windows-amd64.tar.gz", v), + MetaKey: fmt.Sprintf("python/%s/windows-amd64.meta.json", v), + Build: func() ([]byte, error) { + return buildFromMSIs(dirURL, verifier, s.extract) + }, + }) + } + return jobs, nil +} + +// parsePythonFTPVersions returns the x.y.z release folders in python.org's +// FTP index. +func parsePythonFTPVersions(index string) []string { + var versions []string + for _, m := range pythonFTPVersionPattern.FindAllStringSubmatch(index, -1) { + versions = append(versions, m[1]) + } + return versions +} + +// hasSignedMSIs reports whether a release's amd64 folder listing has every +// one of pythonMSIComponents together with its .asc signature. +func hasSignedMSIs(listing string) bool { + for _, c := range pythonMSIComponents { + if !strings.Contains(listing, `href="`+c+`.msi"`) || !strings.Contains(listing, `href="`+c+`.msi.asc"`) { + return false + } + } + return true +} + +// buildFromMSIs downloads pythonMSIComponents from dirURL, verifies each +// against its signature, unpacks them into one install and packs it. +func buildFromMSIs(dirURL string, verifier *signatureVerifier, extract func(msiPath, destDir string) error) ([]byte, error) { + if extract == nil { + extract = runMsiextract + } + + work, err := os.MkdirTemp("", "dtvem-python-msi-") + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(work) }() + + installDir := filepath.Join(work, "install") + for _, c := range pythonMSIComponents { + msi, err := download(dirURL + c + ".msi") + if err != nil { + return nil, err + } + sig, err := download(dirURL + c + ".msi.asc") + if err != nil { + return nil, err + } + if err := verifier.verify(msi, sig); err != nil { + return nil, fmt.Errorf("%s.msi: %w", c, err) + } + + msiPath := filepath.Join(work, c+".msi") + if err := os.WriteFile(msiPath, msi, 0o644); err != nil { + return nil, err + } + if err := extract(msiPath, installDir); err != nil { + return nil, fmt.Errorf("extracting %s.msi: %w", c, err) + } + } + + return archiveDir(installDir, true) +} + +// runMsiextract unpacks an MSI with msiextract from msitools. +func runMsiextract(msiPath, destDir string) error { + if _, err := exec.LookPath("msiextract"); err != nil { + return errors.New("msiextract not found; install msitools (apt-get install msitools)") + } + out, err := exec.Command("msiextract", "-C", destDir, msiPath).CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +// PythonNuGetSource repackages python.org's NuGet packages, which hold a +// full install under tools/. +type PythonNuGetSource struct { + // flatContainerURL and registrationURL point at nuget.org's API; tests + // point them at a local server. + flatContainerURL string + registrationURL string +} + +func (s *PythonNuGetSource) Name() string { + return "python.org NuGet packages" +} + +// FetchVersions returns a job for every release package from +// minWindowsPython onward. Sources registered earlier win for any version +// and platform they also provide. +func (s *PythonNuGetSource) FetchVersions() ([]MirrorJob, error) { + flat := s.flatContainerURL + if flat == "" { + flat = nugetFlatContainer + } + registration := s.registrationURL + if registration == "" { + registration = nugetRegistrationURL + } + + var jobs []MirrorJob + for _, pkg := range pythonNuGetPackages { + body, err := download(flat + pkg.id + "/index.json") + if err != nil { + return nil, fmt.Errorf("listing %s: %w", pkg.id, err) + } + var index struct { + Versions []string `json:"versions"` + } + if err := json.Unmarshal(body, &index); err != nil { + return nil, fmt.Errorf("parsing %s index: %w", pkg.id, err) + } + + for _, v := range index.Versions { + parts, ok := parseVersion(v) + if !ok || compareVersions(parts, minWindowsPython) < 0 { + continue + } + id := pkg.id + nupkgURL := fmt.Sprintf("%s%s/%s/%s.%s.nupkg", flat, id, v, id, v) + leafURL := fmt.Sprintf("%s%s/%s.json", registration, id, v) + jobs = append(jobs, MirrorJob{ + Runtime: "python", + Version: v, + Platform: pkg.platform, + URL: nupkgURL, + R2Key: fmt.Sprintf("python/%s/%s.tar.gz", v, pkg.platform), + MetaKey: fmt.Sprintf("python/%s/%s.meta.json", v, pkg.platform), + Build: func() ([]byte, error) { + return buildFromNuGet(nupkgURL, leafURL) + }, + }) + } + } + return jobs, nil +} + +// buildFromNuGet downloads a package, verifies it against the SHA-512 in +// its nuget.org catalog entry and repackages its tools/ folder. +func buildFromNuGet(nupkgURL, leafURL string) ([]byte, error) { + wantHash, err := nugetPackageHash(leafURL) + if err != nil { + return nil, err + } + pkg, err := download(nupkgURL) + if err != nil { + return nil, err + } + if err := verifySHA512(pkg, wantHash); err != nil { + return nil, fmt.Errorf("%s: %w", nupkgURL, err) + } + return archiveNuGetTools(pkg) +} + +// nugetPackageHash returns the base64 SHA-512 nuget.org records for a +// package, read from the catalog entry its registration leaf points to. +func nugetPackageHash(leafURL string) (string, error) { + body, err := download(leafURL) + if err != nil { + return "", err + } + var leaf struct { + CatalogEntry string `json:"catalogEntry"` + } + if err := json.Unmarshal(body, &leaf); err != nil || leaf.CatalogEntry == "" { + return "", fmt.Errorf("no catalog entry in %s", leafURL) + } + + body, err = download(leaf.CatalogEntry) + if err != nil { + return "", err + } + var entry struct { + PackageHash string `json:"packageHash"` + PackageHashAlgorithm string `json:"packageHashAlgorithm"` + } + if err := json.Unmarshal(body, &entry); err != nil { + return "", fmt.Errorf("parsing %s: %w", leaf.CatalogEntry, err) + } + if !strings.EqualFold(entry.PackageHashAlgorithm, "SHA512") || entry.PackageHash == "" { + return "", fmt.Errorf("%s has no SHA512 package hash", leaf.CatalogEntry) + } + return entry.PackageHash, nil +} + +// verifySHA512 checks data against a base64-encoded SHA-512 digest. +func verifySHA512(data []byte, wantBase64 string) error { + sum := sha512.Sum512(data) + if got := base64.StdEncoding.EncodeToString(sum[:]); got != wantBase64 { + return fmt.Errorf("SHA-512 mismatch: expected %s, got %s", wantBase64, got) + } + return nil +} + +// signatureVerifier checks detached OpenPGP signatures against one pinned +// key. +type signatureVerifier struct { + keyring openpgp.EntityList + fingerprint string +} + +// newPythonWindowsVerifier returns a verifier for python.org's Windows +// release signing key. +func newPythonWindowsVerifier() (*signatureVerifier, error) { + return newSignatureVerifier(pythonWindowsSigningKey, pythonWindowsSigningFingerprint) +} + +// newSignatureVerifier parses an armored public key and confirms it is the +// key with the given fingerprint. +func newSignatureVerifier(armoredKey []byte, fingerprint string) (*signatureVerifier, error) { + keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(armoredKey)) + if err != nil { + return nil, fmt.Errorf("reading signing key: %w", err) + } + if len(keyring) != 1 { + return nil, fmt.Errorf("signing key file holds %d keys, want 1", len(keyring)) + } + if got := strings.ToUpper(hex.EncodeToString(keyring[0].PrimaryKey.Fingerprint)); got != fingerprint { + return nil, fmt.Errorf("signing key fingerprint is %s, want %s", got, fingerprint) + } + return &signatureVerifier{keyring: keyring, fingerprint: fingerprint}, nil +} + +// verify checks that armoredSig is a valid signature of data by the pinned +// key. +func (v *signatureVerifier) verify(data, armoredSig []byte) error { + signer, err := openpgp.CheckArmoredDetachedSignature(v.keyring, bytes.NewReader(data), bytes.NewReader(armoredSig), nil) + if err != nil { + return fmt.Errorf("signature check failed: %w", err) + } + if got := strings.ToUpper(hex.EncodeToString(signer.PrimaryKey.Fingerprint)); got != v.fingerprint { + return fmt.Errorf("signed by %s, want %s", got, v.fingerprint) + } + return nil +} + +// archiveDir packs a full Windows Python install from dir. requireTk also +// demands tkinter, which MSI builds include and NuGet builds do not. +func archiveDir(dir string, requireTk bool) ([]byte, error) { + var files []string + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.Type().IsRegular() { + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + files = append(files, filepath.ToSlash(rel)) + } else if !d.IsDir() { + return fmt.Errorf("unexpected non-regular file %s", p) + } + return nil + }) + if err != nil { + return nil, err + } + if err := validateWindowsPython(files, requireTk); err != nil { + return nil, err + } + + w := newPythonArchive() + sort.Strings(files) + for _, rel := range files { + data, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + if err != nil { + return nil, err + } + if err := w.addFile(rel, data); err != nil { + return nil, err + } + } + return w.close() +} + +// archiveNuGetTools packs the full install under tools/ in a NuGet package, +// dropping the package's own files. +func archiveNuGetTools(pkg []byte) ([]byte, error) { + zr, err := zip.NewReader(bytes.NewReader(pkg), int64(len(pkg))) + if err != nil { + return nil, fmt.Errorf("reading package: %w", err) + } + + contents := make(map[string]*zip.File) + var files []string + for _, f := range zr.File { + name := f.Name + // NuGet stores part names URI-escaped ("script%20(dev).tmpl"). + if unescaped, err := url.PathUnescape(name); err == nil { + name = unescaped + } + rel, ok := strings.CutPrefix(name, "tools/") + if !ok || rel == "" || strings.HasSuffix(rel, "/") { + continue + } + if !isSafeRelPath(rel) { + return nil, fmt.Errorf("unsafe path in package: %s", f.Name) + } + contents[rel] = f + files = append(files, rel) + } + if err := validateWindowsPython(files, false); err != nil { + return nil, err + } + + w := newPythonArchive() + sort.Strings(files) + for _, rel := range files { + data, err := readZipFile(contents[rel]) + if err != nil { + return nil, err + } + if err := w.addFile(rel, data); err != nil { + return nil, err + } + } + return w.close() +} + +func readZipFile(f *zip.File) ([]byte, error) { + rc, err := f.Open() + if err != nil { + return nil, err + } + defer func() { _ = rc.Close() }() + return io.ReadAll(rc) +} + +// isSafeRelPath reports whether p stays inside the directory it is +// relative to. +func isSafeRelPath(p string) bool { + clean := path.Clean(p) + return clean != "." && !path.IsAbs(clean) && clean != ".." && !strings.HasPrefix(clean, "../") && !strings.Contains(clean, ":") +} + +// validateWindowsPython checks that files, relative to an install root, +// make a full Windows Python rather than an embeddable one. +func validateWindowsPython(files []string, requireTk bool) error { + have := make(map[string]bool, len(files)) + importLib := false + for _, f := range files { + have[f] = true + if strings.HasSuffix(f, "._pth") { + return fmt.Errorf("%w: found %s, which would put it in isolated mode", errNotFullBuild, f) + } + if strings.HasPrefix(f, "libs/python") && strings.HasSuffix(f, ".lib") { + importLib = true + } + } + + required := []string{"python.exe", "Lib/os.py", "Lib/venv/__init__.py", "Lib/ensurepip/__init__.py", "include/Python.h"} + if requireTk { + required = append(required, "DLLs/_tkinter.pyd", "Lib/tkinter/__init__.py") + } + var missing []string + for _, r := range required { + if !have[r] { + missing = append(missing, r) + } + } + if !importLib { + missing = append(missing, "libs/pythonXY.lib") + } + if len(missing) > 0 { + return fmt.Errorf("%w, missing: %s", errNotFullBuild, strings.Join(missing, ", ")) + } + return nil +} + +// pythonArchive writes a .tar.gz with every entry under python/, the layout +// of python-build-standalone's Windows archives. +type pythonArchive struct { + buf bytes.Buffer + gz *gzip.Writer + tw *tar.Writer + dirs map[string]bool +} + +func newPythonArchive() *pythonArchive { + a := &pythonArchive{dirs: make(map[string]bool)} + a.gz = gzip.NewWriter(&a.buf) + a.tw = tar.NewWriter(a.gz) + return a +} + +// addFile adds rel, a slash-separated path below the install root, after +// any of its parent directories not yet written. +func (a *pythonArchive) addFile(rel string, data []byte) error { + if err := a.addDir(path.Dir(rel)); err != nil { + return err + } + if err := a.tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: "python/" + rel, + Mode: 0o644, + Size: int64(len(data)), + ModTime: archiveModTime, + Format: tar.FormatPAX, + }); err != nil { + return err + } + _, err := a.tw.Write(data) + return err +} + +func (a *pythonArchive) addDir(rel string) error { + if a.dirs[rel] { + return nil + } + name := "python/" + if rel != "." { + if err := a.addDir(path.Dir(rel)); err != nil { + return err + } + name += rel + "/" + } + a.dirs[rel] = true + return a.tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeDir, + Name: name, + Mode: 0o755, + ModTime: archiveModTime, + Format: tar.FormatPAX, + }) +} + +func (a *pythonArchive) close() ([]byte, error) { + if err := a.tw.Close(); err != nil { + return nil, err + } + if err := a.gz.Close(); err != nil { + return nil, err + } + return a.buf.Bytes(), nil +} + +// download fetches url and returns its body, failing on any non-200 status. +func download(url string) ([]byte, error) { + resp, err := httpGetWithRetry(url, 3) + if err != nil { + return nil, fmt.Errorf("downloading %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("downloading %s: HTTP %d", url, resp.StatusCode) + } + return io.ReadAll(resp.Body) +} + +func fetchText(url string) (string, error) { + body, err := download(url) + return string(body), err +} + +// parseVersion parses an x.y.z release version. +func parseVersion(v string) ([3]int, bool) { + var parts [3]int + if !releaseVersionPattern.MatchString(v) { + return parts, false + } + for i, f := range strings.Split(v, ".") { + n, err := strconv.Atoi(f) + if err != nil { + return parts, false + } + parts[i] = n + } + return parts, true +} + +func compareVersions(a, b [3]int) int { + for i := range a { + if a[i] != b[i] { + return a[i] - b[i] + } + } + return 0 +} diff --git a/scripts/mirror-binaries/python_winbuild_network_test.go b/scripts/mirror-binaries/python_winbuild_network_test.go new file mode 100644 index 0000000..54c01f2 --- /dev/null +++ b/scripts/mirror-binaries/python_winbuild_network_test.go @@ -0,0 +1,137 @@ +package main + +// Tests against the real python.org and nuget.org. They download about +// 50 MB, and the MSI build needs msiextract, so they only run when asked: +// +// MIRROR_NETWORK_TESTS=1 go test -v -run Network ./... +// +// On Debian or Ubuntu, install msiextract with: apt-get install msitools. +// Set MIRROR_NETWORK_OUT to a directory to keep the built archives, e.g. to +// unpack and run them on Windows. + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func requireNetworkTests(t *testing.T) { + t.Helper() + if os.Getenv("MIRROR_NETWORK_TESTS") != "1" { + t.Skip("set MIRROR_NETWORK_TESTS=1 to run tests against python.org and nuget.org") + } +} + +// checkRealArchive confirms an archive holds a full install and keeps it +// in MIRROR_NETWORK_OUT if set. +func checkRealArchive(t *testing.T, name string, archive []byte, requireTk bool) { + t.Helper() + var files []string + for entry := range tarEntries(t, archive) { + if rel := entry[len("python/"):]; rel != "" && rel[len(rel)-1] != '/' { + files = append(files, rel) + } + } + if err := validateWindowsPython(files, requireTk); err != nil { + t.Errorf("%s: %v", name, err) + } + t.Logf("%s: %d files, %.1f MB", name, len(files), float64(len(archive))/1e6) + + if dir := os.Getenv("MIRROR_NETWORK_OUT"); dir != "" { + if err := os.WriteFile(filepath.Join(dir, name), archive, 0o644); err != nil { + t.Errorf("saving %s: %v", name, err) + } + } +} + +func TestNetwork_BuildFromMSIs(t *testing.T) { + requireNetworkTests(t) + if _, err := exec.LookPath("msiextract"); err != nil { + t.Skip("msiextract not installed (apt-get install msitools)") + } + verifier, err := newPythonWindowsVerifier() + if err != nil { + t.Fatalf("newPythonWindowsVerifier: %v", err) + } + + // The oldest and newest signed releases in range, and 3.9.11, which has + // no NuGet package at all. + for _, v := range []string{"3.5.0", "3.9.11", "3.12.2"} { + archive, err := buildFromMSIs(pythonFTPBaseURL+v+"/amd64/", verifier, nil) + if err != nil { + t.Errorf("buildFromMSIs(%s): %v", v, err) + continue + } + checkRealArchive(t, "python-"+v+"-windows-amd64.tar.gz", archive, true) + } +} + +func TestNetwork_BuildFromNuGet(t *testing.T) { + requireNetworkTests(t) + + // The first pythonarm64 release, and one from the 3.11 line. + for _, v := range []string{"3.9.7", "3.11.4"} { + nupkgURL := nugetFlatContainer + "pythonarm64/" + v + "/pythonarm64." + v + ".nupkg" + leafURL := nugetRegistrationURL + "pythonarm64/" + v + ".json" + archive, err := buildFromNuGet(nupkgURL, leafURL) + if err != nil { + t.Errorf("buildFromNuGet(pythonarm64 %s): %v", v, err) + continue + } + checkRealArchive(t, "python-"+v+"-windows-arm64.tar.gz", archive, false) + } + + // x64 NuGet packages are not reliably full installs, which is why the + // NuGet source skips them; the validator must say so. + nupkgURL := nugetFlatContainer + "python/3.5.2/python.3.5.2.nupkg" + leafURL := nugetRegistrationURL + "python/3.5.2.json" + if _, err := buildFromNuGet(nupkgURL, leafURL); !errors.Is(err, errNotFullBuild) { + t.Errorf("buildFromNuGet(python 3.5.2) = %v, want errNotFullBuild", err) + } +} + +func TestNetwork_FetchVersions(t *testing.T) { + requireNetworkTests(t) + + msiJobs, err := (&PythonMSISource{}).FetchVersions() + if err != nil { + t.Fatalf("PythonMSISource.FetchVersions: %v", err) + } + msi := make(map[string]bool) + for _, j := range msiJobs { + msi[j.Version] = true + } + t.Logf("python.org MSIs: %d signed x64 releases", len(msiJobs)) + for _, v := range []string{"3.5.0", "3.9.11", "3.12.2", "3.13.1"} { + if !msi[v] { + t.Errorf("MSI source has no job for %s", v) + } + } + for _, v := range []string{"3.5.2", "3.9.15", "3.14.0"} { + if msi[v] { + t.Errorf("MSI source has a job for %s, which has unsigned or no MSIs", v) + } + } + + nugetJobs, err := (&PythonNuGetSource{}).FetchVersions() + if err != nil { + t.Fatalf("PythonNuGetSource.FetchVersions: %v", err) + } + nuget := make(map[string]bool) + for _, j := range nugetJobs { + nuget[j.Version+"/"+j.Platform] = true + } + t.Logf("python.org NuGet packages: %d release jobs", len(nugetJobs)) + for _, key := range []string{"3.9.7/windows-arm64", "3.11.4/windows-arm64", "3.12.2/windows-arm64"} { + if !nuget[key] { + t.Errorf("NuGet source has no job for %s", key) + } + } + for _, j := range nugetJobs { + if j.Platform != "windows-arm64" { + t.Errorf("NuGet source has a %s job (%s); it should only cover windows-arm64", j.Platform, j.Version) + } + } +} diff --git a/scripts/mirror-binaries/python_winbuild_test.go b/scripts/mirror-binaries/python_winbuild_test.go new file mode 100644 index 0000000..45bb5e3 --- /dev/null +++ b/scripts/mirror-binaries/python_winbuild_test.go @@ -0,0 +1,690 @@ +package main + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "testing" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/packet" +) + +// testSigner is a throwaway OpenPGP key for signing test data. +type testSigner struct { + entity *openpgp.Entity + armoredKey []byte + fingerprint string +} + +func newTestSigner(t *testing.T) *testSigner { + t.Helper() + entity, err := openpgp.NewEntity("dtvem test", "", "test@example.invalid", &packet.Config{Algorithm: packet.PubKeyAlgoEdDSA}) + if err != nil { + t.Fatalf("NewEntity: %v", err) + } + var buf bytes.Buffer + w, err := armor.Encode(&buf, openpgp.PublicKeyType, nil) + if err != nil { + t.Fatalf("armor.Encode: %v", err) + } + if err := entity.Serialize(w); err != nil { + t.Fatalf("Serialize: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("closing armor: %v", err) + } + return &testSigner{ + entity: entity, + armoredKey: buf.Bytes(), + fingerprint: strings.ToUpper(hex.EncodeToString(entity.PrimaryKey.Fingerprint)), + } +} + +func (s *testSigner) sign(t *testing.T, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + if err := openpgp.ArmoredDetachSign(&buf, s.entity, bytes.NewReader(data), nil); err != nil { + t.Fatalf("ArmoredDetachSign: %v", err) + } + return buf.Bytes() +} + +func (s *testSigner) verifier(t *testing.T) *signatureVerifier { + t.Helper() + v, err := newSignatureVerifier(s.armoredKey, s.fingerprint) + if err != nil { + t.Fatalf("newSignatureVerifier: %v", err) + } + return v +} + +func TestPythonWindowsSigningKey(t *testing.T) { + if _, err := newPythonWindowsVerifier(); err != nil { + t.Fatalf("embedded signing key: %v", err) + } + + // A checkout with CRLF line endings must load the same key. + crlf := bytes.ReplaceAll(pythonWindowsSigningKey, []byte("\n"), []byte("\r\n")) + if _, err := newSignatureVerifier(crlf, pythonWindowsSigningFingerprint); err != nil { + t.Errorf("embedded signing key with CRLF line endings: %v", err) + } +} + +func TestNewSignatureVerifier_WrongFingerprint(t *testing.T) { + signer := newTestSigner(t) + if _, err := newSignatureVerifier(signer.armoredKey, pythonWindowsSigningFingerprint); err == nil { + t.Error("newSignatureVerifier() accepted a key with a different fingerprint") + } +} + +func TestSignatureVerifier(t *testing.T) { + signer := newTestSigner(t) + other := newTestSigner(t) + v := signer.verifier(t) + data := []byte("python msi contents") + + if err := v.verify(data, signer.sign(t, data)); err != nil { + t.Errorf("verify(good signature) = %v, want nil", err) + } + if err := v.verify([]byte("tampered contents"), signer.sign(t, data)); err == nil { + t.Error("verify(tampered data) = nil, want error") + } + if err := v.verify(data, other.sign(t, data)); err == nil { + t.Error("verify(signature by another key) = nil, want error") + } + if err := v.verify(data, []byte("not a signature")); err == nil { + t.Error("verify(garbage) = nil, want error") + } +} + +func TestParsePythonFTPVersions(t *testing.T) { + index := `../ +2.7.18/ +3.9.11/ +3.13.0/ +3.14/ +3.13.0a1/ +README.html` + + got := parsePythonFTPVersions(index) + want := []string{"2.7.18", "3.9.11", "3.13.0"} + if !reflect.DeepEqual(got, want) { + t.Errorf("parsePythonFTPVersions() = %v, want %v", got, want) + } +} + +// msiListing returns a python.org amd64 folder listing with the given files. +func msiListing(files ...string) string { + var b strings.Builder + for _, f := range files { + fmt.Fprintf(&b, "%s\n", f, f) + } + return b.String() +} + +func signedMSIFiles() []string { + var files []string + for _, c := range pythonMSIComponents { + files = append(files, c+".msi", c+".msi.asc") + } + return append(files, "doc.msi", "doc.msi.asc", "pip.msi") +} + +func TestHasSignedMSIs(t *testing.T) { + if !hasSignedMSIs(msiListing(signedMSIFiles()...)) { + t.Error("hasSignedMSIs(all components signed) = false, want true") + } + + var unsigned []string + for _, c := range pythonMSIComponents { + unsigned = append(unsigned, c+".msi") + } + if hasSignedMSIs(msiListing(unsigned...)) { + t.Error("hasSignedMSIs(no signatures) = true, want false") + } + + if hasSignedMSIs(msiListing("core.msi", "core.msi.asc", "exe.msi", "exe.msi.asc")) { + t.Error("hasSignedMSIs(missing components) = true, want false") + } +} + +func TestPythonMSISource_FetchVersions(t *testing.T) { + var unsigned []string + for _, c := range pythonMSIComponents { + unsigned = append(unsigned, c+".msi") + } + pages := map[string]string{ + "/": msiListing("3.4.10/", "3.5.2/", "3.9.11/", "3.9.15/", "3.13.1/", "3.14.0/"), + // 3.5.2's MSIs have no signatures. + "/3.5.2/amd64/": msiListing(unsigned...), + "/3.9.11/amd64/": msiListing(signedMSIFiles()...), + "/3.13.1/amd64/": msiListing(signedMSIFiles()...), + // 3.9.15 is a source-only security release: no amd64 folder. + // 3.4.10 predates the MSI installer and 3.14.0 is Sigstore-only; + // neither should be requested. + "/3.14.0/amd64/": msiListing(signedMSIFiles()...), + } + var requested pathRecorder + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requested.record(r.URL.Path) + page, ok := pages[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, page) + })) + defer srv.Close() + + src := &PythonMSISource{baseURL: srv.URL + "/"} + jobs, err := src.FetchVersions() + if err != nil { + t.Fatalf("FetchVersions() error: %v", err) + } + + var versions []string + for _, j := range jobs { + versions = append(versions, j.Version) + if j.Platform != "windows-amd64" || j.Build == nil { + t.Errorf("job %s: platform %q, Build set %v; want windows-amd64 with a Build step", j.Version, j.Platform, j.Build != nil) + } + if want := fmt.Sprintf("python/%s/windows-amd64.tar.gz", j.Version); j.R2Key != want { + t.Errorf("job %s: R2Key %q, want %q", j.Version, j.R2Key, want) + } + if want := fmt.Sprintf("python/%s/windows-amd64.meta.json", j.Version); j.MetaKey != want { + t.Errorf("job %s: MetaKey %q, want %q", j.Version, j.MetaKey, want) + } + if want := srv.URL + "/" + j.Version + "/amd64/"; j.URL != want { + t.Errorf("job %s: URL %q, want %q", j.Version, j.URL, want) + } + } + sort.Strings(versions) + if want := []string{"3.13.1", "3.9.11"}; !reflect.DeepEqual(versions, want) { + t.Errorf("FetchVersions() versions = %v, want %v", versions, want) + } + for _, p := range requested.list() { + if strings.HasPrefix(p, "/3.4.10/") || strings.HasPrefix(p, "/3.14.0/") { + t.Errorf("requested %s, outside the MSI-signed range", p) + } + } +} + +// pathRecorder records requested paths; test servers handle requests on +// several goroutines at once. +type pathRecorder struct { + mu sync.Mutex + paths []string +} + +func (r *pathRecorder) record(p string) { + r.mu.Lock() + defer r.mu.Unlock() + r.paths = append(r.paths, p) +} + +func (r *pathRecorder) list() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.paths...) +} + +// fakeMSIContents is what the fake extractor writes for each component, +// mirroring the real MSIs' contents. +var fakeMSIContents = map[string][]string{ + "core": {"python39.dll", "python3.dll"}, + "exe": {"python.exe", "pythonw.exe", "vcruntime140.dll"}, + "lib": {"Lib/os.py", "Lib/venv/__init__.py", "Lib/venv/scripts/nt/python.exe", "Lib/ensurepip/__init__.py", "Lib/ensurepip/_bundled/pip-22.0.4-py3-none-any.whl", "DLLs/_ssl.pyd"}, + "dev": {"include/Python.h", "libs/python39.lib"}, + "tcltk": {"DLLs/_tkinter.pyd", "DLLs/tcl86t.dll", "Lib/tkinter/__init__.py", "tcl/tcl8.6/init.tcl"}, +} + +// fakeExtract stands in for msiextract: the MSI's content names the +// component, and the component's files are written into destDir. +func fakeExtract(msiPath, destDir string) error { + component, err := os.ReadFile(msiPath) + if err != nil { + return err + } + for _, rel := range fakeMSIContents[strings.TrimPrefix(string(component), "msi:")] { + p := filepath.Join(destDir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return err + } + if err := os.WriteFile(p, []byte("contents of "+rel), 0o644); err != nil { + return err + } + } + return nil +} + +// msiServer serves each component's MSI, whose content is "msi:", +// and its signature by signer. +func msiServer(t *testing.T, signer *testSigner) *httptest.Server { + t.Helper() + files := make(map[string][]byte) + for _, c := range pythonMSIComponents { + msi := []byte("msi:" + c) + files["/"+c+".msi"] = msi + files["/"+c+".msi.asc"] = signer.sign(t, msi) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv +} + +// tarEntries lists the entries of a .tar.gz, directories ending in "/". +func tarEntries(t *testing.T, archive []byte) map[string]string { + t.Helper() + gz, err := gzip.NewReader(bytes.NewReader(archive)) + if err != nil { + t.Fatalf("gzip.NewReader: %v", err) + } + tr := tar.NewReader(gz) + entries := make(map[string]string) + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("reading tar: %v", err) + } + data, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("reading %s: %v", h.Name, err) + } + if !h.ModTime.Equal(archiveModTime) { + t.Errorf("%s: ModTime %v, want %v", h.Name, h.ModTime, archiveModTime) + } + entries[h.Name] = string(data) + } + return entries +} + +func TestBuildFromMSIs(t *testing.T) { + signer := newTestSigner(t) + srv := msiServer(t, signer) + + archive, err := buildFromMSIs(srv.URL+"/", signer.verifier(t), fakeExtract) + if err != nil { + t.Fatalf("buildFromMSIs() error: %v", err) + } + + entries := tarEntries(t, archive) + for _, files := range fakeMSIContents { + for _, rel := range files { + if got, ok := entries["python/"+rel]; !ok || got != "contents of "+rel { + t.Errorf("archive entry python/%s = %q (present %v)", rel, got, ok) + } + } + } + for _, dir := range []string{"python/", "python/Lib/", "python/Lib/venv/scripts/nt/", "python/tcl/tcl8.6/"} { + if _, ok := entries[dir]; !ok { + t.Errorf("archive has no directory entry %s", dir) + } + } + for name := range entries { + if !strings.HasPrefix(name, "python/") { + t.Errorf("archive entry %s is outside python/", name) + } + } + + again, err := buildFromMSIs(srv.URL+"/", signer.verifier(t), fakeExtract) + if err != nil { + t.Fatalf("second buildFromMSIs() error: %v", err) + } + if !bytes.Equal(archive, again) { + t.Error("building the same release twice produced different archives") + } +} + +func TestBuildFromMSIs_BadSignature(t *testing.T) { + srv := msiServer(t, newTestSigner(t)) + trusted := newTestSigner(t) + + if _, err := buildFromMSIs(srv.URL+"/", trusted.verifier(t), fakeExtract); err == nil { + t.Error("buildFromMSIs() accepted MSIs signed by an untrusted key") + } +} + +func TestBuildFromMSIs_ExtractError(t *testing.T) { + signer := newTestSigner(t) + srv := msiServer(t, signer) + failing := func(string, string) error { return fmt.Errorf("msiextract exploded") } + + if _, err := buildFromMSIs(srv.URL+"/", signer.verifier(t), failing); err == nil || !strings.Contains(err.Error(), "msiextract exploded") { + t.Errorf("buildFromMSIs() error = %v, want the extract error", err) + } +} + +func TestValidateWindowsPython(t *testing.T) { + full := []string{"python.exe", "Lib/os.py", "Lib/venv/__init__.py", "Lib/ensurepip/__init__.py", "include/Python.h", "libs/python39.lib"} + withTk := append(append([]string{}, full...), "DLLs/_tkinter.pyd", "Lib/tkinter/__init__.py") + + tests := []struct { + name string + files []string + requireTk bool + wantErr string + }{ + {name: "full install", files: full}, + {name: "full install with tkinter", files: withTk, requireTk: true}, + {name: "tkinter required but missing", files: full, requireTk: true, wantErr: "DLLs/_tkinter.pyd"}, + {name: "embeddable ._pth", files: append(append([]string{}, full...), "python39._pth"), wantErr: "isolated mode"}, + {name: "no venv", files: []string{"python.exe", "Lib/os.py", "Lib/ensurepip/__init__.py", "include/Python.h", "libs/python39.lib"}, wantErr: "Lib/venv/__init__.py"}, + {name: "no import library", files: full[:5], wantErr: "libs/pythonXY.lib"}, + {name: "embeddable layout", files: []string{"python.exe", "python39.zip", "python39.dll"}, wantErr: "missing"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateWindowsPython(tt.files, tt.requireTk) + if tt.wantErr == "" { + if err != nil { + t.Errorf("validateWindowsPython() = %v, want nil", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("validateWindowsPython() = %v, want error mentioning %q", err, tt.wantErr) + } + }) + } +} + +// nupkg builds a NuGet package with the given entries. +func nupkg(t *testing.T, entries map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + w, err := zw.Create(name) + if err != nil { + t.Fatalf("zip Create %s: %v", name, err) + } + if _, err := io.WriteString(w, entries[name]); err != nil { + t.Fatalf("zip write %s: %v", name, err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("zip Close: %v", err) + } + return buf.Bytes() +} + +func fullNuGetEntries() map[string]string { + return map[string]string{ + "[Content_Types].xml": "types", + "_rels/.rels": "rels", + "python.nuspec": "nuspec", + "build/native/python.props": "props", + "package/services/metadata/x": "meta", + ".signature.p7s": "sig", + "tools/": "", + "tools/python.exe": "exe", + "tools/Lib/os.py": "os", + "tools/Lib/venv/__init__.py": "venv", + "tools/Lib/ensurepip/__init__.py": "ensurepip", + "tools/include/Python.h": "header", + "tools/libs/python39.lib": "lib", + "tools/Lib/site-packages/setuptools/script%20(dev).tmpl": "template", + } +} + +func TestArchiveNuGetTools(t *testing.T) { + archive, err := archiveNuGetTools(nupkg(t, fullNuGetEntries())) + if err != nil { + t.Fatalf("archiveNuGetTools() error: %v", err) + } + entries := tarEntries(t, archive) + + for rel, want := range map[string]string{ + "python.exe": "exe", + "Lib/os.py": "os", + "include/Python.h": "header", + "Lib/site-packages/setuptools/script (dev).tmpl": "template", + } { + if got, ok := entries["python/"+rel]; !ok || got != want { + t.Errorf("archive entry python/%s = %q (present %v), want %q", rel, got, ok, want) + } + } + for name := range entries { + if !strings.HasPrefix(name, "python/") { + t.Errorf("archive entry %s is outside python/", name) + } + for _, dropped := range []string{"nuspec", "props", ".rels", "Content_Types", "p7s", "services"} { + if strings.Contains(name, dropped) { + t.Errorf("archive kept NuGet packaging file %s", name) + } + } + } +} + +func TestArchiveNuGetTools_Rejects(t *testing.T) { + embeddable := fullNuGetEntries() + embeddable["tools/python39._pth"] = "python39.zip" + + incomplete := fullNuGetEntries() + delete(incomplete, "tools/Lib/venv/__init__.py") + + unsafe := fullNuGetEntries() + unsafe["tools/../escape.txt"] = "x" + + for name, entries := range map[string]map[string]string{"embeddable": embeddable, "incomplete": incomplete, "unsafe path": unsafe} { + if _, err := archiveNuGetTools(nupkg(t, entries)); err == nil { + t.Errorf("archiveNuGetTools(%s) = nil error, want rejection", name) + } + } + if _, err := archiveNuGetTools([]byte("not a zip")); err == nil { + t.Error("archiveNuGetTools(not a zip) = nil error, want error") + } +} + +func TestVerifySHA512(t *testing.T) { + data := []byte("package") + sum := sha512.Sum512(data) + good := base64.StdEncoding.EncodeToString(sum[:]) + + if err := verifySHA512(data, good); err != nil { + t.Errorf("verifySHA512(matching) = %v, want nil", err) + } + if err := verifySHA512([]byte("tampered"), good); err == nil { + t.Error("verifySHA512(mismatch) = nil, want error") + } +} + +// nugetServer serves python.org-style NuGet packages: flat container +// indexes and packages, registration leaves and catalog entries. +func nugetServer(t *testing.T, indexes map[string][]string, packages map[string][]byte, hashes map[string]string) (*httptest.Server, *pathRecorder) { + t.Helper() + requested := &pathRecorder{} + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p := r.URL.Path + requested.record(p) + switch { + case strings.HasPrefix(p, "/flat/") && strings.HasSuffix(p, "/index.json"): + id := strings.TrimSuffix(strings.TrimPrefix(p, "/flat/"), "/index.json") + versions, ok := indexes[id] + if !ok { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string][]string{"versions": versions}) + case strings.HasPrefix(p, "/flat/") && strings.HasSuffix(p, ".nupkg"): + pkg, ok := packages[filepath.Base(p)] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(pkg) + case strings.HasPrefix(p, "/registration/"): + key := strings.TrimSuffix(strings.TrimPrefix(p, "/registration/"), ".json") + _, _ = fmt.Fprintf(w, `{"catalogEntry": "%s/catalog/%s.json"}`, srv.URL, strings.ReplaceAll(key, "/", ".")) + case strings.HasPrefix(p, "/catalog/"): + key := strings.TrimSuffix(strings.TrimPrefix(p, "/catalog/"), ".json") + hash, ok := hashes[key] + if !ok { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprintf(w, `{"packageHashAlgorithm": "SHA512", "packageHash": "%s"}`, hash) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, requested +} + +func TestPythonNuGetSource_FetchVersions(t *testing.T) { + srv, requested := nugetServer(t, map[string][]string{ + // x64 packages are not reliably full installs and must not be used. + "python": {"3.5.2", "3.13.1"}, + "pythonarm64": {"3.4.4", "3.9.7", "3.13.0-b1", "3.11.4"}, + }, nil, nil) + src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} + + jobs, err := src.FetchVersions() + if err != nil { + t.Fatalf("FetchVersions() error: %v", err) + } + + var got []string + for _, j := range jobs { + got = append(got, j.Version+"/"+j.Platform) + if j.Build == nil { + t.Errorf("job %s/%s has no Build step", j.Version, j.Platform) + } + if want := fmt.Sprintf("python/%s/%s.tar.gz", j.Version, j.Platform); j.R2Key != want { + t.Errorf("job %s: R2Key %q, want %q", j.Version, j.R2Key, want) + } + } + want := []string{"3.9.7/windows-arm64", "3.11.4/windows-arm64"} + if !reflect.DeepEqual(got, want) { + t.Errorf("FetchVersions() = %v, want %v", got, want) + } + if want := srv.URL + "/flat/pythonarm64/3.11.4/pythonarm64.3.11.4.nupkg"; jobs[1].URL != want { + t.Errorf("arm64 job URL = %q, want %q", jobs[1].URL, want) + } + for _, p := range requested.list() { + if strings.HasPrefix(p, "/flat/python/") { + t.Errorf("requested %s; x64 NuGet packages should not be listed", p) + } + } +} + +func TestNotFullBuildErrors(t *testing.T) { + // validateWindowsPython's rejections must reach the mirror loop as + // errNotFullBuild, so those jobs are skipped rather than failed. + if err := validateWindowsPython([]string{"python.exe", "python39.zip"}, false); !errors.Is(err, errNotFullBuild) { + t.Errorf("validateWindowsPython(incomplete) = %v, want errNotFullBuild", err) + } + + embeddable := fullNuGetEntries() + embeddable["tools/python39._pth"] = "python39.zip" + if _, err := archiveNuGetTools(nupkg(t, embeddable)); !errors.Is(err, errNotFullBuild) { + t.Errorf("archiveNuGetTools(embeddable) = %v, want errNotFullBuild", err) + } + + incomplete := MirrorJob{Build: func() ([]byte, error) { + return nil, validateWindowsPython(nil, false) + }} + if _, _, _, err := fetchBinary(incomplete, &Stats{}); !errors.Is(err, errNotFullBuild) { + t.Errorf("fetchBinary(incomplete build) = %v, want errNotFullBuild", err) + } +} + +func TestBuildFromNuGet(t *testing.T) { + pkg := nupkg(t, fullNuGetEntries()) + sum := sha512.Sum512(pkg) + srv, _ := nugetServer(t, + map[string][]string{"python": {}, "pythonarm64": {"3.11.4"}}, + map[string][]byte{"pythonarm64.3.11.4.nupkg": pkg}, + map[string]string{"pythonarm64.3.11.4": base64.StdEncoding.EncodeToString(sum[:])}, + ) + src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} + jobs, err := src.FetchVersions() + if err != nil || len(jobs) != 1 { + t.Fatalf("FetchVersions() = %d jobs, %v; want 1 job", len(jobs), err) + } + + archive, err := jobs[0].Build() + if err != nil { + t.Fatalf("Build() error: %v", err) + } + if _, ok := tarEntries(t, archive)["python/python.exe"]; !ok { + t.Error("archive has no python/python.exe") + } +} + +func TestBuildFromNuGet_HashMismatch(t *testing.T) { + pkg := nupkg(t, fullNuGetEntries()) + other := sha512.Sum512([]byte("a different package")) + srv, _ := nugetServer(t, + map[string][]string{"python": {}, "pythonarm64": {"3.11.4"}}, + map[string][]byte{"pythonarm64.3.11.4.nupkg": pkg}, + map[string]string{"pythonarm64.3.11.4": base64.StdEncoding.EncodeToString(other[:])}, + ) + src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} + jobs, err := src.FetchVersions() + if err != nil || len(jobs) != 1 { + t.Fatalf("FetchVersions() = %d jobs, %v; want 1 job", len(jobs), err) + } + + if _, err := jobs[0].Build(); err == nil || !strings.Contains(err.Error(), "SHA-512 mismatch") { + t.Errorf("Build() error = %v, want a SHA-512 mismatch", err) + } +} + +func TestParseVersion(t *testing.T) { + tests := []struct { + in string + want [3]int + wantOK bool + }{ + {"3.9.11", [3]int{3, 9, 11}, true}, + {"3.13.0-b1", [3]int{}, false}, + {"3.14", [3]int{}, false}, + {"v3.9.11", [3]int{}, false}, + } + for _, tt := range tests { + got, ok := parseVersion(tt.in) + if ok != tt.wantOK || (ok && got != tt.want) { + t.Errorf("parseVersion(%q) = %v, %v; want %v, %v", tt.in, got, ok, tt.want, tt.wantOK) + } + } + + if compareVersions([3]int{3, 9, 11}, [3]int{3, 10, 0}) >= 0 { + t.Error("compareVersions(3.9.11, 3.10.0) should be negative") + } +} diff --git a/scripts/mirror-binaries/sources.go b/scripts/mirror-binaries/sources.go index 72cb993..33a28a7 100644 --- a/scripts/mirror-binaries/sources.go +++ b/scripts/mirror-binaries/sources.go @@ -84,7 +84,10 @@ func getUpstreamSources(runtime string) ([]UpstreamSource, error) { case "python": return []UpstreamSource{ &PythonStandaloneSource{}, - &PythonOfficialSource{}, + // Full Windows builds for versions python-build-standalone + // lacks, in place of python.org's embeddable zips. + &PythonMSISource{}, + &PythonNuGetSource{}, }, nil case "ruby": return []UpstreamSource{ diff --git a/scripts/mirror-binaries/sources_test.go b/scripts/mirror-binaries/sources_test.go index 2ed960a..dde17e8 100644 --- a/scripts/mirror-binaries/sources_test.go +++ b/scripts/mirror-binaries/sources_test.go @@ -138,3 +138,22 @@ func TestCollectJobs(t *testing.T) { }) } } + +// The first source to offer a version and platform wins, so the order sets +// which build a version gets: python-build-standalone where it has one, then +// python.org's MSIs (full x64 builds with tkinter), then its NuGet packages. +func TestGetUpstreamSources_PythonOrder(t *testing.T) { + sources, err := getUpstreamSources("python") + if err != nil { + t.Fatalf("getUpstreamSources(python) error: %v", err) + } + + var got []string + for _, s := range sources { + got = append(got, s.Name()) + } + want := []string{"python-build-standalone", "python.org MSIs", "python.org NuGet packages"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("python sources = %v, want %v", got, want) + } +} From aa812e968e6514b13e7fc0fd22471bed988cdb2b Mon Sep 17 00:00:00 2001 From: "Calvin A. Allen" Date: Thu, 22 Oct 2026 18:46:04 -0400 Subject: [PATCH 2/2] fix(python): disable the ._pth file of embeddable Windows installs python.org's embeddable Windows package ships a pythonXY._pth file that runs Python in isolated mode: a script cannot import modules sitting next to it, PYTHONPATH is ignored, and site-packages is off. The installer used to uncomment "import site" in that file, which brought back site-packages but left the isolation. The installer now renames the file to pythonXY._pth.disabled instead. Python only looks for the exact name, so the install behaves like a regular one, and the rename can be undone. The mirror serves full builds for every affected version except 3.5.2 (its MSIs are unsigned and its NuGet package incomplete), so this is the fallback for that version and any embeddable build still served. A new dtvem doctor check (python-isolated-mode) flags installed Python versions that still have a ._pth file, such as ones installed by older releases. --fix applies the same rename in place; the resolution also recommends reinstalling to get a full build with venv, headers and tkinter. The shared logic lives in a small internal/pth package. getPipURL now uses the version-specific get-pip.py for Python 3.9 as well: the main script requires 3.10+, so pip setup failed for 3.9 installs that rely on the get-pip.py fallback (embeddable builds). --- CLAUDE.md | 1 + src/internal/doctor/check_python_pth.go | 109 +++++++++++++++++++ src/internal/doctor/check_python_pth_test.go | 106 ++++++++++++++++++ src/internal/pth/pth.go | 50 +++++++++ src/internal/pth/pth_test.go | 99 +++++++++++++++++ src/runtimes/python/provider_full.go | 46 +++----- src/runtimes/python/provider_test.go | 96 +--------------- 7 files changed, 382 insertions(+), 125 deletions(-) create mode 100644 src/internal/doctor/check_python_pth.go create mode 100644 src/internal/doctor/check_python_pth_test.go create mode 100644 src/internal/pth/pth.go create mode 100644 src/internal/pth/pth_test.go diff --git a/CLAUDE.md b/CLAUDE.md index fc0faee..3c5b80c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -170,6 +170,7 @@ Maps to runtime provider → Resolves version | `internal/download/` | File downloads with progress | | `internal/manifest/` | Version manifest fetching and caching | | `internal/migration/` | Migration detection and helpers | +| `internal/pth/` | Turns off the `._pth` file of python.org's embeddable Windows Python (installer and `dtvem doctor`) | | `internal/testutil/` | Shared test utility functions | | `internal/constants/` | Platform constants | | `src/cmd/` | CLI commands (one file per command) | diff --git a/src/internal/doctor/check_python_pth.go b/src/internal/doctor/check_python_pth.go new file mode 100644 index 0000000..0fb0e12 --- /dev/null +++ b/src/internal/doctor/check_python_pth.go @@ -0,0 +1,109 @@ +package doctor + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/CodingWithCalvin/dtvem.cli/src/internal/config" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/pth" +) + +// pythonPthCheck flags installed Python versions that still have a ._pth +// file, the mark of python.org's embeddable Windows package. While it is +// there Python runs in isolated mode: a script can't import modules sitting +// next to it, and PYTHONPATH is ignored. +// +// Older dtvem releases installed embeddable builds for some Windows +// versions. The mirror now serves full builds and the installer turns the +// file off, but versions installed before that keep it until fixed. +// +// The fix renames the file to ._pth.disabled, which restores normal +// imports in place. Only a reinstall brings the rest of a full build (venv, +// C headers, tkinter), so the resolution recommends that too. +type pythonPthCheck struct { + pythonVersionsDir func() string +} + +func newPythonPthCheck() *pythonPthCheck { + return &pythonPthCheck{ + pythonVersionsDir: func() string { return filepath.Join(config.DefaultPaths().Versions, "python") }, + } +} + +func (pythonPthCheck) Name() string { return "python-isolated-mode" } + +func (c pythonPthCheck) Run() Finding { + root := c.pythonVersionsDir() + entries, err := os.ReadDir(root) + if errors.Is(err, fs.ErrNotExist) { + return Finding{OK: true, Title: "No Python versions installed"} + } + if err != nil { + return Finding{ + Severity: SeverityWarning, + Title: "Could not list installed Python versions", + Details: []Detail{{Key: "Error", Value: err.Error()}}, + Resolution: "Check that " + root + " is readable.", + } + } + + type isolatedInstall struct { + version string + dir string + files []string + } + var isolated []isolatedInstall + for _, e := range entries { + if !e.IsDir() { + continue + } + dir := filepath.Join(root, e.Name()) + files, err := pth.Find(dir) + if err != nil || len(files) == 0 { + continue + } + isolated = append(isolated, isolatedInstall{version: e.Name(), dir: dir, files: files}) + } + + if len(isolated) == 0 { + return Finding{OK: true, Title: "No installed Python version runs in isolated mode"} + } + + sort.Slice(isolated, func(i, j int) bool { return isolated[i].version < isolated[j].version }) + details := make([]Detail, 0, len(isolated)) + var versions []string + for _, inst := range isolated { + names := make([]string, len(inst.files)) + for i, f := range inst.files { + names[i] = filepath.Base(f) + } + details = append(details, Detail{Key: "Python " + inst.version, Value: strings.Join(names, ", ")}) + versions = append(versions, inst.version) + } + + return Finding{ + Severity: SeverityWarning, + Title: "Python versions run in isolated mode (embeddable builds)", + Details: details, + Resolution: "Rename their ._pth files to ._pth.disabled so scripts can import modules next to them and PYTHONPATH works. " + + "For a full build with venv, C headers and tkinter, reinstall: dtvem uninstall python , then dtvem install python (" + + strings.Join(versions, ", ") + ").", + Fix: func() error { + for _, inst := range isolated { + if _, err := pth.Disable(inst.dir); err != nil { + return fmt.Errorf("Python %s: %w", inst.version, err) + } + } + return nil + }, + } +} + +func init() { + Register(newPythonPthCheck()) +} diff --git a/src/internal/doctor/check_python_pth_test.go b/src/internal/doctor/check_python_pth_test.go new file mode 100644 index 0000000..c21aaf8 --- /dev/null +++ b/src/internal/doctor/check_python_pth_test.go @@ -0,0 +1,106 @@ +package doctor + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// pythonInstalls creates a python versions directory with one install per +// entry, adding a pythonXY._pth file to the versions listed in embeddable. +func pythonInstalls(t *testing.T, versions []string, embeddable map[string]bool) string { + t.Helper() + root := filepath.Join(t.TempDir(), "versions", "python") + for _, v := range versions { + dir := filepath.Join(root, v) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + if err := os.WriteFile(filepath.Join(dir, "python.exe"), nil, 0o644); err != nil { + t.Fatalf("writing python.exe: %v", err) + } + if embeddable[v] { + parts := strings.Split(v, ".") + name := "python" + parts[0] + parts[1] + "._pth" + if err := os.WriteFile(filepath.Join(dir, name), []byte("python39.zip\n.\nimport site\n"), 0o644); err != nil { + t.Fatalf("writing %s: %v", name, err) + } + } + } + return root +} + +func TestPythonPthCheck_NoPythonInstalled(t *testing.T) { + c := pythonPthCheck{pythonVersionsDir: func() string { return filepath.Join(t.TempDir(), "missing") }} + + if f := c.Run(); !f.OK { + t.Errorf("Run() = %+v, want OK", f) + } +} + +func TestPythonPthCheck_FullBuilds(t *testing.T) { + root := pythonInstalls(t, []string{"3.12.12", "3.13.1"}, nil) + c := pythonPthCheck{pythonVersionsDir: func() string { return root }} + + if f := c.Run(); !f.OK { + t.Errorf("Run() = %+v, want OK", f) + } +} + +func TestPythonPthCheck_FlagsAndFixesEmbeddableInstalls(t *testing.T) { + root := pythonInstalls(t, []string{"3.12.12", "3.9.9", "3.8.9"}, map[string]bool{"3.9.9": true, "3.8.9": true}) + c := pythonPthCheck{pythonVersionsDir: func() string { return root }} + + f := c.Run() + if f.OK || f.Severity != SeverityWarning { + t.Fatalf("Run() OK=%v severity=%v, want a warning", f.OK, f.Severity) + } + if !f.Fixable() { + t.Fatal("finding is not fixable, want the rename as a fix") + } + want := []Detail{ + {Key: "Python 3.8.9", Value: "python38._pth"}, + {Key: "Python 3.9.9", Value: "python39._pth"}, + } + if len(f.Details) != len(want) { + t.Fatalf("Details = %v, want %v", f.Details, want) + } + for i := range want { + if f.Details[i] != want[i] { + t.Errorf("Details[%d] = %v, want %v", i, f.Details[i], want[i]) + } + } + for _, s := range []string{"._pth.disabled", "dtvem install python", "3.8.9, 3.9.9"} { + if !strings.Contains(f.Resolution, s) { + t.Errorf("Resolution %q does not mention %q", f.Resolution, s) + } + } + + if err := f.Fix(); err != nil { + t.Fatalf("Fix() error: %v", err) + } + for _, v := range []string{"3.8.9", "3.9.9"} { + parts := strings.Split(v, ".") + pthFile := filepath.Join(root, v, "python"+parts[0]+parts[1]+"._pth") + if _, err := os.Stat(pthFile); !os.IsNotExist(err) { + t.Errorf("%s still exists after Fix()", pthFile) + } + if _, err := os.Stat(pthFile + ".disabled"); err != nil { + t.Errorf("%s.disabled missing after Fix(): %v", pthFile, err) + } + } + + if after := c.Run(); !after.OK { + t.Errorf("Run() after Fix() = %+v, want OK", after) + } +} + +func TestPythonPthCheck_Registered(t *testing.T) { + for _, c := range All() { + if c.Name() == "python-isolated-mode" { + return + } + } + t.Error("python-isolated-mode check is not registered") +} diff --git a/src/internal/pth/pth.go b/src/internal/pth/pth.go new file mode 100644 index 0000000..902fca8 --- /dev/null +++ b/src/internal/pth/pth.go @@ -0,0 +1,50 @@ +// Package pth handles the ._pth file in python.org's embeddable Windows +// Python package. +// +// While a pythonXY._pth file sits next to python.exe, Python builds sys.path +// from that file alone and runs in isolated mode: a script's own directory is +// not on sys.path, PYTHONPATH is ignored, and site-packages is off. That suits +// a runtime embedded in an application, not one people run directly. Full +// builds have no ._pth file. Renaming it to pythonXY._pth.disabled makes an +// embeddable install behave like a regular one, and the rename can be undone. +package pth + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" +) + +// DisabledSuffix is appended to a ._pth file's name to turn it off. Python +// only looks for the exact ._pth name. +const DisabledSuffix = ".disabled" + +// Find returns the ._pth files directly inside a Python install directory. +func Find(installDir string) ([]string, error) { + return filepath.Glob(filepath.Join(installDir, "*._pth")) +} + +// Disable renames every ._pth file in a Python install directory to +// .disabled and returns the files it renamed. A leftover .disabled +// copy from an earlier rename is replaced. +func Disable(installDir string) ([]string, error) { + files, err := Find(installDir) + if err != nil { + return nil, err + } + + var disabled []string + for _, f := range files { + target := f + DisabledSuffix + if err := os.Remove(target); err != nil && !errors.Is(err, fs.ErrNotExist) { + return disabled, fmt.Errorf("removing old %s: %w", filepath.Base(target), err) + } + if err := os.Rename(f, target); err != nil { + return disabled, fmt.Errorf("disabling %s: %w", filepath.Base(f), err) + } + disabled = append(disabled, f) + } + return disabled, nil +} diff --git a/src/internal/pth/pth_test.go b/src/internal/pth/pth_test.go new file mode 100644 index 0000000..59c54e5 --- /dev/null +++ b/src/internal/pth/pth_test.go @@ -0,0 +1,99 @@ +package pth + +import ( + "os" + "path/filepath" + "testing" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("writing %s: %v", path, err) + } +} + +func readFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + return string(data) +} + +func TestFind(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "python39._pth"), "python39.zip\n.\n") + writeFile(t, filepath.Join(dir, "python.exe"), "") + writeFile(t, filepath.Join(dir, "python38._pth.disabled"), "") + if err := os.MkdirAll(filepath.Join(dir, "Lib"), 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(dir, "Lib", "nested._pth"), "") + + got, err := Find(dir) + if err != nil { + t.Fatalf("Find() error: %v", err) + } + if len(got) != 1 || filepath.Base(got[0]) != "python39._pth" { + t.Errorf("Find() = %v, want only python39._pth", got) + } +} + +func TestFind_FullBuild(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "python.exe"), "") + + got, err := Find(dir) + if err != nil || len(got) != 0 { + t.Errorf("Find(full build) = %v, %v; want none", got, err) + } +} + +func TestDisable(t *testing.T) { + dir := t.TempDir() + pthFile := filepath.Join(dir, "python39._pth") + writeFile(t, pthFile, "python39.zip\n.\nimport site\n") + + disabled, err := Disable(dir) + if err != nil { + t.Fatalf("Disable() error: %v", err) + } + if len(disabled) != 1 || disabled[0] != pthFile { + t.Errorf("Disable() = %v, want [%s]", disabled, pthFile) + } + if _, err := os.Stat(pthFile); !os.IsNotExist(err) { + t.Errorf("%s still exists after Disable()", pthFile) + } + if got := readFile(t, pthFile+DisabledSuffix); got != "python39.zip\n.\nimport site\n" { + t.Errorf("disabled file content = %q, want the original", got) + } + + // Nothing left to disable. + again, err := Disable(dir) + if err != nil || len(again) != 0 { + t.Errorf("second Disable() = %v, %v; want nothing", again, err) + } +} + +func TestDisable_ReplacesOldDisabledCopy(t *testing.T) { + dir := t.TempDir() + pthFile := filepath.Join(dir, "python39._pth") + writeFile(t, pthFile, "new") + writeFile(t, pthFile+DisabledSuffix, "old") + + if _, err := Disable(dir); err != nil { + t.Fatalf("Disable() error: %v", err) + } + if got := readFile(t, pthFile+DisabledSuffix); got != "new" { + t.Errorf("disabled file content = %q, want %q", got, "new") + } +} + +func TestDisable_MissingDirectory(t *testing.T) { + disabled, err := Disable(filepath.Join(t.TempDir(), "missing")) + if err != nil || len(disabled) != 0 { + t.Errorf("Disable(missing dir) = %v, %v; want nothing", disabled, err) + } +} diff --git a/src/runtimes/python/provider_full.go b/src/runtimes/python/provider_full.go index 128e0cb..63c74ca 100644 --- a/src/runtimes/python/provider_full.go +++ b/src/runtimes/python/provider_full.go @@ -20,6 +20,7 @@ import ( "github.com/CodingWithCalvin/dtvem.cli/src/internal/constants" "github.com/CodingWithCalvin/dtvem.cli/src/internal/download" "github.com/CodingWithCalvin/dtvem.cli/src/internal/manifest" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/pth" "github.com/CodingWithCalvin/dtvem.cli/src/internal/runtime" "github.com/CodingWithCalvin/dtvem.cli/src/internal/shim" "github.com/CodingWithCalvin/dtvem.cli/src/internal/ui" @@ -224,10 +225,16 @@ func (p *Provider) installPip(version string) error { installPath := config.RuntimeVersionPath("python", version) - // For python.org embeddable packages, enable site-packages first. - // This file doesn't exist in python-build-standalone, so errors are ignored. - pthFile := filepath.Join(installPath, fmt.Sprintf("python%s._pth", strings.Join(strings.Split(version, ".")[:2], ""))) - _ = p.enableSitePackages(pthFile) // Best effort - ignore errors + // python.org embeddable packages ship a ._pth file that runs Python in + // isolated mode: no script directory on sys.path, PYTHONPATH ignored, + // site-packages off. Turning it off makes the install behave like a + // regular one. Full builds have no ._pth file. + if disabled, err := pth.Disable(installPath); err != nil { + ui.Warning("Could not disable the ._pth file in %s: %v", installPath, err) + ui.Info("Python %s will run in isolated mode until it is renamed to ._pth.disabled", version) + } else if len(disabled) > 0 { + ui.Debug("Disabled %v so Python %s runs as a regular install", disabled, version) + } // Step 1: ensurepip bootstraps the pip module on distributions that // ship without one (python.org embeddable). It's a no-op on python- @@ -338,44 +345,21 @@ func (p *Provider) installPipWithGetPip(version, pythonPath, installPath string) } // getPipURL returns the appropriate get-pip.py URL for the given Python version. -// Older Python versions (3.8 and below) require version-specific URLs since the +// Older Python versions (3.9 and below) require version-specific URLs since the // main get-pip.py no longer supports end-of-life Python versions. func (p *Provider) getPipURL(version string) string { parts := strings.Split(version, ".") if len(parts) >= 2 && parts[0] == "3" { + // The main get-pip.py requires Python 3.10+; older versions + // have their own under /pip/3.x/. minor, err := strconv.Atoi(parts[1]) - if err == nil && minor <= 8 { + if err == nil && minor <= 9 { return fmt.Sprintf("https://bootstrap.pypa.io/pip/%s.%s/get-pip.py", parts[0], parts[1]) } } return "https://bootstrap.pypa.io/get-pip.py" } -// enableSitePackages modifies the ._pth file to enable site-packages. -func (p *Provider) enableSitePackages(pthFile string) error { - content, err := os.ReadFile(pthFile) - if err != nil { - return err - } - - lines := strings.Split(string(content), "\n") - found := false - for i, line := range lines { - if strings.Contains(line, "import site") { - lines[i] = "import site" - found = true - break - } - } - - if !found { - lines = append(lines, "import site") - } - - newContent := strings.Join(lines, "\n") - return os.WriteFile(pthFile, []byte(newContent), 0644) -} - // Uninstall removes an installed version. func (p *Provider) Uninstall(version string) error { // TODO: Implement Python uninstallation diff --git a/src/runtimes/python/provider_test.go b/src/runtimes/python/provider_test.go index 6ed63bd..7e79674 100644 --- a/src/runtimes/python/provider_test.go +++ b/src/runtimes/python/provider_test.go @@ -3,7 +3,6 @@ package python import ( "os" "path/filepath" - "strings" "testing" "github.com/CodingWithCalvin/dtvem.cli/src/internal/runtime" @@ -111,9 +110,9 @@ func TestPythonProvider_GetPipURL(t *testing.T) { expectedURL: "https://bootstrap.pypa.io/get-pip.py", }, { - name: "Python 3.9 uses default URL", + name: "Python 3.9 uses version-specific URL", version: "3.9.18", - expectedURL: "https://bootstrap.pypa.io/get-pip.py", + expectedURL: "https://bootstrap.pypa.io/pip/3.9/get-pip.py", }, { name: "Python 3.8 uses version-specific URL", @@ -147,97 +146,6 @@ func TestPythonProvider_GetPipURL(t *testing.T) { } } -// TestPythonProvider_EnableSitePackages tests the ._pth file modification -func TestPythonProvider_EnableSitePackages(t *testing.T) { - provider := NewProvider() - - t.Run("returns error for non-existent file", func(t *testing.T) { - err := provider.enableSitePackages("/nonexistent/path/python311._pth") - if err == nil { - t.Error("enableSitePackages() should return error for non-existent file") - } - }) - - t.Run("uncomments import site line", func(t *testing.T) { - // Create a temp file with commented import site - tempDir := t.TempDir() - pthFile := filepath.Join(tempDir, "python311._pth") - content := "python311.zip\n.\n#import site\n" - if err := os.WriteFile(pthFile, []byte(content), 0644); err != nil { - t.Fatalf("Failed to create test file: %v", err) - } - - err := provider.enableSitePackages(pthFile) - if err != nil { - t.Fatalf("enableSitePackages() returned error: %v", err) - } - - // Read and verify - result, err := os.ReadFile(pthFile) - if err != nil { - t.Fatalf("Failed to read result file: %v", err) - } - - if !strings.Contains(string(result), "import site") { - t.Error("Result should contain 'import site'") - } - if strings.Contains(string(result), "#import site") { - t.Error("Result should not contain commented '#import site'") - } - }) - - t.Run("adds import site if missing", func(t *testing.T) { - // Create a temp file without import site - tempDir := t.TempDir() - pthFile := filepath.Join(tempDir, "python311._pth") - content := "python311.zip\n.\n" - if err := os.WriteFile(pthFile, []byte(content), 0644); err != nil { - t.Fatalf("Failed to create test file: %v", err) - } - - err := provider.enableSitePackages(pthFile) - if err != nil { - t.Fatalf("enableSitePackages() returned error: %v", err) - } - - // Read and verify - result, err := os.ReadFile(pthFile) - if err != nil { - t.Fatalf("Failed to read result file: %v", err) - } - - if !strings.Contains(string(result), "import site") { - t.Error("Result should contain 'import site'") - } - }) - - t.Run("preserves already uncommented import site", func(t *testing.T) { - // Create a temp file with already uncommented import site - tempDir := t.TempDir() - pthFile := filepath.Join(tempDir, "python311._pth") - content := "python311.zip\n.\nimport site\n" - if err := os.WriteFile(pthFile, []byte(content), 0644); err != nil { - t.Fatalf("Failed to create test file: %v", err) - } - - err := provider.enableSitePackages(pthFile) - if err != nil { - t.Fatalf("enableSitePackages() returned error: %v", err) - } - - // Read and verify - should still have import site, not duplicated - result, err := os.ReadFile(pthFile) - if err != nil { - t.Fatalf("Failed to read result file: %v", err) - } - - count := strings.Count(string(result), "import site") - if count != 1 { - t.Errorf("Expected exactly 1 'import site' line, got %d", count) - } - }) -} - // TestPythonProvider_Shims tests the shim configuration func TestPythonProvider_Shims(t *testing.T) { provider := NewProvider()