diff --git a/.github/workflows/mirror-sync.yml b/.github/workflows/mirror-sync.yml index 5487650..a2ab9b1 100644 --- a/.github/workflows/mirror-sync.yml +++ b/.github/workflows/mirror-sync.yml @@ -37,6 +37,11 @@ jobs: with: go-version-file: 'go.mod' + - name: Install msitools + # Full Windows Python builds are assembled from python.org's MSIs + if: matrix.runtime == 'python' + run: sudo apt-get update && sudo apt-get install -y msitools + - name: Build mirror tool run: | cd scripts/mirror-binaries diff --git a/.github/workflows/mirror-tool.yml b/.github/workflows/mirror-tool.yml new file mode 100644 index 0000000..9b9a726 --- /dev/null +++ b/.github/workflows/mirror-tool.yml @@ -0,0 +1,45 @@ +name: Mirror Tool + +# Tests for scripts/mirror-binaries, which is its own Go module and so is +# not covered by the Build & Test workflow. + +on: + pull_request: + branches: [ main ] + paths: + - 'scripts/mirror-binaries/**' + - '.github/workflows/mirror-tool.yml' + push: + branches: [ main ] + paths: + - 'scripts/mirror-binaries/**' + - '.github/workflows/mirror-tool.yml' + +permissions: + contents: read + +jobs: + test: + name: Test mirror tool + runs-on: ubuntu-latest + defaults: + run: + working-directory: scripts/mirror-binaries + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'scripts/mirror-binaries/go.mod' + cache-dependency-path: 'scripts/mirror-binaries/go.sum' + + - name: Vet + run: go vet ./... + + # Network tests (python_winbuild_network_test.go) only run when + # MIRROR_NETWORK_TESTS=1, so this needs neither network nor msitools. + - name: Test + run: go test -race ./... diff --git a/CLAUDE.md b/CLAUDE.md index 137c3cb..3c5b80c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -170,6 +170,7 @@ Maps to runtime provider → Resolves version | `internal/download/` | File downloads with progress | | `internal/manifest/` | Version manifest fetching and caching | | `internal/migration/` | Migration detection and helpers | +| `internal/pth/` | Turns off the `._pth` file of python.org's embeddable Windows Python (installer and `dtvem doctor`) | | `internal/testutil/` | Shared test utility functions | | `internal/constants/` | Platform constants | | `src/cmd/` | CLI commands (one file per command) | @@ -324,7 +325,10 @@ cd src && go test -cover ./... # With coverage | `generate-manifests-from-r2.yml` | Manual/scheduled | Generate version manifests from R2 mirror | | `deploy-manifests.yml` | Push to main (manifests/) | Deploy manifest files to R2 | | `mirror-all.yml` | Manual dispatch | Mirror all runtime binaries to R2 | -| `mirror-sync.yml` | Scheduled | Sync new versions to R2 mirror | +| `mirror-sync.yml` | Scheduled | Sync new versions to R2 mirror (installs msitools for Python) | +| `mirror-tool.yml` | PR, push to main (`scripts/mirror-binaries/`) | Vet and test the mirror tool, a separate Go module | + +**Windows Python builds in the mirror**: python-build-standalone is the first source. For versions it lacks, the mirror assembles full builds instead of using python.org's embeddable zips (`scripts/mirror-binaries/python_winbuild.go`): `windows-amd64` from python.org's per-component MSIs (verified against the pinned Windows signing key in `keys/`, unpacked with `msiextract`), `windows-arm64` from python.org's `pythonarm64` NuGet packages (verified against nuget.org's SHA-512). Both are packed as a `.tar.gz` with everything under `python/`. Builds that turn out not to be full installs are skipped, not published. `MIRROR_NETWORK_TESTS=1 go test -run Network ./...` (needs msitools) builds real archives. ### Reusable Workflows (from `dtvem/.github` repo) diff --git a/scripts/mirror-binaries/go.mod b/scripts/mirror-binaries/go.mod index 04e1475..31dc334 100644 --- a/scripts/mirror-binaries/go.mod +++ b/scripts/mirror-binaries/go.mod @@ -3,6 +3,7 @@ module github.com/CodingWithCalvin/dtvem.cli/scripts/mirror-binaries go 1.23.0 require ( + github.com/ProtonMail/go-crypto v1.5.2 github.com/aws/aws-sdk-go-v2 v1.32.6 github.com/aws/aws-sdk-go-v2/config v1.28.6 github.com/aws/aws-sdk-go-v2/credentials v1.17.47 @@ -24,4 +25,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect github.com/aws/smithy-go v1.22.1 // indirect + github.com/cloudflare/circl v1.6.3 // indirect + golang.org/x/crypto v0.41.0 // indirect + golang.org/x/sys v0.35.0 // indirect ) diff --git a/scripts/mirror-binaries/go.sum b/scripts/mirror-binaries/go.sum index 8c425bd..6563cdb 100644 --- a/scripts/mirror-binaries/go.sum +++ b/scripts/mirror-binaries/go.sum @@ -1,3 +1,5 @@ +github.com/ProtonMail/go-crypto v1.5.2 h1:cucYnvqcY7UOXVD//mSyjeaPY0SSN3v5cDkYPxumINk= +github.com/ProtonMail/go-crypto v1.5.2/go.mod h1:/RaSu30DaKO4RY+XdV/ACcCcZkGr7AhUIduq5sjzzCo= github.com/aws/aws-sdk-go-v2 v1.32.6 h1:7BokKRgRPuGmKkFMhEg/jSul+tB9VvXhcViILtfG8b4= github.com/aws/aws-sdk-go-v2 v1.32.6/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= @@ -34,3 +36,9 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 h1:s4074ZO1Hk8qv65GqNXqDjmkf4HS github.com/aws/aws-sdk-go-v2/service/sts v1.33.2/go.mod h1:mVggCnIWoM09jP71Wh+ea7+5gAp53q+49wDFs1SW5z8= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= +github.com/cloudflare/circl v1.6.3/go.mod h1:2eXP6Qfat4O/Yhh8BznvKnJ+uzEoTQ6jVKJRn81BiS4= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= diff --git a/scripts/mirror-binaries/keys/python-windows-signing.asc b/scripts/mirror-binaries/keys/python-windows-signing.asc new file mode 100644 index 0000000..f0a4a61 --- /dev/null +++ b/scripts/mirror-binaries/keys/python-windows-signing.asc @@ -0,0 +1,252 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Comment: Hostname: +Version: Hockeypuck 2.2 + +xsFNBFUh1AUBEACdUPt6PwJVO23zGZqgtgBeA9JsO22dk3CMzrwPJdUmMd6mcRWa +vl4BoAba66fuC17GvOgGXimKI+iaw5Vt9QI3uSjUjFSfc24J8T7NB/yAr/0zEcex +raHD2dxT/JpE/iY0yWHxRlitvwGSw1Qlq3NnY8tDI1DJEJD+gBuCktvVvu1FfQTw +6bd+aEq0c4sWJHAOnKLuLH0pNFOznnynAFGPGBBsm/YwYc5BP2JVvka775LUjA+W +1h2Sgg3FAUPIm64pc4Pq6mUo6Tulw72xsWMpCL1/5atXNPXT6rJUOB8euTcNMr4l +1O6GKSsiLeLAuvq4bmhOKtLzjWzXnY1gDVoOfdgpD6o4ZHk4xiVsdVE8hCa/ylz8 +1ZwRW2gGo2jP8t3hKciR2i+Qs+6lPNZpeFIxa6Uo9ER1IBgCHHapIR/UdcOFyoS0 +MNn7Ui7DLQNM4gI/G17eG9tfvjW2dl4SgFSYWMq/OtXnPDUBGqFUWsn8adOL2PFL +B7kM5ZRTPc5SnY9hoSGa5E20rJZIXcpy1aygRz/xUjoKwNzAySSEyyIorUxZ8KaH +EEBQSsqwe04MXIENqnDozH0/cvP4JXEDSl8EkzMSCWSoavQSIYD5pQppyFQpGHqa +5CuOA25Ja+sgp2xqahtr3fEqZUknPQSoYlnJbaHnzsGSlRAVWMsklsZibQARAQAB +zUBTdGV2ZSBEb3dlciAoUHl0aG9uIFJlbGVhc2UgU2lnbmluZykgPHN0ZXZlLmRv +d2VyQG1pY3Jvc29mdC5jb20+wsF5BBMBCAAjBQJVIdQFAhsDBwsJCAcDAgEGFQgC +CQoLBBYCAwECHgECF4AACgkQ/GJGQ0hwNOUX3Q//e8D156Hu3oqT+1F0PeNs9W7l +eqUxr9C3mtmh59tKNgQYx6fe2F66Bf3v/FuofmPhavIVqF3WCXQvQ1YMyYb21wCB +vFaxevyAxkOtq+H3/wCMkGPpIgjgglYY8Lqjk41WMNqWEVHCN1MF1T3qbQ0l3CGZ +Bwi7RFJ0uWDE3ZD7mawJrnAgp33CbXr12xw9siWD+L8H1zG2pbDQQyiYurmABXOM +r/vw8ktd631yb7tqWBr7UXfn7O5gRE71E7m4SOONKSLLOk1+jJcVnsV+k8vLyORw +kE9vNR+M6nW3uRncxpiMdRImMdbCW4EgTagoebESWOfwV81Sd3XzPzL+UGGOLpTv +6lDRXQvH9XGZtHrjqN5mgeI2Pko2n3+7dFiM9BrnzuHRwEcCYN7NCzvVn+fs/321 +jjHH773H2hzW4WilpNepLJXP4OzWLzHzrpXjNOmBshj8//2vXG2l+dzojLqTby5Y +cjzuTXxqab0Z+cQBJXAMNUS4RyXIL07SLP81VXxz2iuNbvAPXFyp5iFAt/SuARrz +qT8xLwnUNmWwKs0C5mdVCAq9ecNTX7OrqBESW57qviYDjznDpHswFhC4uS3WUxJZ +6SaWT0w0NN/txr8Q+EfbEfpeUU/FJe5Bftzzz83pH0MxfjYrKwYTxW9pT7dx8N9h +YE0yLlwjEUyhAW/+s+/CwXMEEwEIACcFAlUh1AUJEPxiRkNIcDTlAhsDBwsJCAcD +AgEGFQgCCQoLBBYCAwEAAB7KD/46ODOeFoiSFZhlZuO9ZKLOn5g1RnnnX3/+9RlH +aythd9iPWR6W67TFOBLMgywOsLRtLcRJSb98njONahJ0/eA38Pinf/v30TbnyNlo +15jcS6F9dFGBcqTBemBUHMgxjJTODlc3tJ6/WRIyUPct6gSxU4VGu6gkLYUnZUj0 +1M07t1E7lBKqZqAqgtr1LgpFJIePgbDHFr8dHoSKMW8C6L87mO7IXfNu3ou9JRpn +IEWM7ZxFb+sPNKHuRCWoBmoBa/7ErEd+S0BTLS8FM3yppReGT5cOSZ/3xTI5yeP5 +WjgYQzU4vT8scnvv/YBMGYz4l/o3mUJj/ZF7pKw3OH2MpYs3rJ+VKvWPwaWxtCcS +DundXB4EInrCFkDmZn3PBvNXxlvfO5r8HWfiTNb5a65MvUkGWhb7g0Gt4kbFK0GP +2RS8664eIbsK/tafoA0uHyFGsiC6ou3pT8VV+nkxTb+NuTDDL95jxLvKpzsS7AWK +6HnfuiLyWQg5yuErMK/5dcUHGa2sYqZg7xPsJDy8Ei81+YpOPjqVXDAGnlQnx8ws +9MtINUe5aF78wOZMIiItH1ql6p2ett/eZaUKioHH8p9orhaHHpqEhfeiMl4QhPBe +qZ4lf0AwaryOnmF3GJoYEno4R4uRsqXu2Szfn1XvBY+7ZZ8vT8vS1JNd3sJfrmtT +PCT7hcLBcwQTAQgAJwUCVSHUBQkQ/GJGQ0hwNOUCGwMHCwkIBwMCAQYVCAIJCgsE +FgIDAQAAHsoQADo4M54WiJIVmGVm471kos6fmDVGeedff/71GUdrK2F32I9ZHpbr +tMU4EsyDLA6wtG0txElJv3yeM41qEnT94Dfw+Kd/+/fRNufI2WjXmNxLoX10UYFy +pMF6YFQcyDGMlM4OVze0nr9ZEjJQ9y3qBLFThUa7qCQthSdlSPTUzTu3UTuUEqpm +oCqC2vUuCkUkh4+BsMcWvx0ehIoxbwLovzuY7shd827ei70lGmcgRYztnEVv6w80 +oe5EJagGagFr/sSsR35LQFMtLwUzfKmlF4ZPlw5Jn/fFMjnJ4/laOBhDNTi9Pyxy +e+/9gEwZjPiX+jeZQmP9kXukrDc4fYylizesn5Uq9Y/BpbG0JxIO6d1cHgQiesIW +QOZmfc8G81fGW987mvwdZ+JM1vlrrky9SQZaFvuDQa3iRsUrQY/ZFLzrrh4huwr+ +1p+gDS4fIUayILqi7elPxVX6eTFNv425MMMv3mPEu8qnOxLsBYroed+6IvJZCDnK +4Sswr/l1xQcZraxipmDvE+wkPLwSLzX5ik4+OpVcMAaeVCfHzCz0y0g1R7loXvzA +5kwiIi0fWqXqnZ62395lpQqKgcfyn2iuFocemoSF96IyXhCE8F6pniV/QDBqvI6e +YXcYmhgSejhHi5Gype7ZLN+fVe8Fj7tlny9Py9LUk13ewl+ua1M8JPuFwsFcBBAB +CAAGBQJVtk8DAAoJECngR5uZyKOX2oYP/iEEkhP8h/An/Nj2COyBqg1TcEcNqO3a +vIrHAfn4RR3xRYVwDcFFBacz3gzKzK3ZqgZHu1YNQ92yBaHF2DojcH9ie3+JgTuj +i9FQLw19NlcofwtmUnNebmclOglhUDAhvCTkpoEBbN5DB5C+KrbrdWOa3qGHrJrX +iHtOlFckcPMk9UdQZMT9sIWfe6sLL1FBPXFQ9PMArx+POnn8OxLMCM2BmtSnGV44 +tDL4135LFbZBEEG8TDCO0lV6NN+OWUXsMG7oqBi+YL7wDzm4FON7JDkWpxRxcYhx +CpqoBwR1yvyyqmhVBTCnG+IhUzbNlyq4RF7BIuRDCNQBbCc43F0ak+dFotnGYV3k +XO9LheoM6q3Vr6MGtqDj4jusxg4k6mQltQAK5P0AcxrT41YfpyhHR8J76Svfdd1A +JADevNofEJKFT2FzryCg2GUQa+RH6VMkLctq/TGHsM9GDeM5Rj9TxUvRcZ3HUmYm +SakFYn9ApDqdbXUsS8px3Lo0IAE2TbYHLl/ScFMD6etDLq4DnCGYlt52cP8/+b8w +QWnUmvf0yBUiQBTQZaninSBkLopNFB/gs0UbUhW9/Be4zD8K9EFEpLbp7UNB1rrz +ao+DfAxZuOf5+azdfdHkaLa1Q12Zoxnop6ljyE7iArJRiCgKfkOst7eyQO9TCG3r +v2WOmXH7TpGuwsFcBBABCAAGBQJWUVonAAoJEMYjTPAjlc0KuW0QAJLZ13Hjtxgo +rzLfckdUK9Dr+AOEPgiUiC+OXnyxZkt9JHG5Bx7WqvtBWTYdmWjGq7QtejFI2uGt +jlHo4Od1QYWklZEK1ZP+jPxwOv6OgryIWxVv+Qvp0VgXeYH78LFr467Qt3CmXKOM +IPsmHJZA20h/thDw2TiiB6QcuOmE8/VkNfEiedVlZDVDYmakQQOjcuRhaMnehKVm +WBD3pZszXpr98zq0HVcy1duP5i3o3Cy9rSpiOp4F+F1Yl1AkVFNzB/P1fldmE0Bn +EzP+hxmh6TdC06pJ3+sxLaPavRYkU2olNpMxMYQavEH/nFasJvtZDLdBRjoKU+O1 +QVjHlFqutoEN4eTXwLOA364i4DVEZUwtXSWABKrhYQf/KAUA9Yf3fQo0MpPPIom9 +qSSJ0QQ7CthXH0AWqfYmngRWKAVk8zECcYPkblitcCJtp9KKmubqwWGbURxOUSHQ +z/GS6IKwThZH1xicZUQUA9Rmm++6qIXadEpxKxsODXt4CR/0jYSvs8x5BxESK0WQ +kfec3WJCn2do8jj80rgAVUYoBEFUDwg59IW793G5UwniWMBtpn0XZn8qkjACyiIF +3vcrydpJfEIGlTrCYzX0MyCnqBxTyJuVMN2Aeo4AxXObsxGh+1ks++AOyoRmIwu4 +bsNk7oKLB21saYSSnXcMXytVro2GgHicwsFcBBABCAAGBQJX9rDyAAoJEH2/rXbN +7KDblxEP/15FmB5ibCsng2QvDBVPv77FanYZxbOlF2ZL3+zKDRZLjAy/s4FaDKRR +37U3cgajQKBJ53WpJNtdoyvGFhfIOJEFyGvbjEaiSzOJqgpfcnKP7cCfc/RDAPiS +0l2CtGW05C3brUg2RAyoKR25npMdgtTrFpUS7PT2oReTwGULNDfpviyQZ8OmxBzk +k7+DhOblVH/Z73XrNzgEb5BkhBlETZlGhXIcVrWf0hdavSX49Cv8YLPRrfOeaLnr +3kVyyBqY4hFaCh/3a7IMihTCVa6WQw4EuBlVxvG6txkxYjALOwTz3fW4yPo4L6rb +YCLWXGLFvyECtQSKBuhXq3uHD77viLIQnJJvU/JzivfLWbqL/IS2zQ/ZltneywoS +SNa/x282yiJdJOjooHRTKyZbp9s5rQ8U/U6d7sTmEmltmIIHKTVIyNvLb+gMKAPK +JFWPdy1ozuZHWRpG5YkZ3/3icuzSWJ9CzqILLzFCQpUQBp2UxyxK99loatDCJpV2 +uN5K+qDyU+XcF18sORC7LREM7XMwndT2AZG0B1KQ+Y9z8IDVTopwp23eKXP6lbWE +qqYNoOkyYtV4idet/I4Z9pPovMKKf3cnrYjfJRke+rcGBnsfQGrxAFKVUfa+qn6a +eCgYmxIXby+qNjtN0ULtexnEc10gGM5lIuVOY2d3Qc+TGFI4FGTPwsBcBBABCAAG +BQJYsBphAAoJEEhSKohZ29goZggIALKlgyoecD5v3ulh1eoctRqtCOxkAoENEfPt +3l5x6N8Wq89yHzf10T1rVioEXOHhDi1m37DDoQmRJD0sOYQymq10xDGRYAJjyOf3 +X0pvRkZ+F7T0U4dSV3DasLIHcN26kRwv1yCYsf0QvhgT6EJZKyUNHtV9qrb9u3A1 +Zp6epC/EyT8zMZj+21GzTUrnbnug3Ak9p7+APCZS4Ahh9ZHFuD38MZ7+OwrUd6ot ++6cbb1nnQLSAGQOHSp6EP6ktrnsKzts0L+tzHurxtJgUkR01imJuSFfYpLoZa/L7 +qXNyEpEUTC/SWzRWD9y2QkM7DLzXcaReVAyJr9rix1lDQbEFIqvCwVwEEAEKAAYF +AllTh9gACgkQXLNh5VL7DRAk7Q//X8eUhwEvl/d9Sv2kBNCZFjAW3QmZp2L/sxhS +cJZXrOFzKUdmjap9Xlul1qr6/Wif7YLKbOdNUI7KziEBn+9SEd90XauoVkzU2F0J +n9ILGQfUHAIpocRTKuCwBrncaBozHQwDO3Dk33AhZ6lqTv/AVLRKHQXwigGTBJxK +4cCEZ+VwK9tKk6BrQB48Rm7pg9HF5ey5JGPRWgUnn1v0IJN5ysZ5m9ChYbqF8Vwv +Mw0txmgKgvdDKpXbF/S59Bp4TH/7Dr2DkAeNTcuzTFBaFE+siMgksZIYKZ1VkVoi +N2qQA7ZaA5LQbUom0WdrKZGefFfPt9ESA4wyL3OfxRsmWmd/5Fxrwm1VbzgPoMd1 +Dc5ExlyqnecdGzDui2bmltNqRJd9ytRq6YUGYzXp4qQkWO61CoC3mkm2M8Ex7DGb +UtXhdg0zoa08w9lXuOtHVhY7XlLWjO1Up8cp4DVxsN/wOXtyH1pcleGo4aEsgyU/ +DH57prFLGz7Egp2JhRDHnZmlonWp74G1VLfqkOqZlqTU4mPA827C8qPCx6cMsRvF +S7OEiDBswkFWBKjkUCw4rLC1tBMBCxJWtZlc+Y0LNyOryJ3h6EJmRIHO57oLen34 +5e1WOi4ROOC/wQMErFk7B3P41Lqmrwb8HGuKn3ca+Aw70hVrZ+7Q3RRFTLlOS/vv +107Fqu7CwVwEEAEKAAYFAlp9o2EACgkQw7DeDnjqz17JrRAAsKN/XBIJSzAa1Fa9 +GIWOEEvvKewveA3txkCpAZ1fRoHR6y1B6zufxVmOR+/Vl5d5MjilDWi7NzEcunBD +s0KLCBdR2oKdnWabUzsH1qokbE2dbwJ/CrHUeZwxymtlmkVz/skVz02tL5zPYO4r +TMagT07KrdVofGxpTp6+hDvtixVV5BsREzBG+2bCJDfkNKNSoVaZcikwzdJe94rN +f9RgihxQ3N0MdfyD5LcHuJdcjWy8OCygCfmxTDsCFUNnc2CXn+5DuKksppE4jBvR +KOz0s29zDEIdnJ0h35XFfEJqrVmJT/IH17TFhGiMQKmnYVkZDlsxQcNPYF/RB6Vo +PAb7U9tAIWdcGWJ5dOmwUCSGPlrbaqI1Hb6gcu59fjueXHQrmOM5WGEMdqXsshoS +X8d632p1bQ3pOKz9NGW7mUPUdal3oLvH1I0Gt8EQi/nf/IWXsUX9/SVTHoZbB5Pm +wGZ+Rsafp84vqRrrDD+gz3PiDSBHkz8Y7xBQlXJe47FXf21e+nvoEveF2VckNnCQ +Rz353LVSj/U8mNut5IZJmpTnuI9qgpAUWEBOEvGXIMUL4VzxVDxkInLdhcN+zXYv +1GZG1c7bU45h8AVoqkIwmeZuKi9r2wUtsR9M3rqWqcjE56mxtVnkYW7CrE9EhofW +dWj//WPVmP22zKGAfwa5xj3tkNHCwHMEEAEIAB0WIQRJM0HfZ47qo6KWEUnUDkBi +bqxXGwUCW+0k0wAKCRDUDkBibqxXG/flB/9vlOvtm/QGtYuBHKR1NW+lTDPv6Nkr +d+NGdRisQ5+CiDIzd2mOvHzoMNGY3dQLvwUYt4dxdUVbTlH+0+C8we6FNeNYNBDQ +X/Qt+Q5zmBO43OrfSv4nLSmqoV7bxoSY7lsp0cBD9qE0Wnhnn5xxhQJVhj8h5tUV +7+jXK2pAlPCkSi3RoHXMYFxwDwxBjBkiexMGt7gwsGcsLl0HyHkkM6ut+Z3zBOWn +0xBagBT7TjgkfRKvo1lFpW9OfZC4tkZUCnLjr7VWQola7dvepaHgMJsqHqOl87YX +e1McAdXGnwc4q5d7oG/o1f4RTg2iNWgABOUT8/sa/NNfDoVXlFtHb+2lwsBzBBAB +CAAdFiEEOOn84tqyMDqaxg0YTn3VBQ8zqOEFAlwsCVEACgkQTn3VBQ8zqOFB5AgA +lFVwqF87o7p/pqanExRT/nECrlUqHINpGVi8Fm9/UbFb3VwdoRVZkWi6UWvaOdYJ +AFsXDhtvS1fieNbwtGrE20hZkHy66m8zJp7YwQp2qyKw8bbQeyUpgk57UqgC2nGs +HStdgbvEsFiNgbtF+0bDvh4Lag7wtCcQgehXy5kpyJFBY09q6oEXd8HNXZU9+Vaf +xBuJvaT8U+8GT5VeRiYv57tB5q008qbw0o3eZklTbg5ss8VibOPq6X1Xu5+PoL3l +Xtmd4ak4guCk52KCr3GTZ1oSZaXLB8PiWLdiIG7zLQFYJ8tfqJWbzNZh4Gk/FDh7 +D2ytVcKukXVpG8B9p3Tz3MLBcwQQAQgAHRYhBEW+vuyVCr0Fzw71w1CgTQw7ZRfy +BQJc8pFGAAoJEFCgTQw7ZRfyeywP/0GH+mle8wjIVNH5haH6VWXvfeVRXzpDqObu +KNWy8r2Vb1OrfnNvZGkc0TZz1ofXo5GSEjDb2G/R9ccErtqcv2HP7WYb2kE8Gq43 +pY65U6CXnL60W4a3snNrGgXzaIA54haGGXgpE7xE3uzNshwP3m4YvAVAv4zFp/vj +aOtWyUaq0LujWZSFE2liXSF7J9yCfDGSfAYSqO8giOOJwjlLt4Y8IeW8F2wiP2jT +oFns0g16N/yvM3k79MdDaStWz2NLA5M7oYgEE61rHRkXB48HKTy6HmZ5MsgbyOW+ +e+gEgiRwWAYryNvUvK1viXJPOnGW5z9Z+ss4zwaw6yyvtsH8/APAYxBSKCDXK3k5 +agtKy8L7XkXcFR5dz4xWUMWrQPnf6L8z2MA41xjjIF6Xo748A204vsLiMJReN0EC +H/kitrbYWrFxWfuvQwQbFvj1jKOt9Y3w8gg5K40fOMbaQpMFy/MWZImMbEqnkmrc +FqGwKSI2Fx4EVG7pBRPTUl1Ts/H47nfaXvvph8255hteL5XJoSbwOy9pBvclSdbd +7ldMj012yBpMb5JstFQLsFMRj1uAc0gw93EfaVX1A6Glv4HhRNVGW1nzIVE3KYH5 +zvWX2yC9j+Olc3JjR6uBuL0+fJmS7Pv0PtcLkZcnolpj01qvX1IhY8JdG9DXlWSV +40DLHttywsBzBBABCAAdFiEEXucagSQtOm55NGslTs2ZgDD1eqIFAl08PY0ACgkQ +Ts2ZgDD1eqLA2gf+M440eyOW70xUqt3W+lFG9R31Esd3fTqYZ9XZ2Ui0qqCfv93Y +k6N2XOOhl3jB8xaegG1+DQwh9DdlVd+aLXhFmslCYEWwUNg/uIMecvjMM8/nBCd3 +ncvgDY/K6NS1MOHjkLtxelWG5Q6bcodADmxuq3jN5xaF2qoHhNza4inRHBwdtG7/ +8iHn/M6NHL0GcVAlnCYVC8TwlWssX0MFA96oOWH29ss+3gI9Sq/rSFsYfhSVMxqR +bw8hEBkCt8o9GH982yNLU6NYPoGqZAVyHZppgZQpXUhwjw4FLQE70DdEwKH7pH6T +Xzm/xoy+0rjNhMGgz8YGgHQKEFldsGsqXfHVtsLAcwQQAQgAHRYhBNZmFyqvP0un +BcHvP+z7qpo6w3+bBQJdZnjbAAoJEOz7qpo6w3+bYMAH/35M4ffLUIBaT9i/s67K +8NkLXS8VWzOVDRVRA26702+PrEk+vQ3pJoFZQQYy+PNkl6gDI8PtLMKTqn+/zI7t +/1iO0dX6Vb9xsB0BbRvxodJvjIuW9CJr+ei6EpjXolzoVDQrZ/aE5DIIu8ltmW9M +wW7DeT11VSMddSvhJjFTCJoLcdXGVbqIa6lgPq/nhsTQVN8b06uLWzjO275mft+u +qfBKzDSI0igXtYGojyKvu1zTQqUxjOvLu3/nVyR96x3a37oy6EWZGizqc6JBdLDP +qrMVvo2YWd4+2lI6PYSpg9uesCsQ7jlFNvZ6qveRG6WMBl/dn0yADhRb15ywt5/q +i27CwHMEEAEIAB0WIQRXC6+NpbbWdXnYtNo2y+0KeqAbpAUCXadaawAKCRA2y+0K +eqAbpLs8B/sFH8FM56UgiERSBQ4f+atYB6HHljHqWmsntp2L8tp9BKa7wSvbtvfI +ch+W9tMX4l/prweIUty5PCTC6R1Qxkg46dwlVEJO6eTAdq7ZYKHw5Fv6C78E4cbw +kq6X0EBnPiKgf7DfBEa/gnZvy+gG8ztPSunYZ5tsB8W8UrVRni6EANcJqjlDOqPV +duwmF8FYOOxIunK2aQiNgwEZ1mQACbc5F/vqdvk6MC00I2rDhIsCImE/S5KMHuo7 +knNKZvxg8xifoSx0iOWumpQAiSTBIaHUgaPrmKx1BfwVwLqYSyuzivp8Xy+T1aAw +N7+LADAV8l1h+rf5vw4QtKzdqGQ8SPdbwsBzBBABCAAdFiEEUOXPFISLGVtIWB06 +PF3+q8WXx/8FAl2/fvAACgkQPF3+q8WXx/8u1Qf/eX7GZBYJjfIpKfcwZVAqb4sE +paLlty127Hf0Q+fTIfqngfMBHRUw73H8JyWHG6g4OuBar6Dkf036NVL5tzWz4bbb +/YWc628f02GlX79yofQDSPt7vceo5OboGmGt2sEY6ObGAocVcLoNc1dRtOqgMjHz +kh9YH8/0hZ82FIEY1JbkdJJePTHTX9BokMM0LubVpA+L7DGrGALyrZcqpaZnG7UG +TeRSaPrq/OE/A/Icx0ckWYHyjhhmQihoE2Npp/bQaf+8iN1AOkcUjJ6PTc4+jsO6 +wOoQ0pUDFxwfgUovHO2udRWgmv/BvId2407RI3YwgYKC8uOup3eUtzl17E1QI8LB +igQQAQgANBYhBOPeodrRYC9jnme/5AUuwev5aN01BQJexiFPFhSAAAAAAA0AAHJl +bUBnbnVwZy5vcmcACgkQBS7B6/lo3TWe8xAAxOSgR4BYbOxTi2D3YmvoAQGXSCYq +5u5+sVR6LjcA6aABWFc90USwp3LkJ8BoANioA2j7R5/mAaIQpWAYaelZJ5IPoQmJ +LK7gvYnumZSxo0yMMrTQT3+1pw5fJeWiEO2YUdwtQSYvnFplnYAEFXk5udKDSnkB +HhWtahbyTDT70SrcLSNKE4OVdzBxNnZafy+W9QWiDvGTpdeg/osWW74CviX87P6j +AIxN+MAB+g+/y/gJQANvM8G8zhUFSoYswVMgR41POArfxfcjI+NL7JvKUPk4FaqX +2oBRd/zVXsUFUIOXqATjnskUk15ehYzUjl0zAJ4fL8xNg/5nyPFDdrBwpJg2Iex9 +pnY30xQILOnywG8T7YM63uD9WvJEWD7MVHjOiECLyz/jzWsD3RFlnCF3vWWPkaH2 +4u9ioxgidpsq1IBneVEVSBHT11CvsMAM77xCa+h3yUc5+9naJWdfGv0TGEGUY++w +c+HxY8a8BdbmscSs4XUIEDvnJUDe6v/WcA/GJo8bmaaTceoA032RvSw/HxhAHMEA +jU4ig1eyKbQMJixTyMpIM2qmdqi0p2njQc6sgrV60ioJYqXy18aDeUQZSLZei3Co +Y8afGdsNP6ShS4hWCk91Z4DrvZYIL8EizB9qQfvpVqZSnHieFkDGOApMkzMEAZjx +rKY7QyuzxtsezgHCwI4EEAEIADgWIQRXFpWVsN3CoDQOmQPh6tx80S+Z4wUCX1dh +ZBoUgAAAAAANAARyZW1AZ251cGcub3JnQU5OSQAKCRDh6tx80S+Z41+2CACsZCGk +gpAfqmoCDJWSGbplj6JXvO3awYJ1qByd+lo5p8bBn3fp9yle+YIyNFy9Qfk4jAxF +psxSgZrDfDHSrnBHShc5wv8bqtd1ECcZcnjxX33U3rlR+obNJFnpNX/JyID+PnKT +11tZ/iIetvOeZmkumA2iHt54yo+gZ91SUoV7bzsmYTmGFHDRFpkmyRBLc9mrtXjg +SwbI1lQwYBOvPsIXwEYn5DETP5UJIKqGrXFtbvok7y79ZGITGHLa82Jb9f0DQttz +cmeLM3BOXadbQCUzrmNil6iDKVh6s32CdPQsYkmCyW0kf22t9wX6pd+z3+rPEKFD +SJ1cSaCqxCE9homOwsFzBBABCAAdFiEEhLVxtwVtdcpLm/p2szaYZjC3UiYFAmHM +g6AACgkQszaYZjC3UiZMBA/+O10BV1HEfHny5z+V2SYgmEmNEkwWL/qdR+DJiryc +31qMcvymW7KhXH77XctWaQ5BxJ5oq2s2G7JTRLwvj5NmQF1BR85drj+387ycP71F +GUr+7Ahsq4ZoLU53MAtW2/iIjvDyosVW4CIKylKwVzc7L8as7ze6bhxPZquwMO26 +UHuY7FBEB5qNhBvWVb+QQs7nkPYnno57dX4NWb/uoBGYf2MfdAhLQYhFvpReM7lI +GnUpVv2v9vohBzJpJ+0rVCFrjnhOKP92KJrc56/NqOneH3Sy2EaDV238MhqLjvyt +0OiztAk2V0debyA57piNxJNnlaPMNeBU5+3CEEeC+Q0p3yajIHx4gW+Nm3G6hj5N +CUNn1/v634va3xtBdCdA/xSrzDw9Zv1HUjZ8Xr6U4eB+HOh96x7R0eOXgPPBRS9h +3PCqykDLMP6hgRb9OTtujNgCkc+myM+6KHl3atARV7y4pUYzaRDxZ6oM8E+MazKD +8g4dGppBbwvzn3+J1NRwNuTLo0F6UNO06zV6PPCLmx9Ynif8U70c5M15qlD2Af6f +6lilBjm63lz7Ux455raeZ8AxNKSwssycElwyc5E762b0O984sWrXB2o8y0eWU/KR +EVdoDISrElxeQjcZ2cro1BxNR0i6v85/gr7p+3adVPEtW4ZSyTYW88nCNy/07lLw +PhHCkgQQFgoAOhYhBKnDlZJc49SuizfOuCw7HjER/AkyBQJlL8vRHBSAAAAAAA0A +BnJlbUBnbnVwZy5vcmdQeXRob24ACgkQLDseMRH8CTJFXwD/dbo63+YVYZtXL3DK +2koE2R1UjCW2oTRGcll8BGSl9KsBAOwVc3LE5JleER56DFey9OeRz8L1JwXlUico +6UiLvF8GwpIEEBYKADoWIQSpw5WSXOPUros3zrgsOx4xEfwJMgUCZS/MDxwUgAAA +AAANAAZyZW1AZ251cGcub3JnUHl0aG9uAAoJECw7HjER/Aky2vcBAOnGlGPt6FCx +PyBubpCMQufpwc2xRxj+uu0jQ0lv0SYFAQCC7dTAltf0lEsokgWdAJFQA3jJaOnJ +hEEtz9/MTyZmBcKSBBAWCgA6FiEEqcOVklzj1K6LN864LDseMRH8CTIFAmUvzCMc +FIAAAAAADQAGcmVtQGdudXBnLm9yZ1B5dGhvbgAKCRAsOx4xEfwJMtqfAP4kYQFu +6lSHlm01EQNTwM35Na1GYwr6LQZ+8OysRmu+swD+MI4w6LaFU0Kw0F4ilzzIIlHz +xyq8WE+PyK+5TCsrGAjCwXkEEAEIACMWIQTXwOkHK6SvSGfdeR2iSyV8o7TD2AUC +ZWNX/AWDA8O4gAAKCRCiSyV8o7TD2DGwD/4pju58F0cq+fs4WGWSuY7XR9cuIxI9 +Oylx+4AI8IzPjmL4iBI945Pvi2R0gwbii8AoTN0vs4k363oOq8zs80pfgOf4u7gv +Mc2NrQ7Vw6vhep664u2xCXLbItoFjU25Du++9+40TuprExgCKqT+c51Mr9coxEuN +LKHakewXhWjcGjCxU637n/bQ4CyCoz7orBLKVMbCNj4udkUyGoX3dabajBC+BCmT +c3hjohr2ZM5QtDdPQDHmLgXetPVmv38aWOk+YAIPqAPYG8SVvYOZht1R2RwBr93P +B4g7lvgBOr7JgdEaf0b05d3C7njQV+I6YgP2A7/hAFLCesDcs6W7G/hSb8tFfu5H +Z81jIaQzXIQjQE4xtbSjMFq0mzqa/SFOKN+YAQyPac/nmGTXStJ9Z45CNWy6WU/P +u5PEvXoR8mI9Mt4NMn1clwLRoLsTXhxtHHiLuOj4oe+ks+k+EpXBaNI3nzhjhtaA +IpZFJBsX4A/kbOem3WEMkXTRxwnxA/vJqsIK66yJU/o0BPSfCQRkFCN197DOHeS5 +bJMzES8WcshJy9PvRtnzTf6DaCU6NH8dZsz11K/hR8r1f9GDudPN0xFK/4KeU0nx +G9mv0Ude5UrEYauVeasgPIOp2z1EOX7pa0AL+FeR4VyhVAUGh+/vLEoSaxBF9ppI +qMShYavJrVZIycJ1BBAWCgAdFiEEIJmlt63FmMoAH0rGteH1dc4zC1kFAmXisJAA +CgkQteH1dc4zC1kzLAEAqLSB2vsiUmfwCy/ABOKC4yCT8YzvrowW+D1HBz7iAHIB +AKtEpC3uewF14z7dRN1pxHWwoCFLceKXDnOTAw+WyLgAwsCaBBABCABEFiEEHGnk ++HPqa68ThNVqHbsVQ1nUiDEFAmXt0YMFgwPCZwADBQF4HIY8W14+XStbQC5dbWlj +cm9zb2Z0XC5jb20+JAAACgkQHbsVQ1nUiDGF9Af+JzQ06C1Da14WCDt+FxF2dI1R +NHAtNoWQGc0ro06+N89trzi5C50l3zZL00faBQ4OCyrPk/mLbkTYjzJ+0NnrFIGP +WGKu7HdvoWaDqFX6LenpPOHZ03mFd77jUGng9jbB/UWu9nVdOdOZcy1aQObazh3g +lJmiVR6W9ala7K5FtiIiWYH7sDeMqHb2c8uLI1vzw9fyC7/DHVSpnPjLGk2rlfhy +PLA9uJPbntUBnCtIN8sL5f542F7cJ/KSstkqoItR1nVy7AMlX7scsobzhtTN+Lmt +OPVVSPMDA5/WY7pYLiFm1RluQ5Ycvh2e/G+8qfqS4/mLciwRLoz1bz9I7vICmM7B +TQRVIdQFARAAtmnsZ9A8ovJIJ9RlWeIylEhHRyQifqzgc/r50uDZVPBjewOA462L +jH3+F6zFGEkU+q2aqSe0A0SJPF/Whj6MNYXLoibxi5D4mGkoIao9ExnXt4LXAc6o +gQpY6vFQBJU5Nr8XCefQbm0loa/oy5uK8JHLWCZ2jAossnVpzDwNeN27+B8h5+Oi +fnWhQCTun1xz5EJiyc0yoBmf46zfmU4CMUBsPvrXcLmw4J3wp35qmrHg1tNyPhd7 +VBlikMrgtrWX9IaPZ40dnrGG/WjOFYB3CKxGb0pTCj7GC4ubxo2upeWZqHLmdIVc +7Nzsfp8EcwJbTj+jZ2Zfq6F8y+jesbgh8CaxYn4hEs23aPYRq5H4/buVmZhUw3/A +AL9ZmyX6AtAQ0HktVtQe7ykP7DLsEpeLG+vPJFY363QeDsLHwOoxnZSfGziVlB4N +/KqIkixNWcFTG8GSE1zKcdJVNoW+3MB3+FtMZWUJhH0FyKg5qLaJCtC7Yo5gsddU ++QCqTn6gcZBnMX5j4LaAmW4hh1RXffwwsbfviK5uhXQCeUnbUaokieetDx4s6Kay +6t9ahTRr0r/Z3VWzvr+xATxNWZzixTdezCGOB2ycZ0vq4bKXBuN8CAyOy5X1hf7R +c1BiAVQCILHJDtz0Ak/Hax6DAa2AHnx9YlugHQf000KroLEY+GaxqYEAEQEAAcLB +XwQYAQgACQUCVSHUBQIbDAAKCRD8YkZDSHA05RtyEACdOEmGolL1xG6I+lDVdot6 +oBZqC9e021aLWqCUpWJFDp0m0aTmCfmOI1gTaFjScxhq1W0GPUoJKUZhk3tlVfdS +CtUckI+xuWKEfqJYtvUtTXpK4jDeaZBovJ3KNpJRIynbr1566zCSQJhHiCGWmE/M +5KN3gPsORbCBQXEkONSVsslf1Wm66hU6uqSWUaceD+4fl5LClbck1DPWchAP7+uL +KPEOtORyH6KRTgKl73zYo7xU1K4QMN/1aMjobPkqNvvkXnUNwO7QMz18Nx+WqPc4 +ksJgW1O1aPQ2qL/ARY5jatZ6BBd7iytfz7d6JOh0FOIlmhBqbWd7fEGrLsSA+EjB +GBwW5BnIMmxP1xhjhwrcI18y8kAK5UzdW2hbbAlc2rlsuxEc+xOYh8kGcc+mZ1j/ +aMn4gALsTbSO/0T+YJhfODNnL1dCj7oPbJGmmG6pb/o7P4azBUVC9lHOuV3XlAPj +SmJylnNsV7+PxwPlXlvKgh4S4C4ZPUc/iPetsxXR2djccOoNxVU4CqJBqYKgul/p +UphXkh7QfEKyH+42UETbVhstdBVUazJ6SeUnv9ClVDGsCEhfEZfNOnOoDzJGxDfE +SoAw7ih91vIhTyHHsK83p2HLDMLPptLzx/0AFBfo6MWGGpd2RSnMWNbvh59wiThl +DeI+Das3ln5nsAo67dMYdA== +=IpRU +-----END PGP PUBLIC KEY BLOCK----- diff --git a/scripts/mirror-binaries/main.go b/scripts/mirror-binaries/main.go index 5fbb297..2fecea3 100644 --- a/scripts/mirror-binaries/main.go +++ b/scripts/mirror-binaries/main.go @@ -42,6 +42,11 @@ type MirrorJob struct { UpstreamSHA256 string // Checksum from upstream manifest (may be empty) R2Key string MetaKey string + + // Build, when set, produces the binary to upload in place of + // downloading URL, for archives dtvem assembles from upstream files. + // URL then records where those files came from. + Build func() ([]byte, error) } // Stats tracks mirroring statistics @@ -144,7 +149,10 @@ func main() { withoutChecksum := 0 for _, job := range jobs { checksumStatus := "(will generate checksum)" - if job.UpstreamSHA256 != "" { + if job.Build != nil { + checksumStatus = "(assembled by dtvem)" + withoutChecksum++ + } else if job.UpstreamSHA256 != "" { checksumStatus = "(has upstream checksum)" withChecksum++ } else { @@ -158,18 +166,9 @@ func main() { return } - // Filter jobs if sync-only (check for .meta.json to determine if mirrored) + // Filter jobs if sync-only if *syncOnly && existingKeys != nil { - var filtered []MirrorJob - for _, job := range jobs { - if !existingKeys[job.MetaKey] { - // Not yet mirrored - filtered = append(filtered, job) - } else if builtFromSourceKeys[job.MetaKey] { - // Exists but was built from source — re-mirror from upstream - filtered = append(filtered, job) - } - } + filtered := pendingJobs(jobs, existingKeys, builtFromSourceKeys) skipped := len(jobs) - len(filtered) fmt.Printf("Skipping %d files already in R2, %d remaining\n", skipped, len(filtered)) jobs = filtered @@ -200,6 +199,27 @@ func main() { } } +// pendingJobs returns the jobs a sync-only run still has to mirror: those +// without metadata in R2, and those whose metadata says built-from-source, +// which are re-mirrored from upstream. +// +// A job dtvem assembles (Build set) is also pending until its archive +// exists. Its archive can replace an older one under the same metadata, as +// a full Windows Python .tar.gz replaces an embeddable .zip, so the +// metadata alone does not show it was mirrored. +func pendingJobs(jobs []MirrorJob, existingKeys, builtFromSourceKeys map[string]bool) []MirrorJob { + var pending []MirrorJob + for _, job := range jobs { + switch { + case !existingKeys[job.MetaKey], builtFromSourceKeys[job.MetaKey]: + pending = append(pending, job) + case job.Build != nil && !existingKeys[job.R2Key]: + pending = append(pending, job) + } + } + return pending +} + // exitOnFetchError exits non-zero when any upstream source failed to fetch. // Without this a broken source mirrors nothing while the workflow still // reports success, hiding missing versions indefinitely. @@ -331,12 +351,7 @@ func processJobs(client *s3.Client, jobs []MirrorJob) *Stats { go func() { defer wg.Done() for job := range jobChan { - if err := mirrorFile(client, job, stats); err != nil { - fmt.Fprintf(os.Stderr, "Error mirroring %s: %v\n", job.R2Key, err) - atomic.AddInt64(&stats.Failed, 1) - } else { - atomic.AddInt64(&stats.Mirrored, 1) - } + recordResult(stats, job, mirrorFile(client, job, stats)) } }() } @@ -361,8 +376,8 @@ func mirrorFile(client *s3.Client, job MirrorJob, stats *Stats) error { } err := doMirror(client, job, stats) - if err == nil { - return nil + if err == nil || errors.Is(err, errNotFullBuild) { + return err } lastErr = err } @@ -370,42 +385,77 @@ func mirrorFile(client *s3.Client, job MirrorJob, stats *Stats) error { return lastErr } -func doMirror(client *s3.Client, job MirrorJob, stats *Stats) error { - // Download file +// recordResult counts a finished job. A job whose upstream build is not a +// full install (errNotFullBuild) counts as skipped, not failed. +func recordResult(stats *Stats, job MirrorJob, err error) { + switch { + case err == nil: + atomic.AddInt64(&stats.Mirrored, 1) + case errors.Is(err, errNotFullBuild): + fmt.Fprintf(os.Stderr, "Skipping %s: %v\n", job.R2Key, err) + atomic.AddInt64(&stats.Skipped, 1) + default: + fmt.Fprintf(os.Stderr, "Error mirroring %s: %v\n", job.R2Key, err) + atomic.AddInt64(&stats.Failed, 1) + } +} + +// fetchBinary returns the binary for a job, its SHA-256 and where that +// checksum comes from: "upstream" when it matched one the upstream +// published, "dtvem" when dtvem computed it, including for archives a Build +// step assembled. +func fetchBinary(job MirrorJob, stats *Stats) (body []byte, checksum, checksumSource string, err error) { + if job.Build != nil { + body, err = job.Build() + if err != nil { + return nil, "", "", fmt.Errorf("build failed: %w", err) + } + } else { + body, err = downloadBinary(job.URL) + if err != nil { + return nil, "", "", err + } + } + atomic.AddInt64(&stats.BytesDown, int64(len(body))) + + hash := sha256.Sum256(body) + checksum = hex.EncodeToString(hash[:]) + + if job.Build == nil && job.UpstreamSHA256 != "" { + if checksum != job.UpstreamSHA256 { + return nil, "", "", fmt.Errorf("checksum mismatch: expected %s, got %s", job.UpstreamSHA256, checksum) + } + atomic.AddInt64(&stats.UpstreamChecksum, 1) + return body, checksum, "upstream", nil + } + atomic.AddInt64(&stats.GeneratedChecksum, 1) + return body, checksum, "dtvem", nil +} + +// downloadBinary downloads a binary to mirror. +func downloadBinary(url string) ([]byte, error) { httpClient := &http.Client{Timeout: 10 * time.Minute} - resp, err := httpClient.Get(job.URL) + resp, err := httpClient.Get(url) if err != nil { - return fmt.Errorf("download failed: %w", err) + return nil, fmt.Errorf("download failed: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return fmt.Errorf("download failed: HTTP %d", resp.StatusCode) + return nil, fmt.Errorf("download failed: HTTP %d", resp.StatusCode) } - // Read body and calculate checksum body, err := io.ReadAll(resp.Body) if err != nil { - return fmt.Errorf("read failed: %w", err) + return nil, fmt.Errorf("read failed: %w", err) } + return body, nil +} - atomic.AddInt64(&stats.BytesDown, int64(len(body))) - - // Calculate actual checksum - hash := sha256.Sum256(body) - actualChecksum := hex.EncodeToString(hash[:]) - - // Determine checksum source and verify if upstream provided one - var checksumSource string - if job.UpstreamSHA256 != "" { - if actualChecksum != job.UpstreamSHA256 { - return fmt.Errorf("checksum mismatch: expected %s, got %s", job.UpstreamSHA256, actualChecksum) - } - checksumSource = "upstream" - atomic.AddInt64(&stats.UpstreamChecksum, 1) - } else { - checksumSource = "dtvem" - atomic.AddInt64(&stats.GeneratedChecksum, 1) +func doMirror(client *s3.Client, job MirrorJob, stats *Stats) error { + body, actualChecksum, checksumSource, err := fetchBinary(job, stats) + if err != nil { + return err } // Determine content type diff --git a/scripts/mirror-binaries/main_test.go b/scripts/mirror-binaries/main_test.go new file mode 100644 index 0000000..b16369e --- /dev/null +++ b/scripts/mirror-binaries/main_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "testing" +) + +func TestPendingJobs(t *testing.T) { + build := func() ([]byte, error) { return nil, nil } + job := func(version string, assembled bool) MirrorJob { + j := MirrorJob{ + Version: version, + R2Key: "python/" + version + "/windows-amd64.tar.gz", + MetaKey: "python/" + version + "/windows-amd64.meta.json", + } + if assembled { + j.Build = build + } + return j + } + jobs := []MirrorJob{ + job("3.12.3", false), // archive and metadata mirrored + job("3.9.11", false), // nothing mirrored yet + job("3.8.20", false), // mirrored, but built from source + job("3.7.9", false), // metadata only: not a dtvem build, left alone + job("3.9.9", true), // dtvem build; metadata from the old embeddable .zip only + job("3.12.2", true), // dtvem build, archive and metadata mirrored + } + existing := map[string]bool{ + "python/3.12.3/windows-amd64.tar.gz": true, + "python/3.12.3/windows-amd64.meta.json": true, + "python/3.8.20/windows-amd64.tar.gz": true, + "python/3.8.20/windows-amd64.meta.json": true, + "python/3.7.9/windows-amd64.meta.json": true, + "python/3.9.9/windows-amd64.zip": true, + "python/3.9.9/windows-amd64.meta.json": true, + "python/3.12.2/windows-amd64.tar.gz": true, + "python/3.12.2/windows-amd64.meta.json": true, + } + builtFromSource := map[string]bool{"python/3.8.20/windows-amd64.meta.json": true} + + var got []string + for _, j := range pendingJobs(jobs, existing, builtFromSource) { + got = append(got, j.Version) + } + if want := []string{"3.9.11", "3.8.20", "3.9.9"}; !reflect.DeepEqual(got, want) { + t.Errorf("pendingJobs() = %v, want %v", got, want) + } +} + +func TestRecordResult(t *testing.T) { + stats := &Stats{} + job := MirrorJob{R2Key: "python/3.5.2/windows-arm64.tar.gz"} + + recordResult(stats, job, nil) + recordResult(stats, job, fmt.Errorf("build failed: %w", errNotFullBuild)) + recordResult(stats, job, errors.New("upload failed")) + + if stats.Mirrored != 1 || stats.Skipped != 1 || stats.Failed != 1 { + t.Errorf("recordResult() counted mirrored=%d skipped=%d failed=%d, want 1 each", stats.Mirrored, stats.Skipped, stats.Failed) + } +} + +func sha256Hex(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func TestFetchBinary_Build(t *testing.T) { + archive := []byte("assembled archive") + job := MirrorJob{ + URL: "https://www.python.org/ftp/python/3.9.11/amd64/", + // A Build job's checksum is always dtvem's, even if one were set. + UpstreamSHA256: "ignored", + Build: func() ([]byte, error) { return archive, nil }, + } + + body, checksum, source, err := fetchBinary(job, &Stats{}) + if err != nil { + t.Fatalf("fetchBinary() error: %v", err) + } + if string(body) != string(archive) || checksum != sha256Hex(archive) || source != "dtvem" { + t.Errorf("fetchBinary() = (%q, %s, %s), want (%q, %s, dtvem)", body, checksum, source, archive, sha256Hex(archive)) + } + + job.Build = func() ([]byte, error) { return nil, errors.New("bad signature") } + if _, _, _, err := fetchBinary(job, &Stats{}); err == nil { + t.Error("fetchBinary() with a failing Build = nil error, want error") + } +} + +func TestFetchBinary_Download(t *testing.T) { + payload := []byte("upstream archive") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(payload) + })) + defer srv.Close() + + tests := []struct { + name string + upstream string + wantSource string + wantErr bool + }{ + {name: "matching upstream checksum", upstream: sha256Hex(payload), wantSource: "upstream"}, + {name: "no upstream checksum", wantSource: "dtvem"}, + {name: "mismatched upstream checksum", upstream: sha256Hex([]byte("other")), wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, checksum, source, err := fetchBinary(MirrorJob{URL: srv.URL, UpstreamSHA256: tt.upstream}, &Stats{}) + if tt.wantErr { + if err == nil { + t.Error("fetchBinary() = nil error, want checksum mismatch") + } + return + } + if err != nil || source != tt.wantSource || checksum != sha256Hex(payload) { + t.Errorf("fetchBinary() = (%s, %s, %v), want (%s, %s, nil)", checksum, source, err, sha256Hex(payload), tt.wantSource) + } + }) + } +} diff --git a/scripts/mirror-binaries/python.go b/scripts/mirror-binaries/python.go index 521719e..a1b0f93 100644 --- a/scripts/mirror-binaries/python.go +++ b/scripts/mirror-binaries/python.go @@ -223,27 +223,3 @@ func (s *PythonStandaloneSource) mapTripleToPlatform(triple string) string { return "" } } - -// PythonOfficialSource fetches Python versions from python.org -// This is a fallback for versions not available in python-build-standalone -type PythonOfficialSource struct{} - -func (s *PythonOfficialSource) Name() string { - return "python.org" -} - -func (s *PythonOfficialSource) FetchVersions() ([]MirrorJob, error) { - // Python.org doesn't provide prebuilt binaries for most platforms - // Only Windows installers and source tarballs are available - // For now, we rely primarily on python-build-standalone - // This source can be expanded later if needed - - // The official FTP has a complex structure: - // https://www.python.org/ftp/python/3.12.0/ - // - Python-3.12.0.tar.xz (source) - // - python-3.12.0-amd64.exe (Windows installer - not a portable archive) - // - python-3.12.0-embed-amd64.zip (Windows embeddable - limited use) - - // For now, return empty - python-build-standalone covers our needs - return []MirrorJob{}, nil -} diff --git a/scripts/mirror-binaries/python_winbuild.go b/scripts/mirror-binaries/python_winbuild.go new file mode 100644 index 0000000..b71ed72 --- /dev/null +++ b/scripts/mirror-binaries/python_winbuild.go @@ -0,0 +1,671 @@ +package main + +// Full Windows builds for Python versions python-build-standalone does not +// cover. +// +// python.org's embeddable zips, which the mirror used to fall back to, are a +// trimmed runtime meant to be shipped inside an application: a ._pth file +// puts them in isolated mode (no script directory on sys.path, PYTHONPATH +// ignored), and they have no venv, ensurepip, headers or tkinter. Instead the +// mirror assembles a full build of the same release from official files: +// +// - windows-amd64: python.org's per-component MSIs, verified against the +// Windows release signing key and unpacked with msiextract (msitools), +// so this runs on a Linux runner; +// - windows-arm64: python.org's NuGet packages, verified against +// nuget.org's SHA-512. +// +// A release with neither (3.5.2's MSIs are unsigned) keeps its embeddable +// build, and dtvem's installer disables its ._pth file instead. +// +// Either way the result is a .tar.gz with everything under python/, the +// layout of python-build-standalone's Windows archives, so dtvem installs it +// without changes. + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "crypto/sha512" + _ "embed" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "net/url" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/ProtonMail/go-crypto/openpgp" +) + +const ( + pythonFTPBaseURL = "https://www.python.org/ftp/python/" + nugetFlatContainer = "https://api.nuget.org/v3-flatcontainer/" + nugetRegistrationURL = "https://api.nuget.org/v3/registration5-gz-semver2/" + + // pythonWindowsSigningFingerprint is the key python.org lists for + // "Windows binaries" (Steve Dower) at + // https://www.python.org/downloads/metadata/pgp/. + pythonWindowsSigningFingerprint = "7ED10B6531D7C8E1BC296021FC624643487034E5" + + // msiDiscoveryWorkers bounds concurrent requests to python.org while + // checking which releases have signed MSIs. + msiDiscoveryWorkers = 8 +) + +// pythonWindowsSigningKey is the armored public key for +// pythonWindowsSigningFingerprint, committed so the mirror never depends on +// a keyserver. +// +//go:embed keys/python-windows-signing.asc +var pythonWindowsSigningKey []byte + +// pythonMSIComponents are the MSIs that together make a full install: +// pythonXY.dll, python.exe, the standard library (with venv and ensurepip), +// headers and import library, and tkinter with Tcl/Tk. pip.msi is left out: +// it holds no files and only runs ensurepip, which dtvem does itself. +var pythonMSIComponents = []string{"core", "exe", "lib", "dev", "tcltk"} + +// pythonNuGetPackages maps a platform to python.org's NuGet package for it. +// Only arm64 is listed: python.org has no arm64 MSIs for 3.9 and 3.10, and +// every pythonarm64 release checked is a full install. x64 is covered by +// python-build-standalone and the MSIs, and its NuGet packages are not +// reliably full (3.5.2, 3.5.3 and 3.6.0 lack venv and ensurepip). +var pythonNuGetPackages = []struct { + platform string + id string +}{ + {"windows-arm64", "pythonarm64"}, +} + +// errNotFullBuild marks an upstream build that turned out not to be a full +// install. The mirror skips such a job rather than failing: retrying cannot +// change upstream's files, and failing would fail every scheduled run. +var errNotFullBuild = errors.New("not a full Python install") + +var ( + // minWindowsPython is the first release with python.org's MSI-based + // installer and NuGet packages. + minWindowsPython = [3]int{3, 5, 0} + // firstSigstoreOnlyPython is the first release python.org signs with + // Sigstore only; earlier MSIs carry OpenPGP signatures. + firstSigstoreOnlyPython = [3]int{3, 14, 0} + + pythonFTPVersionPattern = regexp.MustCompile(`href="(\d+\.\d+\.\d+)/"`) + releaseVersionPattern = regexp.MustCompile(`^\d+\.\d+\.\d+$`) + + // archiveModTime is the modification time of every archive entry, so + // rebuilding the same release produces the same archive. + archiveModTime = time.Date(2020, time.January, 1, 0, 0, 0, 0, time.UTC) +) + +// PythonMSISource assembles full windows-amd64 builds from python.org's +// per-component MSIs. +type PythonMSISource struct { + // baseURL is python.org's FTP root; tests point it at a local server. + baseURL string + // extract unpacks one MSI into a directory; nil means msiextract. + extract func(msiPath, destDir string) error +} + +func (s *PythonMSISource) Name() string { + return "python.org MSIs" +} + +func (s *PythonMSISource) ftpBase() string { + if s.baseURL != "" { + return s.baseURL + } + return pythonFTPBaseURL +} + +// FetchVersions returns a job for every release from minWindowsPython up to +// firstSigstoreOnlyPython whose amd64 folder has all of pythonMSIComponents +// with their .asc signatures. Releases without Windows binaries (source-only +// security releases) or without signatures (3.5.2) get no job. +func (s *PythonMSISource) FetchVersions() ([]MirrorJob, error) { + verifier, err := newPythonWindowsVerifier() + if err != nil { + return nil, err + } + + base := s.ftpBase() + index, err := fetchText(base) + if err != nil { + return nil, fmt.Errorf("listing %s: %w", base, err) + } + + var candidates []string + for _, v := range parsePythonFTPVersions(index) { + parts, _ := parseVersion(v) + if compareVersions(parts, minWindowsPython) >= 0 && compareVersions(parts, firstSigstoreOnlyPython) < 0 { + candidates = append(candidates, v) + } + } + + signed := make([]bool, len(candidates)) + var wg sync.WaitGroup + sem := make(chan struct{}, msiDiscoveryWorkers) + for i, v := range candidates { + wg.Add(1) + go func() { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + listing, err := fetchText(base + v + "/amd64/") + signed[i] = err == nil && hasSignedMSIs(listing) + }() + } + wg.Wait() + + var jobs []MirrorJob + for i, v := range candidates { + if !signed[i] { + continue + } + dirURL := base + v + "/amd64/" + jobs = append(jobs, MirrorJob{ + Runtime: "python", + Version: v, + Platform: "windows-amd64", + URL: dirURL, + R2Key: fmt.Sprintf("python/%s/windows-amd64.tar.gz", v), + MetaKey: fmt.Sprintf("python/%s/windows-amd64.meta.json", v), + Build: func() ([]byte, error) { + return buildFromMSIs(dirURL, verifier, s.extract) + }, + }) + } + return jobs, nil +} + +// parsePythonFTPVersions returns the x.y.z release folders in python.org's +// FTP index. +func parsePythonFTPVersions(index string) []string { + var versions []string + for _, m := range pythonFTPVersionPattern.FindAllStringSubmatch(index, -1) { + versions = append(versions, m[1]) + } + return versions +} + +// hasSignedMSIs reports whether a release's amd64 folder listing has every +// one of pythonMSIComponents together with its .asc signature. +func hasSignedMSIs(listing string) bool { + for _, c := range pythonMSIComponents { + if !strings.Contains(listing, `href="`+c+`.msi"`) || !strings.Contains(listing, `href="`+c+`.msi.asc"`) { + return false + } + } + return true +} + +// buildFromMSIs downloads pythonMSIComponents from dirURL, verifies each +// against its signature, unpacks them into one install and packs it. +func buildFromMSIs(dirURL string, verifier *signatureVerifier, extract func(msiPath, destDir string) error) ([]byte, error) { + if extract == nil { + extract = runMsiextract + } + + work, err := os.MkdirTemp("", "dtvem-python-msi-") + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(work) }() + + installDir := filepath.Join(work, "install") + for _, c := range pythonMSIComponents { + msi, err := download(dirURL + c + ".msi") + if err != nil { + return nil, err + } + sig, err := download(dirURL + c + ".msi.asc") + if err != nil { + return nil, err + } + if err := verifier.verify(msi, sig); err != nil { + return nil, fmt.Errorf("%s.msi: %w", c, err) + } + + msiPath := filepath.Join(work, c+".msi") + if err := os.WriteFile(msiPath, msi, 0o644); err != nil { + return nil, err + } + if err := extract(msiPath, installDir); err != nil { + return nil, fmt.Errorf("extracting %s.msi: %w", c, err) + } + } + + return archiveDir(installDir, true) +} + +// runMsiextract unpacks an MSI with msiextract from msitools. +func runMsiextract(msiPath, destDir string) error { + if _, err := exec.LookPath("msiextract"); err != nil { + return errors.New("msiextract not found; install msitools (apt-get install msitools)") + } + out, err := exec.Command("msiextract", "-C", destDir, msiPath).CombinedOutput() + if err != nil { + return fmt.Errorf("%w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +// PythonNuGetSource repackages python.org's NuGet packages, which hold a +// full install under tools/. +type PythonNuGetSource struct { + // flatContainerURL and registrationURL point at nuget.org's API; tests + // point them at a local server. + flatContainerURL string + registrationURL string +} + +func (s *PythonNuGetSource) Name() string { + return "python.org NuGet packages" +} + +// FetchVersions returns a job for every release package from +// minWindowsPython onward. Sources registered earlier win for any version +// and platform they also provide. +func (s *PythonNuGetSource) FetchVersions() ([]MirrorJob, error) { + flat := s.flatContainerURL + if flat == "" { + flat = nugetFlatContainer + } + registration := s.registrationURL + if registration == "" { + registration = nugetRegistrationURL + } + + var jobs []MirrorJob + for _, pkg := range pythonNuGetPackages { + body, err := download(flat + pkg.id + "/index.json") + if err != nil { + return nil, fmt.Errorf("listing %s: %w", pkg.id, err) + } + var index struct { + Versions []string `json:"versions"` + } + if err := json.Unmarshal(body, &index); err != nil { + return nil, fmt.Errorf("parsing %s index: %w", pkg.id, err) + } + + for _, v := range index.Versions { + parts, ok := parseVersion(v) + if !ok || compareVersions(parts, minWindowsPython) < 0 { + continue + } + id := pkg.id + nupkgURL := fmt.Sprintf("%s%s/%s/%s.%s.nupkg", flat, id, v, id, v) + leafURL := fmt.Sprintf("%s%s/%s.json", registration, id, v) + jobs = append(jobs, MirrorJob{ + Runtime: "python", + Version: v, + Platform: pkg.platform, + URL: nupkgURL, + R2Key: fmt.Sprintf("python/%s/%s.tar.gz", v, pkg.platform), + MetaKey: fmt.Sprintf("python/%s/%s.meta.json", v, pkg.platform), + Build: func() ([]byte, error) { + return buildFromNuGet(nupkgURL, leafURL) + }, + }) + } + } + return jobs, nil +} + +// buildFromNuGet downloads a package, verifies it against the SHA-512 in +// its nuget.org catalog entry and repackages its tools/ folder. +func buildFromNuGet(nupkgURL, leafURL string) ([]byte, error) { + wantHash, err := nugetPackageHash(leafURL) + if err != nil { + return nil, err + } + pkg, err := download(nupkgURL) + if err != nil { + return nil, err + } + if err := verifySHA512(pkg, wantHash); err != nil { + return nil, fmt.Errorf("%s: %w", nupkgURL, err) + } + return archiveNuGetTools(pkg) +} + +// nugetPackageHash returns the base64 SHA-512 nuget.org records for a +// package, read from the catalog entry its registration leaf points to. +func nugetPackageHash(leafURL string) (string, error) { + body, err := download(leafURL) + if err != nil { + return "", err + } + var leaf struct { + CatalogEntry string `json:"catalogEntry"` + } + if err := json.Unmarshal(body, &leaf); err != nil || leaf.CatalogEntry == "" { + return "", fmt.Errorf("no catalog entry in %s", leafURL) + } + + body, err = download(leaf.CatalogEntry) + if err != nil { + return "", err + } + var entry struct { + PackageHash string `json:"packageHash"` + PackageHashAlgorithm string `json:"packageHashAlgorithm"` + } + if err := json.Unmarshal(body, &entry); err != nil { + return "", fmt.Errorf("parsing %s: %w", leaf.CatalogEntry, err) + } + if !strings.EqualFold(entry.PackageHashAlgorithm, "SHA512") || entry.PackageHash == "" { + return "", fmt.Errorf("%s has no SHA512 package hash", leaf.CatalogEntry) + } + return entry.PackageHash, nil +} + +// verifySHA512 checks data against a base64-encoded SHA-512 digest. +func verifySHA512(data []byte, wantBase64 string) error { + sum := sha512.Sum512(data) + if got := base64.StdEncoding.EncodeToString(sum[:]); got != wantBase64 { + return fmt.Errorf("SHA-512 mismatch: expected %s, got %s", wantBase64, got) + } + return nil +} + +// signatureVerifier checks detached OpenPGP signatures against one pinned +// key. +type signatureVerifier struct { + keyring openpgp.EntityList + fingerprint string +} + +// newPythonWindowsVerifier returns a verifier for python.org's Windows +// release signing key. +func newPythonWindowsVerifier() (*signatureVerifier, error) { + return newSignatureVerifier(pythonWindowsSigningKey, pythonWindowsSigningFingerprint) +} + +// newSignatureVerifier parses an armored public key and confirms it is the +// key with the given fingerprint. +func newSignatureVerifier(armoredKey []byte, fingerprint string) (*signatureVerifier, error) { + keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(armoredKey)) + if err != nil { + return nil, fmt.Errorf("reading signing key: %w", err) + } + if len(keyring) != 1 { + return nil, fmt.Errorf("signing key file holds %d keys, want 1", len(keyring)) + } + if got := strings.ToUpper(hex.EncodeToString(keyring[0].PrimaryKey.Fingerprint)); got != fingerprint { + return nil, fmt.Errorf("signing key fingerprint is %s, want %s", got, fingerprint) + } + return &signatureVerifier{keyring: keyring, fingerprint: fingerprint}, nil +} + +// verify checks that armoredSig is a valid signature of data by the pinned +// key. +func (v *signatureVerifier) verify(data, armoredSig []byte) error { + signer, err := openpgp.CheckArmoredDetachedSignature(v.keyring, bytes.NewReader(data), bytes.NewReader(armoredSig), nil) + if err != nil { + return fmt.Errorf("signature check failed: %w", err) + } + if got := strings.ToUpper(hex.EncodeToString(signer.PrimaryKey.Fingerprint)); got != v.fingerprint { + return fmt.Errorf("signed by %s, want %s", got, v.fingerprint) + } + return nil +} + +// archiveDir packs a full Windows Python install from dir. requireTk also +// demands tkinter, which MSI builds include and NuGet builds do not. +func archiveDir(dir string, requireTk bool) ([]byte, error) { + var files []string + err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.Type().IsRegular() { + rel, err := filepath.Rel(dir, p) + if err != nil { + return err + } + files = append(files, filepath.ToSlash(rel)) + } else if !d.IsDir() { + return fmt.Errorf("unexpected non-regular file %s", p) + } + return nil + }) + if err != nil { + return nil, err + } + if err := validateWindowsPython(files, requireTk); err != nil { + return nil, err + } + + w := newPythonArchive() + sort.Strings(files) + for _, rel := range files { + data, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + if err != nil { + return nil, err + } + if err := w.addFile(rel, data); err != nil { + return nil, err + } + } + return w.close() +} + +// archiveNuGetTools packs the full install under tools/ in a NuGet package, +// dropping the package's own files. +func archiveNuGetTools(pkg []byte) ([]byte, error) { + zr, err := zip.NewReader(bytes.NewReader(pkg), int64(len(pkg))) + if err != nil { + return nil, fmt.Errorf("reading package: %w", err) + } + + contents := make(map[string]*zip.File) + var files []string + for _, f := range zr.File { + name := f.Name + // NuGet stores part names URI-escaped ("script%20(dev).tmpl"). + if unescaped, err := url.PathUnescape(name); err == nil { + name = unescaped + } + rel, ok := strings.CutPrefix(name, "tools/") + if !ok || rel == "" || strings.HasSuffix(rel, "/") { + continue + } + if !isSafeRelPath(rel) { + return nil, fmt.Errorf("unsafe path in package: %s", f.Name) + } + contents[rel] = f + files = append(files, rel) + } + if err := validateWindowsPython(files, false); err != nil { + return nil, err + } + + w := newPythonArchive() + sort.Strings(files) + for _, rel := range files { + data, err := readZipFile(contents[rel]) + if err != nil { + return nil, err + } + if err := w.addFile(rel, data); err != nil { + return nil, err + } + } + return w.close() +} + +func readZipFile(f *zip.File) ([]byte, error) { + rc, err := f.Open() + if err != nil { + return nil, err + } + defer func() { _ = rc.Close() }() + return io.ReadAll(rc) +} + +// isSafeRelPath reports whether p stays inside the directory it is +// relative to. +func isSafeRelPath(p string) bool { + clean := path.Clean(p) + return clean != "." && !path.IsAbs(clean) && clean != ".." && !strings.HasPrefix(clean, "../") && !strings.Contains(clean, ":") +} + +// validateWindowsPython checks that files, relative to an install root, +// make a full Windows Python rather than an embeddable one. +func validateWindowsPython(files []string, requireTk bool) error { + have := make(map[string]bool, len(files)) + importLib := false + for _, f := range files { + have[f] = true + if strings.HasSuffix(f, "._pth") { + return fmt.Errorf("%w: found %s, which would put it in isolated mode", errNotFullBuild, f) + } + if strings.HasPrefix(f, "libs/python") && strings.HasSuffix(f, ".lib") { + importLib = true + } + } + + required := []string{"python.exe", "Lib/os.py", "Lib/venv/__init__.py", "Lib/ensurepip/__init__.py", "include/Python.h"} + if requireTk { + required = append(required, "DLLs/_tkinter.pyd", "Lib/tkinter/__init__.py") + } + var missing []string + for _, r := range required { + if !have[r] { + missing = append(missing, r) + } + } + if !importLib { + missing = append(missing, "libs/pythonXY.lib") + } + if len(missing) > 0 { + return fmt.Errorf("%w, missing: %s", errNotFullBuild, strings.Join(missing, ", ")) + } + return nil +} + +// pythonArchive writes a .tar.gz with every entry under python/, the layout +// of python-build-standalone's Windows archives. +type pythonArchive struct { + buf bytes.Buffer + gz *gzip.Writer + tw *tar.Writer + dirs map[string]bool +} + +func newPythonArchive() *pythonArchive { + a := &pythonArchive{dirs: make(map[string]bool)} + a.gz = gzip.NewWriter(&a.buf) + a.tw = tar.NewWriter(a.gz) + return a +} + +// addFile adds rel, a slash-separated path below the install root, after +// any of its parent directories not yet written. +func (a *pythonArchive) addFile(rel string, data []byte) error { + if err := a.addDir(path.Dir(rel)); err != nil { + return err + } + if err := a.tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeReg, + Name: "python/" + rel, + Mode: 0o644, + Size: int64(len(data)), + ModTime: archiveModTime, + Format: tar.FormatPAX, + }); err != nil { + return err + } + _, err := a.tw.Write(data) + return err +} + +func (a *pythonArchive) addDir(rel string) error { + if a.dirs[rel] { + return nil + } + name := "python/" + if rel != "." { + if err := a.addDir(path.Dir(rel)); err != nil { + return err + } + name += rel + "/" + } + a.dirs[rel] = true + return a.tw.WriteHeader(&tar.Header{ + Typeflag: tar.TypeDir, + Name: name, + Mode: 0o755, + ModTime: archiveModTime, + Format: tar.FormatPAX, + }) +} + +func (a *pythonArchive) close() ([]byte, error) { + if err := a.tw.Close(); err != nil { + return nil, err + } + if err := a.gz.Close(); err != nil { + return nil, err + } + return a.buf.Bytes(), nil +} + +// download fetches url and returns its body, failing on any non-200 status. +func download(url string) ([]byte, error) { + resp, err := httpGetWithRetry(url, 3) + if err != nil { + return nil, fmt.Errorf("downloading %s: %w", url, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != 200 { + return nil, fmt.Errorf("downloading %s: HTTP %d", url, resp.StatusCode) + } + return io.ReadAll(resp.Body) +} + +func fetchText(url string) (string, error) { + body, err := download(url) + return string(body), err +} + +// parseVersion parses an x.y.z release version. +func parseVersion(v string) ([3]int, bool) { + var parts [3]int + if !releaseVersionPattern.MatchString(v) { + return parts, false + } + for i, f := range strings.Split(v, ".") { + n, err := strconv.Atoi(f) + if err != nil { + return parts, false + } + parts[i] = n + } + return parts, true +} + +func compareVersions(a, b [3]int) int { + for i := range a { + if a[i] != b[i] { + return a[i] - b[i] + } + } + return 0 +} diff --git a/scripts/mirror-binaries/python_winbuild_network_test.go b/scripts/mirror-binaries/python_winbuild_network_test.go new file mode 100644 index 0000000..54c01f2 --- /dev/null +++ b/scripts/mirror-binaries/python_winbuild_network_test.go @@ -0,0 +1,137 @@ +package main + +// Tests against the real python.org and nuget.org. They download about +// 50 MB, and the MSI build needs msiextract, so they only run when asked: +// +// MIRROR_NETWORK_TESTS=1 go test -v -run Network ./... +// +// On Debian or Ubuntu, install msiextract with: apt-get install msitools. +// Set MIRROR_NETWORK_OUT to a directory to keep the built archives, e.g. to +// unpack and run them on Windows. + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func requireNetworkTests(t *testing.T) { + t.Helper() + if os.Getenv("MIRROR_NETWORK_TESTS") != "1" { + t.Skip("set MIRROR_NETWORK_TESTS=1 to run tests against python.org and nuget.org") + } +} + +// checkRealArchive confirms an archive holds a full install and keeps it +// in MIRROR_NETWORK_OUT if set. +func checkRealArchive(t *testing.T, name string, archive []byte, requireTk bool) { + t.Helper() + var files []string + for entry := range tarEntries(t, archive) { + if rel := entry[len("python/"):]; rel != "" && rel[len(rel)-1] != '/' { + files = append(files, rel) + } + } + if err := validateWindowsPython(files, requireTk); err != nil { + t.Errorf("%s: %v", name, err) + } + t.Logf("%s: %d files, %.1f MB", name, len(files), float64(len(archive))/1e6) + + if dir := os.Getenv("MIRROR_NETWORK_OUT"); dir != "" { + if err := os.WriteFile(filepath.Join(dir, name), archive, 0o644); err != nil { + t.Errorf("saving %s: %v", name, err) + } + } +} + +func TestNetwork_BuildFromMSIs(t *testing.T) { + requireNetworkTests(t) + if _, err := exec.LookPath("msiextract"); err != nil { + t.Skip("msiextract not installed (apt-get install msitools)") + } + verifier, err := newPythonWindowsVerifier() + if err != nil { + t.Fatalf("newPythonWindowsVerifier: %v", err) + } + + // The oldest and newest signed releases in range, and 3.9.11, which has + // no NuGet package at all. + for _, v := range []string{"3.5.0", "3.9.11", "3.12.2"} { + archive, err := buildFromMSIs(pythonFTPBaseURL+v+"/amd64/", verifier, nil) + if err != nil { + t.Errorf("buildFromMSIs(%s): %v", v, err) + continue + } + checkRealArchive(t, "python-"+v+"-windows-amd64.tar.gz", archive, true) + } +} + +func TestNetwork_BuildFromNuGet(t *testing.T) { + requireNetworkTests(t) + + // The first pythonarm64 release, and one from the 3.11 line. + for _, v := range []string{"3.9.7", "3.11.4"} { + nupkgURL := nugetFlatContainer + "pythonarm64/" + v + "/pythonarm64." + v + ".nupkg" + leafURL := nugetRegistrationURL + "pythonarm64/" + v + ".json" + archive, err := buildFromNuGet(nupkgURL, leafURL) + if err != nil { + t.Errorf("buildFromNuGet(pythonarm64 %s): %v", v, err) + continue + } + checkRealArchive(t, "python-"+v+"-windows-arm64.tar.gz", archive, false) + } + + // x64 NuGet packages are not reliably full installs, which is why the + // NuGet source skips them; the validator must say so. + nupkgURL := nugetFlatContainer + "python/3.5.2/python.3.5.2.nupkg" + leafURL := nugetRegistrationURL + "python/3.5.2.json" + if _, err := buildFromNuGet(nupkgURL, leafURL); !errors.Is(err, errNotFullBuild) { + t.Errorf("buildFromNuGet(python 3.5.2) = %v, want errNotFullBuild", err) + } +} + +func TestNetwork_FetchVersions(t *testing.T) { + requireNetworkTests(t) + + msiJobs, err := (&PythonMSISource{}).FetchVersions() + if err != nil { + t.Fatalf("PythonMSISource.FetchVersions: %v", err) + } + msi := make(map[string]bool) + for _, j := range msiJobs { + msi[j.Version] = true + } + t.Logf("python.org MSIs: %d signed x64 releases", len(msiJobs)) + for _, v := range []string{"3.5.0", "3.9.11", "3.12.2", "3.13.1"} { + if !msi[v] { + t.Errorf("MSI source has no job for %s", v) + } + } + for _, v := range []string{"3.5.2", "3.9.15", "3.14.0"} { + if msi[v] { + t.Errorf("MSI source has a job for %s, which has unsigned or no MSIs", v) + } + } + + nugetJobs, err := (&PythonNuGetSource{}).FetchVersions() + if err != nil { + t.Fatalf("PythonNuGetSource.FetchVersions: %v", err) + } + nuget := make(map[string]bool) + for _, j := range nugetJobs { + nuget[j.Version+"/"+j.Platform] = true + } + t.Logf("python.org NuGet packages: %d release jobs", len(nugetJobs)) + for _, key := range []string{"3.9.7/windows-arm64", "3.11.4/windows-arm64", "3.12.2/windows-arm64"} { + if !nuget[key] { + t.Errorf("NuGet source has no job for %s", key) + } + } + for _, j := range nugetJobs { + if j.Platform != "windows-arm64" { + t.Errorf("NuGet source has a %s job (%s); it should only cover windows-arm64", j.Platform, j.Version) + } + } +} diff --git a/scripts/mirror-binaries/python_winbuild_test.go b/scripts/mirror-binaries/python_winbuild_test.go new file mode 100644 index 0000000..45bb5e3 --- /dev/null +++ b/scripts/mirror-binaries/python_winbuild_test.go @@ -0,0 +1,690 @@ +package main + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "crypto/sha512" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "sync" + "testing" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/ProtonMail/go-crypto/openpgp/packet" +) + +// testSigner is a throwaway OpenPGP key for signing test data. +type testSigner struct { + entity *openpgp.Entity + armoredKey []byte + fingerprint string +} + +func newTestSigner(t *testing.T) *testSigner { + t.Helper() + entity, err := openpgp.NewEntity("dtvem test", "", "test@example.invalid", &packet.Config{Algorithm: packet.PubKeyAlgoEdDSA}) + if err != nil { + t.Fatalf("NewEntity: %v", err) + } + var buf bytes.Buffer + w, err := armor.Encode(&buf, openpgp.PublicKeyType, nil) + if err != nil { + t.Fatalf("armor.Encode: %v", err) + } + if err := entity.Serialize(w); err != nil { + t.Fatalf("Serialize: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("closing armor: %v", err) + } + return &testSigner{ + entity: entity, + armoredKey: buf.Bytes(), + fingerprint: strings.ToUpper(hex.EncodeToString(entity.PrimaryKey.Fingerprint)), + } +} + +func (s *testSigner) sign(t *testing.T, data []byte) []byte { + t.Helper() + var buf bytes.Buffer + if err := openpgp.ArmoredDetachSign(&buf, s.entity, bytes.NewReader(data), nil); err != nil { + t.Fatalf("ArmoredDetachSign: %v", err) + } + return buf.Bytes() +} + +func (s *testSigner) verifier(t *testing.T) *signatureVerifier { + t.Helper() + v, err := newSignatureVerifier(s.armoredKey, s.fingerprint) + if err != nil { + t.Fatalf("newSignatureVerifier: %v", err) + } + return v +} + +func TestPythonWindowsSigningKey(t *testing.T) { + if _, err := newPythonWindowsVerifier(); err != nil { + t.Fatalf("embedded signing key: %v", err) + } + + // A checkout with CRLF line endings must load the same key. + crlf := bytes.ReplaceAll(pythonWindowsSigningKey, []byte("\n"), []byte("\r\n")) + if _, err := newSignatureVerifier(crlf, pythonWindowsSigningFingerprint); err != nil { + t.Errorf("embedded signing key with CRLF line endings: %v", err) + } +} + +func TestNewSignatureVerifier_WrongFingerprint(t *testing.T) { + signer := newTestSigner(t) + if _, err := newSignatureVerifier(signer.armoredKey, pythonWindowsSigningFingerprint); err == nil { + t.Error("newSignatureVerifier() accepted a key with a different fingerprint") + } +} + +func TestSignatureVerifier(t *testing.T) { + signer := newTestSigner(t) + other := newTestSigner(t) + v := signer.verifier(t) + data := []byte("python msi contents") + + if err := v.verify(data, signer.sign(t, data)); err != nil { + t.Errorf("verify(good signature) = %v, want nil", err) + } + if err := v.verify([]byte("tampered contents"), signer.sign(t, data)); err == nil { + t.Error("verify(tampered data) = nil, want error") + } + if err := v.verify(data, other.sign(t, data)); err == nil { + t.Error("verify(signature by another key) = nil, want error") + } + if err := v.verify(data, []byte("not a signature")); err == nil { + t.Error("verify(garbage) = nil, want error") + } +} + +func TestParsePythonFTPVersions(t *testing.T) { + index := `../ +2.7.18/ +3.9.11/ +3.13.0/ +3.14/ +3.13.0a1/ +README.html` + + got := parsePythonFTPVersions(index) + want := []string{"2.7.18", "3.9.11", "3.13.0"} + if !reflect.DeepEqual(got, want) { + t.Errorf("parsePythonFTPVersions() = %v, want %v", got, want) + } +} + +// msiListing returns a python.org amd64 folder listing with the given files. +func msiListing(files ...string) string { + var b strings.Builder + for _, f := range files { + fmt.Fprintf(&b, "%s\n", f, f) + } + return b.String() +} + +func signedMSIFiles() []string { + var files []string + for _, c := range pythonMSIComponents { + files = append(files, c+".msi", c+".msi.asc") + } + return append(files, "doc.msi", "doc.msi.asc", "pip.msi") +} + +func TestHasSignedMSIs(t *testing.T) { + if !hasSignedMSIs(msiListing(signedMSIFiles()...)) { + t.Error("hasSignedMSIs(all components signed) = false, want true") + } + + var unsigned []string + for _, c := range pythonMSIComponents { + unsigned = append(unsigned, c+".msi") + } + if hasSignedMSIs(msiListing(unsigned...)) { + t.Error("hasSignedMSIs(no signatures) = true, want false") + } + + if hasSignedMSIs(msiListing("core.msi", "core.msi.asc", "exe.msi", "exe.msi.asc")) { + t.Error("hasSignedMSIs(missing components) = true, want false") + } +} + +func TestPythonMSISource_FetchVersions(t *testing.T) { + var unsigned []string + for _, c := range pythonMSIComponents { + unsigned = append(unsigned, c+".msi") + } + pages := map[string]string{ + "/": msiListing("3.4.10/", "3.5.2/", "3.9.11/", "3.9.15/", "3.13.1/", "3.14.0/"), + // 3.5.2's MSIs have no signatures. + "/3.5.2/amd64/": msiListing(unsigned...), + "/3.9.11/amd64/": msiListing(signedMSIFiles()...), + "/3.13.1/amd64/": msiListing(signedMSIFiles()...), + // 3.9.15 is a source-only security release: no amd64 folder. + // 3.4.10 predates the MSI installer and 3.14.0 is Sigstore-only; + // neither should be requested. + "/3.14.0/amd64/": msiListing(signedMSIFiles()...), + } + var requested pathRecorder + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requested.record(r.URL.Path) + page, ok := pages[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = io.WriteString(w, page) + })) + defer srv.Close() + + src := &PythonMSISource{baseURL: srv.URL + "/"} + jobs, err := src.FetchVersions() + if err != nil { + t.Fatalf("FetchVersions() error: %v", err) + } + + var versions []string + for _, j := range jobs { + versions = append(versions, j.Version) + if j.Platform != "windows-amd64" || j.Build == nil { + t.Errorf("job %s: platform %q, Build set %v; want windows-amd64 with a Build step", j.Version, j.Platform, j.Build != nil) + } + if want := fmt.Sprintf("python/%s/windows-amd64.tar.gz", j.Version); j.R2Key != want { + t.Errorf("job %s: R2Key %q, want %q", j.Version, j.R2Key, want) + } + if want := fmt.Sprintf("python/%s/windows-amd64.meta.json", j.Version); j.MetaKey != want { + t.Errorf("job %s: MetaKey %q, want %q", j.Version, j.MetaKey, want) + } + if want := srv.URL + "/" + j.Version + "/amd64/"; j.URL != want { + t.Errorf("job %s: URL %q, want %q", j.Version, j.URL, want) + } + } + sort.Strings(versions) + if want := []string{"3.13.1", "3.9.11"}; !reflect.DeepEqual(versions, want) { + t.Errorf("FetchVersions() versions = %v, want %v", versions, want) + } + for _, p := range requested.list() { + if strings.HasPrefix(p, "/3.4.10/") || strings.HasPrefix(p, "/3.14.0/") { + t.Errorf("requested %s, outside the MSI-signed range", p) + } + } +} + +// pathRecorder records requested paths; test servers handle requests on +// several goroutines at once. +type pathRecorder struct { + mu sync.Mutex + paths []string +} + +func (r *pathRecorder) record(p string) { + r.mu.Lock() + defer r.mu.Unlock() + r.paths = append(r.paths, p) +} + +func (r *pathRecorder) list() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.paths...) +} + +// fakeMSIContents is what the fake extractor writes for each component, +// mirroring the real MSIs' contents. +var fakeMSIContents = map[string][]string{ + "core": {"python39.dll", "python3.dll"}, + "exe": {"python.exe", "pythonw.exe", "vcruntime140.dll"}, + "lib": {"Lib/os.py", "Lib/venv/__init__.py", "Lib/venv/scripts/nt/python.exe", "Lib/ensurepip/__init__.py", "Lib/ensurepip/_bundled/pip-22.0.4-py3-none-any.whl", "DLLs/_ssl.pyd"}, + "dev": {"include/Python.h", "libs/python39.lib"}, + "tcltk": {"DLLs/_tkinter.pyd", "DLLs/tcl86t.dll", "Lib/tkinter/__init__.py", "tcl/tcl8.6/init.tcl"}, +} + +// fakeExtract stands in for msiextract: the MSI's content names the +// component, and the component's files are written into destDir. +func fakeExtract(msiPath, destDir string) error { + component, err := os.ReadFile(msiPath) + if err != nil { + return err + } + for _, rel := range fakeMSIContents[strings.TrimPrefix(string(component), "msi:")] { + p := filepath.Join(destDir, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return err + } + if err := os.WriteFile(p, []byte("contents of "+rel), 0o644); err != nil { + return err + } + } + return nil +} + +// msiServer serves each component's MSI, whose content is "msi:", +// and its signature by signer. +func msiServer(t *testing.T, signer *testSigner) *httptest.Server { + t.Helper() + files := make(map[string][]byte) + for _, c := range pythonMSIComponents { + msi := []byte("msi:" + c) + files["/"+c+".msi"] = msi + files["/"+c+".msi.asc"] = signer.sign(t, msi) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, ok := files[r.URL.Path] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(body) + })) + t.Cleanup(srv.Close) + return srv +} + +// tarEntries lists the entries of a .tar.gz, directories ending in "/". +func tarEntries(t *testing.T, archive []byte) map[string]string { + t.Helper() + gz, err := gzip.NewReader(bytes.NewReader(archive)) + if err != nil { + t.Fatalf("gzip.NewReader: %v", err) + } + tr := tar.NewReader(gz) + entries := make(map[string]string) + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("reading tar: %v", err) + } + data, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("reading %s: %v", h.Name, err) + } + if !h.ModTime.Equal(archiveModTime) { + t.Errorf("%s: ModTime %v, want %v", h.Name, h.ModTime, archiveModTime) + } + entries[h.Name] = string(data) + } + return entries +} + +func TestBuildFromMSIs(t *testing.T) { + signer := newTestSigner(t) + srv := msiServer(t, signer) + + archive, err := buildFromMSIs(srv.URL+"/", signer.verifier(t), fakeExtract) + if err != nil { + t.Fatalf("buildFromMSIs() error: %v", err) + } + + entries := tarEntries(t, archive) + for _, files := range fakeMSIContents { + for _, rel := range files { + if got, ok := entries["python/"+rel]; !ok || got != "contents of "+rel { + t.Errorf("archive entry python/%s = %q (present %v)", rel, got, ok) + } + } + } + for _, dir := range []string{"python/", "python/Lib/", "python/Lib/venv/scripts/nt/", "python/tcl/tcl8.6/"} { + if _, ok := entries[dir]; !ok { + t.Errorf("archive has no directory entry %s", dir) + } + } + for name := range entries { + if !strings.HasPrefix(name, "python/") { + t.Errorf("archive entry %s is outside python/", name) + } + } + + again, err := buildFromMSIs(srv.URL+"/", signer.verifier(t), fakeExtract) + if err != nil { + t.Fatalf("second buildFromMSIs() error: %v", err) + } + if !bytes.Equal(archive, again) { + t.Error("building the same release twice produced different archives") + } +} + +func TestBuildFromMSIs_BadSignature(t *testing.T) { + srv := msiServer(t, newTestSigner(t)) + trusted := newTestSigner(t) + + if _, err := buildFromMSIs(srv.URL+"/", trusted.verifier(t), fakeExtract); err == nil { + t.Error("buildFromMSIs() accepted MSIs signed by an untrusted key") + } +} + +func TestBuildFromMSIs_ExtractError(t *testing.T) { + signer := newTestSigner(t) + srv := msiServer(t, signer) + failing := func(string, string) error { return fmt.Errorf("msiextract exploded") } + + if _, err := buildFromMSIs(srv.URL+"/", signer.verifier(t), failing); err == nil || !strings.Contains(err.Error(), "msiextract exploded") { + t.Errorf("buildFromMSIs() error = %v, want the extract error", err) + } +} + +func TestValidateWindowsPython(t *testing.T) { + full := []string{"python.exe", "Lib/os.py", "Lib/venv/__init__.py", "Lib/ensurepip/__init__.py", "include/Python.h", "libs/python39.lib"} + withTk := append(append([]string{}, full...), "DLLs/_tkinter.pyd", "Lib/tkinter/__init__.py") + + tests := []struct { + name string + files []string + requireTk bool + wantErr string + }{ + {name: "full install", files: full}, + {name: "full install with tkinter", files: withTk, requireTk: true}, + {name: "tkinter required but missing", files: full, requireTk: true, wantErr: "DLLs/_tkinter.pyd"}, + {name: "embeddable ._pth", files: append(append([]string{}, full...), "python39._pth"), wantErr: "isolated mode"}, + {name: "no venv", files: []string{"python.exe", "Lib/os.py", "Lib/ensurepip/__init__.py", "include/Python.h", "libs/python39.lib"}, wantErr: "Lib/venv/__init__.py"}, + {name: "no import library", files: full[:5], wantErr: "libs/pythonXY.lib"}, + {name: "embeddable layout", files: []string{"python.exe", "python39.zip", "python39.dll"}, wantErr: "missing"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateWindowsPython(tt.files, tt.requireTk) + if tt.wantErr == "" { + if err != nil { + t.Errorf("validateWindowsPython() = %v, want nil", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("validateWindowsPython() = %v, want error mentioning %q", err, tt.wantErr) + } + }) + } +} + +// nupkg builds a NuGet package with the given entries. +func nupkg(t *testing.T, entries map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + zw := zip.NewWriter(&buf) + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + w, err := zw.Create(name) + if err != nil { + t.Fatalf("zip Create %s: %v", name, err) + } + if _, err := io.WriteString(w, entries[name]); err != nil { + t.Fatalf("zip write %s: %v", name, err) + } + } + if err := zw.Close(); err != nil { + t.Fatalf("zip Close: %v", err) + } + return buf.Bytes() +} + +func fullNuGetEntries() map[string]string { + return map[string]string{ + "[Content_Types].xml": "types", + "_rels/.rels": "rels", + "python.nuspec": "nuspec", + "build/native/python.props": "props", + "package/services/metadata/x": "meta", + ".signature.p7s": "sig", + "tools/": "", + "tools/python.exe": "exe", + "tools/Lib/os.py": "os", + "tools/Lib/venv/__init__.py": "venv", + "tools/Lib/ensurepip/__init__.py": "ensurepip", + "tools/include/Python.h": "header", + "tools/libs/python39.lib": "lib", + "tools/Lib/site-packages/setuptools/script%20(dev).tmpl": "template", + } +} + +func TestArchiveNuGetTools(t *testing.T) { + archive, err := archiveNuGetTools(nupkg(t, fullNuGetEntries())) + if err != nil { + t.Fatalf("archiveNuGetTools() error: %v", err) + } + entries := tarEntries(t, archive) + + for rel, want := range map[string]string{ + "python.exe": "exe", + "Lib/os.py": "os", + "include/Python.h": "header", + "Lib/site-packages/setuptools/script (dev).tmpl": "template", + } { + if got, ok := entries["python/"+rel]; !ok || got != want { + t.Errorf("archive entry python/%s = %q (present %v), want %q", rel, got, ok, want) + } + } + for name := range entries { + if !strings.HasPrefix(name, "python/") { + t.Errorf("archive entry %s is outside python/", name) + } + for _, dropped := range []string{"nuspec", "props", ".rels", "Content_Types", "p7s", "services"} { + if strings.Contains(name, dropped) { + t.Errorf("archive kept NuGet packaging file %s", name) + } + } + } +} + +func TestArchiveNuGetTools_Rejects(t *testing.T) { + embeddable := fullNuGetEntries() + embeddable["tools/python39._pth"] = "python39.zip" + + incomplete := fullNuGetEntries() + delete(incomplete, "tools/Lib/venv/__init__.py") + + unsafe := fullNuGetEntries() + unsafe["tools/../escape.txt"] = "x" + + for name, entries := range map[string]map[string]string{"embeddable": embeddable, "incomplete": incomplete, "unsafe path": unsafe} { + if _, err := archiveNuGetTools(nupkg(t, entries)); err == nil { + t.Errorf("archiveNuGetTools(%s) = nil error, want rejection", name) + } + } + if _, err := archiveNuGetTools([]byte("not a zip")); err == nil { + t.Error("archiveNuGetTools(not a zip) = nil error, want error") + } +} + +func TestVerifySHA512(t *testing.T) { + data := []byte("package") + sum := sha512.Sum512(data) + good := base64.StdEncoding.EncodeToString(sum[:]) + + if err := verifySHA512(data, good); err != nil { + t.Errorf("verifySHA512(matching) = %v, want nil", err) + } + if err := verifySHA512([]byte("tampered"), good); err == nil { + t.Error("verifySHA512(mismatch) = nil, want error") + } +} + +// nugetServer serves python.org-style NuGet packages: flat container +// indexes and packages, registration leaves and catalog entries. +func nugetServer(t *testing.T, indexes map[string][]string, packages map[string][]byte, hashes map[string]string) (*httptest.Server, *pathRecorder) { + t.Helper() + requested := &pathRecorder{} + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + p := r.URL.Path + requested.record(p) + switch { + case strings.HasPrefix(p, "/flat/") && strings.HasSuffix(p, "/index.json"): + id := strings.TrimSuffix(strings.TrimPrefix(p, "/flat/"), "/index.json") + versions, ok := indexes[id] + if !ok { + http.NotFound(w, r) + return + } + _ = json.NewEncoder(w).Encode(map[string][]string{"versions": versions}) + case strings.HasPrefix(p, "/flat/") && strings.HasSuffix(p, ".nupkg"): + pkg, ok := packages[filepath.Base(p)] + if !ok { + http.NotFound(w, r) + return + } + _, _ = w.Write(pkg) + case strings.HasPrefix(p, "/registration/"): + key := strings.TrimSuffix(strings.TrimPrefix(p, "/registration/"), ".json") + _, _ = fmt.Fprintf(w, `{"catalogEntry": "%s/catalog/%s.json"}`, srv.URL, strings.ReplaceAll(key, "/", ".")) + case strings.HasPrefix(p, "/catalog/"): + key := strings.TrimSuffix(strings.TrimPrefix(p, "/catalog/"), ".json") + hash, ok := hashes[key] + if !ok { + http.NotFound(w, r) + return + } + _, _ = fmt.Fprintf(w, `{"packageHashAlgorithm": "SHA512", "packageHash": "%s"}`, hash) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + return srv, requested +} + +func TestPythonNuGetSource_FetchVersions(t *testing.T) { + srv, requested := nugetServer(t, map[string][]string{ + // x64 packages are not reliably full installs and must not be used. + "python": {"3.5.2", "3.13.1"}, + "pythonarm64": {"3.4.4", "3.9.7", "3.13.0-b1", "3.11.4"}, + }, nil, nil) + src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} + + jobs, err := src.FetchVersions() + if err != nil { + t.Fatalf("FetchVersions() error: %v", err) + } + + var got []string + for _, j := range jobs { + got = append(got, j.Version+"/"+j.Platform) + if j.Build == nil { + t.Errorf("job %s/%s has no Build step", j.Version, j.Platform) + } + if want := fmt.Sprintf("python/%s/%s.tar.gz", j.Version, j.Platform); j.R2Key != want { + t.Errorf("job %s: R2Key %q, want %q", j.Version, j.R2Key, want) + } + } + want := []string{"3.9.7/windows-arm64", "3.11.4/windows-arm64"} + if !reflect.DeepEqual(got, want) { + t.Errorf("FetchVersions() = %v, want %v", got, want) + } + if want := srv.URL + "/flat/pythonarm64/3.11.4/pythonarm64.3.11.4.nupkg"; jobs[1].URL != want { + t.Errorf("arm64 job URL = %q, want %q", jobs[1].URL, want) + } + for _, p := range requested.list() { + if strings.HasPrefix(p, "/flat/python/") { + t.Errorf("requested %s; x64 NuGet packages should not be listed", p) + } + } +} + +func TestNotFullBuildErrors(t *testing.T) { + // validateWindowsPython's rejections must reach the mirror loop as + // errNotFullBuild, so those jobs are skipped rather than failed. + if err := validateWindowsPython([]string{"python.exe", "python39.zip"}, false); !errors.Is(err, errNotFullBuild) { + t.Errorf("validateWindowsPython(incomplete) = %v, want errNotFullBuild", err) + } + + embeddable := fullNuGetEntries() + embeddable["tools/python39._pth"] = "python39.zip" + if _, err := archiveNuGetTools(nupkg(t, embeddable)); !errors.Is(err, errNotFullBuild) { + t.Errorf("archiveNuGetTools(embeddable) = %v, want errNotFullBuild", err) + } + + incomplete := MirrorJob{Build: func() ([]byte, error) { + return nil, validateWindowsPython(nil, false) + }} + if _, _, _, err := fetchBinary(incomplete, &Stats{}); !errors.Is(err, errNotFullBuild) { + t.Errorf("fetchBinary(incomplete build) = %v, want errNotFullBuild", err) + } +} + +func TestBuildFromNuGet(t *testing.T) { + pkg := nupkg(t, fullNuGetEntries()) + sum := sha512.Sum512(pkg) + srv, _ := nugetServer(t, + map[string][]string{"python": {}, "pythonarm64": {"3.11.4"}}, + map[string][]byte{"pythonarm64.3.11.4.nupkg": pkg}, + map[string]string{"pythonarm64.3.11.4": base64.StdEncoding.EncodeToString(sum[:])}, + ) + src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} + jobs, err := src.FetchVersions() + if err != nil || len(jobs) != 1 { + t.Fatalf("FetchVersions() = %d jobs, %v; want 1 job", len(jobs), err) + } + + archive, err := jobs[0].Build() + if err != nil { + t.Fatalf("Build() error: %v", err) + } + if _, ok := tarEntries(t, archive)["python/python.exe"]; !ok { + t.Error("archive has no python/python.exe") + } +} + +func TestBuildFromNuGet_HashMismatch(t *testing.T) { + pkg := nupkg(t, fullNuGetEntries()) + other := sha512.Sum512([]byte("a different package")) + srv, _ := nugetServer(t, + map[string][]string{"python": {}, "pythonarm64": {"3.11.4"}}, + map[string][]byte{"pythonarm64.3.11.4.nupkg": pkg}, + map[string]string{"pythonarm64.3.11.4": base64.StdEncoding.EncodeToString(other[:])}, + ) + src := &PythonNuGetSource{flatContainerURL: srv.URL + "/flat/", registrationURL: srv.URL + "/registration/"} + jobs, err := src.FetchVersions() + if err != nil || len(jobs) != 1 { + t.Fatalf("FetchVersions() = %d jobs, %v; want 1 job", len(jobs), err) + } + + if _, err := jobs[0].Build(); err == nil || !strings.Contains(err.Error(), "SHA-512 mismatch") { + t.Errorf("Build() error = %v, want a SHA-512 mismatch", err) + } +} + +func TestParseVersion(t *testing.T) { + tests := []struct { + in string + want [3]int + wantOK bool + }{ + {"3.9.11", [3]int{3, 9, 11}, true}, + {"3.13.0-b1", [3]int{}, false}, + {"3.14", [3]int{}, false}, + {"v3.9.11", [3]int{}, false}, + } + for _, tt := range tests { + got, ok := parseVersion(tt.in) + if ok != tt.wantOK || (ok && got != tt.want) { + t.Errorf("parseVersion(%q) = %v, %v; want %v, %v", tt.in, got, ok, tt.want, tt.wantOK) + } + } + + if compareVersions([3]int{3, 9, 11}, [3]int{3, 10, 0}) >= 0 { + t.Error("compareVersions(3.9.11, 3.10.0) should be negative") + } +} diff --git a/scripts/mirror-binaries/sources.go b/scripts/mirror-binaries/sources.go index 72cb993..33a28a7 100644 --- a/scripts/mirror-binaries/sources.go +++ b/scripts/mirror-binaries/sources.go @@ -84,7 +84,10 @@ func getUpstreamSources(runtime string) ([]UpstreamSource, error) { case "python": return []UpstreamSource{ &PythonStandaloneSource{}, - &PythonOfficialSource{}, + // Full Windows builds for versions python-build-standalone + // lacks, in place of python.org's embeddable zips. + &PythonMSISource{}, + &PythonNuGetSource{}, }, nil case "ruby": return []UpstreamSource{ diff --git a/scripts/mirror-binaries/sources_test.go b/scripts/mirror-binaries/sources_test.go index 2ed960a..dde17e8 100644 --- a/scripts/mirror-binaries/sources_test.go +++ b/scripts/mirror-binaries/sources_test.go @@ -138,3 +138,22 @@ func TestCollectJobs(t *testing.T) { }) } } + +// The first source to offer a version and platform wins, so the order sets +// which build a version gets: python-build-standalone where it has one, then +// python.org's MSIs (full x64 builds with tkinter), then its NuGet packages. +func TestGetUpstreamSources_PythonOrder(t *testing.T) { + sources, err := getUpstreamSources("python") + if err != nil { + t.Fatalf("getUpstreamSources(python) error: %v", err) + } + + var got []string + for _, s := range sources { + got = append(got, s.Name()) + } + want := []string{"python-build-standalone", "python.org MSIs", "python.org NuGet packages"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("python sources = %v, want %v", got, want) + } +} diff --git a/src/internal/doctor/check_python_pth.go b/src/internal/doctor/check_python_pth.go new file mode 100644 index 0000000..0fb0e12 --- /dev/null +++ b/src/internal/doctor/check_python_pth.go @@ -0,0 +1,109 @@ +package doctor + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/CodingWithCalvin/dtvem.cli/src/internal/config" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/pth" +) + +// pythonPthCheck flags installed Python versions that still have a ._pth +// file, the mark of python.org's embeddable Windows package. While it is +// there Python runs in isolated mode: a script can't import modules sitting +// next to it, and PYTHONPATH is ignored. +// +// Older dtvem releases installed embeddable builds for some Windows +// versions. The mirror now serves full builds and the installer turns the +// file off, but versions installed before that keep it until fixed. +// +// The fix renames the file to ._pth.disabled, which restores normal +// imports in place. Only a reinstall brings the rest of a full build (venv, +// C headers, tkinter), so the resolution recommends that too. +type pythonPthCheck struct { + pythonVersionsDir func() string +} + +func newPythonPthCheck() *pythonPthCheck { + return &pythonPthCheck{ + pythonVersionsDir: func() string { return filepath.Join(config.DefaultPaths().Versions, "python") }, + } +} + +func (pythonPthCheck) Name() string { return "python-isolated-mode" } + +func (c pythonPthCheck) Run() Finding { + root := c.pythonVersionsDir() + entries, err := os.ReadDir(root) + if errors.Is(err, fs.ErrNotExist) { + return Finding{OK: true, Title: "No Python versions installed"} + } + if err != nil { + return Finding{ + Severity: SeverityWarning, + Title: "Could not list installed Python versions", + Details: []Detail{{Key: "Error", Value: err.Error()}}, + Resolution: "Check that " + root + " is readable.", + } + } + + type isolatedInstall struct { + version string + dir string + files []string + } + var isolated []isolatedInstall + for _, e := range entries { + if !e.IsDir() { + continue + } + dir := filepath.Join(root, e.Name()) + files, err := pth.Find(dir) + if err != nil || len(files) == 0 { + continue + } + isolated = append(isolated, isolatedInstall{version: e.Name(), dir: dir, files: files}) + } + + if len(isolated) == 0 { + return Finding{OK: true, Title: "No installed Python version runs in isolated mode"} + } + + sort.Slice(isolated, func(i, j int) bool { return isolated[i].version < isolated[j].version }) + details := make([]Detail, 0, len(isolated)) + var versions []string + for _, inst := range isolated { + names := make([]string, len(inst.files)) + for i, f := range inst.files { + names[i] = filepath.Base(f) + } + details = append(details, Detail{Key: "Python " + inst.version, Value: strings.Join(names, ", ")}) + versions = append(versions, inst.version) + } + + return Finding{ + Severity: SeverityWarning, + Title: "Python versions run in isolated mode (embeddable builds)", + Details: details, + Resolution: "Rename their ._pth files to ._pth.disabled so scripts can import modules next to them and PYTHONPATH works. " + + "For a full build with venv, C headers and tkinter, reinstall: dtvem uninstall python , then dtvem install python (" + + strings.Join(versions, ", ") + ").", + Fix: func() error { + for _, inst := range isolated { + if _, err := pth.Disable(inst.dir); err != nil { + return fmt.Errorf("Python %s: %w", inst.version, err) + } + } + return nil + }, + } +} + +func init() { + Register(newPythonPthCheck()) +} diff --git a/src/internal/doctor/check_python_pth_test.go b/src/internal/doctor/check_python_pth_test.go new file mode 100644 index 0000000..c21aaf8 --- /dev/null +++ b/src/internal/doctor/check_python_pth_test.go @@ -0,0 +1,106 @@ +package doctor + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// pythonInstalls creates a python versions directory with one install per +// entry, adding a pythonXY._pth file to the versions listed in embeddable. +func pythonInstalls(t *testing.T, versions []string, embeddable map[string]bool) string { + t.Helper() + root := filepath.Join(t.TempDir(), "versions", "python") + for _, v := range versions { + dir := filepath.Join(root, v) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir %s: %v", dir, err) + } + if err := os.WriteFile(filepath.Join(dir, "python.exe"), nil, 0o644); err != nil { + t.Fatalf("writing python.exe: %v", err) + } + if embeddable[v] { + parts := strings.Split(v, ".") + name := "python" + parts[0] + parts[1] + "._pth" + if err := os.WriteFile(filepath.Join(dir, name), []byte("python39.zip\n.\nimport site\n"), 0o644); err != nil { + t.Fatalf("writing %s: %v", name, err) + } + } + } + return root +} + +func TestPythonPthCheck_NoPythonInstalled(t *testing.T) { + c := pythonPthCheck{pythonVersionsDir: func() string { return filepath.Join(t.TempDir(), "missing") }} + + if f := c.Run(); !f.OK { + t.Errorf("Run() = %+v, want OK", f) + } +} + +func TestPythonPthCheck_FullBuilds(t *testing.T) { + root := pythonInstalls(t, []string{"3.12.12", "3.13.1"}, nil) + c := pythonPthCheck{pythonVersionsDir: func() string { return root }} + + if f := c.Run(); !f.OK { + t.Errorf("Run() = %+v, want OK", f) + } +} + +func TestPythonPthCheck_FlagsAndFixesEmbeddableInstalls(t *testing.T) { + root := pythonInstalls(t, []string{"3.12.12", "3.9.9", "3.8.9"}, map[string]bool{"3.9.9": true, "3.8.9": true}) + c := pythonPthCheck{pythonVersionsDir: func() string { return root }} + + f := c.Run() + if f.OK || f.Severity != SeverityWarning { + t.Fatalf("Run() OK=%v severity=%v, want a warning", f.OK, f.Severity) + } + if !f.Fixable() { + t.Fatal("finding is not fixable, want the rename as a fix") + } + want := []Detail{ + {Key: "Python 3.8.9", Value: "python38._pth"}, + {Key: "Python 3.9.9", Value: "python39._pth"}, + } + if len(f.Details) != len(want) { + t.Fatalf("Details = %v, want %v", f.Details, want) + } + for i := range want { + if f.Details[i] != want[i] { + t.Errorf("Details[%d] = %v, want %v", i, f.Details[i], want[i]) + } + } + for _, s := range []string{"._pth.disabled", "dtvem install python", "3.8.9, 3.9.9"} { + if !strings.Contains(f.Resolution, s) { + t.Errorf("Resolution %q does not mention %q", f.Resolution, s) + } + } + + if err := f.Fix(); err != nil { + t.Fatalf("Fix() error: %v", err) + } + for _, v := range []string{"3.8.9", "3.9.9"} { + parts := strings.Split(v, ".") + pthFile := filepath.Join(root, v, "python"+parts[0]+parts[1]+"._pth") + if _, err := os.Stat(pthFile); !os.IsNotExist(err) { + t.Errorf("%s still exists after Fix()", pthFile) + } + if _, err := os.Stat(pthFile + ".disabled"); err != nil { + t.Errorf("%s.disabled missing after Fix(): %v", pthFile, err) + } + } + + if after := c.Run(); !after.OK { + t.Errorf("Run() after Fix() = %+v, want OK", after) + } +} + +func TestPythonPthCheck_Registered(t *testing.T) { + for _, c := range All() { + if c.Name() == "python-isolated-mode" { + return + } + } + t.Error("python-isolated-mode check is not registered") +} diff --git a/src/internal/pth/pth.go b/src/internal/pth/pth.go new file mode 100644 index 0000000..902fca8 --- /dev/null +++ b/src/internal/pth/pth.go @@ -0,0 +1,50 @@ +// Package pth handles the ._pth file in python.org's embeddable Windows +// Python package. +// +// While a pythonXY._pth file sits next to python.exe, Python builds sys.path +// from that file alone and runs in isolated mode: a script's own directory is +// not on sys.path, PYTHONPATH is ignored, and site-packages is off. That suits +// a runtime embedded in an application, not one people run directly. Full +// builds have no ._pth file. Renaming it to pythonXY._pth.disabled makes an +// embeddable install behave like a regular one, and the rename can be undone. +package pth + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" +) + +// DisabledSuffix is appended to a ._pth file's name to turn it off. Python +// only looks for the exact ._pth name. +const DisabledSuffix = ".disabled" + +// Find returns the ._pth files directly inside a Python install directory. +func Find(installDir string) ([]string, error) { + return filepath.Glob(filepath.Join(installDir, "*._pth")) +} + +// Disable renames every ._pth file in a Python install directory to +// .disabled and returns the files it renamed. A leftover .disabled +// copy from an earlier rename is replaced. +func Disable(installDir string) ([]string, error) { + files, err := Find(installDir) + if err != nil { + return nil, err + } + + var disabled []string + for _, f := range files { + target := f + DisabledSuffix + if err := os.Remove(target); err != nil && !errors.Is(err, fs.ErrNotExist) { + return disabled, fmt.Errorf("removing old %s: %w", filepath.Base(target), err) + } + if err := os.Rename(f, target); err != nil { + return disabled, fmt.Errorf("disabling %s: %w", filepath.Base(f), err) + } + disabled = append(disabled, f) + } + return disabled, nil +} diff --git a/src/internal/pth/pth_test.go b/src/internal/pth/pth_test.go new file mode 100644 index 0000000..59c54e5 --- /dev/null +++ b/src/internal/pth/pth_test.go @@ -0,0 +1,99 @@ +package pth + +import ( + "os" + "path/filepath" + "testing" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("writing %s: %v", path, err) + } +} + +func readFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + return string(data) +} + +func TestFind(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "python39._pth"), "python39.zip\n.\n") + writeFile(t, filepath.Join(dir, "python.exe"), "") + writeFile(t, filepath.Join(dir, "python38._pth.disabled"), "") + if err := os.MkdirAll(filepath.Join(dir, "Lib"), 0o755); err != nil { + t.Fatal(err) + } + writeFile(t, filepath.Join(dir, "Lib", "nested._pth"), "") + + got, err := Find(dir) + if err != nil { + t.Fatalf("Find() error: %v", err) + } + if len(got) != 1 || filepath.Base(got[0]) != "python39._pth" { + t.Errorf("Find() = %v, want only python39._pth", got) + } +} + +func TestFind_FullBuild(t *testing.T) { + dir := t.TempDir() + writeFile(t, filepath.Join(dir, "python.exe"), "") + + got, err := Find(dir) + if err != nil || len(got) != 0 { + t.Errorf("Find(full build) = %v, %v; want none", got, err) + } +} + +func TestDisable(t *testing.T) { + dir := t.TempDir() + pthFile := filepath.Join(dir, "python39._pth") + writeFile(t, pthFile, "python39.zip\n.\nimport site\n") + + disabled, err := Disable(dir) + if err != nil { + t.Fatalf("Disable() error: %v", err) + } + if len(disabled) != 1 || disabled[0] != pthFile { + t.Errorf("Disable() = %v, want [%s]", disabled, pthFile) + } + if _, err := os.Stat(pthFile); !os.IsNotExist(err) { + t.Errorf("%s still exists after Disable()", pthFile) + } + if got := readFile(t, pthFile+DisabledSuffix); got != "python39.zip\n.\nimport site\n" { + t.Errorf("disabled file content = %q, want the original", got) + } + + // Nothing left to disable. + again, err := Disable(dir) + if err != nil || len(again) != 0 { + t.Errorf("second Disable() = %v, %v; want nothing", again, err) + } +} + +func TestDisable_ReplacesOldDisabledCopy(t *testing.T) { + dir := t.TempDir() + pthFile := filepath.Join(dir, "python39._pth") + writeFile(t, pthFile, "new") + writeFile(t, pthFile+DisabledSuffix, "old") + + if _, err := Disable(dir); err != nil { + t.Fatalf("Disable() error: %v", err) + } + if got := readFile(t, pthFile+DisabledSuffix); got != "new" { + t.Errorf("disabled file content = %q, want %q", got, "new") + } +} + +func TestDisable_MissingDirectory(t *testing.T) { + disabled, err := Disable(filepath.Join(t.TempDir(), "missing")) + if err != nil || len(disabled) != 0 { + t.Errorf("Disable(missing dir) = %v, %v; want nothing", disabled, err) + } +} diff --git a/src/runtimes/python/provider_full.go b/src/runtimes/python/provider_full.go index 128e0cb..63c74ca 100644 --- a/src/runtimes/python/provider_full.go +++ b/src/runtimes/python/provider_full.go @@ -20,6 +20,7 @@ import ( "github.com/CodingWithCalvin/dtvem.cli/src/internal/constants" "github.com/CodingWithCalvin/dtvem.cli/src/internal/download" "github.com/CodingWithCalvin/dtvem.cli/src/internal/manifest" + "github.com/CodingWithCalvin/dtvem.cli/src/internal/pth" "github.com/CodingWithCalvin/dtvem.cli/src/internal/runtime" "github.com/CodingWithCalvin/dtvem.cli/src/internal/shim" "github.com/CodingWithCalvin/dtvem.cli/src/internal/ui" @@ -224,10 +225,16 @@ func (p *Provider) installPip(version string) error { installPath := config.RuntimeVersionPath("python", version) - // For python.org embeddable packages, enable site-packages first. - // This file doesn't exist in python-build-standalone, so errors are ignored. - pthFile := filepath.Join(installPath, fmt.Sprintf("python%s._pth", strings.Join(strings.Split(version, ".")[:2], ""))) - _ = p.enableSitePackages(pthFile) // Best effort - ignore errors + // python.org embeddable packages ship a ._pth file that runs Python in + // isolated mode: no script directory on sys.path, PYTHONPATH ignored, + // site-packages off. Turning it off makes the install behave like a + // regular one. Full builds have no ._pth file. + if disabled, err := pth.Disable(installPath); err != nil { + ui.Warning("Could not disable the ._pth file in %s: %v", installPath, err) + ui.Info("Python %s will run in isolated mode until it is renamed to ._pth.disabled", version) + } else if len(disabled) > 0 { + ui.Debug("Disabled %v so Python %s runs as a regular install", disabled, version) + } // Step 1: ensurepip bootstraps the pip module on distributions that // ship without one (python.org embeddable). It's a no-op on python- @@ -338,44 +345,21 @@ func (p *Provider) installPipWithGetPip(version, pythonPath, installPath string) } // getPipURL returns the appropriate get-pip.py URL for the given Python version. -// Older Python versions (3.8 and below) require version-specific URLs since the +// Older Python versions (3.9 and below) require version-specific URLs since the // main get-pip.py no longer supports end-of-life Python versions. func (p *Provider) getPipURL(version string) string { parts := strings.Split(version, ".") if len(parts) >= 2 && parts[0] == "3" { + // The main get-pip.py requires Python 3.10+; older versions + // have their own under /pip/3.x/. minor, err := strconv.Atoi(parts[1]) - if err == nil && minor <= 8 { + if err == nil && minor <= 9 { return fmt.Sprintf("https://bootstrap.pypa.io/pip/%s.%s/get-pip.py", parts[0], parts[1]) } } return "https://bootstrap.pypa.io/get-pip.py" } -// enableSitePackages modifies the ._pth file to enable site-packages. -func (p *Provider) enableSitePackages(pthFile string) error { - content, err := os.ReadFile(pthFile) - if err != nil { - return err - } - - lines := strings.Split(string(content), "\n") - found := false - for i, line := range lines { - if strings.Contains(line, "import site") { - lines[i] = "import site" - found = true - break - } - } - - if !found { - lines = append(lines, "import site") - } - - newContent := strings.Join(lines, "\n") - return os.WriteFile(pthFile, []byte(newContent), 0644) -} - // Uninstall removes an installed version. func (p *Provider) Uninstall(version string) error { // TODO: Implement Python uninstallation diff --git a/src/runtimes/python/provider_test.go b/src/runtimes/python/provider_test.go index 6ed63bd..7e79674 100644 --- a/src/runtimes/python/provider_test.go +++ b/src/runtimes/python/provider_test.go @@ -3,7 +3,6 @@ package python import ( "os" "path/filepath" - "strings" "testing" "github.com/CodingWithCalvin/dtvem.cli/src/internal/runtime" @@ -111,9 +110,9 @@ func TestPythonProvider_GetPipURL(t *testing.T) { expectedURL: "https://bootstrap.pypa.io/get-pip.py", }, { - name: "Python 3.9 uses default URL", + name: "Python 3.9 uses version-specific URL", version: "3.9.18", - expectedURL: "https://bootstrap.pypa.io/get-pip.py", + expectedURL: "https://bootstrap.pypa.io/pip/3.9/get-pip.py", }, { name: "Python 3.8 uses version-specific URL", @@ -147,97 +146,6 @@ func TestPythonProvider_GetPipURL(t *testing.T) { } } -// TestPythonProvider_EnableSitePackages tests the ._pth file modification -func TestPythonProvider_EnableSitePackages(t *testing.T) { - provider := NewProvider() - - t.Run("returns error for non-existent file", func(t *testing.T) { - err := provider.enableSitePackages("/nonexistent/path/python311._pth") - if err == nil { - t.Error("enableSitePackages() should return error for non-existent file") - } - }) - - t.Run("uncomments import site line", func(t *testing.T) { - // Create a temp file with commented import site - tempDir := t.TempDir() - pthFile := filepath.Join(tempDir, "python311._pth") - content := "python311.zip\n.\n#import site\n" - if err := os.WriteFile(pthFile, []byte(content), 0644); err != nil { - t.Fatalf("Failed to create test file: %v", err) - } - - err := provider.enableSitePackages(pthFile) - if err != nil { - t.Fatalf("enableSitePackages() returned error: %v", err) - } - - // Read and verify - result, err := os.ReadFile(pthFile) - if err != nil { - t.Fatalf("Failed to read result file: %v", err) - } - - if !strings.Contains(string(result), "import site") { - t.Error("Result should contain 'import site'") - } - if strings.Contains(string(result), "#import site") { - t.Error("Result should not contain commented '#import site'") - } - }) - - t.Run("adds import site if missing", func(t *testing.T) { - // Create a temp file without import site - tempDir := t.TempDir() - pthFile := filepath.Join(tempDir, "python311._pth") - content := "python311.zip\n.\n" - if err := os.WriteFile(pthFile, []byte(content), 0644); err != nil { - t.Fatalf("Failed to create test file: %v", err) - } - - err := provider.enableSitePackages(pthFile) - if err != nil { - t.Fatalf("enableSitePackages() returned error: %v", err) - } - - // Read and verify - result, err := os.ReadFile(pthFile) - if err != nil { - t.Fatalf("Failed to read result file: %v", err) - } - - if !strings.Contains(string(result), "import site") { - t.Error("Result should contain 'import site'") - } - }) - - t.Run("preserves already uncommented import site", func(t *testing.T) { - // Create a temp file with already uncommented import site - tempDir := t.TempDir() - pthFile := filepath.Join(tempDir, "python311._pth") - content := "python311.zip\n.\nimport site\n" - if err := os.WriteFile(pthFile, []byte(content), 0644); err != nil { - t.Fatalf("Failed to create test file: %v", err) - } - - err := provider.enableSitePackages(pthFile) - if err != nil { - t.Fatalf("enableSitePackages() returned error: %v", err) - } - - // Read and verify - should still have import site, not duplicated - result, err := os.ReadFile(pthFile) - if err != nil { - t.Fatalf("Failed to read result file: %v", err) - } - - count := strings.Count(string(result), "import site") - if count != 1 { - t.Errorf("Expected exactly 1 'import site' line, got %d", count) - } - }) -} - // TestPythonProvider_Shims tests the shim configuration func TestPythonProvider_Shims(t *testing.T) { provider := NewProvider()