From 8f4d1fc5b7c93140922c37547df37f39b2368859 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 21 Sep 2026 16:53:28 -0700 Subject: [PATCH 1/2] Update the dependency list for nogil dockerfile --- dev/spark-test-image/python-314-nogil/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/spark-test-image/python-314-nogil/Dockerfile b/dev/spark-test-image/python-314-nogil/Dockerfile index 6dea9c2fc35ed..ed12bd27eb5f8 100644 --- a/dev/spark-test-image/python-314-nogil/Dockerfile +++ b/dev/spark-test-image/python-314-nogil/Dockerfile @@ -63,7 +63,7 @@ ENV VIRTUAL_ENV=/opt/spark-venv RUN python3.14t -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# TODO: Add BASIC_PIP_PKGS and CONNECT_PIP_PKGS when it supports Python 3.14 free threaded -# TODO: Add lxml, grpcio, grpcio-status back when they support Python 3.14 free threaded -RUN python3.14t -m pip install 'numpy>=2.1' 'pyarrow>=23.0.0' 'six==1.16.0' 'pandas==2.3.3' 'pystack>=1.6.0' scipy coverage matplotlib openpyxl jinja2 psutil && \ - python3.14t -m pip cache purge +# TODO: Add connect packages when grpcio supports Python 3.14 free threaded +RUN python3.14 -m pip install --group ml_torch --index-url https://download.pytorch.org/whl/cpu && \ + python3.14 -m pip install --group ci_classic_standard && \ + python3.14 -m pip cache purge From 72b643cca31f22e108327b902357fdfff5ab12a5 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 21 Sep 2026 17:04:45 -0700 Subject: [PATCH 2/2] Add build contexts --- .github/workflows/build_infra_images_cache.yml | 2 ++ dev/spark-test-image/python-314-nogil/Dockerfile | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/build_infra_images_cache.yml b/.github/workflows/build_infra_images_cache.yml index 1dd1ca9cd66cb..92c5d5e0d19e4 100644 --- a/.github/workflows/build_infra_images_cache.yml +++ b/.github/workflows/build_infra_images_cache.yml @@ -231,6 +231,8 @@ jobs: uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a with: context: ./dev/spark-test-image/python-314-nogil/ + build-contexts: | + root=./ push: true tags: ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-314-nogil-cache:${{ github.ref_name }}-static cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-314-nogil-cache:${{ github.ref_name }} diff --git a/dev/spark-test-image/python-314-nogil/Dockerfile b/dev/spark-test-image/python-314-nogil/Dockerfile index ed12bd27eb5f8..28b19923d4f6d 100644 --- a/dev/spark-test-image/python-314-nogil/Dockerfile +++ b/dev/spark-test-image/python-314-nogil/Dockerfile @@ -63,6 +63,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv RUN python3.14t -m venv $VIRTUAL_ENV ENV PATH="$VIRTUAL_ENV/bin:$PATH" +COPY --from=root pyproject.toml ./pyproject.toml + +RUN python3.14 -m pip install -U pip + # TODO: Add connect packages when grpcio supports Python 3.14 free threaded RUN python3.14 -m pip install --group ml_torch --index-url https://download.pytorch.org/whl/cpu && \ python3.14 -m pip install --group ci_classic_standard && \