diff --git a/.github/workflows/build_infra_images_cache.yml b/.github/workflows/build_infra_images_cache.yml
index 1dd1ca9cd66cb..1c02d2da15ea3 100644
--- a/.github/workflows/build_infra_images_cache.yml
+++ b/.github/workflows/build_infra_images_cache.yml
@@ -185,6 +185,8 @@ jobs:
         uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
         with:
           context: ./dev/spark-test-image/python-312-pandas-3/
+          build-contexts: |
+            root=./
           push: true
           tags: ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-312-pandas-3-cache:${{ github.ref_name }}-static
           cache-from: type=registry,ref=ghcr.io/apache/spark/apache-spark-github-action-image-pyspark-python-312-pandas-3-cache:${{ github.ref_name }}
diff --git a/dev/spark-test-image/python-312-pandas-3/Dockerfile b/dev/spark-test-image/python-312-pandas-3/Dockerfile
index e2a2c189df15b..d7306d1197888 100644
--- a/dev/spark-test-image/python-312-pandas-3/Dockerfile
+++ b/dev/spark-test-image/python-312-pandas-3/Dockerfile
@@ -62,10 +62,10 @@ ENV VIRTUAL_ENV=/opt/spark-venv
 RUN python3.12 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
-ARG BASIC_PIP_PKGS="numpy pyarrow>=23.0.0 six==1.16.0 pandas>=3 scipy plotly<6.0.0 coverage matplotlib openpyxl memory-profiler>=0.61.0 scikit-learn>=1.3.2"
-ARG CONNECT_PIP_PKGS="grpcio==1.76.0 grpcio-status==1.76.0 protobuf==6.33.5 googleapis-common-protos==1.71.0 zstandard==0.25.0 graphviz==0.20.3"
+COPY --from=root pyproject.toml ./pyproject.toml
 
-RUN python3.12 -m pip install $BASIC_PIP_PKGS unittest-xml-reporting $CONNECT_PIP_PKGS lxml && \
-    python3.12 -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu && \
-    python3.12 -m pip install torcheval && \
+RUN python3.12 -m pip install -U pip
+
+RUN python3.12 -m pip install --group ml_torch --index-url https://download.pytorch.org/whl/cpu && \
+    python3.12 -m pip install --group ci_classic_pandas_3 --group ci_connect_standard && \
     python3.12 -m pip cache purge
diff --git a/pyproject.toml b/pyproject.toml
index 9fcd00e69bf9e..9b4233507f43c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -262,6 +262,19 @@ ci_classic_minimum = [
     "psutil",
 ]
 
+ci_classic_pandas_3 = [
+    {include-group = "internal_pyspark_devtool"},
+    {include-group = "sql"},
+    {include-group = "pandas_on_spark"},
+    {include-group = "pandas_on_spark_extra"},
+    {include-group = "ml"},
+    # torch should be installed with ml_torch group
+    {include-group = "ml_extra_base"},
+    {include-group = "internal_test"},
+    "pyarrow>=23.0.0",
+    "pandas>=3.0.0",
+]
+
 ci_lint = [
     {include-group = "internal_lint"},
     {include-group = "ml_extra_base"},