From 8a1eefbcf8dc842c4205e8bf63a4c13ef090a31c Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 16:54:52 +0530 Subject: [PATCH 01/45] Update print statement from 'Hello' to 'Goodbye' --- .github/workflows/pr-build.yaml | 806 +++++++++++++++++++++----------- 1 file changed, 534 insertions(+), 272 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index c8db120167..33ff432826 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -66,11 +66,11 @@ jobs: if [ -n "$RELEVANT_CHANGES" ]; then echo "should_build=true" >> $GITHUB_OUTPUT - echo "✅ Found relevant changes:" + echo "Found relevant changes:" echo "$RELEVANT_CHANGES" else echo "should_build=false" >> $GITHUB_OUTPUT - echo "⏭️ Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile" + echo "Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile" fi build_info: @@ -78,23 +78,27 @@ jobs: if: needs.check_changes.outputs.should_build == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} outputs: - wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }} - has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }} + wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }} + has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }} has_dockerfile_changes: ${{ steps.set_flags.outputs.has_dockerfile_changes }} - docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }} - build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }} + docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }} + build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }} + # Per-UBI-major named outputs - each is a {script,tested_on} JSON object or "" + script_ubi8: ${{ steps.emit.outputs.script_ubi8 }} + script_ubi9: ${{ steps.emit.outputs.script_ubi9 }} + script_ubi10: ${{ steps.emit.outputs.script_ubi10 }} steps: - name: Checkout code (Pull Request) if: github.event_name == 'pull_request' - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Checkout code (Workflow Dispatch) if: github.event_name == 'workflow_dispatch' - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: ref: refs/pull/${{ inputs.pr_number }}/head @@ -147,7 +151,7 @@ jobs: # Reuse changed files from check_changes job CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}" - # If workflow_dispatch, check_changes did not compute changed_files — fetch them now + # If workflow_dispatch, check_changes did not compute changed_files - fetch them now if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then git fetch origin ${{ github.base_ref }} --depth=1 CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) @@ -189,20 +193,40 @@ jobs: chmod +x ./gha-script/read_buildinfo.sh bash ./gha-script/read_buildinfo.sh + - name: Emit per-UBI outputs + id: emit + run: | + source variable.sh + echo "SCRIPT_UBI8=$SCRIPT_UBI8" + echo "SCRIPT_UBI9=$SCRIPT_UBI9" + echo "SCRIPT_UBI10=$SCRIPT_UBI10" + # Use multiline delimiter syntax so JSON double-quotes are not truncated + { + echo "script_ubi8<<__EOF__" + echo "$SCRIPT_UBI8" + echo "__EOF__" + echo "script_ubi9<<__EOF__" + echo "$SCRIPT_UBI9" + echo "__EOF__" + echo "script_ubi10<<__EOF__" + echo "$SCRIPT_UBI10" + echo "__EOF__" + } >> $GITHUB_OUTPUT + - name: Set job control flags id: set_flags run: | if [ -z "$BUILD_INFO_FILE" ]; then - echo "❌ BUILD_INFO_FILE is not set — cannot determine job control flags" + echo "ERROR: BUILD_INFO_FILE is not set - cannot determine job control flags" exit 1 fi if [ ! -f "$BUILD_INFO_FILE" ]; then - echo "❌ BUILD_INFO_FILE does not exist: $BUILD_INFO_FILE" + echo "ERROR: BUILD_INFO_FILE does not exist: $BUILD_INFO_FILE" exit 1 fi PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) - # Strip trailing slash if present — prevents double-slash in regex (e.g. "m/multidict/" → "m/multidict") + # Strip trailing slash if present - prevents double-slash in regex (e.g. "m/multidict/" -> "m/multidict") PACKAGE_DIR="${PACKAGE_DIR%/}" WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) @@ -222,10 +246,10 @@ jobs: # Set outputs for wheel builds if [ "$WHEEL_BUILD" == "true" ] && [ -n "$SH_SCRIPT_CHANGED" ]; then echo "wheel_build_enabled=true" >> $GITHUB_OUTPUT - echo "✅ Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" + echo "Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" else echo "wheel_build_enabled=false" >> $GITHUB_OUTPUT - echo "⏭️ Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" + echo "Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" fi # Set output for sh changes @@ -239,7 +263,7 @@ jobs: if [ "$DOCKER_BUILD" == "true" ] && [ -n "$DOCKERFILE_CHANGED" ]; then echo "docker_build_enabled=true" >> $GITHUB_OUTPUT echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT - echo "✅ Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" + echo "Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" else echo "docker_build_enabled=false" >> $GITHUB_OUTPUT if [ -n "$DOCKERFILE_CHANGED" ]; then @@ -247,17 +271,17 @@ jobs: else echo "has_dockerfile_changes=false" >> $GITHUB_OUTPUT fi - echo "⏭️ Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" + echo "Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" fi # Set output for build job (runs when build_info.json or .sh scripts change) BUILD_INFO_CHANGED=$(echo "$CHANGED_FILES" | grep 'build_info\.json' || true) if [ -n "$BUILD_INFO_CHANGED" ] || [ -n "$SH_SCRIPT_CHANGED" ]; then echo "build_package_enabled=true" >> $GITHUB_OUTPUT - echo "✅ Build package job will run (build_info.json or .sh scripts changed)" + echo "Build package job will run (build_info.json or .sh scripts changed)" else echo "build_package_enabled=false" >> $GITHUB_OUTPUT - echo "⏭️ Build package job will be skipped (no build_info.json or .sh script changes)" + echo "Build package job will be skipped (no build_info.json or .sh script changes)" fi - name: Create scanner-env.sh @@ -289,420 +313,658 @@ jobs: run: tar -czf package-cache.tar.gz package-cache/ - name: Upload package cache - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: package-cache path: package-cache.tar.gz - build: - needs: build_info - if: needs.build_info.outputs.build_package_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + # --------------------------------------------------------------------------- + # BUILD JOBS - one per UBI major version. + # Runs execute_changed_scripts.py inside the correct UBI container. + # Each job only runs when a script exists for that UBI version. + # All three are parallel (no inter-dependency). + # --------------------------------------------------------------------------- + build_ubi8: + needs: build_info + if: ${{ needs.build_info.outputs.build_package_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Build Package + - name: Build Package (UBI8) run: | source package-cache/variable.sh source package-cache/scanner-env.sh + sudo chown -R $USER:$USER . + echo "--- Executing changed scripts for UBI8 ---" + chmod +x ./gha-script/execute_changed_scripts.py + python3 -u ./gha-script/execute_changed_scripts.py - echo "------------------- variable.sh -----------------------------" - cat package-cache/variable.sh - echo "------------------- scanner-env.sh -----------------------------" - cat package-cache/scanner-env.sh - + build_ubi9: + needs: build_info + if: ${{ needs.build_info.outputs.build_package_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI9) + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh sudo chown -R $USER:$USER . - echo "------------------- Executing changed scripts -----------------------------" + echo "--- Executing changed scripts for UBI9 ---" chmod +x ./gha-script/execute_changed_scripts.py python3 -u ./gha-script/execute_changed_scripts.py -# ===================== WHEEL JOBS ===================== + build_ubi10: + needs: build_info + if: ${{ needs.build_info.outputs.build_package_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI10) + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + sudo chown -R $USER:$USER . + echo "--- Executing changed scripts for UBI10 ---" + chmod +x ./gha-script/execute_changed_scripts.py + python3 -u ./gha-script/execute_changed_scripts.py + # --------------------------------------------------------------------------- + # WHEEL BUILD JOBS - one job per UBI version per Python version. + # All parallel. UBI10 supports py312-py314 only (py310/py311 not supported). + # PR builds do not run scan steps. + # --------------------------------------------------------------------------- - wheel_build_py310: + wheel_build_ubi8_py310: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.10" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh source package-cache/variable.sh source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - + if [ -z "$BUILD_SCRIPT_CHANGED" ]; then echo "Skipping - no .sh changes"; exit 0; fi sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false chmod +x ./gha-script/build_wheels.sh bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - - wheel_build_py311: + wheel_build_ubi8_py311: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.11" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh run: | - sudo apt update -y + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + wheel_build_ubi8_py312: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi - # Check if any .sh build script is modified in this package + wheel_build_ubi8_py313: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi + wheel_build_ubi8_py314: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi + # UBI9 wheel jobs ----------------------------------------------------------- - wheel_build_py312: + wheel_build_ubi9_py310: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.12" - + PYTHON_VERSION: "3.10" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi - # Check if any .sh build script is modified in this package + wheel_build_ubi9_py311: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.11" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi + wheel_build_ubi9_py312: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - wheel_build_py313: + wheel_build_ubi9_py313: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.13" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - wheel_build_py314: + wheel_build_ubi9_py314: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.14" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh + # UBI10 wheel jobs (py312-py314 only - py310/py311 not supported on UBI10) ---- - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 + wheel_build_ubi10_py312: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi - # Check if any .sh build script is modified in this package + wheel_build_ubi10_py313: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi + wheel_build_ubi10_py314: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi + # --------------------------------------------------------------------------- + # build_docker - only when Dockerfile changed and docker_build=true in build_info + # --------------------------------------------------------------------------- build_docker: needs: build_info if: needs.build_info.outputs.docker_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Build Docker Image run: | - echo "===== Sourcing environment =====" ls package-cache - chmod +x package-cache/variable.sh chmod +x package-cache/scanner-env.sh source package-cache/variable.sh source package-cache/scanner-env.sh - BUILD_DOCKER=$(echo "$BUILD_DOCKER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) - if [[ "$BUILD_DOCKER" != "true" ]]; then - echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER" - exit 0 + echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER"; exit 0 fi - if [[ -z "$DOCKERFILE_CHANGED" ]]; then - echo "Skipping Docker build as no Dockerfile changes detected in PR" - exit 0 + echo "Skipping Docker build as no Dockerfile changes detected in PR"; exit 0 fi - echo "Dockerfile change detected, proceeding with build..." - - echo "===== Starting Docker build =====" sudo chown -R $USER:$USER . chmod +x ./gha-script/build_docker.sh bash ./gha-script/build_docker.sh - - echo "===== Docker images after build =====" docker images - - echo "===== Saving Docker image =====" docker save -o package-cache/image.tar "$IMAGE_NAME" - ls -lh package-cache/image.tar From d3fad9314e8f26c23dc28443e84f41a3cc29a9f8 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 16:55:13 +0530 Subject: [PATCH 02/45] Update currency-build.yaml --- .github/workflows/currency-build.yaml | 2176 ++++++++++++++++--------- 1 file changed, 1381 insertions(+), 795 deletions(-) diff --git a/.github/workflows/currency-build.yaml b/.github/workflows/currency-build.yaml index 3783d25c0d..764b4324d1 100644 --- a/.github/workflows/currency-build.yaml +++ b/.github/workflows/currency-build.yaml @@ -1,9 +1,6 @@ name: Currency Build on: workflow_dispatch: - branches: - - master - inputs: package_name: description: 'Name of the package to build' @@ -54,8 +51,14 @@ run-name: > format('Currency Build {0} && Unique ID {1}', inputs.package_name, inputs.unique_id) }} - jobs: + + # --------------------------------------------------------------------------- + # build_info: reads build_info.json, runs read_buildinfo.sh, emits + # per-UBI-major outputs (script_ubi8, script_ubi9, script_ubi10). + # Each output is a JSON object string {"script":"...","tested_on":"..."} + # or an empty string "" when that UBI version has no script for this package. + # --------------------------------------------------------------------------- build_info: runs-on: ubuntu-24.04-ppc64le-p10 env: @@ -67,15 +70,19 @@ jobs: ENABLE_TRIVY: ${{ inputs.enable_trivy }} ENABLE_SYFT: ${{ inputs.enable_syft }} ENABLE_GRYPE: ${{ inputs.enable_grype }} + outputs: + script_ubi8: ${{ steps.emit.outputs.script_ubi8 }} + script_ubi9: ${{ steps.emit.outputs.script_ubi9 }} + script_ubi10: ${{ steps.emit.outputs.script_ubi10 }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Install system packages run: | sudo apt update -y sudo apt install -y jq file - + - name: Install Python dependencies run: | pip3 install --force-reinstall -v "requests==2.31.0" @@ -92,35 +99,50 @@ jobs: cat variable.sh echo "=======================" + - name: Emit per-UBI outputs + id: emit + run: | + source variable.sh + echo "SCRIPT_UBI8=$SCRIPT_UBI8" + echo "SCRIPT_UBI9=$SCRIPT_UBI9" + echo "SCRIPT_UBI10=$SCRIPT_UBI10" + # Use multiline delimiter syntax so JSON double-quotes are not truncated + { + echo "script_ubi8<<__EOF__" + echo "$SCRIPT_UBI8" + echo "__EOF__" + echo "script_ubi9<<__EOF__" + echo "$SCRIPT_UBI9" + echo "__EOF__" + echo "script_ubi10<<__EOF__" + echo "$SCRIPT_UBI10" + echo "__EOF__" + } >> $GITHUB_OUTPUT + - name: Create scanner-env.sh run: | - echo "Creating package-cache........." mkdir package-cache - echo "Creating scanner-env.sh ......." echo "export VALIDATE_BUILD_SCRIPT='${{ inputs.validate_build_script }}'" > scanner-env.sh - echo "export BUILD_DOCKER='${{ inputs.build_docker }}'" >> scanner-env.sh - echo "export PACKAGE_NAME='${{ inputs.package_name }}'" >> scanner-env.sh + echo "export BUILD_DOCKER='${{ inputs.build_docker }}'" >> scanner-env.sh + echo "export PACKAGE_NAME='${{ inputs.package_name }}'" >> scanner-env.sh echo "===== scanner-env.sh =====" cat scanner-env.sh echo "==========================" - echo "Moving scanner-env.sh and variable.sh into package-cache............" - sudo mv variable.sh package-cache - sudo mv scanner-env.sh package-cache - echo "Moved scanner-env.sh and variable.sh into package-cache............" + sudo mv variable.sh package-cache/ + sudo mv scanner-env.sh package-cache/ - name: Archive package cache run: tar -czf package-cache.tar.gz package-cache/ - - name: Upload variable.sh and scanner-env.sh via package-cache - uses: actions/upload-artifact@v6 + - name: Upload package-cache artifact + uses: actions/upload-artifact@v4 with: name: package-cache path: package-cache.tar.gz # --------------------------------------------------------------------------- - # install_scan_tools: Installs grype ONCE (static Go binary — fully portable) - # and caches it as an artifact for all wheel_build_pyXXX jobs. - # Scancode-toolkit is fetched once and distributed as an artifact too. + # install_scan_tools: installs grype + fetches scancode-toolkit source ONCE. + # Skipped entirely when wheel_build=false. # --------------------------------------------------------------------------- install_scan_tools: needs: build_info @@ -128,7 +150,7 @@ jobs: runs-on: ubuntu-24.04-ppc64le-p10 steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Install system dependencies run: | @@ -141,7 +163,7 @@ jobs: bash ./gha-script/scanner-scripts/fetch_scancode_toolkit.sh - name: Upload scancode-toolkit-src artifact - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: scancode-toolkit-src path: scancode-toolkit-src.tar.gz @@ -157,188 +179,242 @@ jobs: echo "[INFO] Checksum verification successful." tar -xzf grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz else - echo "[ERROR] Checksum verification FAILED for grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz." + echo "[ERROR] Checksum verification FAILED." exit 1 fi mkdir -p scan-tools-bin mv grype scan-tools-bin/grype chmod +x scan-tools-bin/grype - echo "Grype version:" scan-tools-bin/grype version - name: Archive grype binary run: tar -czf scan-tools-cache.tar.gz scan-tools-bin/ - name: Upload scan-tools-cache artifact - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: scan-tools-cache path: scan-tools-cache.tar.gz - - build: + # --------------------------------------------------------------------------- + # BUILD JOBS — one per UBI major version. + # Each job runs build_package.sh inside the correct UBI container. + # build_ubi9 always runs (every package has at least a UBI9 script). + # build_ubi8 / build_ubi10 only run when that UBI version has a script. + # All three are independent of each other (fully parallel). + # --------------------------------------------------------------------------- + + build_ubi8: needs: build_info - if: ${{ inputs.validate_build_script == 'true' }} + if: ${{ inputs.validate_build_script == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Build Package + - name: Build Package (UBI8) env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | - echo "Sourcing variable.sh and scanner-env.sh..........................................." source package-cache/variable.sh source package-cache/scanner-env.sh - echo "===== variable.sh =====" - cat package-cache/variable.sh - echo "===== scanner-env.sh =====" - cat package-cache/scanner-env.sh - echo "==============LSCPU================" + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "BUILD_SCRIPT=$BUILD_SCRIPT TESTED_ON=$TESTED_ON" lscpu - echo "===================================" chmod +x ./gha-script/build_package.sh bash ./gha-script/build_package.sh cloned_package=$(ls -td -- */ | head -n 1) - sudo mv "$cloned_package" package-cache + sudo mv "$cloned_package" package-cache/ echo "export CLONED_PACKAGE=\"$cloned_package\"" >> package-cache/scanner-env.sh - echo "-------------updated scanner-env.sh-----------------------" - cat package-cache/scanner-env.sh - echo "---------------------------------------------" - cd package-cache - sudo chown $USER:$USER -R . + cd package-cache && sudo chown $USER:$USER -R . chmod +x ../gha-script/pre_process.sh bash ../gha-script/pre_process.sh cd $GITHUB_WORKSPACE - gzip build_log + gzip build_log && mv build_log.gz build_log_UBI8.gz chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh build_log.gz + bash ./gha-script/upload-scripts/upload_file.sh build_log_UBI8.gz + - name: Fix permissions + run: sudo chown -R $USER:$USER package-cache && sudo chmod -R u+rwX,go+rX,go-w package-cache + - name: Archive updated package-cache + run: tar -czf package-cache.tar.gz package-cache/ + - name: Upload updated package-cache (UBI8) + uses: actions/upload-artifact@v4 + with: + name: updated-package-cache-ubi8 + path: package-cache.tar.gz - - name: Fix permissions on package-cache - run: | - USER=$(whoami) - echo "Current user: $USER" - echo "Fixing ownership and permissions for package-cache" - sudo chown -R $USER:$USER package-cache - sudo chmod -R u+rwX,go+rX,go-w package-cache - - - name: Archive updated package cache + build_ubi9: + needs: build_info + if: ${{ inputs.validate_build_script == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI9) + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "BUILD_SCRIPT=$BUILD_SCRIPT TESTED_ON=$TESTED_ON" + lscpu + chmod +x ./gha-script/build_package.sh + bash ./gha-script/build_package.sh + cloned_package=$(ls -td -- */ | head -n 1) + sudo mv "$cloned_package" package-cache/ + echo "export CLONED_PACKAGE=\"$cloned_package\"" >> package-cache/scanner-env.sh + cd package-cache && sudo chown $USER:$USER -R . + chmod +x ../gha-script/pre_process.sh + bash ../gha-script/pre_process.sh + cd $GITHUB_WORKSPACE + gzip build_log && mv build_log.gz build_log_UBI9.gz + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh build_log_UBI9.gz + - name: Fix permissions + run: sudo chown -R $USER:$USER package-cache && sudo chmod -R u+rwX,go+rX,go-w package-cache + - name: Archive updated package-cache run: tar -czf package-cache.tar.gz package-cache/ + - name: Upload updated package-cache (UBI9) + uses: actions/upload-artifact@v4 + with: + name: updated-package-cache-ubi9 + path: package-cache.tar.gz - - name: Upload updated package-cache - uses: actions/upload-artifact@v6 + build_ubi10: + needs: build_info + if: ${{ inputs.validate_build_script == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI10) + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "BUILD_SCRIPT=$BUILD_SCRIPT TESTED_ON=$TESTED_ON" + lscpu + chmod +x ./gha-script/build_package.sh + bash ./gha-script/build_package.sh + cloned_package=$(ls -td -- */ | head -n 1) + sudo mv "$cloned_package" package-cache/ + echo "export CLONED_PACKAGE=\"$cloned_package\"" >> package-cache/scanner-env.sh + cd package-cache && sudo chown $USER:$USER -R . + chmod +x ../gha-script/pre_process.sh + bash ../gha-script/pre_process.sh + cd $GITHUB_WORKSPACE + gzip build_log && mv build_log.gz build_log_UBI10.gz + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh build_log_UBI10.gz + - name: Fix permissions + run: sudo chown -R $USER:$USER package-cache && sudo chmod -R u+rwX,go+rX,go-w package-cache + - name: Archive updated package-cache + run: tar -czf package-cache.tar.gz package-cache/ + - name: Upload updated package-cache (UBI10) + uses: actions/upload-artifact@v4 with: - name: updated-package-cache + name: updated-package-cache-ubi10 path: package-cache.tar.gz - # --------------------------------------------------------------------------- - # wheel_build_pyXXX jobs: build wheel → run CVE scan (generalized_wheel_scanner) - # → run scancode scan → run grype scan → upload all results to COS. - # Grype is reused from the install_scan_tools artifact (static binary). + # WHEEL BUILD JOBS — one job per UBI version per Python version. + # All 15 jobs are fully parallel (none depend on the build jobs above). + # UBI10 does not support Python 3.10 — those jobs are skipped via if:. + # Each job needs install_scan_tools for grype/scancode artifacts. # --------------------------------------------------------------------------- - wheel_build_py310: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py310: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.10" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh source package-cache/variable.sh source package-cache/scanner-env.sh - - pwd - ls -lh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" chmod +x ./gha-script/build_wheels.sh bash ./gha-script/build_wheels.sh - - WORKSPACE=$(pwd) - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py310_log.gz" - - # Compress and rename the log file - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py310_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py310.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | source package-cache/variable.sh source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | source package-cache/variable.sh @@ -346,909 +422,1441 @@ jobs: export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - mkdir -p wheel_scancode_py310 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py310 - - tar cvzf wheel_scancode_py310.tar.gz wheel_scancode_py310 - chmod +x ./gha-script/upload-scripts/upload_file.sh + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; + tar czf wheel_scancode_py310.tar.gz wheel_scancode_py310 bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py310.tar.gz - mkdir -p grype_wheel_py310 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py310 - - tar cvzf grype_wheel_py310.tar.gz grype_wheel_py310 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; + tar czf grype_wheel_py310.tar.gz grype_wheel_py310 bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py310.tar.gz - - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then + if ls *.whl 1>/dev/null 2>&1; then sudo chmod a+r *.whl WHEEL_FILE=$(ls *.whl) SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) if [ -n "$WHEEL_FILE" ]; then CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." + [ -f "$CVE_REPORT" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" || echo "CVE report not found — skipping." fi - wheel_build_py311: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py311: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.11" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh source package-cache/variable.sh source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" chmod +x ./gha-script/build_wheels.sh bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py311_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" - + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py311_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py311.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - mkdir -p wheel_scancode_py311 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py311/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py311 - - tar cvzf wheel_scancode_py311.tar.gz wheel_scancode_py311 - chmod +x ./gha-script/upload-scripts/upload_file.sh + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py311/ \; + tar czf wheel_scancode_py311.tar.gz wheel_scancode_py311 bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py311.tar.gz - mkdir -p grype_wheel_py311 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py311/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py311 - - tar cvzf grype_wheel_py311.tar.gz grype_wheel_py311 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py311/ \; + tar czf grype_wheel_py311.tar.gz grype_wheel_py311 bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py311.tar.gz - - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_py312: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py312: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py312_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" - + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py312_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py312.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - mkdir -p wheel_scancode_py312 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py312 - - tar cvzf wheel_scancode_py312.tar.gz wheel_scancode_py312 - chmod +x ./gha-script/upload-scripts/upload_file.sh + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; + tar czf wheel_scancode_py312.tar.gz wheel_scancode_py312 bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz - mkdir -p grype_wheel_py312 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py312 - - tar cvzf grype_wheel_py312.tar.gz grype_wheel_py312 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; + tar czf grype_wheel_py312.tar.gz grype_wheel_py312 bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz - - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi - - wheel_build_py313: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + + wheel_build_ubi8_py313: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py313_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" - + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py313.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - mkdir -p wheel_scancode_py313 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py313 - - tar cvzf wheel_scancode_py313.tar.gz wheel_scancode_py313 - chmod +x ./gha-script/upload-scripts/upload_file.sh + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; + tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313 bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz - mkdir -p grype_wheel_py313 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py313 - - tar cvzf grype_wheel_py313.tar.gz grype_wheel_py313 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; + tar czf grype_wheel_py313.tar.gz grype_wheel_py313 bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz - - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_py314: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py314: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py314.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py314 + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; + tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314 + bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz + mkdir -p grype_wheel_py314 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; + tar czf grype_wheel_py314.tar.gz grype_wheel_py314 + bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + + # UBI9 wheel jobs (py310–py314) ------------------------------------------------ + wheel_build_ubi9_py310: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.10" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py310_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py310.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py310; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; + tar czf wheel_scancode_py310.tar.gz wheel_scancode_py310; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py310.tar.gz + mkdir -p grype_wheel_py310; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; + tar czf grype_wheel_py310.tar.gz grype_wheel_py310; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py310.tar.gz + - name: Upload wheel to COS run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - - name: Cache pip downloads for scancode-toolkit + wheel_build_ubi9_py311: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.11" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py311_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py311.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py311; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py311/ \; + tar czf wheel_scancode_py311.tar.gz wheel_scancode_py311; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py311.tar.gz + mkdir -p grype_wheel_py311; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py311/ \; + tar czf grype_wheel_py311.tar.gz grype_wheel_py311; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py311.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + wheel_build_ubi9_py312: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py312_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py312.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py312; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; + tar czf wheel_scancode_py312.tar.gz wheel_scancode_py312; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz + mkdir -p grype_wheel_py312; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; + tar czf grype_wheel_py312.tar.gz grype_wheel_py312; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + wheel_build_ubi9_py313: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py313.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py313; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; + tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz + mkdir -p grype_wheel_py313; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; + tar czf grype_wheel_py313.tar.gz grype_wheel_py313; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + wheel_build_ubi9_py314: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py314.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a + mkdir -p wheel_scancode_py314; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; + tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz + mkdir -p grype_wheel_py314; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; + tar czf grype_wheel_py314.tar.gz grype_wheel_py314; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py314_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" + # UBI10 wheel jobs (py311–py314 only — py310 not supported on UBI10) ----------- - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" + wheel_build_ubi10_py312: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py312_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi10_py312.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py312; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; + tar czf wheel_scancode_py312.tar.gz wheel_scancode_py312; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz + mkdir -p grype_wheel_py312; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; + tar czf grype_wheel_py312.tar.gz grype_wheel_py312; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + + wheel_build_ubi10_py313: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi10_py313.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py313; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; + tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz + mkdir -p grype_wheel_py313; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; + tar czf grype_wheel_py313.tar.gz grype_wheel_py313; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + wheel_build_ubi10_py314: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi10_py314.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - mkdir -p wheel_scancode_py314 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py314 - - tar cvzf wheel_scancode_py314.tar.gz wheel_scancode_py314 - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz - - mkdir -p grype_wheel_py314 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py314 - - tar cvzf grype_wheel_py314.tar.gz grype_wheel_py314 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz - + mkdir -p wheel_scancode_py314; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; + tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz + mkdir -p grype_wheel_py314; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; + tar czf grype_wheel_py314.tar.gz grype_wheel_py314; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true # --------------------------------------------------------------------------- - # wheel_license_extract: Runs after all wheel builds complete. Downloads the - # per-Python-version scancode JSON files from COS and extracts SPDX licenses. + # WHEEL LICENSE EXTRACT — one per UBI version. + # Downloads per-Python-version scancode JSON from COS and extracts licenses. + # Runs after all wheel jobs for that UBI version finish (at least one must succeed). # --------------------------------------------------------------------------- - wheel_license_extract: - if: ${{ inputs.wheel_build == 'true' && (needs.wheel_build_py310.result == 'success' || needs.wheel_build_py311.result == 'success' || needs.wheel_build_py312.result == 'success' || needs.wheel_build_py313.result == 'success' || needs.wheel_build_py314.result == 'success') }} + + wheel_license_extract_ubi8: needs: - - wheel_build_py310 - - wheel_build_py311 - - wheel_build_py312 - - wheel_build_py313 - - wheel_build_py314 + - wheel_build_ubi8_py310 + - wheel_build_ubi8_py311 + - wheel_build_ubi8_py312 + - wheel_build_ubi8_py313 + - wheel_build_ubi8_py314 + if: | + always() && + inputs.wheel_build == 'true' && + needs.build_info.outputs.script_ubi8 != '' && + (needs.wheel_build_ubi8_py310.result == 'success' || + needs.wheel_build_ubi8_py311.result == 'success' || + needs.wheel_build_ubi8_py312.result == 'success' || + needs.wheel_build_ubi8_py313.result == 'success' || + needs.wheel_build_ubi8_py314.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: - - name: Checkout code - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Download scancode JSON outputs from COS + - name: Download scancode JSON outputs from COS and extract licenses env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | source package-cache/variable.sh source package-cache/scanner-env.sh - - echo "Downloading per-version scancode archives from COS..." mkdir -p all_wheel_scancode - chmod +x ./gha-script/download-scripts/download_file.sh for PY_VER in py310 py311 py312 py313 py314; do ARCHIVE="wheel_scancode_${PY_VER}.tar.gz" - echo "Attempting to download $ARCHIVE..." bash ./gha-script/download-scripts/download_file.sh "$ARCHIVE" \ || echo "Warning: $ARCHIVE not found on COS (skipped)" if [ -f "$ARCHIVE" ]; then tar -xzf "$ARCHIVE" find "wheel_scancode_${PY_VER}" -type f -name "*.json" \ -exec cp {} all_wheel_scancode/ \; 2>/dev/null || true - echo "Collected JSON files from $ARCHIVE" fi done + echo "Scancode JSON files collected:"; ls -lh all_wheel_scancode + for JSON_FILE in all_wheel_scancode/*.json; do + [ -f "$JSON_FILE" ] || { echo "No JSON files — skipping."; break; } + echo "--- $JSON_FILE ---" + python3 ./gha-script/licenses_extract_script.py "$JSON_FILE" + done - echo "All scancode JSON files available:" - ls -lh all_wheel_scancode - - - name: Extract licenses from each scancode JSON + wheel_license_extract_ubi9: + needs: + - wheel_build_ubi9_py310 + - wheel_build_ubi9_py311 + - wheel_build_ubi9_py312 + - wheel_build_ubi9_py313 + - wheel_build_ubi9_py314 + if: | + always() && + inputs.wheel_build == 'true' && + needs.build_info.outputs.script_ubi9 != '' && + (needs.wheel_build_ubi9_py310.result == 'success' || + needs.wheel_build_ubi9_py311.result == 'success' || + needs.wheel_build_ubi9_py312.result == 'success' || + needs.wheel_build_ubi9_py313.result == 'success' || + needs.wheel_build_ubi9_py314.result == 'success') + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scancode JSON outputs from COS and extract licenses + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | - echo "===== License extraction per wheel =====" + source package-cache/variable.sh + source package-cache/scanner-env.sh + mkdir -p all_wheel_scancode + chmod +x ./gha-script/download-scripts/download_file.sh + for PY_VER in py310 py311 py312 py313 py314; do + ARCHIVE="wheel_scancode_${PY_VER}.tar.gz" + bash ./gha-script/download-scripts/download_file.sh "$ARCHIVE" \ + || echo "Warning: $ARCHIVE not found on COS (skipped)" + if [ -f "$ARCHIVE" ]; then + tar -xzf "$ARCHIVE" + find "wheel_scancode_${PY_VER}" -type f -name "*.json" \ + -exec cp {} all_wheel_scancode/ \; 2>/dev/null || true + fi + done + echo "Scancode JSON files collected:"; ls -lh all_wheel_scancode for JSON_FILE in all_wheel_scancode/*.json; do - [ -f "$JSON_FILE" ] || { echo "No JSON files found — skipping."; break; } + [ -f "$JSON_FILE" ] || { echo "No JSON files — skipping."; break; } echo "--- $JSON_FILE ---" python3 ./gha-script/licenses_extract_script.py "$JSON_FILE" done - echo "========================================" - - source_scanner: - needs: build - if: ${{ inputs.validate_build_script == 'true' }} + wheel_license_extract_ubi10: + needs: + - wheel_build_ubi10_py312 + - wheel_build_ubi10_py313 + - wheel_build_ubi10_py314 + if: | + always() && + inputs.wheel_build == 'true' && + needs.build_info.outputs.script_ubi10 != '' && + (needs.wheel_build_ubi10_py312.result == 'success' || + needs.wheel_build_ubi10_py313.result == 'success' || + needs.wheel_build_ubi10_py314.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: - name: updated-package-cache - + name: package-cache - name: Extract package cache run: tar -xzf package-cache.tar.gz + - name: Download scancode JSON outputs from COS and extract licenses + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + mkdir -p all_wheel_scancode + chmod +x ./gha-script/download-scripts/download_file.sh + for PY_VER in py312 py313 py314; do + ARCHIVE="wheel_scancode_${PY_VER}.tar.gz" + bash ./gha-script/download-scripts/download_file.sh "$ARCHIVE" \ + || echo "Warning: $ARCHIVE not found on COS (skipped)" + if [ -f "$ARCHIVE" ]; then + tar -xzf "$ARCHIVE" + find "wheel_scancode_${PY_VER}" -type f -name "*.json" \ + -exec cp {} all_wheel_scancode/ \; 2>/dev/null || true + fi + done + echo "Scancode JSON files collected:"; ls -lh all_wheel_scancode + for JSON_FILE in all_wheel_scancode/*.json; do + [ -f "$JSON_FILE" ] || { echo "No JSON files — skipping."; break; } + echo "--- $JSON_FILE ---" + python3 ./gha-script/licenses_extract_script.py "$JSON_FILE" + done + # --------------------------------------------------------------------------- + # source_scanner_ubiN: scans the cloned source tree (trivy / syft / grype). + # One job per UBI version, each depending on its respective build_ubiN job. + # Only runs when the corresponding build_ubiN succeeded and has a script. + # --------------------------------------------------------------------------- + source_scanner_ubi8: + needs: build_ubi8 + if: ${{ always() && inputs.validate_build_script == 'true' && needs.build_ubi8.result == 'success' }} + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download updated package-cache (UBI8) + uses: actions/download-artifact@v4 + with: + name: updated-package-cache-ubi8 + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz - name: Prepare scanner environment run: | - echo "Sourcing variable.sh..." source package-cache/variable.sh source package-cache/scanner-env.sh - echo "---------updated scanner-env.sh cat----------------" cat package-cache/scanner-env.sh - echo "scanner-env.sh created with:" - cat package-cache/scanner-env.sh - - name: Run Trivy Scan if: ${{ inputs.enable_trivy == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/trivy_code_scan.sh + bash ./gha-script/scanner-scripts/trivy_code_scan.sh + mkdir source_trivy + mv package-cache/trivy_source_vulnerabilities_results.json \ + package-cache/trivy_source_sbom_results.cyclonedx source_trivy + - name: Run Syft Scan + if: ${{ inputs.enable_syft == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/syft_code_scan.sh + bash ./gha-script/scanner-scripts/syft_code_scan.sh + mkdir source_syft + mv package-cache/syft_source_sbom_results.json source_syft + - name: Run Grype Scan + if: ${{ inputs.enable_grype == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_code_scan.sh + bash ./gha-script/scanner-scripts/grype_code_scan.sh + mkdir source_grype + mv package-cache/grype_source_sbom_results.json \ + package-cache/grype_source_vulnerabilities_results.json source_grype + - name: Upload Scanner Results + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + mkdir source + source package-cache/variable.sh; source package-cache/scanner-env.sh + find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + tar cvzf source_scanner_ubi8.tar.gz source + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh source_scanner_ubi8.tar.gz + + source_scanner_ubi9: + needs: build_ubi9 + if: ${{ always() && inputs.validate_build_script == 'true' && needs.build_ubi9.result == 'success' }} + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download updated package-cache (UBI9) + uses: actions/download-artifact@v4 + with: + name: updated-package-cache-ubi9 + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz + - name: Prepare scanner environment + run: | source package-cache/variable.sh source package-cache/scanner-env.sh cat package-cache/scanner-env.sh - + - name: Run Trivy Scan + if: ${{ inputs.enable_trivy == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/trivy_code_scan.sh - bash ./gha-script/scanner-scripts/trivy_code_scan.sh; + bash ./gha-script/scanner-scripts/trivy_code_scan.sh mkdir source_trivy - mv package-cache/trivy_source_vulnerabilities_results.json package-cache/trivy_source_sbom_results.cyclonedx source_trivy - + mv package-cache/trivy_source_vulnerabilities_results.json \ + package-cache/trivy_source_sbom_results.cyclonedx source_trivy - name: Run Syft Scan if: ${{ inputs.enable_syft == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/syft_code_scan.sh bash ./gha-script/scanner-scripts/syft_code_scan.sh mkdir source_syft mv package-cache/syft_source_sbom_results.json source_syft - - name: Run Grype Scan if: ${{ inputs.enable_grype == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_code_scan.sh bash ./gha-script/scanner-scripts/grype_code_scan.sh mkdir source_grype - mv package-cache/grype_source_sbom_results.json package-cache/grype_source_vulnerabilities_results.json source_grype - + mv package-cache/grype_source_sbom_results.json \ + package-cache/grype_source_vulnerabilities_results.json source_grype - name: Upload Scanner Results env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | mkdir source + source package-cache/variable.sh; source package-cache/scanner-env.sh + find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + tar cvzf source_scanner_ubi9.tar.gz source + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh source_scanner_ubi9.tar.gz + + source_scanner_ubi10: + needs: build_ubi10 + if: ${{ always() && inputs.validate_build_script == 'true' && needs.build_ubi10.result == 'success' }} + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download updated package-cache (UBI10) + uses: actions/download-artifact@v4 + with: + name: updated-package-cache-ubi10 + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz + - name: Prepare scanner environment + run: | source package-cache/variable.sh source package-cache/scanner-env.sh + cat package-cache/scanner-env.sh + - name: Run Trivy Scan + if: ${{ inputs.enable_trivy == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/trivy_code_scan.sh + bash ./gha-script/scanner-scripts/trivy_code_scan.sh + mkdir source_trivy + mv package-cache/trivy_source_vulnerabilities_results.json \ + package-cache/trivy_source_sbom_results.cyclonedx source_trivy + - name: Run Syft Scan + if: ${{ inputs.enable_syft == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/syft_code_scan.sh + bash ./gha-script/scanner-scripts/syft_code_scan.sh + mkdir source_syft + mv package-cache/syft_source_sbom_results.json source_syft + - name: Run Grype Scan + if: ${{ inputs.enable_grype == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_code_scan.sh + bash ./gha-script/scanner-scripts/grype_code_scan.sh + mkdir source_grype + mv package-cache/grype_source_sbom_results.json \ + package-cache/grype_source_vulnerabilities_results.json source_grype + - name: Upload Scanner Results + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + mkdir source + source package-cache/variable.sh; source package-cache/scanner-env.sh find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true - find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true - tar cvzf source_scanner.tar.gz source + tar cvzf source_scanner_ubi10.tar.gz source chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh source_scanner.tar.gz + bash ./gha-script/upload-scripts/upload_file.sh source_scanner_ubi10.tar.gz + # --------------------------------------------------------------------------- + # build_docker / image_scanner + # --------------------------------------------------------------------------- build_docker: - needs: build_info + needs: build_info if: ${{ inputs.build_docker == 'true' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Build Docker Image run: | - echo "................................Sourcing scanner-env.sh............................." ls package-cache source package-cache/variable.sh source package-cache/scanner-env.sh @@ -1256,21 +1864,17 @@ jobs: bash ./gha-script/build_docker.sh docker save -o package-cache/image.tar "$IMAGE_NAME" docker images - - name: Upload Docker Image env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/upload-scripts/upload_docker_image.sh bash ./gha-script/upload-scripts/upload_docker_image.sh - - name: Archive package cache with image run: tar -czf package-cache.tar.gz package-cache/ - - name: Upload package-cache with image.tar - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: package-cache-with-image path: package-cache.tar.gz @@ -1280,132 +1884,114 @@ jobs: if: ${{ inputs.build_docker == 'true' }} runs-on: ubuntu-24.04-ppc64le-p10 steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache-with-image - - name: Extract package cache run: tar -xzf package-cache.tar.gz - + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz - name: Load Docker Image run: | - echo "Sourcing variable.sh ..........................................." - source package-cache/variable.sh - source package-cache/scanner-env.sh - echo ".......................Loading iamge.tar ............................" + source package-cache/variable.sh; source package-cache/scanner-env.sh docker load -i "package-cache/image.tar" - - name: Run Trivy Image Scan if: ${{ inputs.enable_trivy == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/trivy_image_scan.sh bash ./gha-script/scanner-scripts/trivy_image_scan.sh mkdir image_trivy mv trivy_image_vulnerabilities_results.json trivy_image_sbom_results.cyclonedx image_trivy - - name: Run Syft Image Scan if: ${{ inputs.enable_syft == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./script/scanner-scripts/syft_image_scan.sh - bash ./script/scanner-scripts/syft_image_scan.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/syft_image_scan.sh + bash ./gha-script/scanner-scripts/syft_image_scan.sh mkdir image_syft mv syft_image_sbom_results.json image_syft - - name: Run Grype Image Scan if: ${{ inputs.enable_grype == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_image_scan.sh bash ./gha-script/scanner-scripts/grype_image_scan.sh mkdir image_grype mv grype_image_sbom_results.json grype_image_vulnerabilities_results.json image_grype - - name: Upload Image Scanner Results env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | mkdir image - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh find image_trivy -mindepth 1 -exec mv -t image {} + 2>/dev/null || true - find image_syft -mindepth 1 -exec mv -t image {} + 2>/dev/null || true + find image_syft -mindepth 1 -exec mv -t image {} + 2>/dev/null || true find image_grype -mindepth 1 -exec mv -t image {} + 2>/dev/null || true tar cvzf image_scanner.tar.gz image chmod +x ./gha-script/upload-scripts/upload_file.sh bash ./gha-script/upload-scripts/upload_file.sh image_scanner.tar.gz + # --------------------------------------------------------------------------- + # final_summary: runs after all parallel tracks complete. + # Uses always() so it runs even when optional jobs (ubi8/ubi10) were skipped. + # Requires at least one of build_ubi8/ubi9/ubi10 to have succeeded. + # --------------------------------------------------------------------------- final_summary: - name: Final Summary Stage - needs: - - build - - source_scanner - - wheel_build_py310 - - wheel_build_py311 - - wheel_build_py312 - - wheel_build_py313 - - wheel_build_py314 - - wheel_license_extract - - if: | - always() && - needs.build.result == 'success' && - needs.source_scanner.result == 'success' - - runs-on: ubuntu-24.04-ppc64le-p10 - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Download package-cache - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Create and activate venv - run: | - python -m venv venv - source venv/bin/activate - python -m pip install --upgrade pip - pip install requests deprecated xlsxwriter packaging - - # Export environment variables - export PYTHONPATH=$(pwd):$(pwd)/process_bom/ - - - name: Summarize build results - env : - PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - IAM_WRITER_API_KEY: ${{ secrets.IAM_WRITER_API_KEY }} - SERVICE_INSTANCE_ID: ${{ secrets.SERVICE_INSTANCE_ID }} - run: | - echo "Sourcing variable.sh and scanner-env.sh..........................................." - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - echo "===== variable.sh =====" - cat package-cache/variable.sh - echo "===== scanner-env.sh =====" - cat package-cache/scanner-env.sh - echo "-------------package name : ${PACKAGE_NAME}--------------------" - echo "-------------version : ${VERSION}--------------------" - pwd - chmod +x process_bom/*.py || true - # Export environment variables - export PYTHONPATH="/tmp/_actions-runner-working-dir/build-scripts/build-scripts:/tmp/_actions-runner-working-dir/build-scripts/build-scripts/process_bom" - python -m process_bom.run_currency_processor - + name: Final Summary Stage + needs: + - build_ubi8 + - build_ubi9 + - build_ubi10 + - wheel_license_extract_ubi8 + - wheel_license_extract_ubi9 + - wheel_license_extract_ubi10 + - source_scanner_ubi8 + - source_scanner_ubi9 + - source_scanner_ubi10 + if: | + always() && + (needs.build_ubi8.result == 'success' || + needs.build_ubi9.result == 'success' || + needs.build_ubi10.result == 'success') + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Create and activate venv + run: | + python -m venv venv + source venv/bin/activate + python -m pip install --upgrade pip + pip install requests deprecated xlsxwriter packaging + - name: Summarize build results + env: + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + IAM_WRITER_API_KEY: ${{ secrets.IAM_WRITER_API_KEY }} + SERVICE_INSTANCE_ID: ${{ secrets.SERVICE_INSTANCE_ID }} + run: | + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "===== scanner-env.sh ====="; cat package-cache/scanner-env.sh + echo "Package: ${PACKAGE_NAME} Version: ${VERSION}" + chmod +x process_bom/*.py || true + export PYTHONPATH="/tmp/_actions-runner-working-dir/build-scripts/build-scripts:/tmp/_actions-runner-working-dir/build-scripts/build-scripts/process_bom" + python -m process_bom.run_currency_processor From c529edc1bfdb50c92a1bcd864d68b7ace3bd8e7f Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 16:55:44 +0530 Subject: [PATCH 03/45] Add environment variables for CVE scanning --- gha-script/build_wheels.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gha-script/build_wheels.py b/gha-script/build_wheels.py index 0fdd6f16a0..b3581a7755 100644 --- a/gha-script/build_wheels.py +++ b/gha-script/build_wheels.py @@ -43,6 +43,15 @@ def trigger_build_wheel(wrapper_file, python_version, image_name, file_name, ver "GHA_CURRENCY_SERVICE_ID_API_KEY": os.getenv("GHA_CURRENCY_SERVICE_ID_API_KEY"), "GHA_CURRENCY_SERVICE_ID": os.getenv("GHA_CURRENCY_SERVICE_ID"), "AUDITWHEEL_EXCLUDE": os.getenv("AUDITWHEEL_EXCLUDE", ""), + # Grype is installed on the host runner at scan-tools-bin/grype. + # The workspace is volume-mounted at /home/tester/ inside the + # container, so the binary is reachable at that in-container path. + # Passing GRYPE_BIN lets generalized_wheel_scanner.py find it via + # os.environ without relying on $PATH (which is host-only). + "GRYPE_BIN": "/home/tester/scan-tools-bin/grype", + # Set to "false" by pr-build.yaml to skip the CVE scan in PR builds. + # Defaults to "true" (scan runs) when unset (currency-build.yaml). + "ENABLE_CVE_SCAN": os.getenv("ENABLE_CVE_SCAN", "true"), } ) From 4082a56b1e81525408f147052c807aa077321843 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 16:56:05 +0530 Subject: [PATCH 04/45] Update build_wheels.sh --- gha-script/build_wheels.sh | 114 ++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 46 deletions(-) diff --git a/gha-script/build_wheels.sh b/gha-script/build_wheels.sh index 59d246dbeb..092766c2da 100644 --- a/gha-script/build_wheels.sh +++ b/gha-script/build_wheels.sh @@ -10,61 +10,83 @@ echo "************************************************************************** docker_image="" -# the below function is used for building a custom docker image, it will be called only when non root user build is set to true. -# function accepts one argument, which is the base image value. +# Builds a custom non-root wrapper image on top of the given base image. +# Sets docker_image to "docker_non_root_image" on success. docker_build_non_root() { echo "building docker image for non root user build" docker build --build-arg BASE_IMAGE="$1" -t docker_non_root_image -f gha-script/dockerfile_non_root . docker_image="docker_non_root_image" } -#Below conditions are used to select the base image based on the 2 flags, tested_on and non_root_build. -#A docker_build_non_root function is called when non root build is true. -if [[ "$TESTED_ON" == UBI:9* || "$TESTED_ON" == UBI9* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - docker pull registry.access.redhat.com/ubi9/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi9/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi9/ubi:$ubi_version" - fi -elif [[ "$TESTED_ON" == UBI:10* || "$TESTED_ON" == UBI10* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - # Fallback to 10.0 if no minor version found - ubi_version=${ubi_version:-10.0} - docker pull registry.access.redhat.com/ubi10/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi10/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi10/ubi:$ubi_version" - fi -else - docker pull registry.access.redhat.com/ubi8/ubi:8.7 - docker_image="registry.access.redhat.com/ubi8/ubi:8.7" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi8/ubi:8.7" - fi -fi +# Resolves the correct UBI registry image from the TESTED_ON string and +# sets the global docker_image variable. Supports UBI8, UBI9, UBI10+. +# The TESTED_ON value is normalised before parsing so all of the following +# formats work: "UBI:9.3" "UBI 9.3" "ubi9.3" "UBI: 9.3" "UBI 10" +resolve_docker_image() { + local raw="$1" + local upper + upper=$(echo "$raw" | tr '[:lower:]' '[:upper:]') -# UBI 10 does not ship Python 3.10 or 3.11 in its repos. -# Building 3.10 from source is not supported for UBI 10 wheel builds. -# Skip those Python versions early to avoid false-positive successes. -if [[ "$TESTED_ON" == UBI:10* || "$TESTED_ON" == UBI10* ]]; then - if [[ "$PYTHON_VERSION" == "3.10" || "$PYTHON_VERSION" == "3.11" ]]; then - echo "Skipping wheel build for Python $PYTHON_VERSION — not supported on UBI 10" - exit 0 - fi -fi + # Collapse "UBI : 9.3" / "UBI:9.3" / "UBI 9.3" / "UBI9.3" -> "UBI9.3" + # Uses sed only - no grep -P (absent on ppc64le runners). + local norm + norm=$(echo "$upper" | sed 's/UBI[[:space:]]*[: ][[:space:]]*/UBI/g') -WHEEL_SCRIPT=gha-script/create_wheel_wrapper.sh + # Extract the major UBI version integer immediately after "UBI" + local major + major=$(echo "$norm" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + case "$major" in + ''|*[!0-9]*) major="" ;; + esac + + if [ -z "$major" ]; then + echo "ERROR: Cannot determine UBI major version from TESTED_ON='$raw'" + exit 1 + fi -# Ensure the wrapper script is readable and executable by all users. -# Required for non-root container builds where the mounted volume is owned -# by the host runner user but executed as test_user inside the container. -chmod a+rx "$WHEEL_SCRIPT" + # Extract the full version string e.g. "9.3", "10.3" (first X.Y found) + local full + full=$(echo "$upper" | grep -oE '[0-9]+\.[0-9]+' | head -1) + + if [ -z "$full" ]; then + # No minor version supplied - use per-major defaults + case "$major" in + 8) full="8.7" ;; + 9) full="9.3" ;; + 10) full="10.0" ;; + *) full="${major}.0" ;; + esac + echo "WARNING: No minor version in TESTED_ON='$raw', defaulting to UBI ${full}" + fi + + # Select the registry image by major version + case "$major" in + 8) + docker_image="registry.access.redhat.com/ubi8/ubi:${full}" + ;; + 9) + docker_image="registry.access.redhat.com/ubi9/ubi:${full}" + ;; + 10) + docker_image="registry.access.redhat.com/ubi10/ubi:${full}" + ;; + *) + echo "ERROR: Unsupported UBI major version '${major}' in TESTED_ON='$raw'" + exit 1 + ;; + esac + + echo " Resolved Docker image: $docker_image (TESTED_ON='$raw')" + + if [[ "$NON_ROOT_BUILD" == "true" ]]; then + docker_build_non_root "$docker_image" + fi +} + +resolve_docker_image "$TESTED_ON" +docker pull "$docker_image" + +WHEEL_SCRIPT=gha-script/create_wheel_wrapper.sh # path to post_process_wheel script (suffix addition, license addition, metadata addition) POST_PROCESS_SCRIPT_PATH=gha-script/post_process_wheel.py From 422627644c540da8e3af4f93fc4463712dd1ee87 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 16:56:40 +0530 Subject: [PATCH 05/45] Refactor wheel wrapper script for dependency installation Refactor installation commands to remove unnecessary sudo usage and streamline the installation of dependencies for Python versions. Update CVE scan handling to improve clarity and functionality. --- gha-script/create_wheel_wrapper.sh | 95 ++++++++++++++++-------------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/gha-script/create_wheel_wrapper.sh b/gha-script/create_wheel_wrapper.sh index b699b68279..011666f26f 100644 --- a/gha-script/create_wheel_wrapper.sh +++ b/gha-script/create_wheel_wrapper.sh @@ -7,28 +7,20 @@ EXTRA_ARGS=${3:-""} POST_PROCESS_SCRIPT_PATH=${4:-"post_process_wheel.py"} CURRENT_DIR=$(pwd) -# Use sudo for privileged commands when running as a non-root user. -# dockerfile_non_root grants test_user passwordless sudo, so this is always safe. -# When already root, sudo is not needed (and may not be installed), so skip it. -if [[ "$(id -u)" -ne 0 ]]; then - YUM="sudo yum" - SUDO="sudo" -else - YUM="yum" - SUDO="" -fi +# install git - required by generate_sha() for all Python versions and UBI versions +yum install -y git -# install gcc — select toolset version based on UBI major version +# install gcc - select toolset version based on UBI major version UBI_MAJOR=$(grep -oP '(?<=^VERSION_ID=")[0-9]+' /etc/os-release || grep -oP 'release \K[0-9]+' /etc/redhat-release 2>/dev/null || echo "8") if [[ "$UBI_MAJOR" -ge 10 ]]; then GCC_TOOLSET="gcc-toolset-15" - $YUM install -y "$GCC_TOOLSET" - # On UBI 10, SCL (Software Collections) was dropped — there is no enable script. + yum install -y "$GCC_TOOLSET" + # On UBI 10, SCL (Software Collections) was dropped - there is no enable script. # Activate the toolset by prepending its bin directory to PATH directly. export PATH="/opt/rh/${GCC_TOOLSET}/root/usr/bin:$PATH" else GCC_TOOLSET="gcc-toolset-13" - $YUM install -y "$GCC_TOOLSET" + yum install -y "$GCC_TOOLSET" source /opt/rh/${GCC_TOOLSET}/enable fi gcc --version @@ -48,48 +40,48 @@ install_python_version() { echo case $version in "3.11" | "3.12") - $YUM install -y python${version} python${version}-devel python${version}-pip + yum install -y python${version} python${version}-devel python${version}-pip ;; "3.10") if ! python3.10 --version &>/dev/null; then - $YUM install -y zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel - $YUM install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel + yum install -y sudo zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel + yum install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel wget https://www.python.org/ftp/python/3.10.20/Python-3.10.20.tgz tar xf Python-3.10.20.tgz cd Python-3.10.20 ./configure --prefix=/usr/local --enable-optimizations --enable-shared make -j2 make altinstall - echo "/usr/local/lib" | $SUDO tee /etc/ld.so.conf.d/python-local.conf && $SUDO ldconfig + echo "/usr/local/lib" > /etc/ld.so.conf.d/python-local.conf && ldconfig echo "Completed..." cd .. && rm -rf Python-3.10.20.tgz fi ;; "3.13") if ! python3.13 --version &>/dev/null; then - $YUM install -y zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel - $YUM install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel + yum install -y sudo zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel + yum install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel wget https://www.python.org/ftp/python/3.13.10/Python-3.13.10.tgz tar xzf Python-3.13.10.tgz cd Python-3.13.10 ./configure --prefix=/usr/local --enable-optimizations --enable-shared make -j2 make altinstall - echo "/usr/local/lib" | $SUDO tee /etc/ld.so.conf.d/python-local.conf && $SUDO ldconfig + echo "/usr/local/lib" > /etc/ld.so.conf.d/python-local.conf && ldconfig cd .. && rm -rf Python-3.13.10.tgz fi ;; "3.14") if ! python3.14 --version &>/dev/null; then - $YUM install -y zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel - $YUM install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel + yum install -y sudo zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel + yum install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel wget https://www.python.org/ftp/python/3.14.3/Python-3.14.3.tgz tar xzf Python-3.14.3.tgz cd Python-3.14.3 ./configure --prefix=/usr/local --enable-optimizations --enable-shared make -j2 make altinstall - echo "/usr/local/lib" | $SUDO tee /etc/ld.so.conf.d/python-local.conf && $SUDO ldconfig + echo "/usr/local/lib" > /etc/ld.so.conf.d/python-local.conf && ldconfig cd .. && rm -rf Python-3.14.3.tgz fi ;; @@ -345,16 +337,31 @@ fi cd "$CURRENT_DIR" wheel_final=(*.whl) -echo -echo "============== Running CVE scan on: ${wheel_final} ==============" -echo +# --------------------------------------------------------------------------- +# run_cve_scan: runs generalized_wheel_scanner.py on the built wheel. +# +# To skip CVE scanning when testing wheel creation locally, comment out the +# run_cve_scan call below (search for "run_cve_scan" further down). +# +# In CI the ENABLE_CVE_SCAN env var controls this: +# ENABLE_CVE_SCAN=false -> skip (set by pr-build.yaml) +# unset or "true" -> run (default for currency-build.yaml) +# --------------------------------------------------------------------------- +run_cve_scan() { + local wheel=$1 + local build_script=$2 + + SCANNER_PATH="gha-script/generalized_wheel_scanner.py" + if [ ! -f "$SCANNER_PATH" ]; then + echo "===> WARNING: $SCANNER_PATH not found, skipping CVE scan." + return 0 + fi + + echo + echo "============== Running CVE scan on: ${wheel} ==============" + echo -# Run CVE scanner — passes wheel + original build script so Lane 3 can parse -# git clone/checkout lines for source-built lib names and versions. -# Scanner is non-blocking — failure does not stop the build. -SCANNER_PATH="gha-script/generalized_wheel_scanner.py" -if [ -f "$SCANNER_PATH" ]; then - if python "$SCANNER_PATH" "${wheel_final}" "${BUILD_SCRIPT_PATH}"; then + if python "$SCANNER_PATH" "${wheel}" "${build_script}"; then echo echo "===> CVE scan completed successfully." echo @@ -363,9 +370,7 @@ if [ -f "$SCANNER_PATH" ]; then echo "===> WARNING: CVE scan failed. Continuing build." echo fi -else - echo "===> WARNING: $SCANNER_PATH not found, skipping CVE scan." -fi +} echo echo "============== Generating sha for: ${wheel_final} ==============" @@ -380,9 +385,6 @@ echo echo "=== Post Processing wheel ${wheel_final} with SHA: ${SHA256_VALUE} ===" echo -# Save CVE report name before post-processing renames the wheel -cve_report_old="${wheel_final%.whl}_cve_report.json" - # post processing of wheels (Suffix addition, license addition, metadata addition) if python ${POST_PROCESS_SCRIPT_PATH} ${wheel_final} ${SHA256_VALUE}; then echo @@ -395,12 +397,17 @@ else exit 1 fi -# Rename CVE report to match the post-processed wheel filename +# CVE scan runs after post-processing so the report is named after the final +# wheel filename (with +ppc64leN suffix) from the start - no rename needed. wheel_post_processed=(*.whl) -cve_report_new="${wheel_post_processed[0]%.whl}_cve_report.json" -if [ -f "$cve_report_old" ] && [ "$cve_report_old" != "$cve_report_new" ]; then - mv "$cve_report_old" "$cve_report_new" - echo "===> CVE report renamed: $cve_report_old → $cve_report_new" + +# Call run_cve_scan - comment out this block locally to skip CVE scanning. +if [ "${ENABLE_CVE_SCAN:-true}" = "false" ]; then + echo + echo "===> Skipping CVE scan (ENABLE_CVE_SCAN=false)." + echo +else + run_cve_scan "${wheel_post_processed[0]}" "${BUILD_SCRIPT_PATH}" fi echo From 783405e3fe35b036ff6dbad61b9377c2aadce2eb Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 16:57:35 +0530 Subject: [PATCH 06/45] Refactor read_buildinfo.sh for clarity and functionality Refactor read_buildinfo.sh to improve readability and structure. Added a new function to read the 'Tested on' value from build scripts and updated the handling of build_script to support both string and array formats. --- gha-script/read_buildinfo.sh | 225 ++++++++++++++++++++++++++++++----- 1 file changed, 192 insertions(+), 33 deletions(-) diff --git a/gha-script/read_buildinfo.sh b/gha-script/read_buildinfo.sh index 5c57c5ad88..4785bd4434 100755 --- a/gha-script/read_buildinfo.sh +++ b/gha-script/read_buildinfo.sh @@ -27,7 +27,7 @@ if [ -f $config_file ]; then jsonObj=$config_file build_script=$(jq .build_script $jsonObj) - if $(jq 'has("use_non_root_user")' $jsonObj); then + if $(jq 'has("use_non_root_user")' $jsonObj); then nonRootBuild=$(jq .use_non_root_user $jsonObj) fi @@ -54,11 +54,11 @@ if [ -f $config_file ]; then # Getting specific build_script name and other overrides for version if [[ $(jq --arg ver "$match_version" '.[$ver]' $config_file) != null ]]; then - version_block=".[\"$match_version\"]" # ✅ Properly quoted key for jq + version_block=".[\"$match_version\"]" # Properly quoted key for jq - # version-specific build_script + # version-specific build_script (may be a string or a list) if [[ $(jq -r "$version_block.build_script" $config_file) != "null" ]]; then - build_script=$(jq -r "$version_block.build_script" $config_file) + build_script=$(jq -c "$version_block.build_script" $config_file) fi # version-specific base_docker_image @@ -73,7 +73,7 @@ if [ -f $config_file ]; then "rhel") variant=1 ;; "ubuntu") variant=2 ;; "alpine") variant=3 ;; - *) + *) echo "No valid distro variant, picking default one" variant=1 ;; esac @@ -125,46 +125,205 @@ fi # fi # fi -# Below code is used to get the tested on parameter value from the build script -build_script_with_quotes=$build_script -stripped_build_script=$(echo "$build_script_with_quotes" | sed 's/"//g') -echo $stripped_build_script +# --------------------------------------------------------------------------- +# Helper: read "# Tested on" from a single build script file. +# Normalises the raw value: strip spaces, uppercase, collapse "UBI : 9.3" -> +# "UBI:9.3" so downstream consumers see a consistent format. +# Usage: tested_on=$(read_tested_on "path/to/script.sh") +# --------------------------------------------------------------------------- +read_tested_on() { + local script_file="$1" + local value="" + if [ -f "$script_file" ]; then + while IFS= read -r line; do + if [[ "$line" == "# Tested on"* ]]; then + # Extract everything after the first colon, strip outer whitespace, + # uppercase, then collapse spaces around colons (e.g. "UBI : 9.3" -> "UBI:9.3") + value=$(echo "$line" | cut -d ':' -f 2- \ + | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \ + | tr '[:lower:]' '[:upper:]' \ + | sed 's/[[:space:]]*:[[:space:]]*/:/g') + break + fi + done < "$script_file" + fi + echo "$value" +} +# --------------------------------------------------------------------------- +# Build the BUILD_SCRIPTS_JSON array and set the single-script compat vars. +# +# build_script in build_info.json can be: +# - a string : "pytorch_ubi_9.3.sh" +# - a list : ["pytorch_ubi_8.3.sh", "pytorch_ubi_9.3.sh", "pytorch_ubi_10.3.sh"] +# +# For both cases we produce: +# BUILD_SCRIPTS_JSON - compact JSON array of {script, tested_on} objects, +# consumed by the GHA matrix via fromJson(). +# BUILD_SCRIPT - backward-compat: first (or only) script filename. +# TESTED_ON - backward-compat: first (or only) tested_on value. +# +# NOTE: '# Tested on' is MANDATORY in every build script. The pipeline will +# abort if it is missing - there is no default fallback. +# --------------------------------------------------------------------------- -if [ -f "$stripped_build_script" ]; then +# Derive type from the already-resolved $build_script shell variable. +# This is correct whether build_script came from the top-level field or +# was overridden by a version block - both paths store the final value in +# $build_script as either a bare filename string or a compact JSON array. +script_type=$(echo "$build_script" | jq -r 'type' 2>/dev/null || echo "string") +echo "build_script type (resolved): $script_type" - echo "build script found" - while IFS= read -r line; do - # Check if the line starts with '# Tested on' - if [[ "$line" == "# Tested on"* ]]; then - # Extract the value after the first colon - tested_on=$(echo "$line" | cut -d ':' -f 2- | tr -d '[:space:]' | tr '[:lower:]' '[:upper:]') - break - fi +BUILD_SCRIPTS_JSON="" + +if [ "$script_type" = "array" ]; then + echo "build_script is a list - iterating each entry to read its '# Tested on' value" + + json_array="[" + first=true + first_script="" + first_tested_on="" + + # jq outputs one bare filename per line (no surrounding quotes) + while IFS= read -r script_name; do + # strip any stray quotes/whitespace that jq might leave + script_name=$(echo "$script_name" | tr -d '"' | xargs) + [ -z "$script_name" ] && continue + + raw_tested_on=$(read_tested_on "$script_name") + + if [ -z "$raw_tested_on" ]; then + echo "ERROR: '# Tested on' header is missing in '$script_name'." + echo " Every build script must declare the UBI version it targets, e.g.:" + echo " # Tested on UBI:9.3" + exit 1 + fi + + echo " script='$script_name' tested_on='$raw_tested_on'" + + # Record first entry for backward-compat single-value exports + if [ "$first" = "true" ]; then + first_script="$script_name" + first_tested_on="$raw_tested_on" + first=false + fi + + # Append JSON object - jq --arg safely escapes both values + entry=$(jq -n \ + --arg s "$script_name" \ + --arg t "$raw_tested_on" \ + '{"script":$s,"tested_on":$t}') - done < "$stripped_build_script" + if [ "$json_array" = "[" ]; then + json_array="${json_array}${entry}" + else + json_array="${json_array},${entry}" + fi + + # Iterate the resolved $build_script variable (not the file) so that + # version-block overrides are respected. + done < <(echo "$build_script" | jq -r '.[]') + + json_array="${json_array}]" + BUILD_SCRIPTS_JSON="$json_array" + + # Backward-compat single values - point to the first entry in the list + build_script="$first_script" + tested_on="$first_tested_on" + +else + # Single string - existing behaviour + build_script_with_quotes=$build_script + stripped_build_script=$(echo "$build_script_with_quotes" | sed 's/"//g') + echo "build_script (single): $stripped_build_script" + + tested_on=$(read_tested_on "$stripped_build_script") + + if [ -z "$tested_on" ]; then + echo "ERROR: '# Tested on' header is missing in '$stripped_build_script'." + echo " Every build script must declare the UBI version it targets, e.g.:" + echo " # Tested on UBI:9.3" + exit 1 + fi echo "Tested on value: $tested_on" + + # Wrap single entry into the same JSON array format for consistency + BUILD_SCRIPTS_JSON=$(jq -n \ + --arg s "$stripped_build_script" \ + --arg t "$tested_on" \ + '[{"script":$s,"tested_on":$t}]') + + # Backward-compat: strip quotes from the jq-extracted string value + build_script="$stripped_build_script" fi -# Extract auditwheel exclusions +echo "BUILD_SCRIPTS_JSON: $BUILD_SCRIPTS_JSON" + +# --------------------------------------------------------------------------- +# Bucket scripts by UBI major version -> SCRIPT_UBI8, SCRIPT_UBI9, SCRIPT_UBI10. +# Each is a single {script, tested_on} JSON object (or empty string ""). +# At most one script per UBI major version is expected per package. +# Callers (currency-build.yaml, pr-build.yaml) use these to drive named jobs: +# build_ubi8 / build_ubi9 / build_ubi10 +# wheel_build_ubi8_pyXXX / wheel_build_ubi9_pyXXX / wheel_build_ubi10_pyXXX +# --------------------------------------------------------------------------- +SCRIPT_UBI8="" +SCRIPT_UBI9="" +SCRIPT_UBI10="" + +while IFS= read -r entry; do + [ -z "$entry" ] && continue + t_on=$(echo "$entry" | jq -r '.tested_on') + # Normalise: uppercase, collapse separators around UBI, then extract the + # integer immediately following "UBI". Uses only sed + tr (POSIX) - no + # grep -P needed, so it works on ppc64le runners where grep -P is absent. + t_upper=$(echo "$t_on" | tr '[:lower:]' '[:upper:]') + # Collapse "UBI : 9.3" / "UBI:9.3" / "UBI 9.3" / "UBI9.3" -> "UBI9.3" + t_norm=$(echo "$t_upper" | sed 's/UBI[[:space:]]*[: ][[:space:]]*/UBI/g') + # Extract digits immediately after "UBI" e.g. "UBI10.0" -> "10" + major=$(echo "$t_norm" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + # If sed left non-numeric content (no UBI match), clear it + case "$major" in + ''|*[!0-9]*) major="" ;; + esac + echo " bucket: tested_on='$t_on' major='$major'" + case "$major" in + 8) SCRIPT_UBI8="$entry" ;; + 9) SCRIPT_UBI9="$entry" ;; + 10) SCRIPT_UBI10="$entry" ;; + *) echo "WARNING: Unknown UBI major '$major' in tested_on='$t_on' - skipping bucket" ;; + esac +done < <(echo "$BUILD_SCRIPTS_JSON" | jq -c '.[]') + +echo "SCRIPT_UBI8: $SCRIPT_UBI8" +echo "SCRIPT_UBI9: $SCRIPT_UBI9" +echo "SCRIPT_UBI10: $SCRIPT_UBI10" + +# Extract auditwheel exclusions (unchanged - same pattern as before) AUDITWHEEL_EXCLUDE="" if jq -e 'has("auditwheel_exclude")' "$config_file" >/dev/null; then AUDITWHEEL_EXCLUDE=$(jq -r '.auditwheel_exclude | join(" ")' "$config_file") fi -# Export variables - -echo "export VERSION=$VERSION" > $CUR_DIR/variable.sh -echo "export BUILD_SCRIPT=$build_script" >> $CUR_DIR/variable.sh -echo "export PKG_DIR_PATH=$package_dirpath" >> $CUR_DIR/variable.sh -echo "export IMAGE_NAME=$image_name" >> $CUR_DIR/variable.sh -#echo "export BUILD_DOCKER=$build_docker" >> $CUR_DIR/variable.sh -#echo "export VALIDATE_BUILD_SCRIPT=$validate_build_script" >> $CUR_DIR/variable.sh -echo "export VARIANT=$variant" >> $CUR_DIR/variable.sh -echo "export BASENAME=$basename" >> $CUR_DIR/variable.sh -echo "export NON_ROOT_BUILD=$nonRootBuild" >> $CUR_DIR/variable.sh -echo "export TESTED_ON=$tested_on" >> $CUR_DIR/variable.sh -echo "export AUDITWHEEL_EXCLUDE=\"$AUDITWHEEL_EXCLUDE\"" >> $CUR_DIR/variable.sh +# --------------------------------------------------------------------------- +# Write variable.sh +# JSON objects are single-quote-wrapped so embedded double-quotes survive. +# --------------------------------------------------------------------------- +echo "export VERSION=\"$VERSION\"" > $CUR_DIR/variable.sh +echo "export BUILD_SCRIPT=\"$build_script\"" >> $CUR_DIR/variable.sh +echo "export PKG_DIR_PATH=\"$package_dirpath\"" >> $CUR_DIR/variable.sh +echo "export IMAGE_NAME=\"$image_name\"" >> $CUR_DIR/variable.sh +echo "export VARIANT=\"$variant\"" >> $CUR_DIR/variable.sh +echo "export BASENAME=\"$basename\"" >> $CUR_DIR/variable.sh +echo "export NON_ROOT_BUILD=\"$nonRootBuild\"" >> $CUR_DIR/variable.sh +echo "export TESTED_ON=\"$tested_on\"" >> $CUR_DIR/variable.sh +echo "export AUDITWHEEL_EXCLUDE=\"$AUDITWHEEL_EXCLUDE\"" >> $CUR_DIR/variable.sh +# Full array - kept for any downstream consumer that still needs it +echo "export BUILD_SCRIPTS_JSON='$BUILD_SCRIPTS_JSON'" >> $CUR_DIR/variable.sh +# Per-UBI-major named exports - empty string when that UBI version has no script +echo "export SCRIPT_UBI8='$SCRIPT_UBI8'" >> $CUR_DIR/variable.sh +echo "export SCRIPT_UBI9='$SCRIPT_UBI9'" >> $CUR_DIR/variable.sh +echo "export SCRIPT_UBI10='$SCRIPT_UBI10'" >> $CUR_DIR/variable.sh chmod +x $CUR_DIR/variable.sh cat $CUR_DIR/variable.sh From 6d7638bb2967968e298c7512ef9ce2519c3d3f9b Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 16:57:58 +0530 Subject: [PATCH 07/45] Refactor grype availability check and usage --- gha-script/generalized_wheel_scanner.py | 33 +++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/gha-script/generalized_wheel_scanner.py b/gha-script/generalized_wheel_scanner.py index 116288acbc..aa00c890fa 100644 --- a/gha-script/generalized_wheel_scanner.py +++ b/gha-script/generalized_wheel_scanner.py @@ -54,16 +54,29 @@ def scan(self) -> Dict: print(f"Scanning Wheel: {self.wheel_name}") print(f"{'='*70}\n") - # Step 0: Check grype availability — grype is installed by the workflow - # step before this script runs; if it is missing, grype-dependent lanes - # are skipped gracefully and Lane 3 (NVD) still runs. + # Step 0: Locate grype. + # Grype is installed in the workflow's install_scan_tools job and cached + # as scan-tools-bin/grype in the artifact. The workspace is volume-mounted + # into this container at /home/tester/, so the binary is accessible at + # /home/tester/scan-tools-bin/grype — passed in via the GRYPE_BIN env var + # by build_wheels.py. We honour GRYPE_BIN first; if unset or the path is + # not executable we fall back to shutil.which('grype') for local runs. + # If neither resolves, grype-dependent lanes are skipped gracefully and + # Lane 3 (NVD) still runs. print("Step 0: Checking grype availability...") - grype_available = bool(shutil.which('grype')) - if grype_available: - print(" grype is available.") + grype_bin = os.environ.get("GRYPE_BIN", "").strip() + if grype_bin and os.path.isfile(grype_bin) and os.access(grype_bin, os.X_OK): + grype_available = True + print(f" grype found via GRYPE_BIN: {grype_bin}") else: - print(" WARNING: grype not found. Lane 1, Lane 2 and Phase 1 " - "(wheel direct scan) will be skipped. Lane 3 (NVD) will still run.") + grype_bin = shutil.which('grype') or "" + grype_available = bool(grype_bin) + if grype_available: + print(f" grype found on PATH: {grype_bin}") + else: + print(" WARNING: grype not found. Lane 1, Lane 2 and Phase 1 " + "(wheel direct scan) will be skipped. Lane 3 (NVD) will still run.") + self.grype_bin = grype_bin # used by _scan_wheel_direct / _scan_for_cves self.results['grype_available'] = grype_available # Step 1: Create system library inventory @@ -471,7 +484,7 @@ def _scan_wheel_direct(self, extract_dir: Path) -> Dict: print(f" Running grype on extracted wheel: {extract_dir}") try: result = subprocess.run( - ['grype', f'dir:{extract_dir}', '-o', 'json'], + [self.grype_bin, f'dir:{extract_dir}', '-o', 'json'], capture_output=True, text=True, timeout=300 ) if result.returncode == 0: @@ -512,7 +525,7 @@ def _scan_for_cves(self) -> Dict: try: result = subprocess.run( - ['grype', 'dir:/', '-o', 'json'], + [self.grype_bin, 'dir:/', '-o', 'json'], capture_output=True, text=True, timeout=300 From 21b834126835d62ba5ebf3a2198e97eafe79cdeb Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 16:58:50 +0530 Subject: [PATCH 08/45] Update grype_code_scan.sh --- gha-script/scanner-scripts/grype_code_scan.sh | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/gha-script/scanner-scripts/grype_code_scan.sh b/gha-script/scanner-scripts/grype_code_scan.sh index ceec35eb9d..a865e9f2bb 100644 --- a/gha-script/scanner-scripts/grype_code_scan.sh +++ b/gha-script/scanner-scripts/grype_code_scan.sh @@ -3,18 +3,19 @@ validate_build_script=$VALIDATE_BUILD_SCRIPT cloned_package=$CLONED_PACKAGE +# Use pre-installed grype from the cached artifact. +# $GRYPE_BIN is set by the workflow (points to scan-tools-bin/grype). +if [ -z "$GRYPE_BIN" ]; then + echo "Error: GRYPE_BIN environment variable not set" + exit 1 +fi + cd package-cache -if [ $validate_build_script == true ];then - GRYPE_VERSION=$(curl -s https://api.github.com/repos/anchore/grype/releases/latest | grep -Po '"tag_name": "\K.*?(?=")') - wget https://github.com/anchore/grype/releases/download/$GRYPE_VERSION/grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - tar -xzf grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - chmod +x grype - sudo mv grype /usr/bin - grype --version +if [ $validate_build_script == true ]; then + echo "------------- Using cached grype ---------------" + $GRYPE_BIN version echo "Executing Grype scanner" - sudo grype -q -o cyclonedx-json dir:${cloned_package} > grype_source_sbom_results.json - #cat grype_source_sbom_results.json - sudo grype -q -o json dir:${cloned_package} > grype_source_vulnerabilities_results.json - #cat grype_source_vulnerabilities_results.json + sudo $GRYPE_BIN -q -o cyclonedx-json dir:${cloned_package} > grype_source_sbom_results.json + sudo $GRYPE_BIN -q -o json dir:${cloned_package} > grype_source_vulnerabilities_results.json fi From 962bc2e31928567bb5a08b47e81b6f4619498e3f Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 16:59:09 +0530 Subject: [PATCH 09/45] Update grype_image_scan.sh --- .../scanner-scripts/grype_image_scan.sh | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/gha-script/scanner-scripts/grype_image_scan.sh b/gha-script/scanner-scripts/grype_image_scan.sh index f91b4bbbfd..f9101d0087 100755 --- a/gha-script/scanner-scripts/grype_image_scan.sh +++ b/gha-script/scanner-scripts/grype_image_scan.sh @@ -3,13 +3,17 @@ image_name=$IMAGE_NAME build_docker=$BUILD_DOCKER -if [ $build_docker == true ];then - GRYPE_VERSION=$(curl -s https://api.github.com/repos/anchore/grype/releases/latest | grep -Po '"tag_name": "\K.*?(?=")') - wget https://github.com/anchore/grype/releases/download/$GRYPE_VERSION/grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - tar -xzf grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - chmod +x grype - sudo mv grype /usr/bin - echo "Executing grype scanner" - sudo grype -q -s AllLayers -o cyclonedx-json ${image_name} > grype_image_sbom_results.json - sudo grype -q -s AllLayers -o json ${image_name} > grype_image_vulnerabilities_results.json +# Use pre-installed grype from the cached artifact. +# $GRYPE_BIN is set by the workflow (points to scan-tools-bin/grype). +if [ -z "$GRYPE_BIN" ]; then + echo "Error: GRYPE_BIN environment variable not set" + exit 1 +fi + +if [ $build_docker == true ]; then + echo "------------- Using cached grype ---------------" + $GRYPE_BIN version + echo "Executing grype scanner" + sudo $GRYPE_BIN -q -s AllLayers -o cyclonedx-json ${image_name} > grype_image_sbom_results.json + sudo $GRYPE_BIN -q -s AllLayers -o json ${image_name} > grype_image_vulnerabilities_results.json fi From c0a8f92c291ecf5abe7dfa15911909a7faddc7fc Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 17:01:56 +0530 Subject: [PATCH 10/45] Update build_package.sh --- gha-script/build_package.sh | 101 ++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 32 deletions(-) diff --git a/gha-script/build_package.sh b/gha-script/build_package.sh index d7a28c7baa..63d1317e8e 100755 --- a/gha-script/build_package.sh +++ b/gha-script/build_package.sh @@ -10,44 +10,81 @@ echo "************************************************************************** docker_image="" -# the below function is used for building a custom docker image, it will be called only when non root user build is set to true. -# function accepts one argument, which is the base image value. +# Builds a custom non-root wrapper image on top of the given base image. +# Sets docker_image to "docker_non_root_image" on success. docker_build_non_root() { echo "building docker image for non root user build" docker build --build-arg BASE_IMAGE="$1" -t docker_non_root_image -f gha-script/dockerfile_non_root . docker_image="docker_non_root_image" } -#Below conditions are used to select the base image based on the 2 flags, tested_on and non_root_build. -#A docker_build_non_root function is called when non root build is true. -if [[ "$TESTED_ON" == UBI:9* || "$TESTED_ON" == UBI9* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - docker pull registry.access.redhat.com/ubi9/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi9/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi9/ubi:$ubi_version" - fi -elif [[ "$TESTED_ON" == UBI:10* || "$TESTED_ON" == UBI10* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - # Fallback to 10.0 if no minor version found - ubi_version=${ubi_version:-10.0} - docker pull registry.access.redhat.com/ubi10/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi10/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi10/ubi:$ubi_version" - fi -else - docker pull registry.access.redhat.com/ubi8/ubi:8.7 - docker_image="registry.access.redhat.com/ubi8/ubi:8.7" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi8/ubi:8.7" - fi -fi +# Resolves the correct UBI registry image from the TESTED_ON string and +# sets the global docker_image variable. Supports UBI8, UBI9, UBI10+. +# The TESTED_ON value is normalised before parsing so all of the following +# formats work: "UBI:9.3" "UBI 9.3" "ubi9.3" "UBI: 9.3" "UBI 10" +resolve_docker_image() { + local raw="$1" + local upper + upper=$(echo "$raw" | tr '[:lower:]' '[:upper:]') + + # Collapse "UBI : 9.3" / "UBI:9.3" / "UBI 9.3" / "UBI9.3" -> "UBI9.3" + # Uses sed only - no grep -P (absent on ppc64le runners). + local norm + norm=$(echo "$upper" | sed 's/UBI[[:space:]]*[: ][[:space:]]*/UBI/g') + + # Extract the major UBI version integer immediately after "UBI" + local major + major=$(echo "$norm" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + case "$major" in + ''|*[!0-9]*) major="" ;; + esac + + if [ -z "$major" ]; then + echo "ERROR: Cannot determine UBI major version from TESTED_ON='$raw'" + exit 1 + fi + + # Extract the full version string e.g. "9.3", "10.3" (first X.Y found) + local full + full=$(echo "$upper" | grep -oE '[0-9]+\.[0-9]+' | head -1) + + if [ -z "$full" ]; then + # No minor version supplied - use per-major defaults + case "$major" in + 8) full="8.7" ;; + 9) full="9.3" ;; + 10) full="10.0" ;; + *) full="${major}.0" ;; + esac + echo "WARNING: No minor version in TESTED_ON='$raw', defaulting to UBI ${full}" + fi + + # Select the registry image by major version + case "$major" in + 8) + docker_image="registry.access.redhat.com/ubi8/ubi:${full}" + ;; + 9) + docker_image="registry.access.redhat.com/ubi9/ubi:${full}" + ;; + 10) + docker_image="registry.access.redhat.com/ubi10/ubi:${full}" + ;; + *) + echo "ERROR: Unsupported UBI major version '${major}' in TESTED_ON='$raw'" + exit 1 + ;; + esac + + echo " Resolved Docker image: $docker_image (TESTED_ON='$raw')" + + if [[ "$NON_ROOT_BUILD" == "true" ]]; then + docker_build_non_root "$docker_image" + fi +} + +resolve_docker_image "$TESTED_ON" +docker pull "$docker_image" python3 gha-script/validate_builds_currency.py "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$docker_image" 2>&1 | tee build_log From a7408c67c41d9dc6d40c3dd18b01cd63c8d80d43 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 17:03:15 +0530 Subject: [PATCH 11/45] Change GITHUB_BUILD_SCRIPT_BASE_OWNER to stutiibm --- gha-script/validate_builds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index 95f00c4b8d..0f830d7e8e 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -11,7 +11,7 @@ GITHUB_BUILD_SCRIPT_BASE_REPO = "build-scripts" -GITHUB_BUILD_SCRIPT_BASE_OWNER = "ppc64le" +GITHUB_BUILD_SCRIPT_BASE_OWNER = "stutiibm" HOME = os.getcwd() package_data = {} From 678bd971ee83663d0c89fe712e4fbbf56e0878a0 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 17:45:43 +0530 Subject: [PATCH 12/45] Enhance version selection for changed scripts Added logic to determine version based on changed .sh files. --- .github/workflows/pr-build.yaml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 33ff432826..7a0f238513 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -180,6 +180,37 @@ jobs: PACKAGE_NAME=$(jq -r '.package_name // ""' $BUILD_INFO_FILE) VERSION=$(jq -r '.version // ""' $BUILD_INFO_FILE) + # If any changed .sh file belongs to a non-default version block, use + # that block's version so read_buildinfo.sh loads the correct build_script + # array (e.g. PR adds ubi8 + ubi10 scripts for v6.4.0 while the top-level + # .version field still points to v6.0.2). + CHANGED_SH=$(echo "$CHANGED_FILES" | grep '\.sh$' || true) + if [ -n "$CHANGED_SH" ]; then + # Iterate every string key in build_info.json that is not a known scalar + # field name, then check if its build_script list contains a changed file. + SCALAR_KEYS='package_name|version|package_dir|wheel_build|docker_build|validate_build_script|base_docker_image|base_docker_variant|use_non_root_user|auditwheel_exclude' + VERSION_KEYS=$(jq -r 'keys[] | select(test("^('"$SCALAR_KEYS"')$") | not)' "$BUILD_INFO_FILE" 2>/dev/null || true) + for ver_key in $VERSION_KEYS; do + # Get the build_script value for this version block (string or array) + bs=$(jq -c --arg k "$ver_key" '.[$k].build_script // empty' "$BUILD_INFO_FILE" 2>/dev/null || true) + [ -z "$bs" ] && continue + # Normalise to a newline-separated list of bare script names + if echo "$bs" | jq -e 'type == "array"' >/dev/null 2>&1; then + scripts=$(echo "$bs" | jq -r '.[]') + else + scripts=$(echo "$bs" | tr -d '"') + fi + # Check if any changed .sh matches any script in this version block + for s in $scripts; do + if echo "$CHANGED_SH" | grep -qF "$s"; then + echo "Changed script '$s' belongs to version block '$ver_key' - using that version instead of default '$VERSION'" + VERSION="$ver_key" + break 2 + fi + done + done + fi + echo "BUILD_INFO_FILE=$BUILD_INFO_FILE" >> $GITHUB_ENV echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV echo "VERSION=$VERSION" >> $GITHUB_ENV From db4d89d4f034b482ffdbd943c322aab66a0ac691 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 18:14:11 +0530 Subject: [PATCH 13/45] Update pr-build.yaml --- .github/workflows/pr-build.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 7a0f238513..f06e50f55a 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -379,6 +379,10 @@ jobs: source package-cache/scanner-env.sh sudo chown -R $USER:$USER . echo "--- Executing changed scripts for UBI8 ---" + # Filter CHANGED_FILES to only the script assigned to this UBI job so that + # execute_changed_scripts.py does not also run the UBI10 (or other) scripts + # that are present in the same PR. + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$BUILD_SCRIPT" || true) chmod +x ./gha-script/execute_changed_scripts.py python3 -u ./gha-script/execute_changed_scripts.py @@ -405,6 +409,7 @@ jobs: source package-cache/scanner-env.sh sudo chown -R $USER:$USER . echo "--- Executing changed scripts for UBI9 ---" + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$BUILD_SCRIPT" || true) chmod +x ./gha-script/execute_changed_scripts.py python3 -u ./gha-script/execute_changed_scripts.py @@ -431,6 +436,7 @@ jobs: source package-cache/scanner-env.sh sudo chown -R $USER:$USER . echo "--- Executing changed scripts for UBI10 ---" + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$BUILD_SCRIPT" || true) chmod +x ./gha-script/execute_changed_scripts.py python3 -u ./gha-script/execute_changed_scripts.py From 897f981c8a79a6a2b75a63779c1146cd0b9e19e8 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 18:48:40 +0530 Subject: [PATCH 14/45] Update pr-build.yaml --- .github/workflows/pr-build.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index f06e50f55a..3f951aea1b 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -329,14 +329,16 @@ jobs: DOCKER_BUILD="false" fi - cat < package-cache/scanner-env.sh - export PACKAGE_NAME=$PACKAGE_NAME - export VERSION=$VERSION - export PACKAGE_DIR=$PACKAGE_DIR - export WHEEL_BUILD=$WHEEL_BUILD - export BUILD_DOCKER=$DOCKER_BUILD - export CHANGED_FILES="$CHANGED_FILES" - EOF + # Write scanner-env.sh without leading indentation spaces. + # CHANGED_FILES is written via a separate heredoc so its newlines are preserved. + { + echo "export PACKAGE_NAME=$PACKAGE_NAME" + echo "export VERSION=$VERSION" + echo "export PACKAGE_DIR=$PACKAGE_DIR" + echo "export WHEEL_BUILD=$WHEEL_BUILD" + echo "export BUILD_DOCKER=$DOCKER_BUILD" + printf 'export CHANGED_FILES=%q\n' "$CHANGED_FILES" + } > package-cache/scanner-env.sh mv variable.sh package-cache/ From bee6e2e5d48b9d95deb88e56e9f6e82e5e109049 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 20:13:09 +0530 Subject: [PATCH 15/45] Refactor UBI build scripts to use THIS_SCRIPT variable --- .github/workflows/pr-build.yaml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 3f951aea1b..4bcaf564c8 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -377,14 +377,16 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Build Package (UBI8) run: | + # Capture the GHA job-level BUILD_SCRIPT before sourcing variable.sh, + # which overwrites BUILD_SCRIPT with the first/backward-compat script. + THIS_SCRIPT="$BUILD_SCRIPT" source package-cache/variable.sh source package-cache/scanner-env.sh sudo chown -R $USER:$USER . - echo "--- Executing changed scripts for UBI8 ---" - # Filter CHANGED_FILES to only the script assigned to this UBI job so that - # execute_changed_scripts.py does not also run the UBI10 (or other) scripts - # that are present in the same PR. - export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$BUILD_SCRIPT" || true) + echo "--- Executing changed scripts for UBI8 (script: $THIS_SCRIPT) ---" + # Filter CHANGED_FILES to only the script assigned to this UBI job. + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$THIS_SCRIPT" || true) + export BUILD_SCRIPT="$THIS_SCRIPT" chmod +x ./gha-script/execute_changed_scripts.py python3 -u ./gha-script/execute_changed_scripts.py @@ -407,11 +409,13 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Build Package (UBI9) run: | + THIS_SCRIPT="$BUILD_SCRIPT" source package-cache/variable.sh source package-cache/scanner-env.sh sudo chown -R $USER:$USER . - echo "--- Executing changed scripts for UBI9 ---" - export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$BUILD_SCRIPT" || true) + echo "--- Executing changed scripts for UBI9 (script: $THIS_SCRIPT) ---" + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$THIS_SCRIPT" || true) + export BUILD_SCRIPT="$THIS_SCRIPT" chmod +x ./gha-script/execute_changed_scripts.py python3 -u ./gha-script/execute_changed_scripts.py @@ -434,11 +438,13 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Build Package (UBI10) run: | + THIS_SCRIPT="$BUILD_SCRIPT" source package-cache/variable.sh source package-cache/scanner-env.sh sudo chown -R $USER:$USER . - echo "--- Executing changed scripts for UBI10 ---" - export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$BUILD_SCRIPT" || true) + echo "--- Executing changed scripts for UBI10 (script: $THIS_SCRIPT) ---" + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$THIS_SCRIPT" || true) + export BUILD_SCRIPT="$THIS_SCRIPT" chmod +x ./gha-script/execute_changed_scripts.py python3 -u ./gha-script/execute_changed_scripts.py From 9eb5e01076fae85fb7980373aee79192fe722cd2 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 23:32:38 +0530 Subject: [PATCH 16/45] Delete m/multidict/multidict_ubi_9.3.sh --- m/multidict/multidict_ubi_9.3.sh | 57 -------------------------------- 1 file changed, 57 deletions(-) delete mode 100644 m/multidict/multidict_ubi_9.3.sh diff --git a/m/multidict/multidict_ubi_9.3.sh b/m/multidict/multidict_ubi_9.3.sh deleted file mode 100644 index b920b7edfb..0000000000 --- a/m/multidict/multidict_ubi_9.3.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -e -# ----------------------------------------------------------------------------- -# -# Package : multidict -# Version : 6.0.2 -# Source repo : https://github.com/aio-libs/multidict.git -# Tested on : UBI:9.3 -# Language : Python -# Ci-Check : True -# Script License : Apache License, Version 2 or later -# Maintainer : Aastha Sharma -# -# Disclaimer : This script has been tested in root mode on given -# ========== platform using the mentioned version of the package. -# It may not work as expected with newer versions of the -# package and/or distribution. In such case, please -# contact "Maintainer" of this script. -# -# --------------------------------------------------------------------------- - -# Variables -PACKAGE_NAME=multidict -PACKAGE_VERSION=${1:-v6.0.2} -PACKAGE_URL=https://github.com/aio-libs/multidict.git - -# Install dependencies -yum install -y git gcc gcc-c++ make wget openssl-devel bzip2-devel libffi-devel zlib-devel python-devel python-pip cmake - -# Clone the repository -git clone $PACKAGE_URL -cd $PACKAGE_NAME # Change directory to the cloned repository -git checkout $PACKAGE_VERSION # Checkout the specified version - -# install necessary Python packages -pip install --upgrade pip setuptools wheel -pip install "coverage==7.5.4" "pytest-cov==5.0.0" objgraph psutil pytest-codspeed - -#install -if ! (python3 setup.py install) ; then - echo "------------------$PACKAGE_NAME:Install_fails-------------------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_Fails" - exit 1 -fi - -# Run tests -if ! pytest --deselect=tests/test_mutable_multidict.py::TestCIMutableMultiDict::test_add --ignore=tests/test_circular_imports.py; then - echo "------------------$PACKAGE_NAME:Install_success_but_test_fails---------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_success_but_test_Fails" - exit 2 -else - echo "------------------$PACKAGE_NAME:Install_&_test_both_success-------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Pass | Both_Install_and_Test_Success" - exit 0 -fi From f9c2064a3e81a889bd4a6172901f1d003c6f9dce Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 23:33:36 +0530 Subject: [PATCH 17/45] Delete s/scikit-learn/scikit-learn_1.9.0_ubi_10.1.sh --- s/scikit-learn/scikit-learn_1.9.0_ubi_10.1.sh | 137 ------------------ 1 file changed, 137 deletions(-) delete mode 100755 s/scikit-learn/scikit-learn_1.9.0_ubi_10.1.sh diff --git a/s/scikit-learn/scikit-learn_1.9.0_ubi_10.1.sh b/s/scikit-learn/scikit-learn_1.9.0_ubi_10.1.sh deleted file mode 100755 index 179eac71a0..0000000000 --- a/s/scikit-learn/scikit-learn_1.9.0_ubi_10.1.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -e -# ----------------------------------------------------------------------------- -# -# Package : scikit-learn -# Version : 1.9.0 -# Source repo : https://github.com/scikit-learn/scikit-learn.git -# Tested on : UBI 10.1 -# Language : Python, Cython, C++ -# Ci-Check : True -# Script License: Apache License 2.0 -# Maintainer : Varsha Kumar -# -# Disclaimer: This script has been tested in root mode on given -# ========== platform using the mentioned version of the package. -# It may not work as expected with newer versions of the -# package and/or distribution. In such case, please -# contact "Maintainer" of this script. -# -# ----------------------------------------------------------------------------- - -# Variables -PACKAGE_DIR="scikit-learn" -PACKAGE_NAME="scikit_learn" -PACKAGE_VERSION=${1:-1.9.0} -PACKAGE_URL="https://github.com/scikit-learn/scikit-learn.git" -SOURCE_ROOT="$(pwd)" - -echo "Building ${PACKAGE_NAME} ${PACKAGE_VERSION}" - -# Install system dependencies -dnf install -y gcc-toolset-15 gcc-toolset-15-gcc-c++ gcc-toolset-15-gcc-gfortran \ - git python3.12 python3.12-devel python3.12-pip pkg-config \ - openblas-devel - -export PATH="/opt/rh/gcc-toolset-15/root/usr/bin:$PATH" - -# Verify GCC version (scikit-learn requires >= 8.0) -gcc --version - -# Install build dependencies -# pythran is required by scipy>=1.17 at build time -python3.12 -m pip install \ - "meson-python>=0.17.1,<0.20.0" \ - "meson>=1.9.0" \ - "ninja" \ - "cython>=3.1.2,<3.3.0" \ - "numpy>=2,<2.5.0" \ - "pythran" \ - "pybind11>=2.13.2" \ - "wheel" - -# Build and install scipy separately so the pythran dep is satisfied -python3.12 -m pip install --no-build-isolation "scipy>=1.10.0,<1.18.0" - -# Clone and checkout -rm -rf "$PACKAGE_DIR" -git clone "$PACKAGE_URL" -cd "${PACKAGE_DIR}" -git checkout "${PACKAGE_VERSION}" - -# Build wheel using pip with no-build-isolation -# (meson-python builds require access to already-installed build deps) -python3.12 -m pip wheel . \ - --no-build-isolation \ - --wheel-dir "${SOURCE_ROOT}/dist/" - -WHEEL=$(find "${SOURCE_ROOT}/dist" -name "${PACKAGE_NAME}-*.whl" | head -1) -if [ -z "$WHEEL" ]; then - echo "ERROR: wheel not found after build" - exit 1 -fi -echo "Wheel: $WHEEL" - -# Copy wheel to /home/tester/ to avoid rebuild by wrapper script -mkdir -p /home/tester -cp "$WHEEL" /home/tester/ - -cd "${SOURCE_ROOT}" - -# Install runtime dependencies and wheel -echo "=== Installing Wheel ===" -python3.12 -m pip install \ - "joblib>=1.4.0" \ - "narwhals>=2.0.1" \ - "threadpoolctl>=3.5.0" -python3.12 -m pip install "$WHEEL" - -# Test -echo "=== Running Tests ===" - -# 1. Version check -python3.12 -c "import importlib.metadata; print('version:', importlib.metadata.version('scikit-learn'))" - -# 2. Basic smoke test -python3.12 - <<'EOF' -import sklearn -print("sklearn version:", sklearn.__version__) -assert sklearn.__version__ == "1.9.0", f"Unexpected version: {sklearn.__version__}" - -from sklearn.datasets import load_iris -from sklearn.ensemble import RandomForestClassifier -from sklearn.model_selection import train_test_split -from sklearn.metrics import accuracy_score - -X, y = load_iris(return_X_y=True) -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) - -clf = RandomForestClassifier(n_estimators=10, random_state=42) -clf.fit(X_train, y_train) -preds = clf.predict(X_test) -acc = accuracy_score(y_test, preds) -assert acc > 0.9, f"Accuracy too low: {acc}" -print(f"RandomForest smoke test: OK (accuracy={acc:.2f})") - -from sklearn.linear_model import LogisticRegression -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import StandardScaler - -pipe = Pipeline([("scaler", StandardScaler()), ("lr", LogisticRegression(max_iter=200))]) -pipe.fit(X_train, y_train) -pipe_acc = accuracy_score(y_test, pipe.predict(X_test)) -assert pipe_acc > 0.9, f"Pipeline accuracy too low: {pipe_acc}" -print(f"Pipeline smoke test: OK (accuracy={pipe_acc:.2f})") -EOF - -# 3. Run upstream test suite (core module only to keep CI time reasonable) -echo "=== Running Upstream Tests ===" -python3.12 -m pip install pytest - -# Resolve the installed sklearn path so pytest uses the compiled wheel, -# not the unbuilt source clone (which lacks the _check_build compiled extension). -SKLEARN_PATH=$(python3.12 -c "import sklearn; import os; print(os.path.dirname(sklearn.__file__))") -python3.12 -m pytest "${SKLEARN_PATH}/tests/test_common.py" -x -q --no-header - -echo -e "\n=== Build Complete ===" -echo "Wheel: $WHEEL" - From 0b9f3d4c866226550fbe272aa0db0a9c800316bd Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 23:33:50 +0530 Subject: [PATCH 18/45] Delete s/scikit-learn/scikit-learn_ubi_8.10.sh --- s/scikit-learn/scikit-learn_ubi_8.10.sh | 87 ------------------------- 1 file changed, 87 deletions(-) delete mode 100644 s/scikit-learn/scikit-learn_ubi_8.10.sh diff --git a/s/scikit-learn/scikit-learn_ubi_8.10.sh b/s/scikit-learn/scikit-learn_ubi_8.10.sh deleted file mode 100644 index 876a73d6df..0000000000 --- a/s/scikit-learn/scikit-learn_ubi_8.10.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -e -# ----------------------------------------------------------------------------- -# -# Package : scikit-learn -# Version : 1.5.0 -# Source repo : https://github.com/scikit-learn/scikit-learn.git -# Tested on : UBI 8.10 -# Language : Python, Cython, C++ -# Ci-Check : False -# Script License: Apache License 2.0 -# Maintainer : Salil Verlekar -# -# Disclaimer: This script has been tested in root mode on given -# ========== platform using the mentioned version of the package. -# It may not work as expected with newer versions of the -# package and/or distribution. In such case, please -# contact "Maintainer" of this script. -# -# ---------------------------------------------------------------------------- - -PACKAGE_NAME=scikit-learn -PACKAGE_VERSION=${1:-1.5.0} -PACKAGE_URL=https://github.com/scikit-learn/scikit-learn.git - -yum install -y python3.11 python3.11-pip python3.11-devel gcc gcc-c++ gcc-gfortran gcc-toolset-10 git -yum install -y openblas-devel --enablerepo=codeready-builder-for-rhel-8-ppc64le-rpms - -source /opt/rh/gcc-toolset-10/enable - -OS_NAME=$(cat /etc/os-release | grep ^PRETTY_NAME | cut -d= -f2) - -# clone source repository -git clone $PACKAGE_URL -cd $PACKAGE_NAME -git checkout $PACKAGE_VERSION -git submodule update --init - -# install scikit-learn dependencies and build dependencies -python3.11 -m pip install wheel numpy scipy cython meson-python ninja 'pytest==8.2.2' 'patchelf>=0.11.0' build - -export SKLEARN_SKIP_OPENMP_TEST=1 - -ln -s /usr/bin/python3.11 python -mv python /usr/bin - -# build wheel in /scikit-learn/dist -if ! python3.11 -m build --wheel --no-isolation; then - echo "------------------$PACKAGE_NAME:build_fails---------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_VERSION | $OS_NAME | GitHub | Fail | Build_Fails" - exit 1 -else - echo "------------------$PACKAGE_NAME:build_success-------------------------" - echo "$PACKAGE_VERSION $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_VERSION | $OS_NAME | GitHub | Pass | Build_Success" -fi - -# install wheel, run show and check that the installed scikit-learn has a version number -python3.11 -m pip install dist/scikit_learn-1.5.0-cp311-cp311-linux_ppc64le.whl -python3.11 -m pip show scikit-learn -cd sklearn -python3.11 -c "import sklearn; sklearn.show_versions()" - -if [ $? == 0 ]; then - echo "------------------$PACKAGE_NAME::Install_Success---------------------" - echo "$PACKAGE_VERSION $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | Pass | Install_Success" -else - echo "------------------$PACKAGE_NAME::Install_Fail-------------------------" - echo "$PACKAGE_VERSION $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | Fail | Install_Fail" - exit 2 -fi - -# test using pytest - set below flag as suggested in GitHub forums to resolve ImportPathMismatchError -export PY_IGNORE_IMPORTMISMATCH=1 -if ! pytest tests/test_random_projection.py ; then - echo "------------------$PACKAGE_NAME:test_fails---------------------" - echo "$PACKAGE_URL $PACKAGE_NAME " - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | Github | Fail | Test_Fails" - exit 2 -else - echo "------------------$PACKAGE_NAME:test_success-------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME " - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | Github | Pass | Test_Success" - exit 0 -fi From 483a805be5474bcc14c21f4449d7cd76f08a0173 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Mon, 3 Aug 2026 23:34:19 +0530 Subject: [PATCH 19/45] Delete s/scikit-learn/scikit-learn_1.7.0_ubi_9.3.sh --- s/scikit-learn/scikit-learn_1.7.0_ubi_9.3.sh | 113 ------------------- 1 file changed, 113 deletions(-) delete mode 100644 s/scikit-learn/scikit-learn_1.7.0_ubi_9.3.sh diff --git a/s/scikit-learn/scikit-learn_1.7.0_ubi_9.3.sh b/s/scikit-learn/scikit-learn_1.7.0_ubi_9.3.sh deleted file mode 100644 index a4331d26ae..0000000000 --- a/s/scikit-learn/scikit-learn_1.7.0_ubi_9.3.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash -e -# ----------------------------------------------------------------------------- -# -# Package : scikit-learn -# Version : 1.7.0 -# Source repo : https://github.com/scikit-learn/scikit-learn.git -# Tested on : UBI 9.3 -# Language : Python, Cython, C++ -# Ci-Check : True -# Script License: Apache License 2.0 -# Maintainer : Manya Rusiya -# -# Disclaimer: This script has been tested in root mode on given -# ========== platform using the mentioned version of the package. -# It may not work as expected with newer versions of the -# package and/or distribution. In such case, please -# contact "Maintainer" of this script. -# -# ----------------------------------------------------------------------------- - -PACKAGE_NAME=scikit-learn -PACKAGE_VERSION=${1:-1.7.0} -PACKAGE_URL=https://github.com/scikit-learn/scikit-learn.git -PACKAGE_DIR=scikit-learn - - -yum install -y \ - git gcc gcc-c++ make libtool cmake clang \ - openssl-devel bzip2-devel libffi-devel xz zlib-devel wget \ - python3.11 python3.11-devel python3.11-pip \ - gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc-gfortran \ - libevent-devel openblas-devel - - -# Setup GCC toolset -export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH - -# Create Python 3.11 venv -python3.11 -m venv venv311 -# shellcheck disable=SC1091 -source venv311/bin/activate - -# Upgrade pip & tools -pip install --upgrade pip setuptools wheel - -# Clone OpenBLAS & build -git clone https://github.com/OpenMathLib/OpenBLAS -cd OpenBLAS -git checkout v0.3.29 -git submodule update --init -wget -q https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/python-ecosystem/o/openblas/pyproject.toml -sed -i "s/{PACKAGE_VERSION}/v0.3.29/g" pyproject.toml - -PREFIX=local/openblas -mkdir -p $PREFIX - -export CF="-Wno-unused-parameter -Wno-old-style-declaration" -export USE_OPENMP=1 -declare -a build_opts -build_opts+=(USE_OPENMP=${USE_OPENMP}) -build_opts+=(BINARY=64 DYNAMIC_ARCH=1 TARGET="POWER9" INTERFACE64=0) -build_opts+=(NO_LAPACK=0 USE_THREAD=1 NUM_THREADS=8 NO_AFFINITY=1) - -make -j$(nproc) ${build_opts[@]} CFLAGS="${CF}" prefix=$PREFIX -make install PREFIX="${PREFIX}" ${build_opts[@]} - -OpenBLASInstallPATH=$(pwd)/$PREFIX -export LD_LIBRARY_PATH="${OpenBLASInstallPATH}/lib:$LD_LIBRARY_PATH" -export PKG_CONFIG_PATH="${OpenBLASInstallPATH}/lib/pkgconfig:$PKG_CONFIG_PATH" - -OpenBLASConfigFile=$(find . -name OpenBLASConfig.cmake | head -n1) -OpenBLASPCFile=$(find . -name openblas.pc | head -n1) - -[ -n "$OpenBLASConfigFile" ] && sed -i "/OpenBLAS_INCLUDE_DIRS/c\SET(OpenBLAS_INCLUDE_DIRS ${OpenBLASInstallPATH}/include)" $OpenBLASConfigFile -[ -n "$OpenBLASConfigFile" ] && sed -i "/OpenBLAS_LIBRARIES/c\SET(OpenBLAS_INCLUDE_DIRS ${OpenBLASInstallPATH}/include)" $OpenBLASConfigFile -[ -n "$OpenBLASPCFile" ] && sed -i "s|libdir=local/openblas/lib|libdir=${OpenBLASInstallPATH}/lib|" $OpenBLASPCFile -[ -n "$OpenBLASPCFile" ] && sed -i "s|includedir=local/openblas/include|includedir=${OpenBLASInstallPATH}/include|" $OpenBLASPCFile - -cd .. - -# clone source repository -git clone $PACKAGE_URL -cd $PACKAGE_NAME -git checkout $PACKAGE_VERSION -git submodule update --init - -# Install Python dependencies -pip install numpy==2.0.2 scipy cython meson-python ninja joblib threadpoolctl patchelf pytest - - -# Install -if ! pip install --editable . --no-build-isolation ; then - echo "------------------$PACKAGE_NAME:Install_fails-------------------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_Fails" - exit 1 -fi - -# test using pytest - set below flag as suggested in GitHub forums to resolve ImportPathMismatchError - -export PY_IGNORE_IMPORTMISMATCH=1 -if ! pytest sklearn/tests/test_random_projection.py; then - echo "--------------------$PACKAGE_NAME:Install_success_but_test_fails---------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_success_but_test_Fails" - exit 2 -else - echo "------------------$PACKAGE_NAME:Install_&_test_both_success-------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Pass | Both_Install_and_Test_Success" - exit 0 -fi From d6a03b7b66f8a430a3df80fcc57dc31b5eb1e635 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 00:39:50 +0530 Subject: [PATCH 20/45] Update pr-build.yaml --- .github/workflows/pr-build.yaml | 73 +++++++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 4bcaf564c8..c70ac30fc7 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -180,35 +180,43 @@ jobs: PACKAGE_NAME=$(jq -r '.package_name // ""' $BUILD_INFO_FILE) VERSION=$(jq -r '.version // ""' $BUILD_INFO_FILE) - # If any changed .sh file belongs to a non-default version block, use - # that block's version so read_buildinfo.sh loads the correct build_script - # array (e.g. PR adds ubi8 + ubi10 scripts for v6.4.0 while the top-level - # .version field still points to v6.0.2). + # Pick the version block whose build_script list has the MOST overlap + # with the changed .sh files. This correctly handles the case where a + # script also appears in older single-script blocks (e.g. scikit-learn_ubi_8.10.sh + # is in both 1.5.0 and 1.9.0) - the new block wins because it matches more + # changed files. The wildcard "*" key is always skipped - it is a catch-all + # fallback, not a real version. + # NOTE: if no .sh files changed (build_info.json only), this block is skipped + # entirely and VERSION stays as the default from the .version field. CHANGED_SH=$(echo "$CHANGED_FILES" | grep '\.sh$' || true) if [ -n "$CHANGED_SH" ]; then - # Iterate every string key in build_info.json that is not a known scalar - # field name, then check if its build_script list contains a changed file. SCALAR_KEYS='package_name|version|package_dir|wheel_build|docker_build|validate_build_script|base_docker_image|base_docker_variant|use_non_root_user|auditwheel_exclude' - VERSION_KEYS=$(jq -r 'keys[] | select(test("^('"$SCALAR_KEYS"')$") | not)' "$BUILD_INFO_FILE" 2>/dev/null || true) + VERSION_KEYS=$(jq -r 'keys[] | select(test("^('"$SCALAR_KEYS"')$") | not) | select(. != "*")' "$BUILD_INFO_FILE" 2>/dev/null || true) + best_ver="" + best_count=0 for ver_key in $VERSION_KEYS; do - # Get the build_script value for this version block (string or array) bs=$(jq -c --arg k "$ver_key" '.[$k].build_script // empty' "$BUILD_INFO_FILE" 2>/dev/null || true) [ -z "$bs" ] && continue - # Normalise to a newline-separated list of bare script names if echo "$bs" | jq -e 'type == "array"' >/dev/null 2>&1; then scripts=$(echo "$bs" | jq -r '.[]') else scripts=$(echo "$bs" | tr -d '"') fi - # Check if any changed .sh matches any script in this version block + match_count=0 for s in $scripts; do if echo "$CHANGED_SH" | grep -qF "$s"; then - echo "Changed script '$s' belongs to version block '$ver_key' - using that version instead of default '$VERSION'" - VERSION="$ver_key" - break 2 + match_count=$((match_count + 1)) fi done + if [ "$match_count" -gt "$best_count" ]; then + best_count=$match_count + best_ver="$ver_key" + fi done + if [ -n "$best_ver" ] && [ "$best_count" -gt 0 ]; then + echo "Version block '$best_ver' matched $best_count changed script(s) - using that version instead of default '$VERSION'" + VERSION="$best_ver" + fi fi echo "BUILD_INFO_FILE=$BUILD_INFO_FILE" >> $GITHUB_ENV @@ -228,6 +236,45 @@ jobs: id: emit run: | source variable.sh + + # Safety net: for every changed .sh file, if its UBI major slot is still + # empty (read_buildinfo.sh loaded the wrong or incomplete version block), + # read the script's own "# Tested on" header and fill the slot directly. + # This guarantees every changed script in the PR is tested regardless of + # how build_info.json version blocks are structured or overlap. + # If no .sh files changed (build_info.json only), this loop does nothing. + while IFS= read -r changed_script; do + [[ "$changed_script" != *.sh ]] && continue + [ ! -f "$changed_script" ] && continue + raw_tested_on=$(grep "^# Tested on" "$changed_script" | head -1 \ + | cut -d ':' -f2- \ + | sed 's/^[[:space:]]*//' \ + | tr '[:lower:]' '[:upper:]' \ + | sed 's/[[:space:]]*:[[:space:]]*/:/g; s/UBI[[:space:]]*/UBI/g') + [ -z "$raw_tested_on" ] && continue + major=$(echo "$raw_tested_on" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + case "$major" in ''|*[!0-9]*) continue ;; esac + script_name=$(basename "$changed_script") + entry=$(jq -n --arg s "$script_name" --arg t "$raw_tested_on" '{"script":$s,"tested_on":$t}') + case "$major" in + 8) + if [ -z "$SCRIPT_UBI8" ]; then + echo "Filling SCRIPT_UBI8 from changed script: $script_name" + SCRIPT_UBI8="$entry" + fi ;; + 9) + if [ -z "$SCRIPT_UBI9" ]; then + echo "Filling SCRIPT_UBI9 from changed script: $script_name" + SCRIPT_UBI9="$entry" + fi ;; + 10) + if [ -z "$SCRIPT_UBI10" ]; then + echo "Filling SCRIPT_UBI10 from changed script: $script_name" + SCRIPT_UBI10="$entry" + fi ;; + esac + done <<< "$CHANGED_FILES" + echo "SCRIPT_UBI8=$SCRIPT_UBI8" echo "SCRIPT_UBI9=$SCRIPT_UBI9" echo "SCRIPT_UBI10=$SCRIPT_UBI10" From 994b1bad84f2027840d9dd3626d35faab10c3313 Mon Sep 17 00:00:00 2001 From: stutiibm Date: Tue, 4 Aug 2026 01:33:20 +0530 Subject: [PATCH 21/45] Fix wheel jobs: capture BUILD_SCRIPT and TESTED_ON before source variable.sh variable.sh always exports BUILD_SCRIPT and TESTED_ON for the first/backward- compat script (UBI8), clobbering the correct per-job values set in the GHA env: block before build_wheels.sh runs. This caused all wheel jobs (ubi9_*, ubi10_*) to pull the UBI8 image and run the UBI8 build script. Fix: in every wheel Run build_wheels.sh step, capture THIS_SCRIPT and THIS_TESTED_ON from the GHA env before sourcing variable.sh, then re-export them as BUILD_SCRIPT and TESTED_ON after the source calls. Applied to all 11 wheel_build_ubiN_pyXXX jobs. --- .github/workflows/pr-build.yaml | 84 +++++++++++++++++---------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index c70ac30fc7..e64c7b005d 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -565,7 +565,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -604,7 +608,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -644,47 +652,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh - if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 - sudo chown -R $USER:$USER . - export ENABLE_CVE_SCAN=false - chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh - - name: Verify wheel was produced - run: | - if ls *.whl 1>/dev/null 2>&1; then - echo "Wheel built successfully: $(ls *.whl)" - else - echo "No wheel produced." - exit 1 - fi - - wheel_build_ubi8_py314: - needs: build_info - if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} - continue-on-error: true - env: - PYTHON_VERSION: "3.14" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -725,7 +697,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -764,7 +740,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -803,7 +783,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -843,7 +827,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -883,7 +871,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -924,7 +916,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -964,7 +960,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 @@ -1004,7 +1004,11 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 From e24facdd869bd10ecd8cab38e45f7d3daf191c3e Mon Sep 17 00:00:00 2001 From: stutiibm Date: Tue, 4 Aug 2026 01:42:38 +0530 Subject: [PATCH 22/45] Fix all 12 wheel jobs: capture BUILD_SCRIPT+TESTED_ON before source variable.sh All 12 wheel_build_ubiN_pyXXX jobs (ubi8: py310-313, ubi9: py310-314, ubi10: py312-314) now capture the GHA job-level BUILD_SCRIPT and TESTED_ON before sourcing variable.sh (which overwrites them with the UBI8 backward- compat values), then restore them after. This ensures each wheel job uses the correct UBI image and build script for its assigned UBI version. --- .github/workflows/pr-build.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index e64c7b005d..c329da0707 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -524,8 +524,12 @@ jobs: run: tar -xzf package-cache.tar.gz - name: Run build_wheels.sh run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) if [ -z "$BUILD_SCRIPT_CHANGED" ]; then echo "Skipping - no .sh changes"; exit 0; fi From 1444529f8b67626566dbf821c8a2da0f447a9ecf Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 01:51:22 +0530 Subject: [PATCH 23/45] Update pr-build.yaml From 52d55ecf815465b78207fbc8896098b50286dc8e Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:16:55 +0530 Subject: [PATCH 24/45] Update post_process_wheel.py --- gha-script/post_process_wheel.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gha-script/post_process_wheel.py b/gha-script/post_process_wheel.py index ff6c31dbf0..fc7bf1aa56 100644 --- a/gha-script/post_process_wheel.py +++ b/gha-script/post_process_wheel.py @@ -43,9 +43,15 @@ logger = logging.getLogger(__name__) -# COS configuration -COS_API_KEY = os.environ["GHA_CURRENCY_SERVICE_ID_API_KEY"] -COS_SERVICE_INSTANCE_ID = os.environ["GHA_CURRENCY_SERVICE_ID"] +# COS configuration +# These are only available in currency builds (not PR builds). +# If missing, post-processing is skipped gracefully. +COS_API_KEY = os.environ.get("GHA_CURRENCY_SERVICE_ID_API_KEY", "") +COS_SERVICE_INSTANCE_ID = os.environ.get("GHA_CURRENCY_SERVICE_ID", "") + +if not COS_API_KEY or not COS_SERVICE_INSTANCE_ID: + logger.info("COS credentials not set - skipping post-processing (PR build environment)") + sys.exit(0) COS_ENDPOINT = "https://s3.us.cloud-object-storage.appdomain.cloud" COS_BUCKET = "ose-power-artifacts-production" From 47f4bc22e934227d7cb3b9bf769d0ebaf760eecb Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:47:21 +0530 Subject: [PATCH 25/45] Add UBI10 build and wheel jobs, fix multi-UBI routing in PR workflow - Add script_ubi8, script_ubi9, script_ubi10 as named build_info job outputs, each a JSON object {script, tested_on} or empty string - Add Emit per-UBI outputs step that reads SCRIPT_UBI8/9/10 from variable.sh and emits them as workflow outputs - Add build_ubi8 and build_ubi10 jobs that run only when their respective scripts exist; build_ubi9 runs as before - Add version-block best-match detection: count how many changed scripts each build_info.json version block contains and pick the highest-count block, preventing shared scripts from routing to the wrong alphabetically-earlier block - Add safety-net in emit step: fill any empty UBI slot directly from the changed script's own # Tested on header, guaranteeing every changed script in the PR is tested - Fix BUILD_SCRIPT/TESTED_ON overwrite bug in all build_ubiN and all 12 wheel_build_ubiN_pyXXX jobs: variable.sh always exports the backward-compat UBI8 script, clobbering the GHA job-level env vars before the build runs; fixed by capturing THIS_SCRIPT/THIS_TESTED_ON before source variable.sh and re-exporting after - Fix scanner-env.sh writer: replace indented heredoc with echo and printf %q to preserve CHANGED_FILES newlines correctly - Remove wheel_build_ubi8_py314 (Python 3.14 not supported on UBI8) - Add wheel_build_ubi10_py312, wheel_build_ubi10_py313, wheel_build_ubi10_py314 jobs --- .github/workflows/pr-build.yaml | 926 ++++++++++++++++++++++---------- 1 file changed, 644 insertions(+), 282 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index c8db120167..c329da0707 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -66,11 +66,11 @@ jobs: if [ -n "$RELEVANT_CHANGES" ]; then echo "should_build=true" >> $GITHUB_OUTPUT - echo "✅ Found relevant changes:" + echo "Found relevant changes:" echo "$RELEVANT_CHANGES" else echo "should_build=false" >> $GITHUB_OUTPUT - echo "⏭️ Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile" + echo "Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile" fi build_info: @@ -78,23 +78,27 @@ jobs: if: needs.check_changes.outputs.should_build == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} outputs: - wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }} - has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }} + wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }} + has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }} has_dockerfile_changes: ${{ steps.set_flags.outputs.has_dockerfile_changes }} - docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }} - build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }} + docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }} + build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }} + # Per-UBI-major named outputs - each is a {script,tested_on} JSON object or "" + script_ubi8: ${{ steps.emit.outputs.script_ubi8 }} + script_ubi9: ${{ steps.emit.outputs.script_ubi9 }} + script_ubi10: ${{ steps.emit.outputs.script_ubi10 }} steps: - name: Checkout code (Pull Request) if: github.event_name == 'pull_request' - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Checkout code (Workflow Dispatch) if: github.event_name == 'workflow_dispatch' - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: ref: refs/pull/${{ inputs.pr_number }}/head @@ -147,7 +151,7 @@ jobs: # Reuse changed files from check_changes job CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}" - # If workflow_dispatch, check_changes did not compute changed_files — fetch them now + # If workflow_dispatch, check_changes did not compute changed_files - fetch them now if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then git fetch origin ${{ github.base_ref }} --depth=1 CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) @@ -176,6 +180,45 @@ jobs: PACKAGE_NAME=$(jq -r '.package_name // ""' $BUILD_INFO_FILE) VERSION=$(jq -r '.version // ""' $BUILD_INFO_FILE) + # Pick the version block whose build_script list has the MOST overlap + # with the changed .sh files. This correctly handles the case where a + # script also appears in older single-script blocks (e.g. scikit-learn_ubi_8.10.sh + # is in both 1.5.0 and 1.9.0) - the new block wins because it matches more + # changed files. The wildcard "*" key is always skipped - it is a catch-all + # fallback, not a real version. + # NOTE: if no .sh files changed (build_info.json only), this block is skipped + # entirely and VERSION stays as the default from the .version field. + CHANGED_SH=$(echo "$CHANGED_FILES" | grep '\.sh$' || true) + if [ -n "$CHANGED_SH" ]; then + SCALAR_KEYS='package_name|version|package_dir|wheel_build|docker_build|validate_build_script|base_docker_image|base_docker_variant|use_non_root_user|auditwheel_exclude' + VERSION_KEYS=$(jq -r 'keys[] | select(test("^('"$SCALAR_KEYS"')$") | not) | select(. != "*")' "$BUILD_INFO_FILE" 2>/dev/null || true) + best_ver="" + best_count=0 + for ver_key in $VERSION_KEYS; do + bs=$(jq -c --arg k "$ver_key" '.[$k].build_script // empty' "$BUILD_INFO_FILE" 2>/dev/null || true) + [ -z "$bs" ] && continue + if echo "$bs" | jq -e 'type == "array"' >/dev/null 2>&1; then + scripts=$(echo "$bs" | jq -r '.[]') + else + scripts=$(echo "$bs" | tr -d '"') + fi + match_count=0 + for s in $scripts; do + if echo "$CHANGED_SH" | grep -qF "$s"; then + match_count=$((match_count + 1)) + fi + done + if [ "$match_count" -gt "$best_count" ]; then + best_count=$match_count + best_ver="$ver_key" + fi + done + if [ -n "$best_ver" ] && [ "$best_count" -gt 0 ]; then + echo "Version block '$best_ver' matched $best_count changed script(s) - using that version instead of default '$VERSION'" + VERSION="$best_ver" + fi + fi + echo "BUILD_INFO_FILE=$BUILD_INFO_FILE" >> $GITHUB_ENV echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV echo "VERSION=$VERSION" >> $GITHUB_ENV @@ -189,20 +232,79 @@ jobs: chmod +x ./gha-script/read_buildinfo.sh bash ./gha-script/read_buildinfo.sh + - name: Emit per-UBI outputs + id: emit + run: | + source variable.sh + + # Safety net: for every changed .sh file, if its UBI major slot is still + # empty (read_buildinfo.sh loaded the wrong or incomplete version block), + # read the script's own "# Tested on" header and fill the slot directly. + # This guarantees every changed script in the PR is tested regardless of + # how build_info.json version blocks are structured or overlap. + # If no .sh files changed (build_info.json only), this loop does nothing. + while IFS= read -r changed_script; do + [[ "$changed_script" != *.sh ]] && continue + [ ! -f "$changed_script" ] && continue + raw_tested_on=$(grep "^# Tested on" "$changed_script" | head -1 \ + | cut -d ':' -f2- \ + | sed 's/^[[:space:]]*//' \ + | tr '[:lower:]' '[:upper:]' \ + | sed 's/[[:space:]]*:[[:space:]]*/:/g; s/UBI[[:space:]]*/UBI/g') + [ -z "$raw_tested_on" ] && continue + major=$(echo "$raw_tested_on" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + case "$major" in ''|*[!0-9]*) continue ;; esac + script_name=$(basename "$changed_script") + entry=$(jq -n --arg s "$script_name" --arg t "$raw_tested_on" '{"script":$s,"tested_on":$t}') + case "$major" in + 8) + if [ -z "$SCRIPT_UBI8" ]; then + echo "Filling SCRIPT_UBI8 from changed script: $script_name" + SCRIPT_UBI8="$entry" + fi ;; + 9) + if [ -z "$SCRIPT_UBI9" ]; then + echo "Filling SCRIPT_UBI9 from changed script: $script_name" + SCRIPT_UBI9="$entry" + fi ;; + 10) + if [ -z "$SCRIPT_UBI10" ]; then + echo "Filling SCRIPT_UBI10 from changed script: $script_name" + SCRIPT_UBI10="$entry" + fi ;; + esac + done <<< "$CHANGED_FILES" + + echo "SCRIPT_UBI8=$SCRIPT_UBI8" + echo "SCRIPT_UBI9=$SCRIPT_UBI9" + echo "SCRIPT_UBI10=$SCRIPT_UBI10" + # Use multiline delimiter syntax so JSON double-quotes are not truncated + { + echo "script_ubi8<<__EOF__" + echo "$SCRIPT_UBI8" + echo "__EOF__" + echo "script_ubi9<<__EOF__" + echo "$SCRIPT_UBI9" + echo "__EOF__" + echo "script_ubi10<<__EOF__" + echo "$SCRIPT_UBI10" + echo "__EOF__" + } >> $GITHUB_OUTPUT + - name: Set job control flags id: set_flags run: | if [ -z "$BUILD_INFO_FILE" ]; then - echo "❌ BUILD_INFO_FILE is not set — cannot determine job control flags" + echo "ERROR: BUILD_INFO_FILE is not set - cannot determine job control flags" exit 1 fi if [ ! -f "$BUILD_INFO_FILE" ]; then - echo "❌ BUILD_INFO_FILE does not exist: $BUILD_INFO_FILE" + echo "ERROR: BUILD_INFO_FILE does not exist: $BUILD_INFO_FILE" exit 1 fi PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) - # Strip trailing slash if present — prevents double-slash in regex (e.g. "m/multidict/" → "m/multidict") + # Strip trailing slash if present - prevents double-slash in regex (e.g. "m/multidict/" -> "m/multidict") PACKAGE_DIR="${PACKAGE_DIR%/}" WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) @@ -222,10 +324,10 @@ jobs: # Set outputs for wheel builds if [ "$WHEEL_BUILD" == "true" ] && [ -n "$SH_SCRIPT_CHANGED" ]; then echo "wheel_build_enabled=true" >> $GITHUB_OUTPUT - echo "✅ Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" + echo "Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" else echo "wheel_build_enabled=false" >> $GITHUB_OUTPUT - echo "⏭️ Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" + echo "Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" fi # Set output for sh changes @@ -239,7 +341,7 @@ jobs: if [ "$DOCKER_BUILD" == "true" ] && [ -n "$DOCKERFILE_CHANGED" ]; then echo "docker_build_enabled=true" >> $GITHUB_OUTPUT echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT - echo "✅ Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" + echo "Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" else echo "docker_build_enabled=false" >> $GITHUB_OUTPUT if [ -n "$DOCKERFILE_CHANGED" ]; then @@ -247,17 +349,17 @@ jobs: else echo "has_dockerfile_changes=false" >> $GITHUB_OUTPUT fi - echo "⏭️ Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" + echo "Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" fi # Set output for build job (runs when build_info.json or .sh scripts change) BUILD_INFO_CHANGED=$(echo "$CHANGED_FILES" | grep 'build_info\.json' || true) if [ -n "$BUILD_INFO_CHANGED" ] || [ -n "$SH_SCRIPT_CHANGED" ]; then echo "build_package_enabled=true" >> $GITHUB_OUTPUT - echo "✅ Build package job will run (build_info.json or .sh scripts changed)" + echo "Build package job will run (build_info.json or .sh scripts changed)" else echo "build_package_enabled=false" >> $GITHUB_OUTPUT - echo "⏭️ Build package job will be skipped (no build_info.json or .sh script changes)" + echo "Build package job will be skipped (no build_info.json or .sh script changes)" fi - name: Create scanner-env.sh @@ -274,14 +376,16 @@ jobs: DOCKER_BUILD="false" fi - cat < package-cache/scanner-env.sh - export PACKAGE_NAME=$PACKAGE_NAME - export VERSION=$VERSION - export PACKAGE_DIR=$PACKAGE_DIR - export WHEEL_BUILD=$WHEEL_BUILD - export BUILD_DOCKER=$DOCKER_BUILD - export CHANGED_FILES="$CHANGED_FILES" - EOF + # Write scanner-env.sh without leading indentation spaces. + # CHANGED_FILES is written via a separate heredoc so its newlines are preserved. + { + echo "export PACKAGE_NAME=$PACKAGE_NAME" + echo "export VERSION=$VERSION" + echo "export PACKAGE_DIR=$PACKAGE_DIR" + echo "export WHEEL_BUILD=$WHEEL_BUILD" + echo "export BUILD_DOCKER=$DOCKER_BUILD" + printf 'export CHANGED_FILES=%q\n' "$CHANGED_FILES" + } > package-cache/scanner-env.sh mv variable.sh package-cache/ @@ -289,420 +393,678 @@ jobs: run: tar -czf package-cache.tar.gz package-cache/ - name: Upload package cache - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: package-cache path: package-cache.tar.gz - build: - needs: build_info - if: needs.build_info.outputs.build_package_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + # --------------------------------------------------------------------------- + # BUILD JOBS - one per UBI major version. + # Runs execute_changed_scripts.py inside the correct UBI container. + # Each job only runs when a script exists for that UBI version. + # All three are parallel (no inter-dependency). + # --------------------------------------------------------------------------- + build_ubi8: + needs: build_info + if: ${{ needs.build_info.outputs.build_package_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Build Package + - name: Build Package (UBI8) run: | + # Capture the GHA job-level BUILD_SCRIPT before sourcing variable.sh, + # which overwrites BUILD_SCRIPT with the first/backward-compat script. + THIS_SCRIPT="$BUILD_SCRIPT" source package-cache/variable.sh source package-cache/scanner-env.sh + sudo chown -R $USER:$USER . + echo "--- Executing changed scripts for UBI8 (script: $THIS_SCRIPT) ---" + # Filter CHANGED_FILES to only the script assigned to this UBI job. + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$THIS_SCRIPT" || true) + export BUILD_SCRIPT="$THIS_SCRIPT" + chmod +x ./gha-script/execute_changed_scripts.py + python3 -u ./gha-script/execute_changed_scripts.py - echo "------------------- variable.sh -----------------------------" - cat package-cache/variable.sh - echo "------------------- scanner-env.sh -----------------------------" - cat package-cache/scanner-env.sh - + build_ubi9: + needs: build_info + if: ${{ needs.build_info.outputs.build_package_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI9) + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + source package-cache/variable.sh + source package-cache/scanner-env.sh sudo chown -R $USER:$USER . - echo "------------------- Executing changed scripts -----------------------------" + echo "--- Executing changed scripts for UBI9 (script: $THIS_SCRIPT) ---" + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$THIS_SCRIPT" || true) + export BUILD_SCRIPT="$THIS_SCRIPT" chmod +x ./gha-script/execute_changed_scripts.py python3 -u ./gha-script/execute_changed_scripts.py -# ===================== WHEEL JOBS ===================== + build_ubi10: + needs: build_info + if: ${{ needs.build_info.outputs.build_package_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI10) + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + source package-cache/variable.sh + source package-cache/scanner-env.sh + sudo chown -R $USER:$USER . + echo "--- Executing changed scripts for UBI10 (script: $THIS_SCRIPT) ---" + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$THIS_SCRIPT" || true) + export BUILD_SCRIPT="$THIS_SCRIPT" + chmod +x ./gha-script/execute_changed_scripts.py + python3 -u ./gha-script/execute_changed_scripts.py + # --------------------------------------------------------------------------- + # WHEEL BUILD JOBS - one job per UBI version per Python version. + # All parallel. UBI10 supports py312-py314 only (py310/py311 not supported). + # PR builds do not run scan steps. + # --------------------------------------------------------------------------- - wheel_build_py310: + wheel_build_ubi8_py310: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.10" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" source package-cache/variable.sh source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - + if [ -z "$BUILD_SCRIPT_CHANGED" ]; then echo "Skipping - no .sh changes"; exit 0; fi sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false chmod +x ./gha-script/build_wheels.sh bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - - wheel_build_py311: + wheel_build_ubi8_py311: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.11" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - wheel_build_py312: + wheel_build_ubi8_py312: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.12" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh run: | - sudo apt update -y + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + wheel_build_ubi8_py313: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh + # UBI9 wheel jobs ----------------------------------------------------------- - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 + wheel_build_ubi9_py310: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.10" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi - # Check if any .sh build script is modified in this package + wheel_build_ubi9_py311: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.11" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi + wheel_build_ubi9_py312: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - wheel_build_py313: + wheel_build_ubi9_py313: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.13" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - wheel_build_py314: + wheel_build_ubi9_py314: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.14" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh + # UBI10 wheel jobs (py312-py314 only - py310/py311 not supported on UBI10) ---- - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 + wheel_build_ubi10_py312: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi - # Check if any .sh build script is modified in this package + wheel_build_ubi10_py313: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi + wheel_build_ubi10_py314: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi + # --------------------------------------------------------------------------- + # build_docker - only when Dockerfile changed and docker_build=true in build_info + # --------------------------------------------------------------------------- build_docker: needs: build_info if: needs.build_info.outputs.docker_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Build Docker Image run: | - echo "===== Sourcing environment =====" ls package-cache - chmod +x package-cache/variable.sh chmod +x package-cache/scanner-env.sh source package-cache/variable.sh source package-cache/scanner-env.sh - BUILD_DOCKER=$(echo "$BUILD_DOCKER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) - if [[ "$BUILD_DOCKER" != "true" ]]; then - echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER" - exit 0 + echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER"; exit 0 fi - if [[ -z "$DOCKERFILE_CHANGED" ]]; then - echo "Skipping Docker build as no Dockerfile changes detected in PR" - exit 0 + echo "Skipping Docker build as no Dockerfile changes detected in PR"; exit 0 fi - echo "Dockerfile change detected, proceeding with build..." - - echo "===== Starting Docker build =====" sudo chown -R $USER:$USER . chmod +x ./gha-script/build_docker.sh bash ./gha-script/build_docker.sh - - echo "===== Docker images after build =====" docker images - - echo "===== Saving Docker image =====" docker save -o package-cache/image.tar "$IMAGE_NAME" - ls -lh package-cache/image.tar From 1028f53ea550ee53f91f86429b969a01489ab381 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:47:59 +0530 Subject: [PATCH 26/45] Add UBI10 build and wheel jobs to currency build workflow - Add script_ubi8, script_ubi9, script_ubi10 outputs to build_info job - Add Emit per-UBI outputs step - Add build_ubi8 and build_ubi10 jobs - Add wheel_build_ubi10_py311 through wheel_build_ubi10_py314 jobs - Remove wheel_build_ubi8_py314 (Python 3.14 not supported on UBI8) - Remove branches: master restriction from workflow_dispatch trigger --- .github/workflows/currency-build.yaml | 2176 ++++++++++++++++--------- 1 file changed, 1381 insertions(+), 795 deletions(-) diff --git a/.github/workflows/currency-build.yaml b/.github/workflows/currency-build.yaml index 3783d25c0d..764b4324d1 100644 --- a/.github/workflows/currency-build.yaml +++ b/.github/workflows/currency-build.yaml @@ -1,9 +1,6 @@ name: Currency Build on: workflow_dispatch: - branches: - - master - inputs: package_name: description: 'Name of the package to build' @@ -54,8 +51,14 @@ run-name: > format('Currency Build {0} && Unique ID {1}', inputs.package_name, inputs.unique_id) }} - jobs: + + # --------------------------------------------------------------------------- + # build_info: reads build_info.json, runs read_buildinfo.sh, emits + # per-UBI-major outputs (script_ubi8, script_ubi9, script_ubi10). + # Each output is a JSON object string {"script":"...","tested_on":"..."} + # or an empty string "" when that UBI version has no script for this package. + # --------------------------------------------------------------------------- build_info: runs-on: ubuntu-24.04-ppc64le-p10 env: @@ -67,15 +70,19 @@ jobs: ENABLE_TRIVY: ${{ inputs.enable_trivy }} ENABLE_SYFT: ${{ inputs.enable_syft }} ENABLE_GRYPE: ${{ inputs.enable_grype }} + outputs: + script_ubi8: ${{ steps.emit.outputs.script_ubi8 }} + script_ubi9: ${{ steps.emit.outputs.script_ubi9 }} + script_ubi10: ${{ steps.emit.outputs.script_ubi10 }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Install system packages run: | sudo apt update -y sudo apt install -y jq file - + - name: Install Python dependencies run: | pip3 install --force-reinstall -v "requests==2.31.0" @@ -92,35 +99,50 @@ jobs: cat variable.sh echo "=======================" + - name: Emit per-UBI outputs + id: emit + run: | + source variable.sh + echo "SCRIPT_UBI8=$SCRIPT_UBI8" + echo "SCRIPT_UBI9=$SCRIPT_UBI9" + echo "SCRIPT_UBI10=$SCRIPT_UBI10" + # Use multiline delimiter syntax so JSON double-quotes are not truncated + { + echo "script_ubi8<<__EOF__" + echo "$SCRIPT_UBI8" + echo "__EOF__" + echo "script_ubi9<<__EOF__" + echo "$SCRIPT_UBI9" + echo "__EOF__" + echo "script_ubi10<<__EOF__" + echo "$SCRIPT_UBI10" + echo "__EOF__" + } >> $GITHUB_OUTPUT + - name: Create scanner-env.sh run: | - echo "Creating package-cache........." mkdir package-cache - echo "Creating scanner-env.sh ......." echo "export VALIDATE_BUILD_SCRIPT='${{ inputs.validate_build_script }}'" > scanner-env.sh - echo "export BUILD_DOCKER='${{ inputs.build_docker }}'" >> scanner-env.sh - echo "export PACKAGE_NAME='${{ inputs.package_name }}'" >> scanner-env.sh + echo "export BUILD_DOCKER='${{ inputs.build_docker }}'" >> scanner-env.sh + echo "export PACKAGE_NAME='${{ inputs.package_name }}'" >> scanner-env.sh echo "===== scanner-env.sh =====" cat scanner-env.sh echo "==========================" - echo "Moving scanner-env.sh and variable.sh into package-cache............" - sudo mv variable.sh package-cache - sudo mv scanner-env.sh package-cache - echo "Moved scanner-env.sh and variable.sh into package-cache............" + sudo mv variable.sh package-cache/ + sudo mv scanner-env.sh package-cache/ - name: Archive package cache run: tar -czf package-cache.tar.gz package-cache/ - - name: Upload variable.sh and scanner-env.sh via package-cache - uses: actions/upload-artifact@v6 + - name: Upload package-cache artifact + uses: actions/upload-artifact@v4 with: name: package-cache path: package-cache.tar.gz # --------------------------------------------------------------------------- - # install_scan_tools: Installs grype ONCE (static Go binary — fully portable) - # and caches it as an artifact for all wheel_build_pyXXX jobs. - # Scancode-toolkit is fetched once and distributed as an artifact too. + # install_scan_tools: installs grype + fetches scancode-toolkit source ONCE. + # Skipped entirely when wheel_build=false. # --------------------------------------------------------------------------- install_scan_tools: needs: build_info @@ -128,7 +150,7 @@ jobs: runs-on: ubuntu-24.04-ppc64le-p10 steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Install system dependencies run: | @@ -141,7 +163,7 @@ jobs: bash ./gha-script/scanner-scripts/fetch_scancode_toolkit.sh - name: Upload scancode-toolkit-src artifact - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: scancode-toolkit-src path: scancode-toolkit-src.tar.gz @@ -157,188 +179,242 @@ jobs: echo "[INFO] Checksum verification successful." tar -xzf grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz else - echo "[ERROR] Checksum verification FAILED for grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz." + echo "[ERROR] Checksum verification FAILED." exit 1 fi mkdir -p scan-tools-bin mv grype scan-tools-bin/grype chmod +x scan-tools-bin/grype - echo "Grype version:" scan-tools-bin/grype version - name: Archive grype binary run: tar -czf scan-tools-cache.tar.gz scan-tools-bin/ - name: Upload scan-tools-cache artifact - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: scan-tools-cache path: scan-tools-cache.tar.gz - - build: + # --------------------------------------------------------------------------- + # BUILD JOBS — one per UBI major version. + # Each job runs build_package.sh inside the correct UBI container. + # build_ubi9 always runs (every package has at least a UBI9 script). + # build_ubi8 / build_ubi10 only run when that UBI version has a script. + # All three are independent of each other (fully parallel). + # --------------------------------------------------------------------------- + + build_ubi8: needs: build_info - if: ${{ inputs.validate_build_script == 'true' }} + if: ${{ inputs.validate_build_script == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Build Package + - name: Build Package (UBI8) env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | - echo "Sourcing variable.sh and scanner-env.sh..........................................." source package-cache/variable.sh source package-cache/scanner-env.sh - echo "===== variable.sh =====" - cat package-cache/variable.sh - echo "===== scanner-env.sh =====" - cat package-cache/scanner-env.sh - echo "==============LSCPU================" + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "BUILD_SCRIPT=$BUILD_SCRIPT TESTED_ON=$TESTED_ON" lscpu - echo "===================================" chmod +x ./gha-script/build_package.sh bash ./gha-script/build_package.sh cloned_package=$(ls -td -- */ | head -n 1) - sudo mv "$cloned_package" package-cache + sudo mv "$cloned_package" package-cache/ echo "export CLONED_PACKAGE=\"$cloned_package\"" >> package-cache/scanner-env.sh - echo "-------------updated scanner-env.sh-----------------------" - cat package-cache/scanner-env.sh - echo "---------------------------------------------" - cd package-cache - sudo chown $USER:$USER -R . + cd package-cache && sudo chown $USER:$USER -R . chmod +x ../gha-script/pre_process.sh bash ../gha-script/pre_process.sh cd $GITHUB_WORKSPACE - gzip build_log + gzip build_log && mv build_log.gz build_log_UBI8.gz chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh build_log.gz + bash ./gha-script/upload-scripts/upload_file.sh build_log_UBI8.gz + - name: Fix permissions + run: sudo chown -R $USER:$USER package-cache && sudo chmod -R u+rwX,go+rX,go-w package-cache + - name: Archive updated package-cache + run: tar -czf package-cache.tar.gz package-cache/ + - name: Upload updated package-cache (UBI8) + uses: actions/upload-artifact@v4 + with: + name: updated-package-cache-ubi8 + path: package-cache.tar.gz - - name: Fix permissions on package-cache - run: | - USER=$(whoami) - echo "Current user: $USER" - echo "Fixing ownership and permissions for package-cache" - sudo chown -R $USER:$USER package-cache - sudo chmod -R u+rwX,go+rX,go-w package-cache - - - name: Archive updated package cache + build_ubi9: + needs: build_info + if: ${{ inputs.validate_build_script == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI9) + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "BUILD_SCRIPT=$BUILD_SCRIPT TESTED_ON=$TESTED_ON" + lscpu + chmod +x ./gha-script/build_package.sh + bash ./gha-script/build_package.sh + cloned_package=$(ls -td -- */ | head -n 1) + sudo mv "$cloned_package" package-cache/ + echo "export CLONED_PACKAGE=\"$cloned_package\"" >> package-cache/scanner-env.sh + cd package-cache && sudo chown $USER:$USER -R . + chmod +x ../gha-script/pre_process.sh + bash ../gha-script/pre_process.sh + cd $GITHUB_WORKSPACE + gzip build_log && mv build_log.gz build_log_UBI9.gz + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh build_log_UBI9.gz + - name: Fix permissions + run: sudo chown -R $USER:$USER package-cache && sudo chmod -R u+rwX,go+rX,go-w package-cache + - name: Archive updated package-cache run: tar -czf package-cache.tar.gz package-cache/ + - name: Upload updated package-cache (UBI9) + uses: actions/upload-artifact@v4 + with: + name: updated-package-cache-ubi9 + path: package-cache.tar.gz - - name: Upload updated package-cache - uses: actions/upload-artifact@v6 + build_ubi10: + needs: build_info + if: ${{ inputs.validate_build_script == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI10) + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "BUILD_SCRIPT=$BUILD_SCRIPT TESTED_ON=$TESTED_ON" + lscpu + chmod +x ./gha-script/build_package.sh + bash ./gha-script/build_package.sh + cloned_package=$(ls -td -- */ | head -n 1) + sudo mv "$cloned_package" package-cache/ + echo "export CLONED_PACKAGE=\"$cloned_package\"" >> package-cache/scanner-env.sh + cd package-cache && sudo chown $USER:$USER -R . + chmod +x ../gha-script/pre_process.sh + bash ../gha-script/pre_process.sh + cd $GITHUB_WORKSPACE + gzip build_log && mv build_log.gz build_log_UBI10.gz + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh build_log_UBI10.gz + - name: Fix permissions + run: sudo chown -R $USER:$USER package-cache && sudo chmod -R u+rwX,go+rX,go-w package-cache + - name: Archive updated package-cache + run: tar -czf package-cache.tar.gz package-cache/ + - name: Upload updated package-cache (UBI10) + uses: actions/upload-artifact@v4 with: - name: updated-package-cache + name: updated-package-cache-ubi10 path: package-cache.tar.gz - # --------------------------------------------------------------------------- - # wheel_build_pyXXX jobs: build wheel → run CVE scan (generalized_wheel_scanner) - # → run scancode scan → run grype scan → upload all results to COS. - # Grype is reused from the install_scan_tools artifact (static binary). + # WHEEL BUILD JOBS — one job per UBI version per Python version. + # All 15 jobs are fully parallel (none depend on the build jobs above). + # UBI10 does not support Python 3.10 — those jobs are skipped via if:. + # Each job needs install_scan_tools for grype/scancode artifacts. # --------------------------------------------------------------------------- - wheel_build_py310: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py310: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.10" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh source package-cache/variable.sh source package-cache/scanner-env.sh - - pwd - ls -lh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" chmod +x ./gha-script/build_wheels.sh bash ./gha-script/build_wheels.sh - - WORKSPACE=$(pwd) - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py310_log.gz" - - # Compress and rename the log file - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py310_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py310.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | source package-cache/variable.sh source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | source package-cache/variable.sh @@ -346,909 +422,1441 @@ jobs: export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - mkdir -p wheel_scancode_py310 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py310 - - tar cvzf wheel_scancode_py310.tar.gz wheel_scancode_py310 - chmod +x ./gha-script/upload-scripts/upload_file.sh + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; + tar czf wheel_scancode_py310.tar.gz wheel_scancode_py310 bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py310.tar.gz - mkdir -p grype_wheel_py310 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py310 - - tar cvzf grype_wheel_py310.tar.gz grype_wheel_py310 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; + tar czf grype_wheel_py310.tar.gz grype_wheel_py310 bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py310.tar.gz - - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then + if ls *.whl 1>/dev/null 2>&1; then sudo chmod a+r *.whl WHEEL_FILE=$(ls *.whl) SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) if [ -n "$WHEEL_FILE" ]; then CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." + [ -f "$CVE_REPORT" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" || echo "CVE report not found — skipping." fi - wheel_build_py311: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py311: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.11" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh source package-cache/variable.sh source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" chmod +x ./gha-script/build_wheels.sh bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py311_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" - + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py311_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py311.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - mkdir -p wheel_scancode_py311 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py311/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py311 - - tar cvzf wheel_scancode_py311.tar.gz wheel_scancode_py311 - chmod +x ./gha-script/upload-scripts/upload_file.sh + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py311/ \; + tar czf wheel_scancode_py311.tar.gz wheel_scancode_py311 bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py311.tar.gz - mkdir -p grype_wheel_py311 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py311/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py311 - - tar cvzf grype_wheel_py311.tar.gz grype_wheel_py311 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py311/ \; + tar czf grype_wheel_py311.tar.gz grype_wheel_py311 bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py311.tar.gz - - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_py312: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py312: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py312_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" - + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py312_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py312.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - mkdir -p wheel_scancode_py312 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py312 - - tar cvzf wheel_scancode_py312.tar.gz wheel_scancode_py312 - chmod +x ./gha-script/upload-scripts/upload_file.sh + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; + tar czf wheel_scancode_py312.tar.gz wheel_scancode_py312 bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz - mkdir -p grype_wheel_py312 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py312 - - tar cvzf grype_wheel_py312.tar.gz grype_wheel_py312 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; + tar czf grype_wheel_py312.tar.gz grype_wheel_py312 bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz - - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi - - wheel_build_py313: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + + wheel_build_ubi8_py313: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py313_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" - + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py313.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - mkdir -p wheel_scancode_py313 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py313 - - tar cvzf wheel_scancode_py313.tar.gz wheel_scancode_py313 - chmod +x ./gha-script/upload-scripts/upload_file.sh + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; + tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313 bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz - mkdir -p grype_wheel_py313 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py313 - - tar cvzf grype_wheel_py313.tar.gz grype_wheel_py313 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; + tar czf grype_wheel_py313.tar.gz grype_wheel_py313 bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz - - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_py314: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py314: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py314.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py314 + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; + tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314 + bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz + mkdir -p grype_wheel_py314 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; + tar czf grype_wheel_py314.tar.gz grype_wheel_py314 + bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + + # UBI9 wheel jobs (py310–py314) ------------------------------------------------ + wheel_build_ubi9_py310: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.10" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py310_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py310.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py310; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; + tar czf wheel_scancode_py310.tar.gz wheel_scancode_py310; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py310.tar.gz + mkdir -p grype_wheel_py310; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; + tar czf grype_wheel_py310.tar.gz grype_wheel_py310; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py310.tar.gz + - name: Upload wheel to COS run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - - name: Cache pip downloads for scancode-toolkit + wheel_build_ubi9_py311: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.11" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py311_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py311.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py311; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py311/ \; + tar czf wheel_scancode_py311.tar.gz wheel_scancode_py311; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py311.tar.gz + mkdir -p grype_wheel_py311; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py311/ \; + tar czf grype_wheel_py311.tar.gz grype_wheel_py311; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py311.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + wheel_build_ubi9_py312: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py312_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py312.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py312; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; + tar czf wheel_scancode_py312.tar.gz wheel_scancode_py312; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz + mkdir -p grype_wheel_py312; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; + tar czf grype_wheel_py312.tar.gz grype_wheel_py312; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + wheel_build_ubi9_py313: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py313.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py313; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; + tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz + mkdir -p grype_wheel_py313; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; + tar czf grype_wheel_py313.tar.gz grype_wheel_py313; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + wheel_build_ubi9_py314: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py314.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a + mkdir -p wheel_scancode_py314; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; + tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz + mkdir -p grype_wheel_py314; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; + tar czf grype_wheel_py314.tar.gz grype_wheel_py314; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py314_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" + # UBI10 wheel jobs (py311–py314 only — py310 not supported on UBI10) ----------- - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" + wheel_build_ubi10_py312: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py312_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi10_py312.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py312; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; + tar czf wheel_scancode_py312.tar.gz wheel_scancode_py312; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz + mkdir -p grype_wheel_py312; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; + tar czf grype_wheel_py312.tar.gz grype_wheel_py312; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + + wheel_build_ubi10_py313: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi10_py313.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py313; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; + tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz + mkdir -p grype_wheel_py313; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; + tar czf grype_wheel_py313.tar.gz grype_wheel_py313; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + wheel_build_ubi10_py314: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi10_py314.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - mkdir -p wheel_scancode_py314 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py314 - - tar cvzf wheel_scancode_py314.tar.gz wheel_scancode_py314 - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz - - mkdir -p grype_wheel_py314 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py314 - - tar cvzf grype_wheel_py314.tar.gz grype_wheel_py314 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz - + mkdir -p wheel_scancode_py314; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; + tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz + mkdir -p grype_wheel_py314; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; + tar czf grype_wheel_py314.tar.gz grype_wheel_py314; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true # --------------------------------------------------------------------------- - # wheel_license_extract: Runs after all wheel builds complete. Downloads the - # per-Python-version scancode JSON files from COS and extracts SPDX licenses. + # WHEEL LICENSE EXTRACT — one per UBI version. + # Downloads per-Python-version scancode JSON from COS and extracts licenses. + # Runs after all wheel jobs for that UBI version finish (at least one must succeed). # --------------------------------------------------------------------------- - wheel_license_extract: - if: ${{ inputs.wheel_build == 'true' && (needs.wheel_build_py310.result == 'success' || needs.wheel_build_py311.result == 'success' || needs.wheel_build_py312.result == 'success' || needs.wheel_build_py313.result == 'success' || needs.wheel_build_py314.result == 'success') }} + + wheel_license_extract_ubi8: needs: - - wheel_build_py310 - - wheel_build_py311 - - wheel_build_py312 - - wheel_build_py313 - - wheel_build_py314 + - wheel_build_ubi8_py310 + - wheel_build_ubi8_py311 + - wheel_build_ubi8_py312 + - wheel_build_ubi8_py313 + - wheel_build_ubi8_py314 + if: | + always() && + inputs.wheel_build == 'true' && + needs.build_info.outputs.script_ubi8 != '' && + (needs.wheel_build_ubi8_py310.result == 'success' || + needs.wheel_build_ubi8_py311.result == 'success' || + needs.wheel_build_ubi8_py312.result == 'success' || + needs.wheel_build_ubi8_py313.result == 'success' || + needs.wheel_build_ubi8_py314.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: - - name: Checkout code - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Download scancode JSON outputs from COS + - name: Download scancode JSON outputs from COS and extract licenses env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | source package-cache/variable.sh source package-cache/scanner-env.sh - - echo "Downloading per-version scancode archives from COS..." mkdir -p all_wheel_scancode - chmod +x ./gha-script/download-scripts/download_file.sh for PY_VER in py310 py311 py312 py313 py314; do ARCHIVE="wheel_scancode_${PY_VER}.tar.gz" - echo "Attempting to download $ARCHIVE..." bash ./gha-script/download-scripts/download_file.sh "$ARCHIVE" \ || echo "Warning: $ARCHIVE not found on COS (skipped)" if [ -f "$ARCHIVE" ]; then tar -xzf "$ARCHIVE" find "wheel_scancode_${PY_VER}" -type f -name "*.json" \ -exec cp {} all_wheel_scancode/ \; 2>/dev/null || true - echo "Collected JSON files from $ARCHIVE" fi done + echo "Scancode JSON files collected:"; ls -lh all_wheel_scancode + for JSON_FILE in all_wheel_scancode/*.json; do + [ -f "$JSON_FILE" ] || { echo "No JSON files — skipping."; break; } + echo "--- $JSON_FILE ---" + python3 ./gha-script/licenses_extract_script.py "$JSON_FILE" + done - echo "All scancode JSON files available:" - ls -lh all_wheel_scancode - - - name: Extract licenses from each scancode JSON + wheel_license_extract_ubi9: + needs: + - wheel_build_ubi9_py310 + - wheel_build_ubi9_py311 + - wheel_build_ubi9_py312 + - wheel_build_ubi9_py313 + - wheel_build_ubi9_py314 + if: | + always() && + inputs.wheel_build == 'true' && + needs.build_info.outputs.script_ubi9 != '' && + (needs.wheel_build_ubi9_py310.result == 'success' || + needs.wheel_build_ubi9_py311.result == 'success' || + needs.wheel_build_ubi9_py312.result == 'success' || + needs.wheel_build_ubi9_py313.result == 'success' || + needs.wheel_build_ubi9_py314.result == 'success') + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scancode JSON outputs from COS and extract licenses + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | - echo "===== License extraction per wheel =====" + source package-cache/variable.sh + source package-cache/scanner-env.sh + mkdir -p all_wheel_scancode + chmod +x ./gha-script/download-scripts/download_file.sh + for PY_VER in py310 py311 py312 py313 py314; do + ARCHIVE="wheel_scancode_${PY_VER}.tar.gz" + bash ./gha-script/download-scripts/download_file.sh "$ARCHIVE" \ + || echo "Warning: $ARCHIVE not found on COS (skipped)" + if [ -f "$ARCHIVE" ]; then + tar -xzf "$ARCHIVE" + find "wheel_scancode_${PY_VER}" -type f -name "*.json" \ + -exec cp {} all_wheel_scancode/ \; 2>/dev/null || true + fi + done + echo "Scancode JSON files collected:"; ls -lh all_wheel_scancode for JSON_FILE in all_wheel_scancode/*.json; do - [ -f "$JSON_FILE" ] || { echo "No JSON files found — skipping."; break; } + [ -f "$JSON_FILE" ] || { echo "No JSON files — skipping."; break; } echo "--- $JSON_FILE ---" python3 ./gha-script/licenses_extract_script.py "$JSON_FILE" done - echo "========================================" - - source_scanner: - needs: build - if: ${{ inputs.validate_build_script == 'true' }} + wheel_license_extract_ubi10: + needs: + - wheel_build_ubi10_py312 + - wheel_build_ubi10_py313 + - wheel_build_ubi10_py314 + if: | + always() && + inputs.wheel_build == 'true' && + needs.build_info.outputs.script_ubi10 != '' && + (needs.wheel_build_ubi10_py312.result == 'success' || + needs.wheel_build_ubi10_py313.result == 'success' || + needs.wheel_build_ubi10_py314.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: - name: updated-package-cache - + name: package-cache - name: Extract package cache run: tar -xzf package-cache.tar.gz + - name: Download scancode JSON outputs from COS and extract licenses + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + mkdir -p all_wheel_scancode + chmod +x ./gha-script/download-scripts/download_file.sh + for PY_VER in py312 py313 py314; do + ARCHIVE="wheel_scancode_${PY_VER}.tar.gz" + bash ./gha-script/download-scripts/download_file.sh "$ARCHIVE" \ + || echo "Warning: $ARCHIVE not found on COS (skipped)" + if [ -f "$ARCHIVE" ]; then + tar -xzf "$ARCHIVE" + find "wheel_scancode_${PY_VER}" -type f -name "*.json" \ + -exec cp {} all_wheel_scancode/ \; 2>/dev/null || true + fi + done + echo "Scancode JSON files collected:"; ls -lh all_wheel_scancode + for JSON_FILE in all_wheel_scancode/*.json; do + [ -f "$JSON_FILE" ] || { echo "No JSON files — skipping."; break; } + echo "--- $JSON_FILE ---" + python3 ./gha-script/licenses_extract_script.py "$JSON_FILE" + done + # --------------------------------------------------------------------------- + # source_scanner_ubiN: scans the cloned source tree (trivy / syft / grype). + # One job per UBI version, each depending on its respective build_ubiN job. + # Only runs when the corresponding build_ubiN succeeded and has a script. + # --------------------------------------------------------------------------- + source_scanner_ubi8: + needs: build_ubi8 + if: ${{ always() && inputs.validate_build_script == 'true' && needs.build_ubi8.result == 'success' }} + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download updated package-cache (UBI8) + uses: actions/download-artifact@v4 + with: + name: updated-package-cache-ubi8 + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz - name: Prepare scanner environment run: | - echo "Sourcing variable.sh..." source package-cache/variable.sh source package-cache/scanner-env.sh - echo "---------updated scanner-env.sh cat----------------" cat package-cache/scanner-env.sh - echo "scanner-env.sh created with:" - cat package-cache/scanner-env.sh - - name: Run Trivy Scan if: ${{ inputs.enable_trivy == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/trivy_code_scan.sh + bash ./gha-script/scanner-scripts/trivy_code_scan.sh + mkdir source_trivy + mv package-cache/trivy_source_vulnerabilities_results.json \ + package-cache/trivy_source_sbom_results.cyclonedx source_trivy + - name: Run Syft Scan + if: ${{ inputs.enable_syft == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/syft_code_scan.sh + bash ./gha-script/scanner-scripts/syft_code_scan.sh + mkdir source_syft + mv package-cache/syft_source_sbom_results.json source_syft + - name: Run Grype Scan + if: ${{ inputs.enable_grype == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_code_scan.sh + bash ./gha-script/scanner-scripts/grype_code_scan.sh + mkdir source_grype + mv package-cache/grype_source_sbom_results.json \ + package-cache/grype_source_vulnerabilities_results.json source_grype + - name: Upload Scanner Results + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + mkdir source + source package-cache/variable.sh; source package-cache/scanner-env.sh + find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + tar cvzf source_scanner_ubi8.tar.gz source + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh source_scanner_ubi8.tar.gz + + source_scanner_ubi9: + needs: build_ubi9 + if: ${{ always() && inputs.validate_build_script == 'true' && needs.build_ubi9.result == 'success' }} + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download updated package-cache (UBI9) + uses: actions/download-artifact@v4 + with: + name: updated-package-cache-ubi9 + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz + - name: Prepare scanner environment + run: | source package-cache/variable.sh source package-cache/scanner-env.sh cat package-cache/scanner-env.sh - + - name: Run Trivy Scan + if: ${{ inputs.enable_trivy == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/trivy_code_scan.sh - bash ./gha-script/scanner-scripts/trivy_code_scan.sh; + bash ./gha-script/scanner-scripts/trivy_code_scan.sh mkdir source_trivy - mv package-cache/trivy_source_vulnerabilities_results.json package-cache/trivy_source_sbom_results.cyclonedx source_trivy - + mv package-cache/trivy_source_vulnerabilities_results.json \ + package-cache/trivy_source_sbom_results.cyclonedx source_trivy - name: Run Syft Scan if: ${{ inputs.enable_syft == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/syft_code_scan.sh bash ./gha-script/scanner-scripts/syft_code_scan.sh mkdir source_syft mv package-cache/syft_source_sbom_results.json source_syft - - name: Run Grype Scan if: ${{ inputs.enable_grype == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_code_scan.sh bash ./gha-script/scanner-scripts/grype_code_scan.sh mkdir source_grype - mv package-cache/grype_source_sbom_results.json package-cache/grype_source_vulnerabilities_results.json source_grype - + mv package-cache/grype_source_sbom_results.json \ + package-cache/grype_source_vulnerabilities_results.json source_grype - name: Upload Scanner Results env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | mkdir source + source package-cache/variable.sh; source package-cache/scanner-env.sh + find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + tar cvzf source_scanner_ubi9.tar.gz source + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh source_scanner_ubi9.tar.gz + + source_scanner_ubi10: + needs: build_ubi10 + if: ${{ always() && inputs.validate_build_script == 'true' && needs.build_ubi10.result == 'success' }} + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download updated package-cache (UBI10) + uses: actions/download-artifact@v4 + with: + name: updated-package-cache-ubi10 + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz + - name: Prepare scanner environment + run: | source package-cache/variable.sh source package-cache/scanner-env.sh + cat package-cache/scanner-env.sh + - name: Run Trivy Scan + if: ${{ inputs.enable_trivy == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/trivy_code_scan.sh + bash ./gha-script/scanner-scripts/trivy_code_scan.sh + mkdir source_trivy + mv package-cache/trivy_source_vulnerabilities_results.json \ + package-cache/trivy_source_sbom_results.cyclonedx source_trivy + - name: Run Syft Scan + if: ${{ inputs.enable_syft == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/syft_code_scan.sh + bash ./gha-script/scanner-scripts/syft_code_scan.sh + mkdir source_syft + mv package-cache/syft_source_sbom_results.json source_syft + - name: Run Grype Scan + if: ${{ inputs.enable_grype == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_code_scan.sh + bash ./gha-script/scanner-scripts/grype_code_scan.sh + mkdir source_grype + mv package-cache/grype_source_sbom_results.json \ + package-cache/grype_source_vulnerabilities_results.json source_grype + - name: Upload Scanner Results + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + mkdir source + source package-cache/variable.sh; source package-cache/scanner-env.sh find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true - find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true - tar cvzf source_scanner.tar.gz source + tar cvzf source_scanner_ubi10.tar.gz source chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh source_scanner.tar.gz + bash ./gha-script/upload-scripts/upload_file.sh source_scanner_ubi10.tar.gz + # --------------------------------------------------------------------------- + # build_docker / image_scanner + # --------------------------------------------------------------------------- build_docker: - needs: build_info + needs: build_info if: ${{ inputs.build_docker == 'true' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Build Docker Image run: | - echo "................................Sourcing scanner-env.sh............................." ls package-cache source package-cache/variable.sh source package-cache/scanner-env.sh @@ -1256,21 +1864,17 @@ jobs: bash ./gha-script/build_docker.sh docker save -o package-cache/image.tar "$IMAGE_NAME" docker images - - name: Upload Docker Image env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/upload-scripts/upload_docker_image.sh bash ./gha-script/upload-scripts/upload_docker_image.sh - - name: Archive package cache with image run: tar -czf package-cache.tar.gz package-cache/ - - name: Upload package-cache with image.tar - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: package-cache-with-image path: package-cache.tar.gz @@ -1280,132 +1884,114 @@ jobs: if: ${{ inputs.build_docker == 'true' }} runs-on: ubuntu-24.04-ppc64le-p10 steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache-with-image - - name: Extract package cache run: tar -xzf package-cache.tar.gz - + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz - name: Load Docker Image run: | - echo "Sourcing variable.sh ..........................................." - source package-cache/variable.sh - source package-cache/scanner-env.sh - echo ".......................Loading iamge.tar ............................" + source package-cache/variable.sh; source package-cache/scanner-env.sh docker load -i "package-cache/image.tar" - - name: Run Trivy Image Scan if: ${{ inputs.enable_trivy == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/trivy_image_scan.sh bash ./gha-script/scanner-scripts/trivy_image_scan.sh mkdir image_trivy mv trivy_image_vulnerabilities_results.json trivy_image_sbom_results.cyclonedx image_trivy - - name: Run Syft Image Scan if: ${{ inputs.enable_syft == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./script/scanner-scripts/syft_image_scan.sh - bash ./script/scanner-scripts/syft_image_scan.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/syft_image_scan.sh + bash ./gha-script/scanner-scripts/syft_image_scan.sh mkdir image_syft mv syft_image_sbom_results.json image_syft - - name: Run Grype Image Scan if: ${{ inputs.enable_grype == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_image_scan.sh bash ./gha-script/scanner-scripts/grype_image_scan.sh mkdir image_grype mv grype_image_sbom_results.json grype_image_vulnerabilities_results.json image_grype - - name: Upload Image Scanner Results env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | mkdir image - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh find image_trivy -mindepth 1 -exec mv -t image {} + 2>/dev/null || true - find image_syft -mindepth 1 -exec mv -t image {} + 2>/dev/null || true + find image_syft -mindepth 1 -exec mv -t image {} + 2>/dev/null || true find image_grype -mindepth 1 -exec mv -t image {} + 2>/dev/null || true tar cvzf image_scanner.tar.gz image chmod +x ./gha-script/upload-scripts/upload_file.sh bash ./gha-script/upload-scripts/upload_file.sh image_scanner.tar.gz + # --------------------------------------------------------------------------- + # final_summary: runs after all parallel tracks complete. + # Uses always() so it runs even when optional jobs (ubi8/ubi10) were skipped. + # Requires at least one of build_ubi8/ubi9/ubi10 to have succeeded. + # --------------------------------------------------------------------------- final_summary: - name: Final Summary Stage - needs: - - build - - source_scanner - - wheel_build_py310 - - wheel_build_py311 - - wheel_build_py312 - - wheel_build_py313 - - wheel_build_py314 - - wheel_license_extract - - if: | - always() && - needs.build.result == 'success' && - needs.source_scanner.result == 'success' - - runs-on: ubuntu-24.04-ppc64le-p10 - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Download package-cache - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Create and activate venv - run: | - python -m venv venv - source venv/bin/activate - python -m pip install --upgrade pip - pip install requests deprecated xlsxwriter packaging - - # Export environment variables - export PYTHONPATH=$(pwd):$(pwd)/process_bom/ - - - name: Summarize build results - env : - PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - IAM_WRITER_API_KEY: ${{ secrets.IAM_WRITER_API_KEY }} - SERVICE_INSTANCE_ID: ${{ secrets.SERVICE_INSTANCE_ID }} - run: | - echo "Sourcing variable.sh and scanner-env.sh..........................................." - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - echo "===== variable.sh =====" - cat package-cache/variable.sh - echo "===== scanner-env.sh =====" - cat package-cache/scanner-env.sh - echo "-------------package name : ${PACKAGE_NAME}--------------------" - echo "-------------version : ${VERSION}--------------------" - pwd - chmod +x process_bom/*.py || true - # Export environment variables - export PYTHONPATH="/tmp/_actions-runner-working-dir/build-scripts/build-scripts:/tmp/_actions-runner-working-dir/build-scripts/build-scripts/process_bom" - python -m process_bom.run_currency_processor - + name: Final Summary Stage + needs: + - build_ubi8 + - build_ubi9 + - build_ubi10 + - wheel_license_extract_ubi8 + - wheel_license_extract_ubi9 + - wheel_license_extract_ubi10 + - source_scanner_ubi8 + - source_scanner_ubi9 + - source_scanner_ubi10 + if: | + always() && + (needs.build_ubi8.result == 'success' || + needs.build_ubi9.result == 'success' || + needs.build_ubi10.result == 'success') + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Create and activate venv + run: | + python -m venv venv + source venv/bin/activate + python -m pip install --upgrade pip + pip install requests deprecated xlsxwriter packaging + - name: Summarize build results + env: + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + IAM_WRITER_API_KEY: ${{ secrets.IAM_WRITER_API_KEY }} + SERVICE_INSTANCE_ID: ${{ secrets.SERVICE_INSTANCE_ID }} + run: | + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "===== scanner-env.sh ====="; cat package-cache/scanner-env.sh + echo "Package: ${PACKAGE_NAME} Version: ${VERSION}" + chmod +x process_bom/*.py || true + export PYTHONPATH="/tmp/_actions-runner-working-dir/build-scripts/build-scripts:/tmp/_actions-runner-working-dir/build-scripts/build-scripts/process_bom" + python -m process_bom.run_currency_processor From a182de58fc1d3be94bb1875b7055409c8a84bf53 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:50:08 +0530 Subject: [PATCH 27/45] Add UBI10 support and multi-script bucketing to read_buildinfo.sh - Add read_tested_on() helper to extract and normalise the '# Tested on' header value from any build script file (uppercase, collapse separators) - Add BUILD_SCRIPTS_JSON array construction supporting both string and list values for the build_script field in build_info.json - Add SCRIPT_UBI8, SCRIPT_UBI9, SCRIPT_UBI10 bucketing by parsing the UBI major version from each script's # Tested on header - Support build_script as a JSON array in version blocks (not just string) - Export SCRIPT_UBI8, SCRIPT_UBI9, SCRIPT_UBI10 in variable.sh so downstream jobs can consume the correct per-UBI script --- gha-script/read_buildinfo.sh | 258 ++++++++++++++++++++++++++--------- 1 file changed, 195 insertions(+), 63 deletions(-) diff --git a/gha-script/read_buildinfo.sh b/gha-script/read_buildinfo.sh index 5c57c5ad88..c5f16ce8c6 100755 --- a/gha-script/read_buildinfo.sh +++ b/gha-script/read_buildinfo.sh @@ -27,7 +27,7 @@ if [ -f $config_file ]; then jsonObj=$config_file build_script=$(jq .build_script $jsonObj) - if $(jq 'has("use_non_root_user")' $jsonObj); then + if $(jq 'has("use_non_root_user")' $jsonObj); then nonRootBuild=$(jq .use_non_root_user $jsonObj) fi @@ -54,11 +54,11 @@ if [ -f $config_file ]; then # Getting specific build_script name and other overrides for version if [[ $(jq --arg ver "$match_version" '.[$ver]' $config_file) != null ]]; then - version_block=".[\"$match_version\"]" # ✅ Properly quoted key for jq + version_block=".[\"$match_version\"]" # Properly quoted key for jq - # version-specific build_script + # version-specific build_script (may be a string or a list) if [[ $(jq -r "$version_block.build_script" $config_file) != "null" ]]; then - build_script=$(jq -r "$version_block.build_script" $config_file) + build_script=$(jq -c "$version_block.build_script" $config_file) fi # version-specific base_docker_image @@ -73,7 +73,7 @@ if [ -f $config_file ]; then "rhel") variant=1 ;; "ubuntu") variant=2 ;; "alpine") variant=3 ;; - *) + *) echo "No valid distro variant, picking default one" variant=1 ;; esac @@ -96,75 +96,207 @@ if [ -f $config_file ]; then fi fi -# #Getting specific build_script name for version -# if [[ $(jq --arg ver "$match_version" '.[$ver]' $config_file) != null ]]; then -# if [[ $(jq -r --arg ver "$match_version" '.[$ver].build_script' $config_file) != null ]]; then -# build_script=$(jq -r --arg ver "$match_version" '.[$ver].build_script' $config_file) -# fi -# if [[ $(jq -r --arg ver "$match_version" '.[$ver].base_docker_image' $config_file) != null ]]; then -# basename=$(jq -r --arg ver "$match_version" '.[$ver].base_docker_image' $config_file) -# fi -# if [[ $(jq -r --arg ver "$match_version" '.[$ver].base_docker_variant' $config_file) != null ]]; then -# variant_str=$(jq -r --arg ver "$match_version" '.[$ver].base_docker_variant' $config_file) -# case "$variant_str" in -# "rhel") -# variant=1 -# ;; -# "ubuntu") -# variant=2 -# ;; -# "alpine") -# variant=3 -# ;; -# *) -# echo "No valid distro variant, picking default one" -# variant=1 -# ;; -# esac -# fi -# fi -# fi - -# Below code is used to get the tested on parameter value from the build script -build_script_with_quotes=$build_script -stripped_build_script=$(echo "$build_script_with_quotes" | sed 's/"//g') -echo $stripped_build_script - - -if [ -f "$stripped_build_script" ]; then - - echo "build script found" - while IFS= read -r line; do - # Check if the line starts with '# Tested on' + + +# --------------------------------------------------------------------------- +# Helper: read "# Tested on" from a single build script file. +# Normalises the raw value: strip spaces, uppercase, collapse "UBI : 9.3" -> +# "UBI:9.3" so downstream consumers see a consistent format. +# Usage: tested_on=$(read_tested_on "path/to/script.sh") +# --------------------------------------------------------------------------- +read_tested_on() { + local script_file="$1" + local value="" + if [ -f "$script_file" ]; then + while IFS= read -r line; do if [[ "$line" == "# Tested on"* ]]; then - # Extract the value after the first colon - tested_on=$(echo "$line" | cut -d ':' -f 2- | tr -d '[:space:]' | tr '[:lower:]' '[:upper:]') - break + # Extract everything after the first colon, strip outer whitespace, + # uppercase, then collapse spaces around colons (e.g. "UBI : 9.3" -> "UBI:9.3") + value=$(echo "$line" | cut -d ':' -f 2- \ + | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \ + | tr '[:lower:]' '[:upper:]' \ + | sed 's/[[:space:]]*:[[:space:]]*/:/g') + break fi + done < "$script_file" + fi + echo "$value" +} + +# --------------------------------------------------------------------------- +# Build the BUILD_SCRIPTS_JSON array and set the single-script compat vars. +# +# build_script in build_info.json can be: +# - a string : "pytorch_ubi_9.3.sh" +# - a list : ["pytorch_ubi_8.3.sh", "pytorch_ubi_9.3.sh", "pytorch_ubi_10.3.sh"] +# +# For both cases we produce: +# BUILD_SCRIPTS_JSON - compact JSON array of {script, tested_on} objects, +# consumed by the GHA matrix via fromJson(). +# BUILD_SCRIPT - backward-compat: first (or only) script filename. +# TESTED_ON - backward-compat: first (or only) tested_on value. +# +# NOTE: '# Tested on' is MANDATORY in every build script. The pipeline will +# abort if it is missing - there is no default fallback. +# --------------------------------------------------------------------------- + +# Derive type from the already-resolved $build_script shell variable. +# This is correct whether build_script came from the top-level field or +# was overridden by a version block - both paths store the final value in +# $build_script as either a bare filename string or a compact JSON array. +script_type=$(echo "$build_script" | jq -r 'type' 2>/dev/null || echo "string") +echo "build_script type (resolved): $script_type" + +BUILD_SCRIPTS_JSON="" + +if [ "$script_type" = "array" ]; then + echo "build_script is a list - iterating each entry to read its '# Tested on' value" + + json_array="[" + first=true + first_script="" + first_tested_on="" + + # jq outputs one bare filename per line (no surrounding quotes) + while IFS= read -r script_name; do + # strip any stray quotes/whitespace that jq might leave + script_name=$(echo "$script_name" | tr -d '"' | xargs) + [ -z "$script_name" ] && continue + + raw_tested_on=$(read_tested_on "$script_name") + + if [ -z "$raw_tested_on" ]; then + echo "ERROR: '# Tested on' header is missing in '$script_name'." + echo " Every build script must declare the UBI version it targets, e.g.:" + echo " # Tested on UBI:9.3" + exit 1 + fi + + echo " script='$script_name' tested_on='$raw_tested_on'" + + # Record first entry for backward-compat single-value exports + if [ "$first" = "true" ]; then + first_script="$script_name" + first_tested_on="$raw_tested_on" + first=false + fi + + # Append JSON object - jq --arg safely escapes both values + entry=$(jq -n \ + --arg s "$script_name" \ + --arg t "$raw_tested_on" \ + '{"script":$s,"tested_on":$t}') - done < "$stripped_build_script" + if [ "$json_array" = "[" ]; then + json_array="${json_array}${entry}" + else + json_array="${json_array},${entry}" + fi + + # Iterate the resolved $build_script variable (not the file) so that + # version-block overrides are respected. + done < <(echo "$build_script" | jq -r '.[]') + + json_array="${json_array}]" + BUILD_SCRIPTS_JSON="$json_array" + + # Backward-compat single values - point to the first entry in the list + build_script="$first_script" + tested_on="$first_tested_on" + +else + # Single string - existing behaviour + build_script_with_quotes=$build_script + stripped_build_script=$(echo "$build_script_with_quotes" | sed 's/"//g') + echo "build_script (single): $stripped_build_script" + + tested_on=$(read_tested_on "$stripped_build_script") + + if [ -z "$tested_on" ]; then + echo "ERROR: '# Tested on' header is missing in '$stripped_build_script'." + echo " Every build script must declare the UBI version it targets, e.g.:" + echo " # Tested on UBI:9.3" + exit 1 + fi echo "Tested on value: $tested_on" + + # Wrap single entry into the same JSON array format for consistency + BUILD_SCRIPTS_JSON=$(jq -n \ + --arg s "$stripped_build_script" \ + --arg t "$tested_on" \ + '[{"script":$s,"tested_on":$t}]') + + # Backward-compat: strip quotes from the jq-extracted string value + build_script="$stripped_build_script" fi -# Extract auditwheel exclusions +echo "BUILD_SCRIPTS_JSON: $BUILD_SCRIPTS_JSON" + +# --------------------------------------------------------------------------- +# Bucket scripts by UBI major version -> SCRIPT_UBI8, SCRIPT_UBI9, SCRIPT_UBI10. +# Each is a single {script, tested_on} JSON object (or empty string ""). +# At most one script per UBI major version is expected per package. +# Callers (currency-build.yaml, pr-build.yaml) use these to drive named jobs: +# build_ubi8 / build_ubi9 / build_ubi10 +# wheel_build_ubi8_pyXXX / wheel_build_ubi9_pyXXX / wheel_build_ubi10_pyXXX +# --------------------------------------------------------------------------- +SCRIPT_UBI8="" +SCRIPT_UBI9="" +SCRIPT_UBI10="" + +while IFS= read -r entry; do + [ -z "$entry" ] && continue + t_on=$(echo "$entry" | jq -r '.tested_on') + # Normalise: uppercase, collapse separators around UBI, then extract the + # integer immediately following "UBI". Uses only sed + tr (POSIX) - no + # grep -P needed, so it works on ppc64le runners where grep -P is absent. + t_upper=$(echo "$t_on" | tr '[:lower:]' '[:upper:]') + # Collapse "UBI : 9.3" / "UBI:9.3" / "UBI 9.3" / "UBI9.3" -> "UBI9.3" + t_norm=$(echo "$t_upper" | sed 's/UBI[[:space:]]*[: ][[:space:]]*/UBI/g') + # Extract digits immediately after "UBI" e.g. "UBI10.0" -> "10" + major=$(echo "$t_norm" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + # If sed left non-numeric content (no UBI match), clear it + case "$major" in + ''|*[!0-9]*) major="" ;; + esac + echo " bucket: tested_on='$t_on' major='$major'" + case "$major" in + 8) SCRIPT_UBI8="$entry" ;; + 9) SCRIPT_UBI9="$entry" ;; + 10) SCRIPT_UBI10="$entry" ;; + *) echo "WARNING: Unknown UBI major '$major' in tested_on='$t_on' - skipping bucket" ;; + esac +done < <(echo "$BUILD_SCRIPTS_JSON" | jq -c '.[]') + +echo "SCRIPT_UBI8: $SCRIPT_UBI8" +echo "SCRIPT_UBI9: $SCRIPT_UBI9" +echo "SCRIPT_UBI10: $SCRIPT_UBI10" + +# Extract auditwheel exclusions (unchanged - same pattern as before) AUDITWHEEL_EXCLUDE="" if jq -e 'has("auditwheel_exclude")' "$config_file" >/dev/null; then AUDITWHEEL_EXCLUDE=$(jq -r '.auditwheel_exclude | join(" ")' "$config_file") fi -# Export variables - -echo "export VERSION=$VERSION" > $CUR_DIR/variable.sh -echo "export BUILD_SCRIPT=$build_script" >> $CUR_DIR/variable.sh -echo "export PKG_DIR_PATH=$package_dirpath" >> $CUR_DIR/variable.sh -echo "export IMAGE_NAME=$image_name" >> $CUR_DIR/variable.sh -#echo "export BUILD_DOCKER=$build_docker" >> $CUR_DIR/variable.sh -#echo "export VALIDATE_BUILD_SCRIPT=$validate_build_script" >> $CUR_DIR/variable.sh -echo "export VARIANT=$variant" >> $CUR_DIR/variable.sh -echo "export BASENAME=$basename" >> $CUR_DIR/variable.sh -echo "export NON_ROOT_BUILD=$nonRootBuild" >> $CUR_DIR/variable.sh -echo "export TESTED_ON=$tested_on" >> $CUR_DIR/variable.sh -echo "export AUDITWHEEL_EXCLUDE=\"$AUDITWHEEL_EXCLUDE\"" >> $CUR_DIR/variable.sh +# --------------------------------------------------------------------------- +# Write variable.sh +# JSON objects are single-quote-wrapped so embedded double-quotes survive. +# --------------------------------------------------------------------------- +echo "export VERSION=\"$VERSION\"" > $CUR_DIR/variable.sh +echo "export BUILD_SCRIPT=\"$build_script\"" >> $CUR_DIR/variable.sh +echo "export PKG_DIR_PATH=\"$package_dirpath\"" >> $CUR_DIR/variable.sh +echo "export IMAGE_NAME=\"$image_name\"" >> $CUR_DIR/variable.sh +echo "export VARIANT=\"$variant\"" >> $CUR_DIR/variable.sh +echo "export BASENAME=\"$basename\"" >> $CUR_DIR/variable.sh +echo "export NON_ROOT_BUILD=\"$nonRootBuild\"" >> $CUR_DIR/variable.sh +echo "export TESTED_ON=\"$tested_on\"" >> $CUR_DIR/variable.sh +echo "export AUDITWHEEL_EXCLUDE=\"$AUDITWHEEL_EXCLUDE\"" >> $CUR_DIR/variable.sh +# Full array - kept for any downstream consumer that still needs it +echo "export BUILD_SCRIPTS_JSON='$BUILD_SCRIPTS_JSON'" >> $CUR_DIR/variable.sh +# Per-UBI-major named exports - empty string when that UBI version has no script +echo "export SCRIPT_UBI8='$SCRIPT_UBI8'" >> $CUR_DIR/variable.sh +echo "export SCRIPT_UBI9='$SCRIPT_UBI9'" >> $CUR_DIR/variable.sh +echo "export SCRIPT_UBI10='$SCRIPT_UBI10'" >> $CUR_DIR/variable.sh chmod +x $CUR_DIR/variable.sh cat $CUR_DIR/variable.sh From 7986eeee643afa1f89c5fdb093260c4825c0b0b4 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:50:48 +0530 Subject: [PATCH 28/45] Generalise UBI image resolution in build_package.sh to support UBI10 - Replace hard-coded UBI8/UBI9/UBI10 if/elif chain with a resolve_docker_image() function that parses UBI major version from the TESTED_ON variable using sed (no grep -P which is absent on ppc64le runners) - Supports all TESTED_ON formats: UBI:8.10, UBI 9.3, UBI:10.0, ubi10.0, UBI : 10.0 etc. - Resolves correct registry image for UBI8, UBI9 and UBI10 --- gha-script/build_package.sh | 101 ++++++++++++++++++++++++------------ 1 file changed, 69 insertions(+), 32 deletions(-) diff --git a/gha-script/build_package.sh b/gha-script/build_package.sh index d7a28c7baa..63d1317e8e 100755 --- a/gha-script/build_package.sh +++ b/gha-script/build_package.sh @@ -10,44 +10,81 @@ echo "************************************************************************** docker_image="" -# the below function is used for building a custom docker image, it will be called only when non root user build is set to true. -# function accepts one argument, which is the base image value. +# Builds a custom non-root wrapper image on top of the given base image. +# Sets docker_image to "docker_non_root_image" on success. docker_build_non_root() { echo "building docker image for non root user build" docker build --build-arg BASE_IMAGE="$1" -t docker_non_root_image -f gha-script/dockerfile_non_root . docker_image="docker_non_root_image" } -#Below conditions are used to select the base image based on the 2 flags, tested_on and non_root_build. -#A docker_build_non_root function is called when non root build is true. -if [[ "$TESTED_ON" == UBI:9* || "$TESTED_ON" == UBI9* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - docker pull registry.access.redhat.com/ubi9/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi9/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi9/ubi:$ubi_version" - fi -elif [[ "$TESTED_ON" == UBI:10* || "$TESTED_ON" == UBI10* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - # Fallback to 10.0 if no minor version found - ubi_version=${ubi_version:-10.0} - docker pull registry.access.redhat.com/ubi10/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi10/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi10/ubi:$ubi_version" - fi -else - docker pull registry.access.redhat.com/ubi8/ubi:8.7 - docker_image="registry.access.redhat.com/ubi8/ubi:8.7" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi8/ubi:8.7" - fi -fi +# Resolves the correct UBI registry image from the TESTED_ON string and +# sets the global docker_image variable. Supports UBI8, UBI9, UBI10+. +# The TESTED_ON value is normalised before parsing so all of the following +# formats work: "UBI:9.3" "UBI 9.3" "ubi9.3" "UBI: 9.3" "UBI 10" +resolve_docker_image() { + local raw="$1" + local upper + upper=$(echo "$raw" | tr '[:lower:]' '[:upper:]') + + # Collapse "UBI : 9.3" / "UBI:9.3" / "UBI 9.3" / "UBI9.3" -> "UBI9.3" + # Uses sed only - no grep -P (absent on ppc64le runners). + local norm + norm=$(echo "$upper" | sed 's/UBI[[:space:]]*[: ][[:space:]]*/UBI/g') + + # Extract the major UBI version integer immediately after "UBI" + local major + major=$(echo "$norm" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + case "$major" in + ''|*[!0-9]*) major="" ;; + esac + + if [ -z "$major" ]; then + echo "ERROR: Cannot determine UBI major version from TESTED_ON='$raw'" + exit 1 + fi + + # Extract the full version string e.g. "9.3", "10.3" (first X.Y found) + local full + full=$(echo "$upper" | grep -oE '[0-9]+\.[0-9]+' | head -1) + + if [ -z "$full" ]; then + # No minor version supplied - use per-major defaults + case "$major" in + 8) full="8.7" ;; + 9) full="9.3" ;; + 10) full="10.0" ;; + *) full="${major}.0" ;; + esac + echo "WARNING: No minor version in TESTED_ON='$raw', defaulting to UBI ${full}" + fi + + # Select the registry image by major version + case "$major" in + 8) + docker_image="registry.access.redhat.com/ubi8/ubi:${full}" + ;; + 9) + docker_image="registry.access.redhat.com/ubi9/ubi:${full}" + ;; + 10) + docker_image="registry.access.redhat.com/ubi10/ubi:${full}" + ;; + *) + echo "ERROR: Unsupported UBI major version '${major}' in TESTED_ON='$raw'" + exit 1 + ;; + esac + + echo " Resolved Docker image: $docker_image (TESTED_ON='$raw')" + + if [[ "$NON_ROOT_BUILD" == "true" ]]; then + docker_build_non_root "$docker_image" + fi +} + +resolve_docker_image "$TESTED_ON" +docker pull "$docker_image" python3 gha-script/validate_builds_currency.py "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$docker_image" 2>&1 | tee build_log From 94a3a43970188d75c889e92d58a051fcc8359c56 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:51:26 +0530 Subject: [PATCH 29/45] Generalise UBI image resolution in build_wheels.sh to support UBI10 - Replace hard-coded UBI8/UBI9/UBI10 if/elif chain with the same resolve_docker_image() function used in build_package.sh - Parses UBI major version from TESTED_ON using sed only - Resolves correct registry image for UBI8, UBI9 and UBI10 --- gha-script/build_wheels.sh | 114 ++++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 46 deletions(-) diff --git a/gha-script/build_wheels.sh b/gha-script/build_wheels.sh index 59d246dbeb..092766c2da 100644 --- a/gha-script/build_wheels.sh +++ b/gha-script/build_wheels.sh @@ -10,61 +10,83 @@ echo "************************************************************************** docker_image="" -# the below function is used for building a custom docker image, it will be called only when non root user build is set to true. -# function accepts one argument, which is the base image value. +# Builds a custom non-root wrapper image on top of the given base image. +# Sets docker_image to "docker_non_root_image" on success. docker_build_non_root() { echo "building docker image for non root user build" docker build --build-arg BASE_IMAGE="$1" -t docker_non_root_image -f gha-script/dockerfile_non_root . docker_image="docker_non_root_image" } -#Below conditions are used to select the base image based on the 2 flags, tested_on and non_root_build. -#A docker_build_non_root function is called when non root build is true. -if [[ "$TESTED_ON" == UBI:9* || "$TESTED_ON" == UBI9* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - docker pull registry.access.redhat.com/ubi9/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi9/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi9/ubi:$ubi_version" - fi -elif [[ "$TESTED_ON" == UBI:10* || "$TESTED_ON" == UBI10* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - # Fallback to 10.0 if no minor version found - ubi_version=${ubi_version:-10.0} - docker pull registry.access.redhat.com/ubi10/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi10/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi10/ubi:$ubi_version" - fi -else - docker pull registry.access.redhat.com/ubi8/ubi:8.7 - docker_image="registry.access.redhat.com/ubi8/ubi:8.7" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi8/ubi:8.7" - fi -fi +# Resolves the correct UBI registry image from the TESTED_ON string and +# sets the global docker_image variable. Supports UBI8, UBI9, UBI10+. +# The TESTED_ON value is normalised before parsing so all of the following +# formats work: "UBI:9.3" "UBI 9.3" "ubi9.3" "UBI: 9.3" "UBI 10" +resolve_docker_image() { + local raw="$1" + local upper + upper=$(echo "$raw" | tr '[:lower:]' '[:upper:]') -# UBI 10 does not ship Python 3.10 or 3.11 in its repos. -# Building 3.10 from source is not supported for UBI 10 wheel builds. -# Skip those Python versions early to avoid false-positive successes. -if [[ "$TESTED_ON" == UBI:10* || "$TESTED_ON" == UBI10* ]]; then - if [[ "$PYTHON_VERSION" == "3.10" || "$PYTHON_VERSION" == "3.11" ]]; then - echo "Skipping wheel build for Python $PYTHON_VERSION — not supported on UBI 10" - exit 0 - fi -fi + # Collapse "UBI : 9.3" / "UBI:9.3" / "UBI 9.3" / "UBI9.3" -> "UBI9.3" + # Uses sed only - no grep -P (absent on ppc64le runners). + local norm + norm=$(echo "$upper" | sed 's/UBI[[:space:]]*[: ][[:space:]]*/UBI/g') -WHEEL_SCRIPT=gha-script/create_wheel_wrapper.sh + # Extract the major UBI version integer immediately after "UBI" + local major + major=$(echo "$norm" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + case "$major" in + ''|*[!0-9]*) major="" ;; + esac + + if [ -z "$major" ]; then + echo "ERROR: Cannot determine UBI major version from TESTED_ON='$raw'" + exit 1 + fi -# Ensure the wrapper script is readable and executable by all users. -# Required for non-root container builds where the mounted volume is owned -# by the host runner user but executed as test_user inside the container. -chmod a+rx "$WHEEL_SCRIPT" + # Extract the full version string e.g. "9.3", "10.3" (first X.Y found) + local full + full=$(echo "$upper" | grep -oE '[0-9]+\.[0-9]+' | head -1) + + if [ -z "$full" ]; then + # No minor version supplied - use per-major defaults + case "$major" in + 8) full="8.7" ;; + 9) full="9.3" ;; + 10) full="10.0" ;; + *) full="${major}.0" ;; + esac + echo "WARNING: No minor version in TESTED_ON='$raw', defaulting to UBI ${full}" + fi + + # Select the registry image by major version + case "$major" in + 8) + docker_image="registry.access.redhat.com/ubi8/ubi:${full}" + ;; + 9) + docker_image="registry.access.redhat.com/ubi9/ubi:${full}" + ;; + 10) + docker_image="registry.access.redhat.com/ubi10/ubi:${full}" + ;; + *) + echo "ERROR: Unsupported UBI major version '${major}' in TESTED_ON='$raw'" + exit 1 + ;; + esac + + echo " Resolved Docker image: $docker_image (TESTED_ON='$raw')" + + if [[ "$NON_ROOT_BUILD" == "true" ]]; then + docker_build_non_root "$docker_image" + fi +} + +resolve_docker_image "$TESTED_ON" +docker pull "$docker_image" + +WHEEL_SCRIPT=gha-script/create_wheel_wrapper.sh # path to post_process_wheel script (suffix addition, license addition, metadata addition) POST_PROCESS_SCRIPT_PATH=gha-script/post_process_wheel.py From 15248494a737a4963910a020e43bccf626162513 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:51:59 +0530 Subject: [PATCH 30/45] Pass GRYPE_BIN and ENABLE_CVE_SCAN into wheel build container - Pass GRYPE_BIN environment variable into the Docker container so generalized_wheel_scanner.py can locate the pre-cached grype binary mounted at /home/tester/scan-tools-bin/grype - Pass ENABLE_CVE_SCAN environment variable so PR builds can set it to false and skip the CVE scan step inside the container --- gha-script/build_wheels.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gha-script/build_wheels.py b/gha-script/build_wheels.py index 0fdd6f16a0..b3581a7755 100644 --- a/gha-script/build_wheels.py +++ b/gha-script/build_wheels.py @@ -43,6 +43,15 @@ def trigger_build_wheel(wrapper_file, python_version, image_name, file_name, ver "GHA_CURRENCY_SERVICE_ID_API_KEY": os.getenv("GHA_CURRENCY_SERVICE_ID_API_KEY"), "GHA_CURRENCY_SERVICE_ID": os.getenv("GHA_CURRENCY_SERVICE_ID"), "AUDITWHEEL_EXCLUDE": os.getenv("AUDITWHEEL_EXCLUDE", ""), + # Grype is installed on the host runner at scan-tools-bin/grype. + # The workspace is volume-mounted at /home/tester/ inside the + # container, so the binary is reachable at that in-container path. + # Passing GRYPE_BIN lets generalized_wheel_scanner.py find it via + # os.environ without relying on $PATH (which is host-only). + "GRYPE_BIN": "/home/tester/scan-tools-bin/grype", + # Set to "false" by pr-build.yaml to skip the CVE scan in PR builds. + # Defaults to "true" (scan runs) when unset (currency-build.yaml). + "ENABLE_CVE_SCAN": os.getenv("ENABLE_CVE_SCAN", "true"), } ) From 0d5a163936e58cc1b11fd90d0dd5593f7077f9a1 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:54:27 +0530 Subject: [PATCH 31/45] Add UBI10 GCC toolset support in create_wheel_wrapper.sh - Select GCC toolset version based on UBI major version: gcc-toolset-15 for UBI10+, gcc-toolset-13 for UBI8/UBI9 - On UBI10, SCL (Software Collections) was dropped so no enable script is sourced; PATH is updated directly instead --- gha-script/create_wheel_wrapper.sh | 95 ++++++++++++++++-------------- 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/gha-script/create_wheel_wrapper.sh b/gha-script/create_wheel_wrapper.sh index b699b68279..011666f26f 100644 --- a/gha-script/create_wheel_wrapper.sh +++ b/gha-script/create_wheel_wrapper.sh @@ -7,28 +7,20 @@ EXTRA_ARGS=${3:-""} POST_PROCESS_SCRIPT_PATH=${4:-"post_process_wheel.py"} CURRENT_DIR=$(pwd) -# Use sudo for privileged commands when running as a non-root user. -# dockerfile_non_root grants test_user passwordless sudo, so this is always safe. -# When already root, sudo is not needed (and may not be installed), so skip it. -if [[ "$(id -u)" -ne 0 ]]; then - YUM="sudo yum" - SUDO="sudo" -else - YUM="yum" - SUDO="" -fi +# install git - required by generate_sha() for all Python versions and UBI versions +yum install -y git -# install gcc — select toolset version based on UBI major version +# install gcc - select toolset version based on UBI major version UBI_MAJOR=$(grep -oP '(?<=^VERSION_ID=")[0-9]+' /etc/os-release || grep -oP 'release \K[0-9]+' /etc/redhat-release 2>/dev/null || echo "8") if [[ "$UBI_MAJOR" -ge 10 ]]; then GCC_TOOLSET="gcc-toolset-15" - $YUM install -y "$GCC_TOOLSET" - # On UBI 10, SCL (Software Collections) was dropped — there is no enable script. + yum install -y "$GCC_TOOLSET" + # On UBI 10, SCL (Software Collections) was dropped - there is no enable script. # Activate the toolset by prepending its bin directory to PATH directly. export PATH="/opt/rh/${GCC_TOOLSET}/root/usr/bin:$PATH" else GCC_TOOLSET="gcc-toolset-13" - $YUM install -y "$GCC_TOOLSET" + yum install -y "$GCC_TOOLSET" source /opt/rh/${GCC_TOOLSET}/enable fi gcc --version @@ -48,48 +40,48 @@ install_python_version() { echo case $version in "3.11" | "3.12") - $YUM install -y python${version} python${version}-devel python${version}-pip + yum install -y python${version} python${version}-devel python${version}-pip ;; "3.10") if ! python3.10 --version &>/dev/null; then - $YUM install -y zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel - $YUM install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel + yum install -y sudo zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel + yum install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel wget https://www.python.org/ftp/python/3.10.20/Python-3.10.20.tgz tar xf Python-3.10.20.tgz cd Python-3.10.20 ./configure --prefix=/usr/local --enable-optimizations --enable-shared make -j2 make altinstall - echo "/usr/local/lib" | $SUDO tee /etc/ld.so.conf.d/python-local.conf && $SUDO ldconfig + echo "/usr/local/lib" > /etc/ld.so.conf.d/python-local.conf && ldconfig echo "Completed..." cd .. && rm -rf Python-3.10.20.tgz fi ;; "3.13") if ! python3.13 --version &>/dev/null; then - $YUM install -y zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel - $YUM install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel + yum install -y sudo zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel + yum install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel wget https://www.python.org/ftp/python/3.13.10/Python-3.13.10.tgz tar xzf Python-3.13.10.tgz cd Python-3.13.10 ./configure --prefix=/usr/local --enable-optimizations --enable-shared make -j2 make altinstall - echo "/usr/local/lib" | $SUDO tee /etc/ld.so.conf.d/python-local.conf && $SUDO ldconfig + echo "/usr/local/lib" > /etc/ld.so.conf.d/python-local.conf && ldconfig cd .. && rm -rf Python-3.13.10.tgz fi ;; "3.14") if ! python3.14 --version &>/dev/null; then - $YUM install -y zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel - $YUM install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel + yum install -y sudo zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel + yum install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel wget https://www.python.org/ftp/python/3.14.3/Python-3.14.3.tgz tar xzf Python-3.14.3.tgz cd Python-3.14.3 ./configure --prefix=/usr/local --enable-optimizations --enable-shared make -j2 make altinstall - echo "/usr/local/lib" | $SUDO tee /etc/ld.so.conf.d/python-local.conf && $SUDO ldconfig + echo "/usr/local/lib" > /etc/ld.so.conf.d/python-local.conf && ldconfig cd .. && rm -rf Python-3.14.3.tgz fi ;; @@ -345,16 +337,31 @@ fi cd "$CURRENT_DIR" wheel_final=(*.whl) -echo -echo "============== Running CVE scan on: ${wheel_final} ==============" -echo +# --------------------------------------------------------------------------- +# run_cve_scan: runs generalized_wheel_scanner.py on the built wheel. +# +# To skip CVE scanning when testing wheel creation locally, comment out the +# run_cve_scan call below (search for "run_cve_scan" further down). +# +# In CI the ENABLE_CVE_SCAN env var controls this: +# ENABLE_CVE_SCAN=false -> skip (set by pr-build.yaml) +# unset or "true" -> run (default for currency-build.yaml) +# --------------------------------------------------------------------------- +run_cve_scan() { + local wheel=$1 + local build_script=$2 + + SCANNER_PATH="gha-script/generalized_wheel_scanner.py" + if [ ! -f "$SCANNER_PATH" ]; then + echo "===> WARNING: $SCANNER_PATH not found, skipping CVE scan." + return 0 + fi + + echo + echo "============== Running CVE scan on: ${wheel} ==============" + echo -# Run CVE scanner — passes wheel + original build script so Lane 3 can parse -# git clone/checkout lines for source-built lib names and versions. -# Scanner is non-blocking — failure does not stop the build. -SCANNER_PATH="gha-script/generalized_wheel_scanner.py" -if [ -f "$SCANNER_PATH" ]; then - if python "$SCANNER_PATH" "${wheel_final}" "${BUILD_SCRIPT_PATH}"; then + if python "$SCANNER_PATH" "${wheel}" "${build_script}"; then echo echo "===> CVE scan completed successfully." echo @@ -363,9 +370,7 @@ if [ -f "$SCANNER_PATH" ]; then echo "===> WARNING: CVE scan failed. Continuing build." echo fi -else - echo "===> WARNING: $SCANNER_PATH not found, skipping CVE scan." -fi +} echo echo "============== Generating sha for: ${wheel_final} ==============" @@ -380,9 +385,6 @@ echo echo "=== Post Processing wheel ${wheel_final} with SHA: ${SHA256_VALUE} ===" echo -# Save CVE report name before post-processing renames the wheel -cve_report_old="${wheel_final%.whl}_cve_report.json" - # post processing of wheels (Suffix addition, license addition, metadata addition) if python ${POST_PROCESS_SCRIPT_PATH} ${wheel_final} ${SHA256_VALUE}; then echo @@ -395,12 +397,17 @@ else exit 1 fi -# Rename CVE report to match the post-processed wheel filename +# CVE scan runs after post-processing so the report is named after the final +# wheel filename (with +ppc64leN suffix) from the start - no rename needed. wheel_post_processed=(*.whl) -cve_report_new="${wheel_post_processed[0]%.whl}_cve_report.json" -if [ -f "$cve_report_old" ] && [ "$cve_report_old" != "$cve_report_new" ]; then - mv "$cve_report_old" "$cve_report_new" - echo "===> CVE report renamed: $cve_report_old → $cve_report_new" + +# Call run_cve_scan - comment out this block locally to skip CVE scanning. +if [ "${ENABLE_CVE_SCAN:-true}" = "false" ]; then + echo + echo "===> Skipping CVE scan (ENABLE_CVE_SCAN=false)." + echo +else + run_cve_scan "${wheel_post_processed[0]}" "${BUILD_SCRIPT_PATH}" fi echo From c25fd5c72b9ed6e852c063e979cbac51945b1a73 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:55:31 +0530 Subject: [PATCH 32/45] Resolve grype binary via GRYPE_BIN env var in wheel scanner - Locate grype using GRYPE_BIN environment variable (set by build_wheels.py to the pre-cached binary path) with fallback to shutil.which('grype') for local/manual runs - Previously relied solely on shutil.which which fails inside the container because grype is not on the container PATH --- gha-script/generalized_wheel_scanner.py | 33 +++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/gha-script/generalized_wheel_scanner.py b/gha-script/generalized_wheel_scanner.py index 116288acbc..aa00c890fa 100644 --- a/gha-script/generalized_wheel_scanner.py +++ b/gha-script/generalized_wheel_scanner.py @@ -54,16 +54,29 @@ def scan(self) -> Dict: print(f"Scanning Wheel: {self.wheel_name}") print(f"{'='*70}\n") - # Step 0: Check grype availability — grype is installed by the workflow - # step before this script runs; if it is missing, grype-dependent lanes - # are skipped gracefully and Lane 3 (NVD) still runs. + # Step 0: Locate grype. + # Grype is installed in the workflow's install_scan_tools job and cached + # as scan-tools-bin/grype in the artifact. The workspace is volume-mounted + # into this container at /home/tester/, so the binary is accessible at + # /home/tester/scan-tools-bin/grype — passed in via the GRYPE_BIN env var + # by build_wheels.py. We honour GRYPE_BIN first; if unset or the path is + # not executable we fall back to shutil.which('grype') for local runs. + # If neither resolves, grype-dependent lanes are skipped gracefully and + # Lane 3 (NVD) still runs. print("Step 0: Checking grype availability...") - grype_available = bool(shutil.which('grype')) - if grype_available: - print(" grype is available.") + grype_bin = os.environ.get("GRYPE_BIN", "").strip() + if grype_bin and os.path.isfile(grype_bin) and os.access(grype_bin, os.X_OK): + grype_available = True + print(f" grype found via GRYPE_BIN: {grype_bin}") else: - print(" WARNING: grype not found. Lane 1, Lane 2 and Phase 1 " - "(wheel direct scan) will be skipped. Lane 3 (NVD) will still run.") + grype_bin = shutil.which('grype') or "" + grype_available = bool(grype_bin) + if grype_available: + print(f" grype found on PATH: {grype_bin}") + else: + print(" WARNING: grype not found. Lane 1, Lane 2 and Phase 1 " + "(wheel direct scan) will be skipped. Lane 3 (NVD) will still run.") + self.grype_bin = grype_bin # used by _scan_wheel_direct / _scan_for_cves self.results['grype_available'] = grype_available # Step 1: Create system library inventory @@ -471,7 +484,7 @@ def _scan_wheel_direct(self, extract_dir: Path) -> Dict: print(f" Running grype on extracted wheel: {extract_dir}") try: result = subprocess.run( - ['grype', f'dir:{extract_dir}', '-o', 'json'], + [self.grype_bin, f'dir:{extract_dir}', '-o', 'json'], capture_output=True, text=True, timeout=300 ) if result.returncode == 0: @@ -512,7 +525,7 @@ def _scan_for_cves(self) -> Dict: try: result = subprocess.run( - ['grype', 'dir:/', '-o', 'json'], + [self.grype_bin, 'dir:/', '-o', 'json'], capture_output=True, text=True, timeout=300 From ddc26be7dc473aca31d3901e88e52829195f6847 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:56:06 +0530 Subject: [PATCH 33/45] Improve COS configuration handling Updated COS configuration to handle missing credentials gracefully in non-currency builds. --- gha-script/post_process_wheel.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/gha-script/post_process_wheel.py b/gha-script/post_process_wheel.py index ff6c31dbf0..fc7bf1aa56 100644 --- a/gha-script/post_process_wheel.py +++ b/gha-script/post_process_wheel.py @@ -43,9 +43,15 @@ logger = logging.getLogger(__name__) -# COS configuration -COS_API_KEY = os.environ["GHA_CURRENCY_SERVICE_ID_API_KEY"] -COS_SERVICE_INSTANCE_ID = os.environ["GHA_CURRENCY_SERVICE_ID"] +# COS configuration +# These are only available in currency builds (not PR builds). +# If missing, post-processing is skipped gracefully. +COS_API_KEY = os.environ.get("GHA_CURRENCY_SERVICE_ID_API_KEY", "") +COS_SERVICE_INSTANCE_ID = os.environ.get("GHA_CURRENCY_SERVICE_ID", "") + +if not COS_API_KEY or not COS_SERVICE_INSTANCE_ID: + logger.info("COS credentials not set - skipping post-processing (PR build environment)") + sys.exit(0) COS_ENDPOINT = "https://s3.us.cloud-object-storage.appdomain.cloud" COS_BUCKET = "ose-power-artifacts-production" From cc1b950a6d2dcccbfd1d3b234c1fd0e7833b0d26 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:57:12 +0530 Subject: [PATCH 34/45] Use pre-cached grype binary in grype_code_scan.sh - Remove runtime grype download (wget + tar + install on every run) - Use $GRYPE_BIN environment variable pointing to the pre-cached binary from the workflow's install_scan_tools artifact - Fail fast with a clear error if GRYPE_BIN is not set --- gha-script/scanner-scripts/grype_code_scan.sh | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/gha-script/scanner-scripts/grype_code_scan.sh b/gha-script/scanner-scripts/grype_code_scan.sh index ceec35eb9d..a865e9f2bb 100644 --- a/gha-script/scanner-scripts/grype_code_scan.sh +++ b/gha-script/scanner-scripts/grype_code_scan.sh @@ -3,18 +3,19 @@ validate_build_script=$VALIDATE_BUILD_SCRIPT cloned_package=$CLONED_PACKAGE +# Use pre-installed grype from the cached artifact. +# $GRYPE_BIN is set by the workflow (points to scan-tools-bin/grype). +if [ -z "$GRYPE_BIN" ]; then + echo "Error: GRYPE_BIN environment variable not set" + exit 1 +fi + cd package-cache -if [ $validate_build_script == true ];then - GRYPE_VERSION=$(curl -s https://api.github.com/repos/anchore/grype/releases/latest | grep -Po '"tag_name": "\K.*?(?=")') - wget https://github.com/anchore/grype/releases/download/$GRYPE_VERSION/grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - tar -xzf grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - chmod +x grype - sudo mv grype /usr/bin - grype --version +if [ $validate_build_script == true ]; then + echo "------------- Using cached grype ---------------" + $GRYPE_BIN version echo "Executing Grype scanner" - sudo grype -q -o cyclonedx-json dir:${cloned_package} > grype_source_sbom_results.json - #cat grype_source_sbom_results.json - sudo grype -q -o json dir:${cloned_package} > grype_source_vulnerabilities_results.json - #cat grype_source_vulnerabilities_results.json + sudo $GRYPE_BIN -q -o cyclonedx-json dir:${cloned_package} > grype_source_sbom_results.json + sudo $GRYPE_BIN -q -o json dir:${cloned_package} > grype_source_vulnerabilities_results.json fi From 9b034ab8b7e2d405e087938f656ae36c7520d370 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:57:38 +0530 Subject: [PATCH 35/45] Use pre-cached grype binary in grype_image_scan.sh - Remove runtime grype download (wget + tar + install on every run) - Use $GRYPE_BIN environment variable pointing to the pre-cached binary from the workflow's install_scan_tools artifact - Fail fast with a clear error if GRYPE_BIN is not set --- .../scanner-scripts/grype_image_scan.sh | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/gha-script/scanner-scripts/grype_image_scan.sh b/gha-script/scanner-scripts/grype_image_scan.sh index f91b4bbbfd..f9101d0087 100755 --- a/gha-script/scanner-scripts/grype_image_scan.sh +++ b/gha-script/scanner-scripts/grype_image_scan.sh @@ -3,13 +3,17 @@ image_name=$IMAGE_NAME build_docker=$BUILD_DOCKER -if [ $build_docker == true ];then - GRYPE_VERSION=$(curl -s https://api.github.com/repos/anchore/grype/releases/latest | grep -Po '"tag_name": "\K.*?(?=")') - wget https://github.com/anchore/grype/releases/download/$GRYPE_VERSION/grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - tar -xzf grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - chmod +x grype - sudo mv grype /usr/bin - echo "Executing grype scanner" - sudo grype -q -s AllLayers -o cyclonedx-json ${image_name} > grype_image_sbom_results.json - sudo grype -q -s AllLayers -o json ${image_name} > grype_image_vulnerabilities_results.json +# Use pre-installed grype from the cached artifact. +# $GRYPE_BIN is set by the workflow (points to scan-tools-bin/grype). +if [ -z "$GRYPE_BIN" ]; then + echo "Error: GRYPE_BIN environment variable not set" + exit 1 +fi + +if [ $build_docker == true ]; then + echo "------------- Using cached grype ---------------" + $GRYPE_BIN version + echo "Executing grype scanner" + sudo $GRYPE_BIN -q -s AllLayers -o cyclonedx-json ${image_name} > grype_image_sbom_results.json + sudo $GRYPE_BIN -q -s AllLayers -o json ${image_name} > grype_image_vulnerabilities_results.json fi From 3c7b0280695c255b7bd35ffd858a63a188bd4b34 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 02:58:26 +0530 Subject: [PATCH 36/45] Restore GITHUB_BUILD_SCRIPT_BASE_OWNER to ppc64le - Change GITHUB_BUILD_SCRIPT_BASE_OWNER from stutiibm back to ppc64le so PR validation fetches file lists from the correct upstream repo From 388a3dc0da3b8c561632ed50a2818a3748dfd55c Mon Sep 17 00:00:00 2001 From: stutiibm Date: Tue, 4 Aug 2026 11:47:19 +0530 Subject: [PATCH 37/45] Restore GITHUB_BUILD_SCRIPT_BASE_OWNER to ppc64le --- gha-script/validate_builds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index 0f830d7e8e..95f00c4b8d 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -11,7 +11,7 @@ GITHUB_BUILD_SCRIPT_BASE_REPO = "build-scripts" -GITHUB_BUILD_SCRIPT_BASE_OWNER = "stutiibm" +GITHUB_BUILD_SCRIPT_BASE_OWNER = "ppc64le" HOME = os.getcwd() package_data = {} From 08fbf1fdf8def4424bed3df07ab21ea1d8807eb5 Mon Sep 17 00:00:00 2001 From: stutiibm Date: Tue, 4 Aug 2026 11:47:25 +0530 Subject: [PATCH 38/45] Remove wheel_build_ubi8_py314: Python 3.14 not supported on UBI8 --- .github/workflows/currency-build.yaml | 95 +-------------------------- 1 file changed, 2 insertions(+), 93 deletions(-) diff --git a/.github/workflows/currency-build.yaml b/.github/workflows/currency-build.yaml index 764b4324d1..9a9f1b0bca 100644 --- a/.github/workflows/currency-build.yaml +++ b/.github/workflows/currency-build.yaml @@ -719,96 +719,7 @@ jobs: WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_ubi8_py314: - needs: [build_info, install_scan_tools] - if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} - continue-on-error: true - env: - PYTHON_VERSION: "3.14" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v4 - with: - name: scan-tools-cache - - name: Extract scan tools - run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - name: Cache pip (scancode) - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: scancode-pip-ppc64le-v32.4.0 - - name: Download scancode-toolkit-src - uses: actions/download-artifact@v4 - with: - name: scancode-toolkit-src - - name: Extract scancode-toolkit source - run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" - export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" - chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" - [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py314.gz" - gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" - bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - - name: Run scancode scan on wheel - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh - bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - - name: Run grype scan on wheel - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" - chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh - bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - - name: Upload wheel scan results - run: | - source package-cache/variable.sh - mkdir -p wheel_scancode_py314 - find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; - tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314 - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz - mkdir -p grype_wheel_py314 - find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; - tar czf grype_wheel_py314.tar.gz grype_wheel_py314 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz - - name: Upload wheel to COS - run: | - if ls *.whl 1>/dev/null 2>&1; then - sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) - chmod +x ./gha-script/upload-scripts/upload_wheel.sh - bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE - else - echo "No wheel file found!"; exit 1 - fi - - name: Upload CVE report - run: | - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - - # UBI9 wheel jobs (py310–py314) ------------------------------------------------ + # UBI9 wheel jobs (py310-py314) ------------------------------------------------ wheel_build_ubi9_py310: needs: [build_info, install_scan_tools] @@ -1500,7 +1411,6 @@ jobs: - wheel_build_ubi8_py311 - wheel_build_ubi8_py312 - wheel_build_ubi8_py313 - - wheel_build_ubi8_py314 if: | always() && inputs.wheel_build == 'true' && @@ -1508,8 +1418,7 @@ jobs: (needs.wheel_build_ubi8_py310.result == 'success' || needs.wheel_build_ubi8_py311.result == 'success' || needs.wheel_build_ubi8_py312.result == 'success' || - needs.wheel_build_ubi8_py313.result == 'success' || - needs.wheel_build_ubi8_py314.result == 'success') + needs.wheel_build_ubi8_py313.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: - uses: actions/checkout@v4 From b4f179c3b320d85b1d415b9f79cfb4fc9211a65f Mon Sep 17 00:00:00 2001 From: stutiibm Date: Tue, 4 Aug 2026 12:08:20 +0530 Subject: [PATCH 39/45] Remove wheel_build_ubi8_py313: Python 3.13 not supported on UBI8 --- .github/workflows/currency-build.yaml | 93 +-------------------------- .github/workflows/pr-build.yaml | 44 ------------- 2 files changed, 1 insertion(+), 136 deletions(-) diff --git a/.github/workflows/currency-build.yaml b/.github/workflows/currency-build.yaml index 9a9f1b0bca..4d33686b40 100644 --- a/.github/workflows/currency-build.yaml +++ b/.github/workflows/currency-build.yaml @@ -630,95 +630,6 @@ jobs: WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_ubi8_py313: - needs: [build_info, install_scan_tools] - if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} - continue-on-error: true - env: - PYTHON_VERSION: "3.13" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v4 - with: - name: scan-tools-cache - - name: Extract scan tools - run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - name: Cache pip (scancode) - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: scancode-pip-ppc64le-v32.4.0 - - name: Download scancode-toolkit-src - uses: actions/download-artifact@v4 - with: - name: scancode-toolkit-src - - name: Extract scancode-toolkit source - run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" - export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" - chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" - [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py313.gz" - gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" - bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - - name: Run scancode scan on wheel - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh - bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - - name: Run grype scan on wheel - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" - chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh - bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - - name: Upload wheel scan results - run: | - source package-cache/variable.sh - mkdir -p wheel_scancode_py313 - find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; - tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313 - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz - mkdir -p grype_wheel_py313 - find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; - tar czf grype_wheel_py313.tar.gz grype_wheel_py313 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz - - name: Upload wheel to COS - run: | - if ls *.whl 1>/dev/null 2>&1; then - sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) - chmod +x ./gha-script/upload-scripts/upload_wheel.sh - bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE - else - echo "No wheel file found!"; exit 1 - fi - - name: Upload CVE report - run: | - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - # UBI9 wheel jobs (py310-py314) ------------------------------------------------ wheel_build_ubi9_py310: @@ -1410,15 +1321,13 @@ jobs: - wheel_build_ubi8_py310 - wheel_build_ubi8_py311 - wheel_build_ubi8_py312 - - wheel_build_ubi8_py313 if: | always() && inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' && (needs.wheel_build_ubi8_py310.result == 'success' || needs.wheel_build_ubi8_py311.result == 'success' || - needs.wheel_build_ubi8_py312.result == 'success' || - needs.wheel_build_ubi8_py313.result == 'success') + needs.wheel_build_ubi8_py312.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index c329da0707..22ef9dc3da 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -632,50 +632,6 @@ jobs: exit 1 fi - wheel_build_ubi8_py313: - needs: build_info - if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} - continue-on-error: true - env: - PYTHON_VERSION: "3.13" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh - run: | - THIS_SCRIPT="$BUILD_SCRIPT" - THIS_TESTED_ON="$TESTED_ON" - source package-cache/variable.sh; source package-cache/scanner-env.sh - export BUILD_SCRIPT="$THIS_SCRIPT" - export TESTED_ON="$THIS_TESTED_ON" - if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 - sudo chown -R $USER:$USER . - export ENABLE_CVE_SCAN=false - chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh - - name: Verify wheel was produced - run: | - if ls *.whl 1>/dev/null 2>&1; then - echo "Wheel built successfully: $(ls *.whl)" - else - echo "No wheel produced." - exit 1 - fi - # UBI9 wheel jobs ----------------------------------------------------------- wheel_build_ubi9_py310: From b6d4c3f20555be45144af6aef56e51889287909b Mon Sep 17 00:00:00 2001 From: stutiibm Date: Tue, 4 Aug 2026 12:16:18 +0530 Subject: [PATCH 40/45] Remove wheel_build_ubi8_py310: Python 3.10 not supported on UBI8 --- .github/workflows/currency-build.yaml | 101 +------------------------- .github/workflows/pr-build.yaml | 45 ------------ 2 files changed, 1 insertion(+), 145 deletions(-) diff --git a/.github/workflows/currency-build.yaml b/.github/workflows/currency-build.yaml index 4d33686b40..3dc8768df3 100644 --- a/.github/workflows/currency-build.yaml +++ b/.github/workflows/currency-build.yaml @@ -355,103 +355,6 @@ jobs: # Each job needs install_scan_tools for grype/scancode artifacts. # --------------------------------------------------------------------------- - wheel_build_ubi8_py310: - needs: [build_info, install_scan_tools] - if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} - env: - PYTHON_VERSION: "3.10" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v4 - with: - name: scan-tools-cache - - name: Extract scan tools - run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - name: Cache pip (scancode) - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: scancode-pip-ppc64le-v32.4.0 - - name: Download scancode-toolkit-src - uses: actions/download-artifact@v4 - with: - name: scancode-toolkit-src - - name: Extract scancode-toolkit source - run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh - run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh - export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" - export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py310_log}.gz" - [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py310.gz" - gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" - bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - - name: Run scancode scan on wheel - run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh - bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - - name: Run grype scan on wheel - run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh - export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" - chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh - bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - - name: Upload wheel scan results - run: | - source package-cache/variable.sh - mkdir -p wheel_scancode_py310 - find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; - tar czf wheel_scancode_py310.tar.gz wheel_scancode_py310 - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py310.tar.gz - mkdir -p grype_wheel_py310 - find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; - tar czf grype_wheel_py310.tar.gz grype_wheel_py310 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py310.tar.gz - - name: Upload wheel to COS - run: | - if ls *.whl 1>/dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) - chmod +x ./gha-script/upload-scripts/upload_wheel.sh - bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE - else - echo "No wheel file found!"; exit 1 - fi - - name: Upload CVE report - run: | - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - [ -f "$CVE_REPORT" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" || echo "CVE report not found — skipping." - fi - wheel_build_ubi8_py311: needs: [build_info, install_scan_tools] if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} @@ -1318,15 +1221,13 @@ jobs: wheel_license_extract_ubi8: needs: - - wheel_build_ubi8_py310 - wheel_build_ubi8_py311 - wheel_build_ubi8_py312 if: | always() && inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' && - (needs.wheel_build_ubi8_py310.result == 'success' || - needs.wheel_build_ubi8_py311.result == 'success' || + (needs.wheel_build_ubi8_py311.result == 'success' || needs.wheel_build_ubi8_py312.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index 22ef9dc3da..a37971a1df 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -501,51 +501,6 @@ jobs: # PR builds do not run scan steps. # --------------------------------------------------------------------------- - wheel_build_ubi8_py310: - needs: build_info - if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} - env: - PYTHON_VERSION: "3.10" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh - run: | - THIS_SCRIPT="$BUILD_SCRIPT" - THIS_TESTED_ON="$TESTED_ON" - source package-cache/variable.sh - source package-cache/scanner-env.sh - export BUILD_SCRIPT="$THIS_SCRIPT" - export TESTED_ON="$THIS_TESTED_ON" - if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - if [ -z "$BUILD_SCRIPT_CHANGED" ]; then echo "Skipping - no .sh changes"; exit 0; fi - sudo chown -R $USER:$USER . - export ENABLE_CVE_SCAN=false - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - name: Verify wheel was produced - run: | - if ls *.whl 1>/dev/null 2>&1; then - echo "Wheel built successfully: $(ls *.whl)" - else - echo "No wheel produced." - exit 1 - fi - wheel_build_ubi8_py311: needs: build_info if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} From c57336f10a33c8e4bffc02242a75ef6746e0a066 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 12:18:28 +0530 Subject: [PATCH 41/45] Removed python3.10, 3.13, 3.14 support for wheel_build ubi8 --- .github/workflows/currency-build.yaml | 287 +------------------------- 1 file changed, 3 insertions(+), 284 deletions(-) diff --git a/.github/workflows/currency-build.yaml b/.github/workflows/currency-build.yaml index 764b4324d1..3dc8768df3 100644 --- a/.github/workflows/currency-build.yaml +++ b/.github/workflows/currency-build.yaml @@ -355,103 +355,6 @@ jobs: # Each job needs install_scan_tools for grype/scancode artifacts. # --------------------------------------------------------------------------- - wheel_build_ubi8_py310: - needs: [build_info, install_scan_tools] - if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} - env: - PYTHON_VERSION: "3.10" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v4 - with: - name: scan-tools-cache - - name: Extract scan tools - run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - name: Cache pip (scancode) - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: scancode-pip-ppc64le-v32.4.0 - - name: Download scancode-toolkit-src - uses: actions/download-artifact@v4 - with: - name: scancode-toolkit-src - - name: Extract scancode-toolkit source - run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh - run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh - export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" - export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py310_log}.gz" - [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py310.gz" - gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" - bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - - name: Run scancode scan on wheel - run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh - bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - - name: Run grype scan on wheel - run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh - export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" - chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh - bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - - name: Upload wheel scan results - run: | - source package-cache/variable.sh - mkdir -p wheel_scancode_py310 - find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; - tar czf wheel_scancode_py310.tar.gz wheel_scancode_py310 - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py310.tar.gz - mkdir -p grype_wheel_py310 - find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; - tar czf grype_wheel_py310.tar.gz grype_wheel_py310 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py310.tar.gz - - name: Upload wheel to COS - run: | - if ls *.whl 1>/dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) - chmod +x ./gha-script/upload-scripts/upload_wheel.sh - bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE - else - echo "No wheel file found!"; exit 1 - fi - - name: Upload CVE report - run: | - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - [ -f "$CVE_REPORT" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" || echo "CVE report not found — skipping." - fi - wheel_build_ubi8_py311: needs: [build_info, install_scan_tools] if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} @@ -630,185 +533,7 @@ jobs: WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_ubi8_py313: - needs: [build_info, install_scan_tools] - if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} - continue-on-error: true - env: - PYTHON_VERSION: "3.13" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v4 - with: - name: scan-tools-cache - - name: Extract scan tools - run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - name: Cache pip (scancode) - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: scancode-pip-ppc64le-v32.4.0 - - name: Download scancode-toolkit-src - uses: actions/download-artifact@v4 - with: - name: scancode-toolkit-src - - name: Extract scancode-toolkit source - run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" - export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" - chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" - [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py313.gz" - gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" - bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - - name: Run scancode scan on wheel - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh - bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - - name: Run grype scan on wheel - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" - chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh - bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - - name: Upload wheel scan results - run: | - source package-cache/variable.sh - mkdir -p wheel_scancode_py313 - find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; - tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313 - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz - mkdir -p grype_wheel_py313 - find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; - tar czf grype_wheel_py313.tar.gz grype_wheel_py313 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz - - name: Upload wheel to COS - run: | - if ls *.whl 1>/dev/null 2>&1; then - sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) - chmod +x ./gha-script/upload-scripts/upload_wheel.sh - bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE - else - echo "No wheel file found!"; exit 1 - fi - - name: Upload CVE report - run: | - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - - wheel_build_ubi8_py314: - needs: [build_info, install_scan_tools] - if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} - continue-on-error: true - env: - PYTHON_VERSION: "3.14" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v4 - with: - name: scan-tools-cache - - name: Extract scan tools - run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - name: Cache pip (scancode) - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: scancode-pip-ppc64le-v32.4.0 - - name: Download scancode-toolkit-src - uses: actions/download-artifact@v4 - with: - name: scancode-toolkit-src - - name: Extract scancode-toolkit source - run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" - export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" - chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" - [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py314.gz" - gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" - bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - - name: Run scancode scan on wheel - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh - bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - - name: Run grype scan on wheel - run: | - source package-cache/variable.sh; source package-cache/scanner-env.sh - export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" - chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh - bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - - name: Upload wheel scan results - run: | - source package-cache/variable.sh - mkdir -p wheel_scancode_py314 - find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; - tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314 - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz - mkdir -p grype_wheel_py314 - find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; - tar czf grype_wheel_py314.tar.gz grype_wheel_py314 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz - - name: Upload wheel to COS - run: | - if ls *.whl 1>/dev/null 2>&1; then - sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) - chmod +x ./gha-script/upload-scripts/upload_wheel.sh - bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE - else - echo "No wheel file found!"; exit 1 - fi - - name: Upload CVE report - run: | - WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - - # UBI9 wheel jobs (py310–py314) ------------------------------------------------ + # UBI9 wheel jobs (py310-py314) ------------------------------------------------ wheel_build_ubi9_py310: needs: [build_info, install_scan_tools] @@ -1496,20 +1221,14 @@ jobs: wheel_license_extract_ubi8: needs: - - wheel_build_ubi8_py310 - wheel_build_ubi8_py311 - wheel_build_ubi8_py312 - - wheel_build_ubi8_py313 - - wheel_build_ubi8_py314 if: | always() && inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' && - (needs.wheel_build_ubi8_py310.result == 'success' || - needs.wheel_build_ubi8_py311.result == 'success' || - needs.wheel_build_ubi8_py312.result == 'success' || - needs.wheel_build_ubi8_py313.result == 'success' || - needs.wheel_build_ubi8_py314.result == 'success') + (needs.wheel_build_ubi8_py311.result == 'success' || + needs.wheel_build_ubi8_py312.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: - uses: actions/checkout@v4 From 7e875f92832d0f8595f5fe29ad82af933de2544b Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 12:19:19 +0530 Subject: [PATCH 42/45] Remove UBI8 wheel build jobs for Python 3.10 and 3.13 Removed wheel build jobs for Python 3.10 and 3.13 in UBI8. --- .github/workflows/pr-build.yaml | 89 --------------------------------- 1 file changed, 89 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index c329da0707..a37971a1df 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -501,51 +501,6 @@ jobs: # PR builds do not run scan steps. # --------------------------------------------------------------------------- - wheel_build_ubi8_py310: - needs: build_info - if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} - env: - PYTHON_VERSION: "3.10" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh - run: | - THIS_SCRIPT="$BUILD_SCRIPT" - THIS_TESTED_ON="$TESTED_ON" - source package-cache/variable.sh - source package-cache/scanner-env.sh - export BUILD_SCRIPT="$THIS_SCRIPT" - export TESTED_ON="$THIS_TESTED_ON" - if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - if [ -z "$BUILD_SCRIPT_CHANGED" ]; then echo "Skipping - no .sh changes"; exit 0; fi - sudo chown -R $USER:$USER . - export ENABLE_CVE_SCAN=false - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - name: Verify wheel was produced - run: | - if ls *.whl 1>/dev/null 2>&1; then - echo "Wheel built successfully: $(ls *.whl)" - else - echo "No wheel produced." - exit 1 - fi - wheel_build_ubi8_py311: needs: build_info if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} @@ -632,50 +587,6 @@ jobs: exit 1 fi - wheel_build_ubi8_py313: - needs: build_info - if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} - runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} - continue-on-error: true - env: - PYTHON_VERSION: "3.13" - BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} - TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: sudo apt update -y - - name: Download package-cache - uses: actions/download-artifact@v4 - with: - name: package-cache - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh - run: | - THIS_SCRIPT="$BUILD_SCRIPT" - THIS_TESTED_ON="$TESTED_ON" - source package-cache/variable.sh; source package-cache/scanner-env.sh - export BUILD_SCRIPT="$THIS_SCRIPT" - export TESTED_ON="$THIS_TESTED_ON" - if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi - BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 - sudo chown -R $USER:$USER . - export ENABLE_CVE_SCAN=false - chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh - - name: Verify wheel was produced - run: | - if ls *.whl 1>/dev/null 2>&1; then - echo "Wheel built successfully: $(ls *.whl)" - else - echo "No wheel produced." - exit 1 - fi - # UBI9 wheel jobs ----------------------------------------------------------- wheel_build_ubi9_py310: From b3074b0dd11131852786c10e0341b9276fa7f800 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 13:05:35 +0530 Subject: [PATCH 43/45] Update validate_builds.py --- gha-script/validate_builds.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index 95f00c4b8d..0f830d7e8e 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -11,7 +11,7 @@ GITHUB_BUILD_SCRIPT_BASE_REPO = "build-scripts" -GITHUB_BUILD_SCRIPT_BASE_OWNER = "ppc64le" +GITHUB_BUILD_SCRIPT_BASE_OWNER = "stutiibm" HOME = os.getcwd() package_data = {} From 9015b53bee428e5e9ffd14f68ecaab7ef2e852a6 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 14:23:01 +0530 Subject: [PATCH 44/45] Update build_info.json --- d/duckdb/build_info.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/d/duckdb/build_info.json b/d/duckdb/build_info.json index f15ddf64b5..4c66bc4abe 100644 --- a/d/duckdb/build_info.json +++ b/d/duckdb/build_info.json @@ -5,13 +5,13 @@ "version": "v1.5.4", "default_branch": "main", "package_dir": "d/duckdb", - "build_script": "duckdb_1.5.4_ubi_9.6.sh", + "build_script": "duckdb_1.5.4_ubi_10.2.sh", "docker_build": false, "validate_build_script": true, "wheel_build" : true, "use_non_root_user": false, "1.5.4": { - "build_script": "duckdb_1.5.4_ubi_9.6.sh" + "build_script": "duckdb_1.5.4_ubi_10.2.sh" }, "*": { "build_script": "duckdb_ubi_9.6.sh" From 2478d06b6652f4aa777c5a2875c0884a526d7900 Mon Sep 17 00:00:00 2001 From: Stuti Ravikiran Wali Date: Tue, 4 Aug 2026 14:24:22 +0530 Subject: [PATCH 45/45] Update and rename duckdb_1.5.4_ubi_9.6.sh to duckdb_1.5.4_ubi_10.2.sh --- ....4_ubi_9.6.sh => duckdb_1.5.4_ubi_10.2.sh} | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) rename d/duckdb/{duckdb_1.5.4_ubi_9.6.sh => duckdb_1.5.4_ubi_10.2.sh} (65%) diff --git a/d/duckdb/duckdb_1.5.4_ubi_9.6.sh b/d/duckdb/duckdb_1.5.4_ubi_10.2.sh similarity index 65% rename from d/duckdb/duckdb_1.5.4_ubi_9.6.sh rename to d/duckdb/duckdb_1.5.4_ubi_10.2.sh index 7db8e3caef..118a8bde0d 100755 --- a/d/duckdb/duckdb_1.5.4_ubi_9.6.sh +++ b/d/duckdb/duckdb_1.5.4_ubi_10.2.sh @@ -4,7 +4,7 @@ # Package : duckdb # Version : v1.5.4 # Source repo : https://github.com/duckdb/duckdb-python.git -# Tested on : UBI:9.6 +# Tested on : UBI:10.2 # Language : Python, C++ # Ci-Check : True # Script License: Apache License, Version 2 or later @@ -23,42 +23,41 @@ PACKAGE_VERSION=${1:-v1.5.4} PACKAGE_DIR=duckdb-python PACKAGE_URL=https://github.com/duckdb/duckdb-python.git PYTHON_VERSION=3.12 +SOURCE_ROOT="$(pwd)" # Install necessary system packages -dnf install -y gcc-toolset-13 make cmake ninja-build libomp-devel git python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel +dnf install -y \ + gcc-toolset-15 \ + cmake \ + ninja-build \ + python3.12 \ + python3.12-devel \ + python3.12-pip -# Enable GCC toolset -source /opt/rh/gcc-toolset-13/enable -export CXX=/opt/rh/gcc-toolset-13/root/usr/bin/g++ +export PATH="/opt/rh/gcc-toolset-15/root/usr/bin:$PATH" +gcc --version +python3.12 -m pip install --upgrade pip setuptools -python${PYTHON_VERSION} -m pip install build wheel setuptools ninja pybind11 +# -- Build wheel -------------------------------------------------------------- +python3.12 -m pip wheel . --no-deps -w "${CURRENT_DIR}/dist/" -# Clone the repository -git clone ${PACKAGE_URL} -cd ${PACKAGE_NAME} -git checkout ${PACKAGE_VERSION} - -git submodule update --init --recursive - -export DUCKDB_BUILD_PYTHON=1 -export DUCKDB_BUILD_STATIC=1 - -echo "Building duckdb wheel..." -if ! python${PYTHON_VERSION} -m build --wheel; then - echo "------------------$PACKAGE_NAME: build_fail------------------" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Build_Fail" +WHEEL=$(find "${CURRENT_DIR}/dist" -name "duckdb-*.whl" | head -1) +if [ -z "$WHEEL" ]; then + echo "ERROR: wheel not found after build" exit 1 fi +echo "Wheel: $WHEEL" -echo "Installing duckdb wheel..." -WHEEL_FILE=$(find dist -name "*.whl" | head -n1) -if [ -n "$WHEEL_FILE" ]; then - python${PYTHON_VERSION} -m pip install "$WHEEL_FILE" +# Copy wheel to /home/tester so the wrapper script can locate it without rebuilding +if [ -d /home/tester ]; then + cp "${WHEEL}" /home/tester/ fi -# Run tests -cd / +cd "${SOURCE_ROOT}" + +# -- Install ------------------------------------------------------------------ +pip3.12 install "$WHEEL" if ! python${PYTHON_VERSION} - <