diff --git a/.github/workflows/currency-build.yaml b/.github/workflows/currency-build.yaml index 3783d25c0d..3dc8768df3 100644 --- a/.github/workflows/currency-build.yaml +++ b/.github/workflows/currency-build.yaml @@ -1,9 +1,6 @@ name: Currency Build on: workflow_dispatch: - branches: - - master - inputs: package_name: description: 'Name of the package to build' @@ -54,8 +51,14 @@ run-name: > format('Currency Build {0} && Unique ID {1}', inputs.package_name, inputs.unique_id) }} - jobs: + + # --------------------------------------------------------------------------- + # build_info: reads build_info.json, runs read_buildinfo.sh, emits + # per-UBI-major outputs (script_ubi8, script_ubi9, script_ubi10). + # Each output is a JSON object string {"script":"...","tested_on":"..."} + # or an empty string "" when that UBI version has no script for this package. + # --------------------------------------------------------------------------- build_info: runs-on: ubuntu-24.04-ppc64le-p10 env: @@ -67,15 +70,19 @@ jobs: ENABLE_TRIVY: ${{ inputs.enable_trivy }} ENABLE_SYFT: ${{ inputs.enable_syft }} ENABLE_GRYPE: ${{ inputs.enable_grype }} + outputs: + script_ubi8: ${{ steps.emit.outputs.script_ubi8 }} + script_ubi9: ${{ steps.emit.outputs.script_ubi9 }} + script_ubi10: ${{ steps.emit.outputs.script_ubi10 }} steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Install system packages run: | sudo apt update -y sudo apt install -y jq file - + - name: Install Python dependencies run: | pip3 install --force-reinstall -v "requests==2.31.0" @@ -92,35 +99,50 @@ jobs: cat variable.sh echo "=======================" + - name: Emit per-UBI outputs + id: emit + run: | + source variable.sh + echo "SCRIPT_UBI8=$SCRIPT_UBI8" + echo "SCRIPT_UBI9=$SCRIPT_UBI9" + echo "SCRIPT_UBI10=$SCRIPT_UBI10" + # Use multiline delimiter syntax so JSON double-quotes are not truncated + { + echo "script_ubi8<<__EOF__" + echo "$SCRIPT_UBI8" + echo "__EOF__" + echo "script_ubi9<<__EOF__" + echo "$SCRIPT_UBI9" + echo "__EOF__" + echo "script_ubi10<<__EOF__" + echo "$SCRIPT_UBI10" + echo "__EOF__" + } >> $GITHUB_OUTPUT + - name: Create scanner-env.sh run: | - echo "Creating package-cache........." mkdir package-cache - echo "Creating scanner-env.sh ......." echo "export VALIDATE_BUILD_SCRIPT='${{ inputs.validate_build_script }}'" > scanner-env.sh - echo "export BUILD_DOCKER='${{ inputs.build_docker }}'" >> scanner-env.sh - echo "export PACKAGE_NAME='${{ inputs.package_name }}'" >> scanner-env.sh + echo "export BUILD_DOCKER='${{ inputs.build_docker }}'" >> scanner-env.sh + echo "export PACKAGE_NAME='${{ inputs.package_name }}'" >> scanner-env.sh echo "===== scanner-env.sh =====" cat scanner-env.sh echo "==========================" - echo "Moving scanner-env.sh and variable.sh into package-cache............" - sudo mv variable.sh package-cache - sudo mv scanner-env.sh package-cache - echo "Moved scanner-env.sh and variable.sh into package-cache............" + sudo mv variable.sh package-cache/ + sudo mv scanner-env.sh package-cache/ - name: Archive package cache run: tar -czf package-cache.tar.gz package-cache/ - - name: Upload variable.sh and scanner-env.sh via package-cache - uses: actions/upload-artifact@v6 + - name: Upload package-cache artifact + uses: actions/upload-artifact@v4 with: name: package-cache path: package-cache.tar.gz # --------------------------------------------------------------------------- - # install_scan_tools: Installs grype ONCE (static Go binary — fully portable) - # and caches it as an artifact for all wheel_build_pyXXX jobs. - # Scancode-toolkit is fetched once and distributed as an artifact too. + # install_scan_tools: installs grype + fetches scancode-toolkit source ONCE. + # Skipped entirely when wheel_build=false. # --------------------------------------------------------------------------- install_scan_tools: needs: build_info @@ -128,7 +150,7 @@ jobs: runs-on: ubuntu-24.04-ppc64le-p10 steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: Install system dependencies run: | @@ -141,7 +163,7 @@ jobs: bash ./gha-script/scanner-scripts/fetch_scancode_toolkit.sh - name: Upload scancode-toolkit-src artifact - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: scancode-toolkit-src path: scancode-toolkit-src.tar.gz @@ -157,1098 +179,1403 @@ jobs: echo "[INFO] Checksum verification successful." tar -xzf grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz else - echo "[ERROR] Checksum verification FAILED for grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz." + echo "[ERROR] Checksum verification FAILED." exit 1 fi mkdir -p scan-tools-bin mv grype scan-tools-bin/grype chmod +x scan-tools-bin/grype - echo "Grype version:" scan-tools-bin/grype version - name: Archive grype binary run: tar -czf scan-tools-cache.tar.gz scan-tools-bin/ - name: Upload scan-tools-cache artifact - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: scan-tools-cache path: scan-tools-cache.tar.gz - - build: + # --------------------------------------------------------------------------- + # BUILD JOBS — one per UBI major version. + # Each job runs build_package.sh inside the correct UBI container. + # build_ubi9 always runs (every package has at least a UBI9 script). + # build_ubi8 / build_ubi10 only run when that UBI version has a script. + # All three are independent of each other (fully parallel). + # --------------------------------------------------------------------------- + + build_ubi8: needs: build_info - if: ${{ inputs.validate_build_script == 'true' }} + if: ${{ inputs.validate_build_script == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Build Package + - name: Build Package (UBI8) env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | - echo "Sourcing variable.sh and scanner-env.sh..........................................." source package-cache/variable.sh source package-cache/scanner-env.sh - echo "===== variable.sh =====" - cat package-cache/variable.sh - echo "===== scanner-env.sh =====" - cat package-cache/scanner-env.sh - echo "==============LSCPU================" + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "BUILD_SCRIPT=$BUILD_SCRIPT TESTED_ON=$TESTED_ON" lscpu - echo "===================================" chmod +x ./gha-script/build_package.sh bash ./gha-script/build_package.sh cloned_package=$(ls -td -- */ | head -n 1) - sudo mv "$cloned_package" package-cache + sudo mv "$cloned_package" package-cache/ echo "export CLONED_PACKAGE=\"$cloned_package\"" >> package-cache/scanner-env.sh - echo "-------------updated scanner-env.sh-----------------------" - cat package-cache/scanner-env.sh - echo "---------------------------------------------" - cd package-cache - sudo chown $USER:$USER -R . + cd package-cache && sudo chown $USER:$USER -R . chmod +x ../gha-script/pre_process.sh bash ../gha-script/pre_process.sh cd $GITHUB_WORKSPACE - gzip build_log + gzip build_log && mv build_log.gz build_log_UBI8.gz chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh build_log.gz + bash ./gha-script/upload-scripts/upload_file.sh build_log_UBI8.gz + - name: Fix permissions + run: sudo chown -R $USER:$USER package-cache && sudo chmod -R u+rwX,go+rX,go-w package-cache + - name: Archive updated package-cache + run: tar -czf package-cache.tar.gz package-cache/ + - name: Upload updated package-cache (UBI8) + uses: actions/upload-artifact@v4 + with: + name: updated-package-cache-ubi8 + path: package-cache.tar.gz - - name: Fix permissions on package-cache - run: | - USER=$(whoami) - echo "Current user: $USER" - echo "Fixing ownership and permissions for package-cache" - sudo chown -R $USER:$USER package-cache - sudo chmod -R u+rwX,go+rX,go-w package-cache - - - name: Archive updated package cache + build_ubi9: + needs: build_info + if: ${{ inputs.validate_build_script == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI9) + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "BUILD_SCRIPT=$BUILD_SCRIPT TESTED_ON=$TESTED_ON" + lscpu + chmod +x ./gha-script/build_package.sh + bash ./gha-script/build_package.sh + cloned_package=$(ls -td -- */ | head -n 1) + sudo mv "$cloned_package" package-cache/ + echo "export CLONED_PACKAGE=\"$cloned_package\"" >> package-cache/scanner-env.sh + cd package-cache && sudo chown $USER:$USER -R . + chmod +x ../gha-script/pre_process.sh + bash ../gha-script/pre_process.sh + cd $GITHUB_WORKSPACE + gzip build_log && mv build_log.gz build_log_UBI9.gz + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh build_log_UBI9.gz + - name: Fix permissions + run: sudo chown -R $USER:$USER package-cache && sudo chmod -R u+rwX,go+rX,go-w package-cache + - name: Archive updated package-cache run: tar -czf package-cache.tar.gz package-cache/ + - name: Upload updated package-cache (UBI9) + uses: actions/upload-artifact@v4 + with: + name: updated-package-cache-ubi9 + path: package-cache.tar.gz - - name: Upload updated package-cache - uses: actions/upload-artifact@v6 + build_ubi10: + needs: build_info + if: ${{ inputs.validate_build_script == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI10) + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "BUILD_SCRIPT=$BUILD_SCRIPT TESTED_ON=$TESTED_ON" + lscpu + chmod +x ./gha-script/build_package.sh + bash ./gha-script/build_package.sh + cloned_package=$(ls -td -- */ | head -n 1) + sudo mv "$cloned_package" package-cache/ + echo "export CLONED_PACKAGE=\"$cloned_package\"" >> package-cache/scanner-env.sh + cd package-cache && sudo chown $USER:$USER -R . + chmod +x ../gha-script/pre_process.sh + bash ../gha-script/pre_process.sh + cd $GITHUB_WORKSPACE + gzip build_log && mv build_log.gz build_log_UBI10.gz + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh build_log_UBI10.gz + - name: Fix permissions + run: sudo chown -R $USER:$USER package-cache && sudo chmod -R u+rwX,go+rX,go-w package-cache + - name: Archive updated package-cache + run: tar -czf package-cache.tar.gz package-cache/ + - name: Upload updated package-cache (UBI10) + uses: actions/upload-artifact@v4 with: - name: updated-package-cache + name: updated-package-cache-ubi10 path: package-cache.tar.gz - # --------------------------------------------------------------------------- - # wheel_build_pyXXX jobs: build wheel → run CVE scan (generalized_wheel_scanner) - # → run scancode scan → run grype scan → upload all results to COS. - # Grype is reused from the install_scan_tools artifact (static binary). + # WHEEL BUILD JOBS — one job per UBI version per Python version. + # All 15 jobs are fully parallel (none depend on the build jobs above). + # UBI10 does not support Python 3.10 — those jobs are skipped via if:. + # Each job needs install_scan_tools for grype/scancode artifacts. # --------------------------------------------------------------------------- - wheel_build_py310: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py311: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.10" + PYTHON_VERSION: "3.11" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh source package-cache/variable.sh source package-cache/scanner-env.sh - - pwd - ls -lh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" chmod +x ./gha-script/build_wheels.sh bash ./gha-script/build_wheels.sh - - WORKSPACE=$(pwd) - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py310_log.gz" - - # Compress and rename the log file - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py311_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py311.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - mkdir -p wheel_scancode_py310 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py310 - - tar cvzf wheel_scancode_py310.tar.gz wheel_scancode_py310 - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py310.tar.gz - - mkdir -p grype_wheel_py310 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py310 - - tar cvzf grype_wheel_py310.tar.gz grype_wheel_py310 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py310.tar.gz - + mkdir -p wheel_scancode_py311 + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py311/ \; + tar czf wheel_scancode_py311.tar.gz wheel_scancode_py311 + bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py311.tar.gz + mkdir -p grype_wheel_py311 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py311/ \; + tar czf grype_wheel_py311.tar.gz grype_wheel_py311 + bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py311.tar.gz - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_py311: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi8_py312: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi8 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.11" + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py311_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" - + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi8).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py312_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi8_py312.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - mkdir -p wheel_scancode_py311 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py311/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py311 - - tar cvzf wheel_scancode_py311.tar.gz wheel_scancode_py311 - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py311.tar.gz - - mkdir -p grype_wheel_py311 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py311/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py311 - - tar cvzf grype_wheel_py311.tar.gz grype_wheel_py311 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py311.tar.gz - + mkdir -p wheel_scancode_py312 + find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; + tar czf wheel_scancode_py312.tar.gz wheel_scancode_py312 + bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz + mkdir -p grype_wheel_py312 + find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; + tar czf grype_wheel_py312.tar.gz grype_wheel_py312 + bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_py312: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + # UBI9 wheel jobs (py310-py314) ------------------------------------------------ + + wheel_build_ubi9_py310: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.12" + PYTHON_VERSION: "3.10" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py312_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" - + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py310_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py310.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - mkdir -p wheel_scancode_py312 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py312 - - tar cvzf wheel_scancode_py312.tar.gz wheel_scancode_py312 - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz - - mkdir -p grype_wheel_py312 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py312 - - tar cvzf grype_wheel_py312.tar.gz grype_wheel_py312 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz - + mkdir -p wheel_scancode_py310; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py310/ \; + tar czf wheel_scancode_py310.tar.gz wheel_scancode_py310; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py310.tar.gz + mkdir -p grype_wheel_py310; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py310/ \; + tar czf grype_wheel_py310.tar.gz grype_wheel_py310; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py310.tar.gz - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi - - wheel_build_py313: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + + wheel_build_ubi9_py311: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} - continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.13" + PYTHON_VERSION: "3.11" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py311_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py311.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py311; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py311/ \; + tar czf wheel_scancode_py311.tar.gz wheel_scancode_py311; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py311.tar.gz + mkdir -p grype_wheel_py311; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py311/ \; + tar czf grype_wheel_py311.tar.gz grype_wheel_py311; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py311.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - - name: Cache pip downloads for scancode-toolkit + wheel_build_ubi9_py312: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py312_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py312.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py313_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" + mkdir -p wheel_scancode_py312; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; + tar czf wheel_scancode_py312.tar.gz wheel_scancode_py312; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz + mkdir -p grype_wheel_py312; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; + tar czf grype_wheel_py312.tar.gz grype_wheel_py312; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true + wheel_build_ubi9_py313: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py313.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - mkdir -p wheel_scancode_py313 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py313 - - tar cvzf wheel_scancode_py313.tar.gz wheel_scancode_py313 - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz + mkdir -p wheel_scancode_py313; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; + tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz + mkdir -p grype_wheel_py313; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; + tar czf grype_wheel_py313.tar.gz grype_wheel_py313; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - mkdir -p grype_wheel_py313 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py313 + wheel_build_ubi9_py314: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi9).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi9_py314.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py314; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; + tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz + mkdir -p grype_wheel_py314; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; + tar czf grype_wheel_py314.tar.gz grype_wheel_py314; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE + else + echo "No wheel file found!"; exit 1 + fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - tar cvzf grype_wheel_py313.tar.gz grype_wheel_py313 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz + # UBI10 wheel jobs (py311–py314 only — py310 not supported on UBI10) ----------- + wheel_build_ubi10_py312: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py312_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi10_py312.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py312; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py312/ \; + tar czf wheel_scancode_py312.tar.gz wheel_scancode_py312; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py312.tar.gz + mkdir -p grype_wheel_py312; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py312/ \; + tar czf grype_wheel_py312.tar.gz grype_wheel_py312; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py312.tar.gz - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + wheel_build_ubi10_py313: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: scancode-pip-ppc64le-v32.4.0 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 + with: + name: scancode-toolkit-src + - name: Extract scancode-toolkit source + run: tar -xzf scancode-toolkit-src.tar.gz + - name: Run build_wheels.sh run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py313_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi10_py313.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" + - name: Run scancode scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh + bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh + - name: Run grype scan on wheel + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh + bash ./gha-script/scanner-scripts/grype_wheel_scan.sh + - name: Upload wheel scan results + run: | + source package-cache/variable.sh + mkdir -p wheel_scancode_py313; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py313/ \; + tar czf wheel_scancode_py313.tar.gz wheel_scancode_py313; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py313.tar.gz + mkdir -p grype_wheel_py313; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py313/ \; + tar czf grype_wheel_py313.tar.gz grype_wheel_py313; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py313.tar.gz + - name: Upload wheel to COS + run: | + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) + chmod +x ./gha-script/upload-scripts/upload_wheel.sh + bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "WARNING: No wheel found — skipping CVE report upload." + echo "No wheel file found!"; exit 1 fi + - name: Upload CVE report + run: | + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true - wheel_build_py314: - needs: - - build_info - - install_scan_tools - if: ${{ inputs.wheel_build == 'true' }} + wheel_build_ubi10_py314: + needs: [build_info, install_scan_tools] + if: ${{ inputs.wheel_build == 'true' && needs.build_info.outputs.script_ubi10 != '' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - + VERSION: ${{ inputs.version }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install system dependencies - run: | - sudo apt update -y - + run: sudo apt update -y - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Download scan-tools-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: scan-tools-cache - - name: Extract scan tools - run: | - tar -xzf scan-tools-cache.tar.gz - echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH - - - name: Cache pip downloads for scancode-toolkit + run: tar -xzf scan-tools-cache.tar.gz && echo "$GITHUB_WORKSPACE/scan-tools-bin" >> $GITHUB_PATH + - name: Cache pip (scancode) uses: actions/cache@v4 with: path: ~/.cache/pip key: scancode-pip-ppc64le-v32.4.0 - - - name: Download scancode-toolkit-src artifact - uses: actions/download-artifact@v7 + - name: Download scancode-toolkit-src + uses: actions/download-artifact@v4 with: name: scancode-toolkit-src - - name: Extract scancode-toolkit source run: tar -xzf scancode-toolkit-src.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - echo "===== Checking files before log compression =====" - pwd - ls -lh - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - - echo "===== AFTER BUILD: checking for wheel_build_log =====" - WORKSPACE=$(pwd) - echo "Workspace = $WORKSPACE" - ls -lh - - echo "----- specifically checking for wheel_build_log -----" - ls -lh "$WORKSPACE/wheel_build_log" || echo "wheel_build_log NOT found!" - - # Use wheel name for log naming - WHEEL_FILE=$(ls "$WORKSPACE"/*.whl | head -n 1) - WHEEL_BASE=$(basename "${WHEEL_FILE%.whl}") - LOG_NAME="${WHEEL_BASE}_py314_log.gz" - - echo "Found wheel_build_log — compressing it" - gzip "$WORKSPACE/wheel_build_log" - mv "$WORKSPACE/wheel_build_log.gz" "$WORKSPACE/$LOG_NAME" - - echo "Final log file to upload: $LOG_NAME" - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh $LOG_NAME - ls -lh "$WORKSPACE/$LOG_NAME" - + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="${{ fromJson(needs.build_info.outputs.script_ubi10).script }}" + export TESTED_ON="${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }}" + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) + LOG_NAME="${WHEEL_FILE:+${WHEEL_FILE%.whl}_py314_log}.gz" + [ -z "$WHEEL_FILE" ] && LOG_NAME="wheel_build_log_ubi10_py314.gz" + gzip wheel_build_log && mv wheel_build_log.gz "$LOG_NAME" + bash ./gha-script/upload-scripts/upload_file.sh "$LOG_NAME" - name: Run scancode scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/scancode_wheel_scan.sh bash ./gha-script/scanner-scripts/scancode_wheel_scan.sh - echo "-------------------------scancode outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - name: Run grype scan on wheel run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_wheel_scan.sh bash ./gha-script/scanner-scripts/grype_wheel_scan.sh - echo "-------------------------grype outputs-----------------------------------------" - ls *.json 2>/dev/null || echo "No JSON output files yet" - - - name: Collect and upload wheel scan results to COS + - name: Upload wheel scan results run: | source package-cache/variable.sh - source package-cache/scanner-env.sh - - mkdir -p wheel_scancode_py314 - find . -maxdepth 1 -type f -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; - echo "Scancode outputs:" - ls -lh wheel_scancode_py314 - - tar cvzf wheel_scancode_py314.tar.gz wheel_scancode_py314 - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz - - mkdir -p grype_wheel_py314 - find . -maxdepth 1 -type f -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; - echo "Grype outputs:" - ls -lh grype_wheel_py314 - - tar cvzf grype_wheel_py314.tar.gz grype_wheel_py314 - bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz - + mkdir -p wheel_scancode_py314; find . -maxdepth 1 -name "*_output.json" ! -name "*_grype_output.json" -exec cp {} wheel_scancode_py314/ \; + tar czf wheel_scancode_py314.tar.gz wheel_scancode_py314; bash ./gha-script/upload-scripts/upload_file.sh wheel_scancode_py314.tar.gz + mkdir -p grype_wheel_py314; find . -maxdepth 1 -name "*_grype_output.json" -exec cp {} grype_wheel_py314/ \; + tar czf grype_wheel_py314.tar.gz grype_wheel_py314; bash ./gha-script/upload-scripts/upload_file.sh grype_wheel_py314.tar.gz - name: Upload wheel to COS run: | - if ls *.whl 1> /dev/null 2>&1; then - sudo chmod a+r *.whl - WHEEL_FILE=$(ls *.whl) - SHA256_VALUE=$(cat sha256.sha) + if ls *.whl 1>/dev/null 2>&1; then + sudo chmod a+r *.whl; WHEEL_FILE=$(ls *.whl); SHA256_VALUE=$(cat sha256.sha) chmod +x ./gha-script/upload-scripts/upload_wheel.sh bash ./gha-script/upload-scripts/upload_wheel.sh $WHEEL_FILE $SHA256_VALUE else - echo "No wheel file found!" - exit 1 + echo "No wheel file found!"; exit 1 fi - - - name: Upload CVE report to COS - env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + - name: Upload CVE report run: | WHEEL_FILE=$(ls *.whl 2>/dev/null | head -n 1) - if [ -n "$WHEEL_FILE" ]; then - CVE_REPORT="${WHEEL_FILE%.whl}_cve_report.json" - if [ -f "$CVE_REPORT" ]; then - echo "Uploading CVE report: $CVE_REPORT" - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh "$CVE_REPORT" - else - echo "WARNING: CVE report not found: $CVE_REPORT — skipping upload." - fi - else - echo "WARNING: No wheel found — skipping CVE report upload." - fi + [ -n "$WHEEL_FILE" ] && CVE="${WHEEL_FILE%.whl}_cve_report.json" && [ -f "$CVE" ] && bash ./gha-script/upload-scripts/upload_file.sh "$CVE" || true # --------------------------------------------------------------------------- - # wheel_license_extract: Runs after all wheel builds complete. Downloads the - # per-Python-version scancode JSON files from COS and extracts SPDX licenses. + # WHEEL LICENSE EXTRACT — one per UBI version. + # Downloads per-Python-version scancode JSON from COS and extracts licenses. + # Runs after all wheel jobs for that UBI version finish (at least one must succeed). # --------------------------------------------------------------------------- - wheel_license_extract: - if: ${{ inputs.wheel_build == 'true' && (needs.wheel_build_py310.result == 'success' || needs.wheel_build_py311.result == 'success' || needs.wheel_build_py312.result == 'success' || needs.wheel_build_py313.result == 'success' || needs.wheel_build_py314.result == 'success') }} + + wheel_license_extract_ubi8: needs: - - wheel_build_py310 - - wheel_build_py311 - - wheel_build_py312 - - wheel_build_py313 - - wheel_build_py314 + - wheel_build_ubi8_py311 + - wheel_build_ubi8_py312 + if: | + always() && + inputs.wheel_build == 'true' && + needs.build_info.outputs.script_ubi8 != '' && + (needs.wheel_build_ubi8_py311.result == 'success' || + needs.wheel_build_ubi8_py312.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: - - name: Checkout code - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Download scancode JSON outputs from COS + - name: Download scancode JSON outputs from COS and extract licenses env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | source package-cache/variable.sh source package-cache/scanner-env.sh - - echo "Downloading per-version scancode archives from COS..." mkdir -p all_wheel_scancode - chmod +x ./gha-script/download-scripts/download_file.sh for PY_VER in py310 py311 py312 py313 py314; do ARCHIVE="wheel_scancode_${PY_VER}.tar.gz" - echo "Attempting to download $ARCHIVE..." bash ./gha-script/download-scripts/download_file.sh "$ARCHIVE" \ || echo "Warning: $ARCHIVE not found on COS (skipped)" if [ -f "$ARCHIVE" ]; then tar -xzf "$ARCHIVE" find "wheel_scancode_${PY_VER}" -type f -name "*.json" \ -exec cp {} all_wheel_scancode/ \; 2>/dev/null || true - echo "Collected JSON files from $ARCHIVE" fi done + echo "Scancode JSON files collected:"; ls -lh all_wheel_scancode + for JSON_FILE in all_wheel_scancode/*.json; do + [ -f "$JSON_FILE" ] || { echo "No JSON files — skipping."; break; } + echo "--- $JSON_FILE ---" + python3 ./gha-script/licenses_extract_script.py "$JSON_FILE" + done - echo "All scancode JSON files available:" - ls -lh all_wheel_scancode - - - name: Extract licenses from each scancode JSON + wheel_license_extract_ubi9: + needs: + - wheel_build_ubi9_py310 + - wheel_build_ubi9_py311 + - wheel_build_ubi9_py312 + - wheel_build_ubi9_py313 + - wheel_build_ubi9_py314 + if: | + always() && + inputs.wheel_build == 'true' && + needs.build_info.outputs.script_ubi9 != '' && + (needs.wheel_build_ubi9_py310.result == 'success' || + needs.wheel_build_ubi9_py311.result == 'success' || + needs.wheel_build_ubi9_py312.result == 'success' || + needs.wheel_build_ubi9_py313.result == 'success' || + needs.wheel_build_ubi9_py314.result == 'success') + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scancode JSON outputs from COS and extract licenses + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | - echo "===== License extraction per wheel =====" + source package-cache/variable.sh + source package-cache/scanner-env.sh + mkdir -p all_wheel_scancode + chmod +x ./gha-script/download-scripts/download_file.sh + for PY_VER in py310 py311 py312 py313 py314; do + ARCHIVE="wheel_scancode_${PY_VER}.tar.gz" + bash ./gha-script/download-scripts/download_file.sh "$ARCHIVE" \ + || echo "Warning: $ARCHIVE not found on COS (skipped)" + if [ -f "$ARCHIVE" ]; then + tar -xzf "$ARCHIVE" + find "wheel_scancode_${PY_VER}" -type f -name "*.json" \ + -exec cp {} all_wheel_scancode/ \; 2>/dev/null || true + fi + done + echo "Scancode JSON files collected:"; ls -lh all_wheel_scancode for JSON_FILE in all_wheel_scancode/*.json; do - [ -f "$JSON_FILE" ] || { echo "No JSON files found — skipping."; break; } + [ -f "$JSON_FILE" ] || { echo "No JSON files — skipping."; break; } echo "--- $JSON_FILE ---" python3 ./gha-script/licenses_extract_script.py "$JSON_FILE" done - echo "========================================" - - source_scanner: - needs: build - if: ${{ inputs.validate_build_script == 'true' }} + wheel_license_extract_ubi10: + needs: + - wheel_build_ubi10_py312 + - wheel_build_ubi10_py313 + - wheel_build_ubi10_py314 + if: | + always() && + inputs.wheel_build == 'true' && + needs.build_info.outputs.script_ubi10 != '' && + (needs.wheel_build_ubi10_py312.result == 'success' || + needs.wheel_build_ubi10_py313.result == 'success' || + needs.wheel_build_ubi10_py314.result == 'success') runs-on: ubuntu-24.04-ppc64le-p10 steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: - name: updated-package-cache - + name: package-cache - name: Extract package cache run: tar -xzf package-cache.tar.gz + - name: Download scancode JSON outputs from COS and extract licenses + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + source package-cache/variable.sh + source package-cache/scanner-env.sh + mkdir -p all_wheel_scancode + chmod +x ./gha-script/download-scripts/download_file.sh + for PY_VER in py312 py313 py314; do + ARCHIVE="wheel_scancode_${PY_VER}.tar.gz" + bash ./gha-script/download-scripts/download_file.sh "$ARCHIVE" \ + || echo "Warning: $ARCHIVE not found on COS (skipped)" + if [ -f "$ARCHIVE" ]; then + tar -xzf "$ARCHIVE" + find "wheel_scancode_${PY_VER}" -type f -name "*.json" \ + -exec cp {} all_wheel_scancode/ \; 2>/dev/null || true + fi + done + echo "Scancode JSON files collected:"; ls -lh all_wheel_scancode + for JSON_FILE in all_wheel_scancode/*.json; do + [ -f "$JSON_FILE" ] || { echo "No JSON files — skipping."; break; } + echo "--- $JSON_FILE ---" + python3 ./gha-script/licenses_extract_script.py "$JSON_FILE" + done + # --------------------------------------------------------------------------- + # source_scanner_ubiN: scans the cloned source tree (trivy / syft / grype). + # One job per UBI version, each depending on its respective build_ubiN job. + # Only runs when the corresponding build_ubiN succeeded and has a script. + # --------------------------------------------------------------------------- + source_scanner_ubi8: + needs: build_ubi8 + if: ${{ always() && inputs.validate_build_script == 'true' && needs.build_ubi8.result == 'success' }} + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download updated package-cache (UBI8) + uses: actions/download-artifact@v4 + with: + name: updated-package-cache-ubi8 + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz - name: Prepare scanner environment run: | - echo "Sourcing variable.sh..." source package-cache/variable.sh source package-cache/scanner-env.sh - echo "---------updated scanner-env.sh cat----------------" cat package-cache/scanner-env.sh - echo "scanner-env.sh created with:" - cat package-cache/scanner-env.sh - - name: Run Trivy Scan if: ${{ inputs.enable_trivy == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/trivy_code_scan.sh + bash ./gha-script/scanner-scripts/trivy_code_scan.sh + mkdir source_trivy + mv package-cache/trivy_source_vulnerabilities_results.json \ + package-cache/trivy_source_sbom_results.cyclonedx source_trivy + - name: Run Syft Scan + if: ${{ inputs.enable_syft == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/syft_code_scan.sh + bash ./gha-script/scanner-scripts/syft_code_scan.sh + mkdir source_syft + mv package-cache/syft_source_sbom_results.json source_syft + - name: Run Grype Scan + if: ${{ inputs.enable_grype == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_code_scan.sh + bash ./gha-script/scanner-scripts/grype_code_scan.sh + mkdir source_grype + mv package-cache/grype_source_sbom_results.json \ + package-cache/grype_source_vulnerabilities_results.json source_grype + - name: Upload Scanner Results + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + mkdir source + source package-cache/variable.sh; source package-cache/scanner-env.sh + find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + tar cvzf source_scanner_ubi8.tar.gz source + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh source_scanner_ubi8.tar.gz + + source_scanner_ubi9: + needs: build_ubi9 + if: ${{ always() && inputs.validate_build_script == 'true' && needs.build_ubi9.result == 'success' }} + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download updated package-cache (UBI9) + uses: actions/download-artifact@v4 + with: + name: updated-package-cache-ubi9 + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz + - name: Prepare scanner environment + run: | source package-cache/variable.sh source package-cache/scanner-env.sh cat package-cache/scanner-env.sh - + - name: Run Trivy Scan + if: ${{ inputs.enable_trivy == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/trivy_code_scan.sh - bash ./gha-script/scanner-scripts/trivy_code_scan.sh; + bash ./gha-script/scanner-scripts/trivy_code_scan.sh mkdir source_trivy - mv package-cache/trivy_source_vulnerabilities_results.json package-cache/trivy_source_sbom_results.cyclonedx source_trivy - + mv package-cache/trivy_source_vulnerabilities_results.json \ + package-cache/trivy_source_sbom_results.cyclonedx source_trivy - name: Run Syft Scan if: ${{ inputs.enable_syft == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/syft_code_scan.sh bash ./gha-script/scanner-scripts/syft_code_scan.sh mkdir source_syft mv package-cache/syft_source_sbom_results.json source_syft - - name: Run Grype Scan if: ${{ inputs.enable_grype == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_code_scan.sh bash ./gha-script/scanner-scripts/grype_code_scan.sh mkdir source_grype - mv package-cache/grype_source_sbom_results.json package-cache/grype_source_vulnerabilities_results.json source_grype - + mv package-cache/grype_source_sbom_results.json \ + package-cache/grype_source_vulnerabilities_results.json source_grype - name: Upload Scanner Results env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | mkdir source + source package-cache/variable.sh; source package-cache/scanner-env.sh + find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + tar cvzf source_scanner_ubi9.tar.gz source + chmod +x ./gha-script/upload-scripts/upload_file.sh + bash ./gha-script/upload-scripts/upload_file.sh source_scanner_ubi9.tar.gz + + source_scanner_ubi10: + needs: build_ubi10 + if: ${{ always() && inputs.validate_build_script == 'true' && needs.build_ubi10.result == 'success' }} + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download updated package-cache (UBI10) + uses: actions/download-artifact@v4 + with: + name: updated-package-cache-ubi10 + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz + - name: Prepare scanner environment + run: | source package-cache/variable.sh source package-cache/scanner-env.sh + cat package-cache/scanner-env.sh + - name: Run Trivy Scan + if: ${{ inputs.enable_trivy == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/trivy_code_scan.sh + bash ./gha-script/scanner-scripts/trivy_code_scan.sh + mkdir source_trivy + mv package-cache/trivy_source_vulnerabilities_results.json \ + package-cache/trivy_source_sbom_results.cyclonedx source_trivy + - name: Run Syft Scan + if: ${{ inputs.enable_syft == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/syft_code_scan.sh + bash ./gha-script/scanner-scripts/syft_code_scan.sh + mkdir source_syft + mv package-cache/syft_source_sbom_results.json source_syft + - name: Run Grype Scan + if: ${{ inputs.enable_grype == 'true' }} + run: | + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" + chmod +x ./gha-script/scanner-scripts/grype_code_scan.sh + bash ./gha-script/scanner-scripts/grype_code_scan.sh + mkdir source_grype + mv package-cache/grype_source_sbom_results.json \ + package-cache/grype_source_vulnerabilities_results.json source_grype + - name: Upload Scanner Results + env: + GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} + run: | + mkdir source + source package-cache/variable.sh; source package-cache/scanner-env.sh find source_trivy -mindepth 1 -exec mv -t source {} + 2>/dev/null || true - find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true + find source_syft -mindepth 1 -exec mv -t source {} + 2>/dev/null || true find source_grype -mindepth 1 -exec mv -t source {} + 2>/dev/null || true - tar cvzf source_scanner.tar.gz source + tar cvzf source_scanner_ubi10.tar.gz source chmod +x ./gha-script/upload-scripts/upload_file.sh - bash ./gha-script/upload-scripts/upload_file.sh source_scanner.tar.gz + bash ./gha-script/upload-scripts/upload_file.sh source_scanner_ubi10.tar.gz + # --------------------------------------------------------------------------- + # build_docker / image_scanner + # --------------------------------------------------------------------------- build_docker: - needs: build_info + needs: build_info if: ${{ inputs.build_docker == 'true' }} runs-on: ${{ inputs.large-runner-label != '' && inputs.large-runner-label || 'ubuntu-24.04-ppc64le-p10' }} steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Build Docker Image run: | - echo "................................Sourcing scanner-env.sh............................." ls package-cache source package-cache/variable.sh source package-cache/scanner-env.sh @@ -1256,21 +1583,17 @@ jobs: bash ./gha-script/build_docker.sh docker save -o package-cache/image.tar "$IMAGE_NAME" docker images - - name: Upload Docker Image env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/upload-scripts/upload_docker_image.sh bash ./gha-script/upload-scripts/upload_docker_image.sh - - name: Archive package cache with image run: tar -czf package-cache.tar.gz package-cache/ - - name: Upload package-cache with image.tar - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: package-cache-with-image path: package-cache.tar.gz @@ -1280,132 +1603,114 @@ jobs: if: ${{ inputs.build_docker == 'true' }} runs-on: ubuntu-24.04-ppc64le-p10 steps: - - uses: actions/checkout@v6 - + - uses: actions/checkout@v4 - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache-with-image - - name: Extract package cache run: tar -xzf package-cache.tar.gz - + - name: Download scan-tools-cache + if: ${{ inputs.enable_grype == 'true' }} + uses: actions/download-artifact@v4 + with: + name: scan-tools-cache + - name: Extract scan tools + if: ${{ inputs.enable_grype == 'true' }} + run: tar -xzf scan-tools-cache.tar.gz - name: Load Docker Image run: | - echo "Sourcing variable.sh ..........................................." - source package-cache/variable.sh - source package-cache/scanner-env.sh - echo ".......................Loading iamge.tar ............................" + source package-cache/variable.sh; source package-cache/scanner-env.sh docker load -i "package-cache/image.tar" - - name: Run Trivy Image Scan if: ${{ inputs.enable_trivy == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh chmod +x ./gha-script/scanner-scripts/trivy_image_scan.sh bash ./gha-script/scanner-scripts/trivy_image_scan.sh mkdir image_trivy mv trivy_image_vulnerabilities_results.json trivy_image_sbom_results.cyclonedx image_trivy - - name: Run Syft Image Scan if: ${{ inputs.enable_syft == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh - chmod +x ./script/scanner-scripts/syft_image_scan.sh - bash ./script/scanner-scripts/syft_image_scan.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + chmod +x ./gha-script/scanner-scripts/syft_image_scan.sh + bash ./gha-script/scanner-scripts/syft_image_scan.sh mkdir image_syft mv syft_image_sbom_results.json image_syft - - name: Run Grype Image Scan if: ${{ inputs.enable_grype == 'true' }} run: | - echo ".................Sourcing variable.sh and scanner-env.sh...................." - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh + export GRYPE_BIN="$GITHUB_WORKSPACE/scan-tools-bin/grype" chmod +x ./gha-script/scanner-scripts/grype_image_scan.sh bash ./gha-script/scanner-scripts/grype_image_scan.sh mkdir image_grype mv grype_image_sbom_results.json grype_image_vulnerabilities_results.json image_grype - - name: Upload Image Scanner Results env: GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} run: | mkdir image - source package-cache/variable.sh - source package-cache/scanner-env.sh + source package-cache/variable.sh; source package-cache/scanner-env.sh find image_trivy -mindepth 1 -exec mv -t image {} + 2>/dev/null || true - find image_syft -mindepth 1 -exec mv -t image {} + 2>/dev/null || true + find image_syft -mindepth 1 -exec mv -t image {} + 2>/dev/null || true find image_grype -mindepth 1 -exec mv -t image {} + 2>/dev/null || true tar cvzf image_scanner.tar.gz image chmod +x ./gha-script/upload-scripts/upload_file.sh bash ./gha-script/upload-scripts/upload_file.sh image_scanner.tar.gz + # --------------------------------------------------------------------------- + # final_summary: runs after all parallel tracks complete. + # Uses always() so it runs even when optional jobs (ubi8/ubi10) were skipped. + # Requires at least one of build_ubi8/ubi9/ubi10 to have succeeded. + # --------------------------------------------------------------------------- final_summary: - name: Final Summary Stage - needs: - - build - - source_scanner - - wheel_build_py310 - - wheel_build_py311 - - wheel_build_py312 - - wheel_build_py313 - - wheel_build_py314 - - wheel_license_extract - - if: | - always() && - needs.build.result == 'success' && - needs.source_scanner.result == 'success' - - runs-on: ubuntu-24.04-ppc64le-p10 - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Download package-cache - uses: actions/download-artifact@v7 - with: - name: package-cache - - - name: Extract package cache - run: tar -xzf package-cache.tar.gz - - - name: Create and activate venv - run: | - python -m venv venv - source venv/bin/activate - python -m pip install --upgrade pip - pip install requests deprecated xlsxwriter packaging - - # Export environment variables - export PYTHONPATH=$(pwd):$(pwd)/process_bom/ - - - name: Summarize build results - env : - PACKAGE_NAME: ${{ inputs.package_name }} - VERSION: ${{ inputs.version }} - IAM_WRITER_API_KEY: ${{ secrets.IAM_WRITER_API_KEY }} - SERVICE_INSTANCE_ID: ${{ secrets.SERVICE_INSTANCE_ID }} - run: | - echo "Sourcing variable.sh and scanner-env.sh..........................................." - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - echo "===== variable.sh =====" - cat package-cache/variable.sh - echo "===== scanner-env.sh =====" - cat package-cache/scanner-env.sh - echo "-------------package name : ${PACKAGE_NAME}--------------------" - echo "-------------version : ${VERSION}--------------------" - pwd - chmod +x process_bom/*.py || true - # Export environment variables - export PYTHONPATH="/tmp/_actions-runner-working-dir/build-scripts/build-scripts:/tmp/_actions-runner-working-dir/build-scripts/build-scripts/process_bom" - python -m process_bom.run_currency_processor - + name: Final Summary Stage + needs: + - build_ubi8 + - build_ubi9 + - build_ubi10 + - wheel_license_extract_ubi8 + - wheel_license_extract_ubi9 + - wheel_license_extract_ubi10 + - source_scanner_ubi8 + - source_scanner_ubi9 + - source_scanner_ubi10 + if: | + always() && + (needs.build_ubi8.result == 'success' || + needs.build_ubi9.result == 'success' || + needs.build_ubi10.result == 'success') + runs-on: ubuntu-24.04-ppc64le-p10 + steps: + - uses: actions/checkout@v4 + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Create and activate venv + run: | + python -m venv venv + source venv/bin/activate + python -m pip install --upgrade pip + pip install requests deprecated xlsxwriter packaging + - name: Summarize build results + env: + PACKAGE_NAME: ${{ inputs.package_name }} + VERSION: ${{ inputs.version }} + IAM_WRITER_API_KEY: ${{ secrets.IAM_WRITER_API_KEY }} + SERVICE_INSTANCE_ID: ${{ secrets.SERVICE_INSTANCE_ID }} + run: | + chmod +x package-cache/variable.sh + chmod +x package-cache/scanner-env.sh + source package-cache/variable.sh + source package-cache/scanner-env.sh + echo "===== variable.sh ====="; cat package-cache/variable.sh + echo "===== scanner-env.sh ====="; cat package-cache/scanner-env.sh + echo "Package: ${PACKAGE_NAME} Version: ${VERSION}" + chmod +x process_bom/*.py || true + export PYTHONPATH="/tmp/_actions-runner-working-dir/build-scripts/build-scripts:/tmp/_actions-runner-working-dir/build-scripts/build-scripts/process_bom" + python -m process_bom.run_currency_processor diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index c8db120167..a37971a1df 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -66,11 +66,11 @@ jobs: if [ -n "$RELEVANT_CHANGES" ]; then echo "should_build=true" >> $GITHUB_OUTPUT - echo "✅ Found relevant changes:" + echo "Found relevant changes:" echo "$RELEVANT_CHANGES" else echo "should_build=false" >> $GITHUB_OUTPUT - echo "⏭️ Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile" + echo "Skipping PR build CI check - no changes related to build_info.json, build scripts (.sh), or Dockerfile" fi build_info: @@ -78,23 +78,27 @@ jobs: if: needs.check_changes.outputs.should_build == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} outputs: - wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }} - has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }} + wheel_build_enabled: ${{ steps.set_flags.outputs.wheel_build_enabled }} + has_sh_changes: ${{ steps.set_flags.outputs.has_sh_changes }} has_dockerfile_changes: ${{ steps.set_flags.outputs.has_dockerfile_changes }} - docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }} - build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }} + docker_build_enabled: ${{ steps.set_flags.outputs.docker_build_enabled }} + build_package_enabled: ${{ steps.set_flags.outputs.build_package_enabled }} + # Per-UBI-major named outputs - each is a {script,tested_on} JSON object or "" + script_ubi8: ${{ steps.emit.outputs.script_ubi8 }} + script_ubi9: ${{ steps.emit.outputs.script_ubi9 }} + script_ubi10: ${{ steps.emit.outputs.script_ubi10 }} steps: - name: Checkout code (Pull Request) if: github.event_name == 'pull_request' - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - name: Checkout code (Workflow Dispatch) if: github.event_name == 'workflow_dispatch' - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: ref: refs/pull/${{ inputs.pr_number }}/head @@ -147,7 +151,7 @@ jobs: # Reuse changed files from check_changes job CHANGED_FILES="${{ needs.check_changes.outputs.changed_files }}" - # If workflow_dispatch, check_changes did not compute changed_files — fetch them now + # If workflow_dispatch, check_changes did not compute changed_files - fetch them now if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then git fetch origin ${{ github.base_ref }} --depth=1 CHANGED_FILES=$(git diff --name-only origin/${{ github.base_ref }}...HEAD) @@ -176,6 +180,45 @@ jobs: PACKAGE_NAME=$(jq -r '.package_name // ""' $BUILD_INFO_FILE) VERSION=$(jq -r '.version // ""' $BUILD_INFO_FILE) + # Pick the version block whose build_script list has the MOST overlap + # with the changed .sh files. This correctly handles the case where a + # script also appears in older single-script blocks (e.g. scikit-learn_ubi_8.10.sh + # is in both 1.5.0 and 1.9.0) - the new block wins because it matches more + # changed files. The wildcard "*" key is always skipped - it is a catch-all + # fallback, not a real version. + # NOTE: if no .sh files changed (build_info.json only), this block is skipped + # entirely and VERSION stays as the default from the .version field. + CHANGED_SH=$(echo "$CHANGED_FILES" | grep '\.sh$' || true) + if [ -n "$CHANGED_SH" ]; then + SCALAR_KEYS='package_name|version|package_dir|wheel_build|docker_build|validate_build_script|base_docker_image|base_docker_variant|use_non_root_user|auditwheel_exclude' + VERSION_KEYS=$(jq -r 'keys[] | select(test("^('"$SCALAR_KEYS"')$") | not) | select(. != "*")' "$BUILD_INFO_FILE" 2>/dev/null || true) + best_ver="" + best_count=0 + for ver_key in $VERSION_KEYS; do + bs=$(jq -c --arg k "$ver_key" '.[$k].build_script // empty' "$BUILD_INFO_FILE" 2>/dev/null || true) + [ -z "$bs" ] && continue + if echo "$bs" | jq -e 'type == "array"' >/dev/null 2>&1; then + scripts=$(echo "$bs" | jq -r '.[]') + else + scripts=$(echo "$bs" | tr -d '"') + fi + match_count=0 + for s in $scripts; do + if echo "$CHANGED_SH" | grep -qF "$s"; then + match_count=$((match_count + 1)) + fi + done + if [ "$match_count" -gt "$best_count" ]; then + best_count=$match_count + best_ver="$ver_key" + fi + done + if [ -n "$best_ver" ] && [ "$best_count" -gt 0 ]; then + echo "Version block '$best_ver' matched $best_count changed script(s) - using that version instead of default '$VERSION'" + VERSION="$best_ver" + fi + fi + echo "BUILD_INFO_FILE=$BUILD_INFO_FILE" >> $GITHUB_ENV echo "PACKAGE_NAME=$PACKAGE_NAME" >> $GITHUB_ENV echo "VERSION=$VERSION" >> $GITHUB_ENV @@ -189,20 +232,79 @@ jobs: chmod +x ./gha-script/read_buildinfo.sh bash ./gha-script/read_buildinfo.sh + - name: Emit per-UBI outputs + id: emit + run: | + source variable.sh + + # Safety net: for every changed .sh file, if its UBI major slot is still + # empty (read_buildinfo.sh loaded the wrong or incomplete version block), + # read the script's own "# Tested on" header and fill the slot directly. + # This guarantees every changed script in the PR is tested regardless of + # how build_info.json version blocks are structured or overlap. + # If no .sh files changed (build_info.json only), this loop does nothing. + while IFS= read -r changed_script; do + [[ "$changed_script" != *.sh ]] && continue + [ ! -f "$changed_script" ] && continue + raw_tested_on=$(grep "^# Tested on" "$changed_script" | head -1 \ + | cut -d ':' -f2- \ + | sed 's/^[[:space:]]*//' \ + | tr '[:lower:]' '[:upper:]' \ + | sed 's/[[:space:]]*:[[:space:]]*/:/g; s/UBI[[:space:]]*/UBI/g') + [ -z "$raw_tested_on" ] && continue + major=$(echo "$raw_tested_on" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + case "$major" in ''|*[!0-9]*) continue ;; esac + script_name=$(basename "$changed_script") + entry=$(jq -n --arg s "$script_name" --arg t "$raw_tested_on" '{"script":$s,"tested_on":$t}') + case "$major" in + 8) + if [ -z "$SCRIPT_UBI8" ]; then + echo "Filling SCRIPT_UBI8 from changed script: $script_name" + SCRIPT_UBI8="$entry" + fi ;; + 9) + if [ -z "$SCRIPT_UBI9" ]; then + echo "Filling SCRIPT_UBI9 from changed script: $script_name" + SCRIPT_UBI9="$entry" + fi ;; + 10) + if [ -z "$SCRIPT_UBI10" ]; then + echo "Filling SCRIPT_UBI10 from changed script: $script_name" + SCRIPT_UBI10="$entry" + fi ;; + esac + done <<< "$CHANGED_FILES" + + echo "SCRIPT_UBI8=$SCRIPT_UBI8" + echo "SCRIPT_UBI9=$SCRIPT_UBI9" + echo "SCRIPT_UBI10=$SCRIPT_UBI10" + # Use multiline delimiter syntax so JSON double-quotes are not truncated + { + echo "script_ubi8<<__EOF__" + echo "$SCRIPT_UBI8" + echo "__EOF__" + echo "script_ubi9<<__EOF__" + echo "$SCRIPT_UBI9" + echo "__EOF__" + echo "script_ubi10<<__EOF__" + echo "$SCRIPT_UBI10" + echo "__EOF__" + } >> $GITHUB_OUTPUT + - name: Set job control flags id: set_flags run: | if [ -z "$BUILD_INFO_FILE" ]; then - echo "❌ BUILD_INFO_FILE is not set — cannot determine job control flags" + echo "ERROR: BUILD_INFO_FILE is not set - cannot determine job control flags" exit 1 fi if [ ! -f "$BUILD_INFO_FILE" ]; then - echo "❌ BUILD_INFO_FILE does not exist: $BUILD_INFO_FILE" + echo "ERROR: BUILD_INFO_FILE does not exist: $BUILD_INFO_FILE" exit 1 fi PACKAGE_DIR=$(jq -r '.package_dir // ""' $BUILD_INFO_FILE) - # Strip trailing slash if present — prevents double-slash in regex (e.g. "m/multidict/" → "m/multidict") + # Strip trailing slash if present - prevents double-slash in regex (e.g. "m/multidict/" -> "m/multidict") PACKAGE_DIR="${PACKAGE_DIR%/}" WHEEL_BUILD=$(jq -r '.wheel_build // "false"' $BUILD_INFO_FILE) @@ -222,10 +324,10 @@ jobs: # Set outputs for wheel builds if [ "$WHEEL_BUILD" == "true" ] && [ -n "$SH_SCRIPT_CHANGED" ]; then echo "wheel_build_enabled=true" >> $GITHUB_OUTPUT - echo "✅ Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" + echo "Wheel builds will run (WHEEL_BUILD=true and .sh scripts changed)" else echo "wheel_build_enabled=false" >> $GITHUB_OUTPUT - echo "⏭️ Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" + echo "Wheel builds will be skipped (WHEEL_BUILD=$WHEEL_BUILD, .sh changes: ${SH_SCRIPT_CHANGED:-none})" fi # Set output for sh changes @@ -239,7 +341,7 @@ jobs: if [ "$DOCKER_BUILD" == "true" ] && [ -n "$DOCKERFILE_CHANGED" ]; then echo "docker_build_enabled=true" >> $GITHUB_OUTPUT echo "has_dockerfile_changes=true" >> $GITHUB_OUTPUT - echo "✅ Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" + echo "Docker build will run (BUILD_DOCKER=true and Dockerfile changed)" else echo "docker_build_enabled=false" >> $GITHUB_OUTPUT if [ -n "$DOCKERFILE_CHANGED" ]; then @@ -247,17 +349,17 @@ jobs: else echo "has_dockerfile_changes=false" >> $GITHUB_OUTPUT fi - echo "⏭️ Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" + echo "Docker build will be skipped (BUILD_DOCKER=$DOCKER_BUILD, Dockerfile changes: ${DOCKERFILE_CHANGED:-none})" fi # Set output for build job (runs when build_info.json or .sh scripts change) BUILD_INFO_CHANGED=$(echo "$CHANGED_FILES" | grep 'build_info\.json' || true) if [ -n "$BUILD_INFO_CHANGED" ] || [ -n "$SH_SCRIPT_CHANGED" ]; then echo "build_package_enabled=true" >> $GITHUB_OUTPUT - echo "✅ Build package job will run (build_info.json or .sh scripts changed)" + echo "Build package job will run (build_info.json or .sh scripts changed)" else echo "build_package_enabled=false" >> $GITHUB_OUTPUT - echo "⏭️ Build package job will be skipped (no build_info.json or .sh script changes)" + echo "Build package job will be skipped (no build_info.json or .sh script changes)" fi - name: Create scanner-env.sh @@ -274,14 +376,16 @@ jobs: DOCKER_BUILD="false" fi - cat < package-cache/scanner-env.sh - export PACKAGE_NAME=$PACKAGE_NAME - export VERSION=$VERSION - export PACKAGE_DIR=$PACKAGE_DIR - export WHEEL_BUILD=$WHEEL_BUILD - export BUILD_DOCKER=$DOCKER_BUILD - export CHANGED_FILES="$CHANGED_FILES" - EOF + # Write scanner-env.sh without leading indentation spaces. + # CHANGED_FILES is written via a separate heredoc so its newlines are preserved. + { + echo "export PACKAGE_NAME=$PACKAGE_NAME" + echo "export VERSION=$VERSION" + echo "export PACKAGE_DIR=$PACKAGE_DIR" + echo "export WHEEL_BUILD=$WHEEL_BUILD" + echo "export BUILD_DOCKER=$DOCKER_BUILD" + printf 'export CHANGED_FILES=%q\n' "$CHANGED_FILES" + } > package-cache/scanner-env.sh mv variable.sh package-cache/ @@ -289,420 +393,589 @@ jobs: run: tar -czf package-cache.tar.gz package-cache/ - name: Upload package cache - uses: actions/upload-artifact@v6 + uses: actions/upload-artifact@v4 with: name: package-cache path: package-cache.tar.gz - build: - needs: build_info - if: needs.build_info.outputs.build_package_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + # --------------------------------------------------------------------------- + # BUILD JOBS - one per UBI major version. + # Runs execute_changed_scripts.py inside the correct UBI container. + # Each job only runs when a script exists for that UBI version. + # All three are parallel (no inter-dependency). + # --------------------------------------------------------------------------- + build_ubi8: + needs: build_info + if: ${{ needs.build_info.outputs.build_package_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Build Package + - name: Build Package (UBI8) run: | + # Capture the GHA job-level BUILD_SCRIPT before sourcing variable.sh, + # which overwrites BUILD_SCRIPT with the first/backward-compat script. + THIS_SCRIPT="$BUILD_SCRIPT" source package-cache/variable.sh source package-cache/scanner-env.sh - - echo "------------------- variable.sh -----------------------------" - cat package-cache/variable.sh - echo "------------------- scanner-env.sh -----------------------------" - cat package-cache/scanner-env.sh - sudo chown -R $USER:$USER . - echo "------------------- Executing changed scripts -----------------------------" + echo "--- Executing changed scripts for UBI8 (script: $THIS_SCRIPT) ---" + # Filter CHANGED_FILES to only the script assigned to this UBI job. + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$THIS_SCRIPT" || true) + export BUILD_SCRIPT="$THIS_SCRIPT" chmod +x ./gha-script/execute_changed_scripts.py python3 -u ./gha-script/execute_changed_scripts.py -# ===================== WHEEL JOBS ===================== - - - wheel_build_py310: + build_ubi9: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false + if: ${{ needs.build_info.outputs.build_package_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} - PYTHON_VERSION: "3.10" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: - fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - - name: Install system dependencies + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Build Package (UBI9) run: | - sudo apt update -y + THIS_SCRIPT="$BUILD_SCRIPT" + source package-cache/variable.sh + source package-cache/scanner-env.sh + sudo chown -R $USER:$USER . + echo "--- Executing changed scripts for UBI9 (script: $THIS_SCRIPT) ---" + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$THIS_SCRIPT" || true) + export BUILD_SCRIPT="$THIS_SCRIPT" + chmod +x ./gha-script/execute_changed_scripts.py + python3 -u ./gha-script/execute_changed_scripts.py - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + build_ubi10: + needs: build_info + if: ${{ needs.build_info.outputs.build_package_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - - name: Run build_wheels.sh + - name: Build Package (UBI10) run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh + THIS_SCRIPT="$BUILD_SCRIPT" source package-cache/variable.sh source package-cache/scanner-env.sh + sudo chown -R $USER:$USER . + echo "--- Executing changed scripts for UBI10 (script: $THIS_SCRIPT) ---" + export CHANGED_FILES=$(echo "$CHANGED_FILES" | grep -F "$THIS_SCRIPT" || true) + export BUILD_SCRIPT="$THIS_SCRIPT" + chmod +x ./gha-script/execute_changed_scripts.py + python3 -u ./gha-script/execute_changed_scripts.py - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi + # --------------------------------------------------------------------------- + # WHEEL BUILD JOBS - one job per UBI version per Python version. + # All parallel. UBI10 supports py312-py314 only (py310/py311 not supported). + # PR builds do not run scan steps. + # --------------------------------------------------------------------------- - # Check if any .sh build script is modified in this package + wheel_build_ubi8_py311: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.11" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi + wheel_build_ubi8_py312: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi8 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi8).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi8).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a + # UBI9 wheel jobs ----------------------------------------------------------- + wheel_build_ubi9_py310: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.10" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - wheel_build_py311: + wheel_build_ubi9_py311: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.11" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh run: | - sudo apt update -y + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + wheel_build_ubi9_py312: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + env: + PYTHON_VERSION: "3.12" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi - # Check if any .sh build script is modified in this package + wheel_build_ubi9_py313: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.13" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 + sudo chown -R $USER:$USER . + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 fi + wheel_build_ubi9_py314: + needs: build_info + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi9 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} + continue-on-error: true + env: + PYTHON_VERSION: "3.14" + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi9).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi9).tested_on }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Install system dependencies + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 + with: + name: package-cache + - name: Extract package cache + run: tar -xzf package-cache.tar.gz + - name: Run build_wheels.sh + run: | + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi + BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi + # UBI10 wheel jobs (py312-py314 only - py310/py311 not supported on UBI10) ---- - wheel_build_py312: + wheel_build_ubi10_py312: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - continue-on-error: false + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.12" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - wheel_build_py313: + wheel_build_ubi10_py313: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.13" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi - wheel_build_py314: + wheel_build_ubi10_py314: needs: build_info - if: needs.build_info.outputs.wheel_build_enabled == 'true' - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} + if: ${{ needs.build_info.outputs.wheel_build_enabled == 'true' && needs.build_info.outputs.script_ubi10 != '' }} + runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.large-runner || 'ubuntu-24.04-ppc64le-p10' }} continue-on-error: true env: - GHA_CURRENCY_SERVICE_ID_API_KEY: ${{ secrets.GHA_CURRENCY_SERVICE_ID_API_KEY }} - GHA_CURRENCY_SERVICE_ID: ${{ secrets.GHA_CURRENCY_SERVICE_ID }} PYTHON_VERSION: "3.14" - + BUILD_SCRIPT: ${{ fromJson(needs.build_info.outputs.script_ubi10).script }} + TESTED_ON: ${{ fromJson(needs.build_info.outputs.script_ubi10).tested_on }} steps: - - name: Checkout code - uses: actions/checkout@v6 + - uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.sha }} - - name: Install system dependencies - run: | - sudo apt update -y - - - name: Download package-cache from previous step - uses: actions/download-artifact@v7 + run: sudo apt update -y + - name: Download package-cache + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Run build_wheels.sh run: | - ls - echo "---------------------updated cache-----------------------" - ls package-cache - - chmod +x package-cache/variable.sh - chmod +x package-cache/scanner-env.sh - source package-cache/variable.sh - source package-cache/scanner-env.sh - - # CONTROL FLAG HERE - if [ "$WHEEL_BUILD" != "true" ]; then - echo "Skipping wheel build as WHEEL_BUILD=false" - exit 0 - fi - - # Check if any .sh build script is modified in this package + THIS_SCRIPT="$BUILD_SCRIPT" + THIS_TESTED_ON="$TESTED_ON" + source package-cache/variable.sh; source package-cache/scanner-env.sh + export BUILD_SCRIPT="$THIS_SCRIPT" + export TESTED_ON="$THIS_TESTED_ON" + if [ "$WHEEL_BUILD" != "true" ]; then echo "Skipping - WHEEL_BUILD=false"; exit 0; fi BUILD_SCRIPT_CHANGED=$(echo "$CHANGED_FILES" | grep -E "^$PACKAGE_DIR/.*\.sh$" || true) - - if [[ -z "$BUILD_SCRIPT_CHANGED" ]]; then - echo "Skipping wheel build as no .sh build script changes detected" - exit 0 - fi - + [ -z "$BUILD_SCRIPT_CHANGED" ] && echo "Skipping - no .sh changes" && exit 0 sudo chown -R $USER:$USER . - chmod +x ./gha-script/build_wheels.sh - bash ./gha-script/build_wheels.sh - - echo "===========after execution ==================" - sudo apt update -y - sudo lsb_release -a 2>/dev/null || echo "lsb_release not available" - sudo uname -a - + export ENABLE_CVE_SCAN=false + chmod +x ./gha-script/build_wheels.sh; bash ./gha-script/build_wheels.sh + - name: Verify wheel was produced + run: | + if ls *.whl 1>/dev/null 2>&1; then + echo "Wheel built successfully: $(ls *.whl)" + else + echo "No wheel produced." + exit 1 + fi + # --------------------------------------------------------------------------- + # build_docker - only when Dockerfile changed and docker_build=true in build_info + # --------------------------------------------------------------------------- build_docker: needs: build_info if: needs.build_info.outputs.docker_build_enabled == 'true' runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04-ppc64le-p10' || inputs.large-runner }} - steps: - name: Checkout code - uses: actions/checkout@v6 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Download package-cache - uses: actions/download-artifact@v7 + uses: actions/download-artifact@v4 with: name: package-cache - - name: Extract package cache run: tar -xzf package-cache.tar.gz - - name: Build Docker Image run: | - echo "===== Sourcing environment =====" ls package-cache - chmod +x package-cache/variable.sh chmod +x package-cache/scanner-env.sh source package-cache/variable.sh source package-cache/scanner-env.sh - BUILD_DOCKER=$(echo "$BUILD_DOCKER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') DOCKERFILE_CHANGED=$(echo "$CHANGED_FILES" | grep -i 'Dockerfile' || true) - if [[ "$BUILD_DOCKER" != "true" ]]; then - echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER" - exit 0 + echo "Skipping Docker build as BUILD_DOCKER=$BUILD_DOCKER"; exit 0 fi - if [[ -z "$DOCKERFILE_CHANGED" ]]; then - echo "Skipping Docker build as no Dockerfile changes detected in PR" - exit 0 + echo "Skipping Docker build as no Dockerfile changes detected in PR"; exit 0 fi - echo "Dockerfile change detected, proceeding with build..." - - echo "===== Starting Docker build =====" sudo chown -R $USER:$USER . chmod +x ./gha-script/build_docker.sh bash ./gha-script/build_docker.sh - - echo "===== Docker images after build =====" docker images - - echo "===== Saving Docker image =====" docker save -o package-cache/image.tar "$IMAGE_NAME" - ls -lh package-cache/image.tar diff --git a/d/duckdb/build_info.json b/d/duckdb/build_info.json index f15ddf64b5..4c66bc4abe 100644 --- a/d/duckdb/build_info.json +++ b/d/duckdb/build_info.json @@ -5,13 +5,13 @@ "version": "v1.5.4", "default_branch": "main", "package_dir": "d/duckdb", - "build_script": "duckdb_1.5.4_ubi_9.6.sh", + "build_script": "duckdb_1.5.4_ubi_10.2.sh", "docker_build": false, "validate_build_script": true, "wheel_build" : true, "use_non_root_user": false, "1.5.4": { - "build_script": "duckdb_1.5.4_ubi_9.6.sh" + "build_script": "duckdb_1.5.4_ubi_10.2.sh" }, "*": { "build_script": "duckdb_ubi_9.6.sh" diff --git a/d/duckdb/duckdb_1.5.4_ubi_9.6.sh b/d/duckdb/duckdb_1.5.4_ubi_10.2.sh similarity index 65% rename from d/duckdb/duckdb_1.5.4_ubi_9.6.sh rename to d/duckdb/duckdb_1.5.4_ubi_10.2.sh index 7db8e3caef..118a8bde0d 100755 --- a/d/duckdb/duckdb_1.5.4_ubi_9.6.sh +++ b/d/duckdb/duckdb_1.5.4_ubi_10.2.sh @@ -4,7 +4,7 @@ # Package : duckdb # Version : v1.5.4 # Source repo : https://github.com/duckdb/duckdb-python.git -# Tested on : UBI:9.6 +# Tested on : UBI:10.2 # Language : Python, C++ # Ci-Check : True # Script License: Apache License, Version 2 or later @@ -23,42 +23,41 @@ PACKAGE_VERSION=${1:-v1.5.4} PACKAGE_DIR=duckdb-python PACKAGE_URL=https://github.com/duckdb/duckdb-python.git PYTHON_VERSION=3.12 +SOURCE_ROOT="$(pwd)" # Install necessary system packages -dnf install -y gcc-toolset-13 make cmake ninja-build libomp-devel git python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel +dnf install -y \ + gcc-toolset-15 \ + cmake \ + ninja-build \ + python3.12 \ + python3.12-devel \ + python3.12-pip -# Enable GCC toolset -source /opt/rh/gcc-toolset-13/enable -export CXX=/opt/rh/gcc-toolset-13/root/usr/bin/g++ +export PATH="/opt/rh/gcc-toolset-15/root/usr/bin:$PATH" +gcc --version +python3.12 -m pip install --upgrade pip setuptools -python${PYTHON_VERSION} -m pip install build wheel setuptools ninja pybind11 +# -- Build wheel -------------------------------------------------------------- +python3.12 -m pip wheel . --no-deps -w "${CURRENT_DIR}/dist/" -# Clone the repository -git clone ${PACKAGE_URL} -cd ${PACKAGE_NAME} -git checkout ${PACKAGE_VERSION} - -git submodule update --init --recursive - -export DUCKDB_BUILD_PYTHON=1 -export DUCKDB_BUILD_STATIC=1 - -echo "Building duckdb wheel..." -if ! python${PYTHON_VERSION} -m build --wheel; then - echo "------------------$PACKAGE_NAME: build_fail------------------" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Build_Fail" +WHEEL=$(find "${CURRENT_DIR}/dist" -name "duckdb-*.whl" | head -1) +if [ -z "$WHEEL" ]; then + echo "ERROR: wheel not found after build" exit 1 fi +echo "Wheel: $WHEEL" -echo "Installing duckdb wheel..." -WHEEL_FILE=$(find dist -name "*.whl" | head -n1) -if [ -n "$WHEEL_FILE" ]; then - python${PYTHON_VERSION} -m pip install "$WHEEL_FILE" +# Copy wheel to /home/tester so the wrapper script can locate it without rebuilding +if [ -d /home/tester ]; then + cp "${WHEEL}" /home/tester/ fi -# Run tests -cd / +cd "${SOURCE_ROOT}" + +# -- Install ------------------------------------------------------------------ +pip3.12 install "$WHEEL" if ! python${PYTHON_VERSION} - < "UBI9.3" + # Uses sed only - no grep -P (absent on ppc64le runners). + local norm + norm=$(echo "$upper" | sed 's/UBI[[:space:]]*[: ][[:space:]]*/UBI/g') + + # Extract the major UBI version integer immediately after "UBI" + local major + major=$(echo "$norm" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + case "$major" in + ''|*[!0-9]*) major="" ;; + esac + + if [ -z "$major" ]; then + echo "ERROR: Cannot determine UBI major version from TESTED_ON='$raw'" + exit 1 + fi + + # Extract the full version string e.g. "9.3", "10.3" (first X.Y found) + local full + full=$(echo "$upper" | grep -oE '[0-9]+\.[0-9]+' | head -1) + + if [ -z "$full" ]; then + # No minor version supplied - use per-major defaults + case "$major" in + 8) full="8.7" ;; + 9) full="9.3" ;; + 10) full="10.0" ;; + *) full="${major}.0" ;; + esac + echo "WARNING: No minor version in TESTED_ON='$raw', defaulting to UBI ${full}" + fi + + # Select the registry image by major version + case "$major" in + 8) + docker_image="registry.access.redhat.com/ubi8/ubi:${full}" + ;; + 9) + docker_image="registry.access.redhat.com/ubi9/ubi:${full}" + ;; + 10) + docker_image="registry.access.redhat.com/ubi10/ubi:${full}" + ;; + *) + echo "ERROR: Unsupported UBI major version '${major}' in TESTED_ON='$raw'" + exit 1 + ;; + esac + + echo " Resolved Docker image: $docker_image (TESTED_ON='$raw')" + + if [[ "$NON_ROOT_BUILD" == "true" ]]; then + docker_build_non_root "$docker_image" + fi +} + +resolve_docker_image "$TESTED_ON" +docker pull "$docker_image" python3 gha-script/validate_builds_currency.py "$PKG_DIR_PATH$BUILD_SCRIPT" "$VERSION" "$docker_image" 2>&1 | tee build_log diff --git a/gha-script/build_wheels.py b/gha-script/build_wheels.py index 0fdd6f16a0..b3581a7755 100644 --- a/gha-script/build_wheels.py +++ b/gha-script/build_wheels.py @@ -43,6 +43,15 @@ def trigger_build_wheel(wrapper_file, python_version, image_name, file_name, ver "GHA_CURRENCY_SERVICE_ID_API_KEY": os.getenv("GHA_CURRENCY_SERVICE_ID_API_KEY"), "GHA_CURRENCY_SERVICE_ID": os.getenv("GHA_CURRENCY_SERVICE_ID"), "AUDITWHEEL_EXCLUDE": os.getenv("AUDITWHEEL_EXCLUDE", ""), + # Grype is installed on the host runner at scan-tools-bin/grype. + # The workspace is volume-mounted at /home/tester/ inside the + # container, so the binary is reachable at that in-container path. + # Passing GRYPE_BIN lets generalized_wheel_scanner.py find it via + # os.environ without relying on $PATH (which is host-only). + "GRYPE_BIN": "/home/tester/scan-tools-bin/grype", + # Set to "false" by pr-build.yaml to skip the CVE scan in PR builds. + # Defaults to "true" (scan runs) when unset (currency-build.yaml). + "ENABLE_CVE_SCAN": os.getenv("ENABLE_CVE_SCAN", "true"), } ) diff --git a/gha-script/build_wheels.sh b/gha-script/build_wheels.sh index 59d246dbeb..092766c2da 100644 --- a/gha-script/build_wheels.sh +++ b/gha-script/build_wheels.sh @@ -10,61 +10,83 @@ echo "************************************************************************** docker_image="" -# the below function is used for building a custom docker image, it will be called only when non root user build is set to true. -# function accepts one argument, which is the base image value. +# Builds a custom non-root wrapper image on top of the given base image. +# Sets docker_image to "docker_non_root_image" on success. docker_build_non_root() { echo "building docker image for non root user build" docker build --build-arg BASE_IMAGE="$1" -t docker_non_root_image -f gha-script/dockerfile_non_root . docker_image="docker_non_root_image" } -#Below conditions are used to select the base image based on the 2 flags, tested_on and non_root_build. -#A docker_build_non_root function is called when non root build is true. -if [[ "$TESTED_ON" == UBI:9* || "$TESTED_ON" == UBI9* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - docker pull registry.access.redhat.com/ubi9/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi9/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi9/ubi:$ubi_version" - fi -elif [[ "$TESTED_ON" == UBI:10* || "$TESTED_ON" == UBI10* ]]; -then - ubi_version=$(echo "$TESTED_ON" | grep -oE '[0-9]+\.[0-9]+') - # Fallback to 10.0 if no minor version found - ubi_version=${ubi_version:-10.0} - docker pull registry.access.redhat.com/ubi10/ubi:$ubi_version - docker_image="registry.access.redhat.com/ubi10/ubi:$ubi_version" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi10/ubi:$ubi_version" - fi -else - docker pull registry.access.redhat.com/ubi8/ubi:8.7 - docker_image="registry.access.redhat.com/ubi8/ubi:8.7" - if [[ "$NON_ROOT_BUILD" == "true" ]]; - then - docker_build_non_root "registry.access.redhat.com/ubi8/ubi:8.7" - fi -fi +# Resolves the correct UBI registry image from the TESTED_ON string and +# sets the global docker_image variable. Supports UBI8, UBI9, UBI10+. +# The TESTED_ON value is normalised before parsing so all of the following +# formats work: "UBI:9.3" "UBI 9.3" "ubi9.3" "UBI: 9.3" "UBI 10" +resolve_docker_image() { + local raw="$1" + local upper + upper=$(echo "$raw" | tr '[:lower:]' '[:upper:]') -# UBI 10 does not ship Python 3.10 or 3.11 in its repos. -# Building 3.10 from source is not supported for UBI 10 wheel builds. -# Skip those Python versions early to avoid false-positive successes. -if [[ "$TESTED_ON" == UBI:10* || "$TESTED_ON" == UBI10* ]]; then - if [[ "$PYTHON_VERSION" == "3.10" || "$PYTHON_VERSION" == "3.11" ]]; then - echo "Skipping wheel build for Python $PYTHON_VERSION — not supported on UBI 10" - exit 0 - fi -fi + # Collapse "UBI : 9.3" / "UBI:9.3" / "UBI 9.3" / "UBI9.3" -> "UBI9.3" + # Uses sed only - no grep -P (absent on ppc64le runners). + local norm + norm=$(echo "$upper" | sed 's/UBI[[:space:]]*[: ][[:space:]]*/UBI/g') -WHEEL_SCRIPT=gha-script/create_wheel_wrapper.sh + # Extract the major UBI version integer immediately after "UBI" + local major + major=$(echo "$norm" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + case "$major" in + ''|*[!0-9]*) major="" ;; + esac + + if [ -z "$major" ]; then + echo "ERROR: Cannot determine UBI major version from TESTED_ON='$raw'" + exit 1 + fi -# Ensure the wrapper script is readable and executable by all users. -# Required for non-root container builds where the mounted volume is owned -# by the host runner user but executed as test_user inside the container. -chmod a+rx "$WHEEL_SCRIPT" + # Extract the full version string e.g. "9.3", "10.3" (first X.Y found) + local full + full=$(echo "$upper" | grep -oE '[0-9]+\.[0-9]+' | head -1) + + if [ -z "$full" ]; then + # No minor version supplied - use per-major defaults + case "$major" in + 8) full="8.7" ;; + 9) full="9.3" ;; + 10) full="10.0" ;; + *) full="${major}.0" ;; + esac + echo "WARNING: No minor version in TESTED_ON='$raw', defaulting to UBI ${full}" + fi + + # Select the registry image by major version + case "$major" in + 8) + docker_image="registry.access.redhat.com/ubi8/ubi:${full}" + ;; + 9) + docker_image="registry.access.redhat.com/ubi9/ubi:${full}" + ;; + 10) + docker_image="registry.access.redhat.com/ubi10/ubi:${full}" + ;; + *) + echo "ERROR: Unsupported UBI major version '${major}' in TESTED_ON='$raw'" + exit 1 + ;; + esac + + echo " Resolved Docker image: $docker_image (TESTED_ON='$raw')" + + if [[ "$NON_ROOT_BUILD" == "true" ]]; then + docker_build_non_root "$docker_image" + fi +} + +resolve_docker_image "$TESTED_ON" +docker pull "$docker_image" + +WHEEL_SCRIPT=gha-script/create_wheel_wrapper.sh # path to post_process_wheel script (suffix addition, license addition, metadata addition) POST_PROCESS_SCRIPT_PATH=gha-script/post_process_wheel.py diff --git a/gha-script/create_wheel_wrapper.sh b/gha-script/create_wheel_wrapper.sh index b699b68279..011666f26f 100644 --- a/gha-script/create_wheel_wrapper.sh +++ b/gha-script/create_wheel_wrapper.sh @@ -7,28 +7,20 @@ EXTRA_ARGS=${3:-""} POST_PROCESS_SCRIPT_PATH=${4:-"post_process_wheel.py"} CURRENT_DIR=$(pwd) -# Use sudo for privileged commands when running as a non-root user. -# dockerfile_non_root grants test_user passwordless sudo, so this is always safe. -# When already root, sudo is not needed (and may not be installed), so skip it. -if [[ "$(id -u)" -ne 0 ]]; then - YUM="sudo yum" - SUDO="sudo" -else - YUM="yum" - SUDO="" -fi +# install git - required by generate_sha() for all Python versions and UBI versions +yum install -y git -# install gcc — select toolset version based on UBI major version +# install gcc - select toolset version based on UBI major version UBI_MAJOR=$(grep -oP '(?<=^VERSION_ID=")[0-9]+' /etc/os-release || grep -oP 'release \K[0-9]+' /etc/redhat-release 2>/dev/null || echo "8") if [[ "$UBI_MAJOR" -ge 10 ]]; then GCC_TOOLSET="gcc-toolset-15" - $YUM install -y "$GCC_TOOLSET" - # On UBI 10, SCL (Software Collections) was dropped — there is no enable script. + yum install -y "$GCC_TOOLSET" + # On UBI 10, SCL (Software Collections) was dropped - there is no enable script. # Activate the toolset by prepending its bin directory to PATH directly. export PATH="/opt/rh/${GCC_TOOLSET}/root/usr/bin:$PATH" else GCC_TOOLSET="gcc-toolset-13" - $YUM install -y "$GCC_TOOLSET" + yum install -y "$GCC_TOOLSET" source /opt/rh/${GCC_TOOLSET}/enable fi gcc --version @@ -48,48 +40,48 @@ install_python_version() { echo case $version in "3.11" | "3.12") - $YUM install -y python${version} python${version}-devel python${version}-pip + yum install -y python${version} python${version}-devel python${version}-pip ;; "3.10") if ! python3.10 --version &>/dev/null; then - $YUM install -y zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel - $YUM install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel + yum install -y sudo zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel + yum install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel wget https://www.python.org/ftp/python/3.10.20/Python-3.10.20.tgz tar xf Python-3.10.20.tgz cd Python-3.10.20 ./configure --prefix=/usr/local --enable-optimizations --enable-shared make -j2 make altinstall - echo "/usr/local/lib" | $SUDO tee /etc/ld.so.conf.d/python-local.conf && $SUDO ldconfig + echo "/usr/local/lib" > /etc/ld.so.conf.d/python-local.conf && ldconfig echo "Completed..." cd .. && rm -rf Python-3.10.20.tgz fi ;; "3.13") if ! python3.13 --version &>/dev/null; then - $YUM install -y zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel - $YUM install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel + yum install -y sudo zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel + yum install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel wget https://www.python.org/ftp/python/3.13.10/Python-3.13.10.tgz tar xzf Python-3.13.10.tgz cd Python-3.13.10 ./configure --prefix=/usr/local --enable-optimizations --enable-shared make -j2 make altinstall - echo "/usr/local/lib" | $SUDO tee /etc/ld.so.conf.d/python-local.conf && $SUDO ldconfig + echo "/usr/local/lib" > /etc/ld.so.conf.d/python-local.conf && ldconfig cd .. && rm -rf Python-3.13.10.tgz fi ;; "3.14") if ! python3.14 --version &>/dev/null; then - $YUM install -y zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel - $YUM install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel + yum install -y sudo zlib-devel wget ncurses git make cmake openssl-devel xz xz-devel + yum install -y libffi libffi-devel sqlite sqlite-devel sqlite-libs bzip2-devel wget https://www.python.org/ftp/python/3.14.3/Python-3.14.3.tgz tar xzf Python-3.14.3.tgz cd Python-3.14.3 ./configure --prefix=/usr/local --enable-optimizations --enable-shared make -j2 make altinstall - echo "/usr/local/lib" | $SUDO tee /etc/ld.so.conf.d/python-local.conf && $SUDO ldconfig + echo "/usr/local/lib" > /etc/ld.so.conf.d/python-local.conf && ldconfig cd .. && rm -rf Python-3.14.3.tgz fi ;; @@ -345,16 +337,31 @@ fi cd "$CURRENT_DIR" wheel_final=(*.whl) -echo -echo "============== Running CVE scan on: ${wheel_final} ==============" -echo +# --------------------------------------------------------------------------- +# run_cve_scan: runs generalized_wheel_scanner.py on the built wheel. +# +# To skip CVE scanning when testing wheel creation locally, comment out the +# run_cve_scan call below (search for "run_cve_scan" further down). +# +# In CI the ENABLE_CVE_SCAN env var controls this: +# ENABLE_CVE_SCAN=false -> skip (set by pr-build.yaml) +# unset or "true" -> run (default for currency-build.yaml) +# --------------------------------------------------------------------------- +run_cve_scan() { + local wheel=$1 + local build_script=$2 + + SCANNER_PATH="gha-script/generalized_wheel_scanner.py" + if [ ! -f "$SCANNER_PATH" ]; then + echo "===> WARNING: $SCANNER_PATH not found, skipping CVE scan." + return 0 + fi + + echo + echo "============== Running CVE scan on: ${wheel} ==============" + echo -# Run CVE scanner — passes wheel + original build script so Lane 3 can parse -# git clone/checkout lines for source-built lib names and versions. -# Scanner is non-blocking — failure does not stop the build. -SCANNER_PATH="gha-script/generalized_wheel_scanner.py" -if [ -f "$SCANNER_PATH" ]; then - if python "$SCANNER_PATH" "${wheel_final}" "${BUILD_SCRIPT_PATH}"; then + if python "$SCANNER_PATH" "${wheel}" "${build_script}"; then echo echo "===> CVE scan completed successfully." echo @@ -363,9 +370,7 @@ if [ -f "$SCANNER_PATH" ]; then echo "===> WARNING: CVE scan failed. Continuing build." echo fi -else - echo "===> WARNING: $SCANNER_PATH not found, skipping CVE scan." -fi +} echo echo "============== Generating sha for: ${wheel_final} ==============" @@ -380,9 +385,6 @@ echo echo "=== Post Processing wheel ${wheel_final} with SHA: ${SHA256_VALUE} ===" echo -# Save CVE report name before post-processing renames the wheel -cve_report_old="${wheel_final%.whl}_cve_report.json" - # post processing of wheels (Suffix addition, license addition, metadata addition) if python ${POST_PROCESS_SCRIPT_PATH} ${wheel_final} ${SHA256_VALUE}; then echo @@ -395,12 +397,17 @@ else exit 1 fi -# Rename CVE report to match the post-processed wheel filename +# CVE scan runs after post-processing so the report is named after the final +# wheel filename (with +ppc64leN suffix) from the start - no rename needed. wheel_post_processed=(*.whl) -cve_report_new="${wheel_post_processed[0]%.whl}_cve_report.json" -if [ -f "$cve_report_old" ] && [ "$cve_report_old" != "$cve_report_new" ]; then - mv "$cve_report_old" "$cve_report_new" - echo "===> CVE report renamed: $cve_report_old → $cve_report_new" + +# Call run_cve_scan - comment out this block locally to skip CVE scanning. +if [ "${ENABLE_CVE_SCAN:-true}" = "false" ]; then + echo + echo "===> Skipping CVE scan (ENABLE_CVE_SCAN=false)." + echo +else + run_cve_scan "${wheel_post_processed[0]}" "${BUILD_SCRIPT_PATH}" fi echo diff --git a/gha-script/generalized_wheel_scanner.py b/gha-script/generalized_wheel_scanner.py index 116288acbc..aa00c890fa 100644 --- a/gha-script/generalized_wheel_scanner.py +++ b/gha-script/generalized_wheel_scanner.py @@ -54,16 +54,29 @@ def scan(self) -> Dict: print(f"Scanning Wheel: {self.wheel_name}") print(f"{'='*70}\n") - # Step 0: Check grype availability — grype is installed by the workflow - # step before this script runs; if it is missing, grype-dependent lanes - # are skipped gracefully and Lane 3 (NVD) still runs. + # Step 0: Locate grype. + # Grype is installed in the workflow's install_scan_tools job and cached + # as scan-tools-bin/grype in the artifact. The workspace is volume-mounted + # into this container at /home/tester/, so the binary is accessible at + # /home/tester/scan-tools-bin/grype — passed in via the GRYPE_BIN env var + # by build_wheels.py. We honour GRYPE_BIN first; if unset or the path is + # not executable we fall back to shutil.which('grype') for local runs. + # If neither resolves, grype-dependent lanes are skipped gracefully and + # Lane 3 (NVD) still runs. print("Step 0: Checking grype availability...") - grype_available = bool(shutil.which('grype')) - if grype_available: - print(" grype is available.") + grype_bin = os.environ.get("GRYPE_BIN", "").strip() + if grype_bin and os.path.isfile(grype_bin) and os.access(grype_bin, os.X_OK): + grype_available = True + print(f" grype found via GRYPE_BIN: {grype_bin}") else: - print(" WARNING: grype not found. Lane 1, Lane 2 and Phase 1 " - "(wheel direct scan) will be skipped. Lane 3 (NVD) will still run.") + grype_bin = shutil.which('grype') or "" + grype_available = bool(grype_bin) + if grype_available: + print(f" grype found on PATH: {grype_bin}") + else: + print(" WARNING: grype not found. Lane 1, Lane 2 and Phase 1 " + "(wheel direct scan) will be skipped. Lane 3 (NVD) will still run.") + self.grype_bin = grype_bin # used by _scan_wheel_direct / _scan_for_cves self.results['grype_available'] = grype_available # Step 1: Create system library inventory @@ -471,7 +484,7 @@ def _scan_wheel_direct(self, extract_dir: Path) -> Dict: print(f" Running grype on extracted wheel: {extract_dir}") try: result = subprocess.run( - ['grype', f'dir:{extract_dir}', '-o', 'json'], + [self.grype_bin, f'dir:{extract_dir}', '-o', 'json'], capture_output=True, text=True, timeout=300 ) if result.returncode == 0: @@ -512,7 +525,7 @@ def _scan_for_cves(self) -> Dict: try: result = subprocess.run( - ['grype', 'dir:/', '-o', 'json'], + [self.grype_bin, 'dir:/', '-o', 'json'], capture_output=True, text=True, timeout=300 diff --git a/gha-script/post_process_wheel.py b/gha-script/post_process_wheel.py index ff6c31dbf0..fc7bf1aa56 100644 --- a/gha-script/post_process_wheel.py +++ b/gha-script/post_process_wheel.py @@ -43,9 +43,15 @@ logger = logging.getLogger(__name__) -# COS configuration -COS_API_KEY = os.environ["GHA_CURRENCY_SERVICE_ID_API_KEY"] -COS_SERVICE_INSTANCE_ID = os.environ["GHA_CURRENCY_SERVICE_ID"] +# COS configuration +# These are only available in currency builds (not PR builds). +# If missing, post-processing is skipped gracefully. +COS_API_KEY = os.environ.get("GHA_CURRENCY_SERVICE_ID_API_KEY", "") +COS_SERVICE_INSTANCE_ID = os.environ.get("GHA_CURRENCY_SERVICE_ID", "") + +if not COS_API_KEY or not COS_SERVICE_INSTANCE_ID: + logger.info("COS credentials not set - skipping post-processing (PR build environment)") + sys.exit(0) COS_ENDPOINT = "https://s3.us.cloud-object-storage.appdomain.cloud" COS_BUCKET = "ose-power-artifacts-production" diff --git a/gha-script/read_buildinfo.sh b/gha-script/read_buildinfo.sh index 5c57c5ad88..c5f16ce8c6 100755 --- a/gha-script/read_buildinfo.sh +++ b/gha-script/read_buildinfo.sh @@ -27,7 +27,7 @@ if [ -f $config_file ]; then jsonObj=$config_file build_script=$(jq .build_script $jsonObj) - if $(jq 'has("use_non_root_user")' $jsonObj); then + if $(jq 'has("use_non_root_user")' $jsonObj); then nonRootBuild=$(jq .use_non_root_user $jsonObj) fi @@ -54,11 +54,11 @@ if [ -f $config_file ]; then # Getting specific build_script name and other overrides for version if [[ $(jq --arg ver "$match_version" '.[$ver]' $config_file) != null ]]; then - version_block=".[\"$match_version\"]" # ✅ Properly quoted key for jq + version_block=".[\"$match_version\"]" # Properly quoted key for jq - # version-specific build_script + # version-specific build_script (may be a string or a list) if [[ $(jq -r "$version_block.build_script" $config_file) != "null" ]]; then - build_script=$(jq -r "$version_block.build_script" $config_file) + build_script=$(jq -c "$version_block.build_script" $config_file) fi # version-specific base_docker_image @@ -73,7 +73,7 @@ if [ -f $config_file ]; then "rhel") variant=1 ;; "ubuntu") variant=2 ;; "alpine") variant=3 ;; - *) + *) echo "No valid distro variant, picking default one" variant=1 ;; esac @@ -96,75 +96,207 @@ if [ -f $config_file ]; then fi fi -# #Getting specific build_script name for version -# if [[ $(jq --arg ver "$match_version" '.[$ver]' $config_file) != null ]]; then -# if [[ $(jq -r --arg ver "$match_version" '.[$ver].build_script' $config_file) != null ]]; then -# build_script=$(jq -r --arg ver "$match_version" '.[$ver].build_script' $config_file) -# fi -# if [[ $(jq -r --arg ver "$match_version" '.[$ver].base_docker_image' $config_file) != null ]]; then -# basename=$(jq -r --arg ver "$match_version" '.[$ver].base_docker_image' $config_file) -# fi -# if [[ $(jq -r --arg ver "$match_version" '.[$ver].base_docker_variant' $config_file) != null ]]; then -# variant_str=$(jq -r --arg ver "$match_version" '.[$ver].base_docker_variant' $config_file) -# case "$variant_str" in -# "rhel") -# variant=1 -# ;; -# "ubuntu") -# variant=2 -# ;; -# "alpine") -# variant=3 -# ;; -# *) -# echo "No valid distro variant, picking default one" -# variant=1 -# ;; -# esac -# fi -# fi -# fi - -# Below code is used to get the tested on parameter value from the build script -build_script_with_quotes=$build_script -stripped_build_script=$(echo "$build_script_with_quotes" | sed 's/"//g') -echo $stripped_build_script - - -if [ -f "$stripped_build_script" ]; then - - echo "build script found" - while IFS= read -r line; do - # Check if the line starts with '# Tested on' + + +# --------------------------------------------------------------------------- +# Helper: read "# Tested on" from a single build script file. +# Normalises the raw value: strip spaces, uppercase, collapse "UBI : 9.3" -> +# "UBI:9.3" so downstream consumers see a consistent format. +# Usage: tested_on=$(read_tested_on "path/to/script.sh") +# --------------------------------------------------------------------------- +read_tested_on() { + local script_file="$1" + local value="" + if [ -f "$script_file" ]; then + while IFS= read -r line; do if [[ "$line" == "# Tested on"* ]]; then - # Extract the value after the first colon - tested_on=$(echo "$line" | cut -d ':' -f 2- | tr -d '[:space:]' | tr '[:lower:]' '[:upper:]') - break + # Extract everything after the first colon, strip outer whitespace, + # uppercase, then collapse spaces around colons (e.g. "UBI : 9.3" -> "UBI:9.3") + value=$(echo "$line" | cut -d ':' -f 2- \ + | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \ + | tr '[:lower:]' '[:upper:]' \ + | sed 's/[[:space:]]*:[[:space:]]*/:/g') + break fi + done < "$script_file" + fi + echo "$value" +} + +# --------------------------------------------------------------------------- +# Build the BUILD_SCRIPTS_JSON array and set the single-script compat vars. +# +# build_script in build_info.json can be: +# - a string : "pytorch_ubi_9.3.sh" +# - a list : ["pytorch_ubi_8.3.sh", "pytorch_ubi_9.3.sh", "pytorch_ubi_10.3.sh"] +# +# For both cases we produce: +# BUILD_SCRIPTS_JSON - compact JSON array of {script, tested_on} objects, +# consumed by the GHA matrix via fromJson(). +# BUILD_SCRIPT - backward-compat: first (or only) script filename. +# TESTED_ON - backward-compat: first (or only) tested_on value. +# +# NOTE: '# Tested on' is MANDATORY in every build script. The pipeline will +# abort if it is missing - there is no default fallback. +# --------------------------------------------------------------------------- + +# Derive type from the already-resolved $build_script shell variable. +# This is correct whether build_script came from the top-level field or +# was overridden by a version block - both paths store the final value in +# $build_script as either a bare filename string or a compact JSON array. +script_type=$(echo "$build_script" | jq -r 'type' 2>/dev/null || echo "string") +echo "build_script type (resolved): $script_type" + +BUILD_SCRIPTS_JSON="" + +if [ "$script_type" = "array" ]; then + echo "build_script is a list - iterating each entry to read its '# Tested on' value" + + json_array="[" + first=true + first_script="" + first_tested_on="" + + # jq outputs one bare filename per line (no surrounding quotes) + while IFS= read -r script_name; do + # strip any stray quotes/whitespace that jq might leave + script_name=$(echo "$script_name" | tr -d '"' | xargs) + [ -z "$script_name" ] && continue + + raw_tested_on=$(read_tested_on "$script_name") + + if [ -z "$raw_tested_on" ]; then + echo "ERROR: '# Tested on' header is missing in '$script_name'." + echo " Every build script must declare the UBI version it targets, e.g.:" + echo " # Tested on UBI:9.3" + exit 1 + fi + + echo " script='$script_name' tested_on='$raw_tested_on'" + + # Record first entry for backward-compat single-value exports + if [ "$first" = "true" ]; then + first_script="$script_name" + first_tested_on="$raw_tested_on" + first=false + fi + + # Append JSON object - jq --arg safely escapes both values + entry=$(jq -n \ + --arg s "$script_name" \ + --arg t "$raw_tested_on" \ + '{"script":$s,"tested_on":$t}') - done < "$stripped_build_script" + if [ "$json_array" = "[" ]; then + json_array="${json_array}${entry}" + else + json_array="${json_array},${entry}" + fi + + # Iterate the resolved $build_script variable (not the file) so that + # version-block overrides are respected. + done < <(echo "$build_script" | jq -r '.[]') + + json_array="${json_array}]" + BUILD_SCRIPTS_JSON="$json_array" + + # Backward-compat single values - point to the first entry in the list + build_script="$first_script" + tested_on="$first_tested_on" + +else + # Single string - existing behaviour + build_script_with_quotes=$build_script + stripped_build_script=$(echo "$build_script_with_quotes" | sed 's/"//g') + echo "build_script (single): $stripped_build_script" + + tested_on=$(read_tested_on "$stripped_build_script") + + if [ -z "$tested_on" ]; then + echo "ERROR: '# Tested on' header is missing in '$stripped_build_script'." + echo " Every build script must declare the UBI version it targets, e.g.:" + echo " # Tested on UBI:9.3" + exit 1 + fi echo "Tested on value: $tested_on" + + # Wrap single entry into the same JSON array format for consistency + BUILD_SCRIPTS_JSON=$(jq -n \ + --arg s "$stripped_build_script" \ + --arg t "$tested_on" \ + '[{"script":$s,"tested_on":$t}]') + + # Backward-compat: strip quotes from the jq-extracted string value + build_script="$stripped_build_script" fi -# Extract auditwheel exclusions +echo "BUILD_SCRIPTS_JSON: $BUILD_SCRIPTS_JSON" + +# --------------------------------------------------------------------------- +# Bucket scripts by UBI major version -> SCRIPT_UBI8, SCRIPT_UBI9, SCRIPT_UBI10. +# Each is a single {script, tested_on} JSON object (or empty string ""). +# At most one script per UBI major version is expected per package. +# Callers (currency-build.yaml, pr-build.yaml) use these to drive named jobs: +# build_ubi8 / build_ubi9 / build_ubi10 +# wheel_build_ubi8_pyXXX / wheel_build_ubi9_pyXXX / wheel_build_ubi10_pyXXX +# --------------------------------------------------------------------------- +SCRIPT_UBI8="" +SCRIPT_UBI9="" +SCRIPT_UBI10="" + +while IFS= read -r entry; do + [ -z "$entry" ] && continue + t_on=$(echo "$entry" | jq -r '.tested_on') + # Normalise: uppercase, collapse separators around UBI, then extract the + # integer immediately following "UBI". Uses only sed + tr (POSIX) - no + # grep -P needed, so it works on ppc64le runners where grep -P is absent. + t_upper=$(echo "$t_on" | tr '[:lower:]' '[:upper:]') + # Collapse "UBI : 9.3" / "UBI:9.3" / "UBI 9.3" / "UBI9.3" -> "UBI9.3" + t_norm=$(echo "$t_upper" | sed 's/UBI[[:space:]]*[: ][[:space:]]*/UBI/g') + # Extract digits immediately after "UBI" e.g. "UBI10.0" -> "10" + major=$(echo "$t_norm" | sed 's/.*UBI\([0-9][0-9]*\).*/\1/') + # If sed left non-numeric content (no UBI match), clear it + case "$major" in + ''|*[!0-9]*) major="" ;; + esac + echo " bucket: tested_on='$t_on' major='$major'" + case "$major" in + 8) SCRIPT_UBI8="$entry" ;; + 9) SCRIPT_UBI9="$entry" ;; + 10) SCRIPT_UBI10="$entry" ;; + *) echo "WARNING: Unknown UBI major '$major' in tested_on='$t_on' - skipping bucket" ;; + esac +done < <(echo "$BUILD_SCRIPTS_JSON" | jq -c '.[]') + +echo "SCRIPT_UBI8: $SCRIPT_UBI8" +echo "SCRIPT_UBI9: $SCRIPT_UBI9" +echo "SCRIPT_UBI10: $SCRIPT_UBI10" + +# Extract auditwheel exclusions (unchanged - same pattern as before) AUDITWHEEL_EXCLUDE="" if jq -e 'has("auditwheel_exclude")' "$config_file" >/dev/null; then AUDITWHEEL_EXCLUDE=$(jq -r '.auditwheel_exclude | join(" ")' "$config_file") fi -# Export variables - -echo "export VERSION=$VERSION" > $CUR_DIR/variable.sh -echo "export BUILD_SCRIPT=$build_script" >> $CUR_DIR/variable.sh -echo "export PKG_DIR_PATH=$package_dirpath" >> $CUR_DIR/variable.sh -echo "export IMAGE_NAME=$image_name" >> $CUR_DIR/variable.sh -#echo "export BUILD_DOCKER=$build_docker" >> $CUR_DIR/variable.sh -#echo "export VALIDATE_BUILD_SCRIPT=$validate_build_script" >> $CUR_DIR/variable.sh -echo "export VARIANT=$variant" >> $CUR_DIR/variable.sh -echo "export BASENAME=$basename" >> $CUR_DIR/variable.sh -echo "export NON_ROOT_BUILD=$nonRootBuild" >> $CUR_DIR/variable.sh -echo "export TESTED_ON=$tested_on" >> $CUR_DIR/variable.sh -echo "export AUDITWHEEL_EXCLUDE=\"$AUDITWHEEL_EXCLUDE\"" >> $CUR_DIR/variable.sh +# --------------------------------------------------------------------------- +# Write variable.sh +# JSON objects are single-quote-wrapped so embedded double-quotes survive. +# --------------------------------------------------------------------------- +echo "export VERSION=\"$VERSION\"" > $CUR_DIR/variable.sh +echo "export BUILD_SCRIPT=\"$build_script\"" >> $CUR_DIR/variable.sh +echo "export PKG_DIR_PATH=\"$package_dirpath\"" >> $CUR_DIR/variable.sh +echo "export IMAGE_NAME=\"$image_name\"" >> $CUR_DIR/variable.sh +echo "export VARIANT=\"$variant\"" >> $CUR_DIR/variable.sh +echo "export BASENAME=\"$basename\"" >> $CUR_DIR/variable.sh +echo "export NON_ROOT_BUILD=\"$nonRootBuild\"" >> $CUR_DIR/variable.sh +echo "export TESTED_ON=\"$tested_on\"" >> $CUR_DIR/variable.sh +echo "export AUDITWHEEL_EXCLUDE=\"$AUDITWHEEL_EXCLUDE\"" >> $CUR_DIR/variable.sh +# Full array - kept for any downstream consumer that still needs it +echo "export BUILD_SCRIPTS_JSON='$BUILD_SCRIPTS_JSON'" >> $CUR_DIR/variable.sh +# Per-UBI-major named exports - empty string when that UBI version has no script +echo "export SCRIPT_UBI8='$SCRIPT_UBI8'" >> $CUR_DIR/variable.sh +echo "export SCRIPT_UBI9='$SCRIPT_UBI9'" >> $CUR_DIR/variable.sh +echo "export SCRIPT_UBI10='$SCRIPT_UBI10'" >> $CUR_DIR/variable.sh chmod +x $CUR_DIR/variable.sh cat $CUR_DIR/variable.sh diff --git a/gha-script/scanner-scripts/grype_code_scan.sh b/gha-script/scanner-scripts/grype_code_scan.sh index ceec35eb9d..a865e9f2bb 100644 --- a/gha-script/scanner-scripts/grype_code_scan.sh +++ b/gha-script/scanner-scripts/grype_code_scan.sh @@ -3,18 +3,19 @@ validate_build_script=$VALIDATE_BUILD_SCRIPT cloned_package=$CLONED_PACKAGE +# Use pre-installed grype from the cached artifact. +# $GRYPE_BIN is set by the workflow (points to scan-tools-bin/grype). +if [ -z "$GRYPE_BIN" ]; then + echo "Error: GRYPE_BIN environment variable not set" + exit 1 +fi + cd package-cache -if [ $validate_build_script == true ];then - GRYPE_VERSION=$(curl -s https://api.github.com/repos/anchore/grype/releases/latest | grep -Po '"tag_name": "\K.*?(?=")') - wget https://github.com/anchore/grype/releases/download/$GRYPE_VERSION/grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - tar -xzf grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - chmod +x grype - sudo mv grype /usr/bin - grype --version +if [ $validate_build_script == true ]; then + echo "------------- Using cached grype ---------------" + $GRYPE_BIN version echo "Executing Grype scanner" - sudo grype -q -o cyclonedx-json dir:${cloned_package} > grype_source_sbom_results.json - #cat grype_source_sbom_results.json - sudo grype -q -o json dir:${cloned_package} > grype_source_vulnerabilities_results.json - #cat grype_source_vulnerabilities_results.json + sudo $GRYPE_BIN -q -o cyclonedx-json dir:${cloned_package} > grype_source_sbom_results.json + sudo $GRYPE_BIN -q -o json dir:${cloned_package} > grype_source_vulnerabilities_results.json fi diff --git a/gha-script/scanner-scripts/grype_image_scan.sh b/gha-script/scanner-scripts/grype_image_scan.sh index f91b4bbbfd..f9101d0087 100755 --- a/gha-script/scanner-scripts/grype_image_scan.sh +++ b/gha-script/scanner-scripts/grype_image_scan.sh @@ -3,13 +3,17 @@ image_name=$IMAGE_NAME build_docker=$BUILD_DOCKER -if [ $build_docker == true ];then - GRYPE_VERSION=$(curl -s https://api.github.com/repos/anchore/grype/releases/latest | grep -Po '"tag_name": "\K.*?(?=")') - wget https://github.com/anchore/grype/releases/download/$GRYPE_VERSION/grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - tar -xzf grype_${GRYPE_VERSION#v}_linux_ppc64le.tar.gz - chmod +x grype - sudo mv grype /usr/bin - echo "Executing grype scanner" - sudo grype -q -s AllLayers -o cyclonedx-json ${image_name} > grype_image_sbom_results.json - sudo grype -q -s AllLayers -o json ${image_name} > grype_image_vulnerabilities_results.json +# Use pre-installed grype from the cached artifact. +# $GRYPE_BIN is set by the workflow (points to scan-tools-bin/grype). +if [ -z "$GRYPE_BIN" ]; then + echo "Error: GRYPE_BIN environment variable not set" + exit 1 +fi + +if [ $build_docker == true ]; then + echo "------------- Using cached grype ---------------" + $GRYPE_BIN version + echo "Executing grype scanner" + sudo $GRYPE_BIN -q -s AllLayers -o cyclonedx-json ${image_name} > grype_image_sbom_results.json + sudo $GRYPE_BIN -q -s AllLayers -o json ${image_name} > grype_image_vulnerabilities_results.json fi diff --git a/gha-script/validate_builds.py b/gha-script/validate_builds.py index 95f00c4b8d..0f830d7e8e 100644 --- a/gha-script/validate_builds.py +++ b/gha-script/validate_builds.py @@ -11,7 +11,7 @@ GITHUB_BUILD_SCRIPT_BASE_REPO = "build-scripts" -GITHUB_BUILD_SCRIPT_BASE_OWNER = "ppc64le" +GITHUB_BUILD_SCRIPT_BASE_OWNER = "stutiibm" HOME = os.getcwd() package_data = {} diff --git a/m/multidict/multidict_ubi_9.3.sh b/m/multidict/multidict_ubi_9.3.sh deleted file mode 100644 index b920b7edfb..0000000000 --- a/m/multidict/multidict_ubi_9.3.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -e -# ----------------------------------------------------------------------------- -# -# Package : multidict -# Version : 6.0.2 -# Source repo : https://github.com/aio-libs/multidict.git -# Tested on : UBI:9.3 -# Language : Python -# Ci-Check : True -# Script License : Apache License, Version 2 or later -# Maintainer : Aastha Sharma -# -# Disclaimer : This script has been tested in root mode on given -# ========== platform using the mentioned version of the package. -# It may not work as expected with newer versions of the -# package and/or distribution. In such case, please -# contact "Maintainer" of this script. -# -# --------------------------------------------------------------------------- - -# Variables -PACKAGE_NAME=multidict -PACKAGE_VERSION=${1:-v6.0.2} -PACKAGE_URL=https://github.com/aio-libs/multidict.git - -# Install dependencies -yum install -y git gcc gcc-c++ make wget openssl-devel bzip2-devel libffi-devel zlib-devel python-devel python-pip cmake - -# Clone the repository -git clone $PACKAGE_URL -cd $PACKAGE_NAME # Change directory to the cloned repository -git checkout $PACKAGE_VERSION # Checkout the specified version - -# install necessary Python packages -pip install --upgrade pip setuptools wheel -pip install "coverage==7.5.4" "pytest-cov==5.0.0" objgraph psutil pytest-codspeed - -#install -if ! (python3 setup.py install) ; then - echo "------------------$PACKAGE_NAME:Install_fails-------------------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_Fails" - exit 1 -fi - -# Run tests -if ! pytest --deselect=tests/test_mutable_multidict.py::TestCIMutableMultiDict::test_add --ignore=tests/test_circular_imports.py; then - echo "------------------$PACKAGE_NAME:Install_success_but_test_fails---------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_success_but_test_Fails" - exit 2 -else - echo "------------------$PACKAGE_NAME:Install_&_test_both_success-------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Pass | Both_Install_and_Test_Success" - exit 0 -fi diff --git a/s/scikit-learn/scikit-learn_1.7.0_ubi_9.3.sh b/s/scikit-learn/scikit-learn_1.7.0_ubi_9.3.sh deleted file mode 100644 index a4331d26ae..0000000000 --- a/s/scikit-learn/scikit-learn_1.7.0_ubi_9.3.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/bash -e -# ----------------------------------------------------------------------------- -# -# Package : scikit-learn -# Version : 1.7.0 -# Source repo : https://github.com/scikit-learn/scikit-learn.git -# Tested on : UBI 9.3 -# Language : Python, Cython, C++ -# Ci-Check : True -# Script License: Apache License 2.0 -# Maintainer : Manya Rusiya -# -# Disclaimer: This script has been tested in root mode on given -# ========== platform using the mentioned version of the package. -# It may not work as expected with newer versions of the -# package and/or distribution. In such case, please -# contact "Maintainer" of this script. -# -# ----------------------------------------------------------------------------- - -PACKAGE_NAME=scikit-learn -PACKAGE_VERSION=${1:-1.7.0} -PACKAGE_URL=https://github.com/scikit-learn/scikit-learn.git -PACKAGE_DIR=scikit-learn - - -yum install -y \ - git gcc gcc-c++ make libtool cmake clang \ - openssl-devel bzip2-devel libffi-devel xz zlib-devel wget \ - python3.11 python3.11-devel python3.11-pip \ - gcc-toolset-13-gcc gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc-gfortran \ - libevent-devel openblas-devel - - -# Setup GCC toolset -export PATH=/opt/rh/gcc-toolset-13/root/usr/bin:$PATH -export LD_LIBRARY_PATH=/opt/rh/gcc-toolset-13/root/usr/lib64:$LD_LIBRARY_PATH - -# Create Python 3.11 venv -python3.11 -m venv venv311 -# shellcheck disable=SC1091 -source venv311/bin/activate - -# Upgrade pip & tools -pip install --upgrade pip setuptools wheel - -# Clone OpenBLAS & build -git clone https://github.com/OpenMathLib/OpenBLAS -cd OpenBLAS -git checkout v0.3.29 -git submodule update --init -wget -q https://raw.githubusercontent.com/ppc64le/build-scripts/refs/heads/python-ecosystem/o/openblas/pyproject.toml -sed -i "s/{PACKAGE_VERSION}/v0.3.29/g" pyproject.toml - -PREFIX=local/openblas -mkdir -p $PREFIX - -export CF="-Wno-unused-parameter -Wno-old-style-declaration" -export USE_OPENMP=1 -declare -a build_opts -build_opts+=(USE_OPENMP=${USE_OPENMP}) -build_opts+=(BINARY=64 DYNAMIC_ARCH=1 TARGET="POWER9" INTERFACE64=0) -build_opts+=(NO_LAPACK=0 USE_THREAD=1 NUM_THREADS=8 NO_AFFINITY=1) - -make -j$(nproc) ${build_opts[@]} CFLAGS="${CF}" prefix=$PREFIX -make install PREFIX="${PREFIX}" ${build_opts[@]} - -OpenBLASInstallPATH=$(pwd)/$PREFIX -export LD_LIBRARY_PATH="${OpenBLASInstallPATH}/lib:$LD_LIBRARY_PATH" -export PKG_CONFIG_PATH="${OpenBLASInstallPATH}/lib/pkgconfig:$PKG_CONFIG_PATH" - -OpenBLASConfigFile=$(find . -name OpenBLASConfig.cmake | head -n1) -OpenBLASPCFile=$(find . -name openblas.pc | head -n1) - -[ -n "$OpenBLASConfigFile" ] && sed -i "/OpenBLAS_INCLUDE_DIRS/c\SET(OpenBLAS_INCLUDE_DIRS ${OpenBLASInstallPATH}/include)" $OpenBLASConfigFile -[ -n "$OpenBLASConfigFile" ] && sed -i "/OpenBLAS_LIBRARIES/c\SET(OpenBLAS_INCLUDE_DIRS ${OpenBLASInstallPATH}/include)" $OpenBLASConfigFile -[ -n "$OpenBLASPCFile" ] && sed -i "s|libdir=local/openblas/lib|libdir=${OpenBLASInstallPATH}/lib|" $OpenBLASPCFile -[ -n "$OpenBLASPCFile" ] && sed -i "s|includedir=local/openblas/include|includedir=${OpenBLASInstallPATH}/include|" $OpenBLASPCFile - -cd .. - -# clone source repository -git clone $PACKAGE_URL -cd $PACKAGE_NAME -git checkout $PACKAGE_VERSION -git submodule update --init - -# Install Python dependencies -pip install numpy==2.0.2 scipy cython meson-python ninja joblib threadpoolctl patchelf pytest - - -# Install -if ! pip install --editable . --no-build-isolation ; then - echo "------------------$PACKAGE_NAME:Install_fails-------------------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_Fails" - exit 1 -fi - -# test using pytest - set below flag as suggested in GitHub forums to resolve ImportPathMismatchError - -export PY_IGNORE_IMPORTMISMATCH=1 -if ! pytest sklearn/tests/test_random_projection.py; then - echo "--------------------$PACKAGE_NAME:Install_success_but_test_fails---------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Fail | Install_success_but_test_Fails" - exit 2 -else - echo "------------------$PACKAGE_NAME:Install_&_test_both_success-------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | GitHub | Pass | Both_Install_and_Test_Success" - exit 0 -fi diff --git a/s/scikit-learn/scikit-learn_1.9.0_ubi_10.1.sh b/s/scikit-learn/scikit-learn_1.9.0_ubi_10.1.sh deleted file mode 100755 index 179eac71a0..0000000000 --- a/s/scikit-learn/scikit-learn_1.9.0_ubi_10.1.sh +++ /dev/null @@ -1,137 +0,0 @@ -#!/bin/bash -e -# ----------------------------------------------------------------------------- -# -# Package : scikit-learn -# Version : 1.9.0 -# Source repo : https://github.com/scikit-learn/scikit-learn.git -# Tested on : UBI 10.1 -# Language : Python, Cython, C++ -# Ci-Check : True -# Script License: Apache License 2.0 -# Maintainer : Varsha Kumar -# -# Disclaimer: This script has been tested in root mode on given -# ========== platform using the mentioned version of the package. -# It may not work as expected with newer versions of the -# package and/or distribution. In such case, please -# contact "Maintainer" of this script. -# -# ----------------------------------------------------------------------------- - -# Variables -PACKAGE_DIR="scikit-learn" -PACKAGE_NAME="scikit_learn" -PACKAGE_VERSION=${1:-1.9.0} -PACKAGE_URL="https://github.com/scikit-learn/scikit-learn.git" -SOURCE_ROOT="$(pwd)" - -echo "Building ${PACKAGE_NAME} ${PACKAGE_VERSION}" - -# Install system dependencies -dnf install -y gcc-toolset-15 gcc-toolset-15-gcc-c++ gcc-toolset-15-gcc-gfortran \ - git python3.12 python3.12-devel python3.12-pip pkg-config \ - openblas-devel - -export PATH="/opt/rh/gcc-toolset-15/root/usr/bin:$PATH" - -# Verify GCC version (scikit-learn requires >= 8.0) -gcc --version - -# Install build dependencies -# pythran is required by scipy>=1.17 at build time -python3.12 -m pip install \ - "meson-python>=0.17.1,<0.20.0" \ - "meson>=1.9.0" \ - "ninja" \ - "cython>=3.1.2,<3.3.0" \ - "numpy>=2,<2.5.0" \ - "pythran" \ - "pybind11>=2.13.2" \ - "wheel" - -# Build and install scipy separately so the pythran dep is satisfied -python3.12 -m pip install --no-build-isolation "scipy>=1.10.0,<1.18.0" - -# Clone and checkout -rm -rf "$PACKAGE_DIR" -git clone "$PACKAGE_URL" -cd "${PACKAGE_DIR}" -git checkout "${PACKAGE_VERSION}" - -# Build wheel using pip with no-build-isolation -# (meson-python builds require access to already-installed build deps) -python3.12 -m pip wheel . \ - --no-build-isolation \ - --wheel-dir "${SOURCE_ROOT}/dist/" - -WHEEL=$(find "${SOURCE_ROOT}/dist" -name "${PACKAGE_NAME}-*.whl" | head -1) -if [ -z "$WHEEL" ]; then - echo "ERROR: wheel not found after build" - exit 1 -fi -echo "Wheel: $WHEEL" - -# Copy wheel to /home/tester/ to avoid rebuild by wrapper script -mkdir -p /home/tester -cp "$WHEEL" /home/tester/ - -cd "${SOURCE_ROOT}" - -# Install runtime dependencies and wheel -echo "=== Installing Wheel ===" -python3.12 -m pip install \ - "joblib>=1.4.0" \ - "narwhals>=2.0.1" \ - "threadpoolctl>=3.5.0" -python3.12 -m pip install "$WHEEL" - -# Test -echo "=== Running Tests ===" - -# 1. Version check -python3.12 -c "import importlib.metadata; print('version:', importlib.metadata.version('scikit-learn'))" - -# 2. Basic smoke test -python3.12 - <<'EOF' -import sklearn -print("sklearn version:", sklearn.__version__) -assert sklearn.__version__ == "1.9.0", f"Unexpected version: {sklearn.__version__}" - -from sklearn.datasets import load_iris -from sklearn.ensemble import RandomForestClassifier -from sklearn.model_selection import train_test_split -from sklearn.metrics import accuracy_score - -X, y = load_iris(return_X_y=True) -X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) - -clf = RandomForestClassifier(n_estimators=10, random_state=42) -clf.fit(X_train, y_train) -preds = clf.predict(X_test) -acc = accuracy_score(y_test, preds) -assert acc > 0.9, f"Accuracy too low: {acc}" -print(f"RandomForest smoke test: OK (accuracy={acc:.2f})") - -from sklearn.linear_model import LogisticRegression -from sklearn.pipeline import Pipeline -from sklearn.preprocessing import StandardScaler - -pipe = Pipeline([("scaler", StandardScaler()), ("lr", LogisticRegression(max_iter=200))]) -pipe.fit(X_train, y_train) -pipe_acc = accuracy_score(y_test, pipe.predict(X_test)) -assert pipe_acc > 0.9, f"Pipeline accuracy too low: {pipe_acc}" -print(f"Pipeline smoke test: OK (accuracy={pipe_acc:.2f})") -EOF - -# 3. Run upstream test suite (core module only to keep CI time reasonable) -echo "=== Running Upstream Tests ===" -python3.12 -m pip install pytest - -# Resolve the installed sklearn path so pytest uses the compiled wheel, -# not the unbuilt source clone (which lacks the _check_build compiled extension). -SKLEARN_PATH=$(python3.12 -c "import sklearn; import os; print(os.path.dirname(sklearn.__file__))") -python3.12 -m pytest "${SKLEARN_PATH}/tests/test_common.py" -x -q --no-header - -echo -e "\n=== Build Complete ===" -echo "Wheel: $WHEEL" - diff --git a/s/scikit-learn/scikit-learn_ubi_8.10.sh b/s/scikit-learn/scikit-learn_ubi_8.10.sh deleted file mode 100644 index 876a73d6df..0000000000 --- a/s/scikit-learn/scikit-learn_ubi_8.10.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -e -# ----------------------------------------------------------------------------- -# -# Package : scikit-learn -# Version : 1.5.0 -# Source repo : https://github.com/scikit-learn/scikit-learn.git -# Tested on : UBI 8.10 -# Language : Python, Cython, C++ -# Ci-Check : False -# Script License: Apache License 2.0 -# Maintainer : Salil Verlekar -# -# Disclaimer: This script has been tested in root mode on given -# ========== platform using the mentioned version of the package. -# It may not work as expected with newer versions of the -# package and/or distribution. In such case, please -# contact "Maintainer" of this script. -# -# ---------------------------------------------------------------------------- - -PACKAGE_NAME=scikit-learn -PACKAGE_VERSION=${1:-1.5.0} -PACKAGE_URL=https://github.com/scikit-learn/scikit-learn.git - -yum install -y python3.11 python3.11-pip python3.11-devel gcc gcc-c++ gcc-gfortran gcc-toolset-10 git -yum install -y openblas-devel --enablerepo=codeready-builder-for-rhel-8-ppc64le-rpms - -source /opt/rh/gcc-toolset-10/enable - -OS_NAME=$(cat /etc/os-release | grep ^PRETTY_NAME | cut -d= -f2) - -# clone source repository -git clone $PACKAGE_URL -cd $PACKAGE_NAME -git checkout $PACKAGE_VERSION -git submodule update --init - -# install scikit-learn dependencies and build dependencies -python3.11 -m pip install wheel numpy scipy cython meson-python ninja 'pytest==8.2.2' 'patchelf>=0.11.0' build - -export SKLEARN_SKIP_OPENMP_TEST=1 - -ln -s /usr/bin/python3.11 python -mv python /usr/bin - -# build wheel in /scikit-learn/dist -if ! python3.11 -m build --wheel --no-isolation; then - echo "------------------$PACKAGE_NAME:build_fails---------------------" - echo "$PACKAGE_URL $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_VERSION | $OS_NAME | GitHub | Fail | Build_Fails" - exit 1 -else - echo "------------------$PACKAGE_NAME:build_success-------------------------" - echo "$PACKAGE_VERSION $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_VERSION | $OS_NAME | GitHub | Pass | Build_Success" -fi - -# install wheel, run show and check that the installed scikit-learn has a version number -python3.11 -m pip install dist/scikit_learn-1.5.0-cp311-cp311-linux_ppc64le.whl -python3.11 -m pip show scikit-learn -cd sklearn -python3.11 -c "import sklearn; sklearn.show_versions()" - -if [ $? == 0 ]; then - echo "------------------$PACKAGE_NAME::Install_Success---------------------" - echo "$PACKAGE_VERSION $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | Pass | Install_Success" -else - echo "------------------$PACKAGE_NAME::Install_Fail-------------------------" - echo "$PACKAGE_VERSION $PACKAGE_NAME" - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | Fail | Install_Fail" - exit 2 -fi - -# test using pytest - set below flag as suggested in GitHub forums to resolve ImportPathMismatchError -export PY_IGNORE_IMPORTMISMATCH=1 -if ! pytest tests/test_random_projection.py ; then - echo "------------------$PACKAGE_NAME:test_fails---------------------" - echo "$PACKAGE_URL $PACKAGE_NAME " - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | Github | Fail | Test_Fails" - exit 2 -else - echo "------------------$PACKAGE_NAME:test_success-------------------------" - echo "$PACKAGE_URL $PACKAGE_NAME " - echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | Github | Pass | Test_Success" - exit 0 -fi