From e32a44d5d9601aca2e16469595c3a2a1009195f3 Mon Sep 17 00:00:00 2001 From: ASDAlexander77 Date: Sun, 27 Sep 2026 17:14:29 +0100 Subject: [PATCH] Linux CI and release: official LLVM package instead of the Dropbox archive The Linux test workflow and the Linux release job downloaded a hand-built LLVM tarball from Dropbox, which now answers "Link Temporarily Disabled" (traffic limit), so every Linux check fails in the first minute. The official LLVM-22.1.8-Linux-X64.tar.xz - unlike the Windows package - has MLIR as well as LLVM, Clang, LLD, lld/wasm-ld and the resource directory, so on Linux it is all that is needed. The new scripts/install_llvm_prebuilt_release.sh unpacks it into 3rdParty/llvm/release, the prefix the custom build made, so nothing that looks there changes. Nothing is compiled. The package is 12 GB unpacked, mostly tools. flang, lldb and bolt are left out (none of the LLVM, Clang and MLIR CMake packages references them; every file those do reference must exist, or find_package fails), leaving about 9.6 GB. That is too big for the actions cache, so the Linux jobs drop their 3rdParty caches and download from GitHub every run; GC was rebuilt every run already. The package's CMake targets link zlib, zstd and libxml2, now installed with ninja-build. The release tarball stages only clang's resource headers, as on Windows: the official package also has compiler-rt's libraries under lib/clang//lib (16 MB of headers vs 109 MB). Verified in WSL (Ubuntu, GCC 15) on a fresh clone of main: the script installs in 8 seconds from the downloaded package; tslang configured with the default paths builds and passes 2949/2949 with the test scripts on the official lld and libraries. Package needs GLIBC 2.34 / GLIBCXX 3.4.30, which ubuntu-22.04 and later have. Co-Authored-By: Claude Opus 5.5 --- .../workflows/cmake-test-release-linux.yml | 50 ++++----------- .github/workflows/create-release.yml | 56 ++++------------- scripts/install_llvm_prebuilt_release.sh | 63 +++++++++++++++++++ 3 files changed, 87 insertions(+), 82 deletions(-) create mode 100755 scripts/install_llvm_prebuilt_release.sh diff --git a/.github/workflows/cmake-test-release-linux.yml b/.github/workflows/cmake-test-release-linux.yml index 5293e8768..836b1888e 100644 --- a/.github/workflows/cmake-test-release-linux.yml +++ b/.github/workflows/cmake-test-release-linux.yml @@ -13,9 +13,9 @@ on: env: # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) BUILD_TYPE: Release - CACHE_VERSION: v10 - LLVM_VERSION: 22 - LLVM_TARGZFILE: "https://www.dropbox.com/scl/fi/ym6xvv8ib6yyewgf47ene/llvm_22_1_8_release.tgz?rlkey=jwkrrp82lql46qd1oysg9p736&st=p828hd39&dl=1" + # The official LLVM release package, installed where a custom LLVM build would be - see + # scripts/install_llvm_prebuilt_release.sh. + LLVM_FULL_VERSION: "22.1.8" GC_VERSION: "8.2.12" LIBATOMIC_OPS_VERSION: "7.10.0" @@ -34,27 +34,8 @@ jobs: - uses: actions/checkout@v4 - - name: Cache Build 3rd Party - id: cache-3rd-party - uses: actions/cache@v4 - with: - path: __build/llvm/ninja - key: ${{ runner.os }}-build-llvm-${{ env.LLVM_VERSION }}-${{ env.CACHE_VERSION }} - - - name: Cache 3rd Party binaries - id: cache-3rd-party-binary - uses: actions/cache@v4 - with: - path: 3rdParty - key: ${{ runner.os }}-3rd-party-${{ env.LLVM_VERSION }}-${{ env.CACHE_VERSION }} - - - name: Cache 3rd Party binaries - LLVM ${{ env.LLVM_VERSION }} Release - id: cache-3rd-party-binary-llvm-release - uses: actions/cache@v4 - with: - path: llvm_${{ env.LLVM_VERSION }}_release.tgz - key: ${{ runner.os }}-3rd-party-llvm-${{ env.LLVM_VERSION }}-release-${{ env.CACHE_VERSION }} - + # No 3rdParty cache: the LLVM prefix is about 9.6 GB, too big for the cache, and downloading it + # from GitHub each run is quick; GC is rebuilt every run anyway. - name: Delete __Build TS folders continue-on-error: true run: rm -R ./__build/tslang/ninja/release @@ -64,21 +45,11 @@ jobs: run: mkdir -p ./__build/tslang/ninja/release; mkdir -p ./__build/gc/ninja/release shell: sh - - name: Download 3rd party - LLVM - continue-on-error: true - shell: sh - run: | - if test ! -f "llvm_${{ env.LLVM_VERSION }}_release.tgz"; then - curl --insecure -L "${{ env.LLVM_TARGZFILE }}" --output llvm_${{ env.LLVM_VERSION }}_release.tgz - fi - - - name: UnZip 3rd party - LLVM + # Downloads and unpacks the official LLVM package - nothing is compiled. + - name: Install 3rd party - LLVM ${{ env.LLVM_FULL_VERSION }} continue-on-error: false shell: sh - run: | - if test -f "llvm_${{ env.LLVM_VERSION }}_release.tgz"; then - file llvm_${{ env.LLVM_VERSION }}_release.tgz; mkdir -p ./3rdParty/llvm; tar -xf llvm_${{ env.LLVM_VERSION }}_release.tgz -C ./3rdParty/llvm/ - fi + run: sh ./scripts/install_llvm_prebuilt_release.sh ${{ env.LLVM_FULL_VERSION }} && df -h . - name: Download 3rd party source - GC continue-on-error: true @@ -105,10 +76,11 @@ jobs: shell: sh run: cp -a ./3rdParty/libatomic_ops-${{ env.LIBATOMIC_OPS_VERSION }}/ ./3rdParty/gc-${{ env.GC_VERSION }}/libatomic_ops/ - - name: Install Ninja + # zlib, zstd and libxml2: the official LLVM package's CMake targets link against them. + - name: Install Ninja and LLVM's link dependencies continue-on-error: false shell: sh - run: sudo apt install ninja-build + run: sudo apt-get update && sudo apt-get install -y ninja-build zlib1g-dev libzstd-dev libxml2-dev # - name: Install Clang 12 # continue-on-error: false diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index c40971b88..3c2944a6d 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -13,11 +13,9 @@ env: # tslang-v0.0-pre-alpha83-windows-x64.zip, tslang-v0.0-pre-alpha83-ubuntu-24.04-x64.tar.gz ASSET_NAME: tslang-${{ github.ref_name }} CACHE_VERSION: v10 - LLVM_VERSION: 22 - # Windows: the official LLVM release package plus a prebuilt MLIR archive, installed where a - # custom LLVM build would be - see scripts/install_llvm_prebuilt_release.ps1. + # The official LLVM release package (plus, on Windows, a prebuilt MLIR archive), installed where + # a custom LLVM build would be - see scripts/install_llvm_prebuilt_release.{ps1,sh}. LLVM_FULL_VERSION: "22.1.8" - LLVM_TARGZFILE: "https://www.dropbox.com/scl/fi/ym6xvv8ib6yyewgf47ene/llvm_22_1_8_release.tgz?rlkey=jwkrrp82lql46qd1oysg9p736&st=p828hd39&dl=1" GC_VERSION: "8.2.12" LIBATOMIC_OPS_VERSION: "7.10.0" @@ -244,28 +242,8 @@ jobs: steps: - uses: actions/checkout@v4 - # keyed by the Ubuntu release, not just the OS: 3rdParty holds the GC built on that release - - name: Cache Build 3rd Party - id: cache-3rd-party - uses: actions/cache@v4 - with: - path: __build/llvm/ninja - key: ${{ matrix.os }}-build-llvm-${{ env.LLVM_VERSION }}-${{ env.CACHE_VERSION }} - - - name: Cache 3rd Party binaries - id: cache-3rd-party-binary - uses: actions/cache@v4 - with: - path: 3rdParty - key: ${{ matrix.os }}-3rd-party-${{ env.LLVM_VERSION }}-${{ env.CACHE_VERSION }} - - - name: Cache 3rd Party binaries - LLVM ${{ env.LLVM_VERSION }} Release - id: cache-3rd-party-binary-llvm-release - uses: actions/cache@v4 - with: - path: llvm_${{ env.LLVM_VERSION }}_release.tgz - key: ${{ matrix.os }}-3rd-party-llvm-${{ env.LLVM_VERSION }}-release-${{ env.CACHE_VERSION }} - + # No 3rdParty cache: the LLVM prefix is about 9.6 GB, too big for the cache, and downloading + # it from GitHub each run is quick; GC is rebuilt every run anyway. - name: Retrieve version run: echo "TAG_NAME=$(git describe --abbrev=0 --tags)" >> $GITHUB_OUTPUT id: version @@ -279,21 +257,11 @@ jobs: run: mkdir -p ./__build/tslang/ninja/release; mkdir -p ./__build/gc/ninja/release shell: sh - - name: Download 3rd party - LLVM - continue-on-error: true - shell: sh - run: | - if test ! -f "llvm_${{ env.LLVM_VERSION }}_release.tgz"; then - curl --insecure -L "${{ env.LLVM_TARGZFILE }}" --output llvm_${{ env.LLVM_VERSION }}_release.tgz - fi - - - name: UnZip 3rd party - LLVM + # Downloads and unpacks the official LLVM package - nothing is compiled. + - name: Install 3rd party - LLVM ${{ env.LLVM_FULL_VERSION }} continue-on-error: false shell: sh - run: | - if test -f "llvm_${{ env.LLVM_VERSION }}_release.tgz"; then - file llvm_${{ env.LLVM_VERSION }}_release.tgz; mkdir -p ./3rdParty/llvm; tar -xf llvm_${{ env.LLVM_VERSION }}_release.tgz -C ./3rdParty/llvm/ - fi + run: sh ./scripts/install_llvm_prebuilt_release.sh ${{ env.LLVM_FULL_VERSION }} && df -h . - name: Download 3rd party source - GC continue-on-error: true @@ -323,7 +291,8 @@ jobs: - name: Install Ninja continue-on-error: false shell: sh - run: sudo apt-get update && sudo apt-get install -y ninja-build libcurl4-openssl-dev + # zlib, zstd and libxml2: the official LLVM package's CMake targets link against them. + run: sudo apt-get update && sudo apt-get install -y ninja-build libcurl4-openssl-dev zlib1g-dev libzstd-dev libxml2-dev # - name: Install Clang 12 # continue-on-error: false @@ -431,9 +400,10 @@ jobs: # defaultlib/*.d.ts, defaultlib/generics/ cp -r ../TypeScriptCompilerDefaultLib/__build/defaultlib ./__stage/ # tsbindgen looks for clang's resource directory next to itself as lib/clang/ - # (findResourceDir's "release package" candidate). - mkdir -p ./__stage/lib - cp -r ../3rdParty/llvm/release/lib/clang ./__stage/lib/ + # (findResourceDir's "release package" candidate) - its headers only: the official LLVM + # package also puts compiler-rt's libraries under lib/clang//lib, which neither + # tsbindgen nor tslang uses. + for d in ../3rdParty/llvm/release/lib/clang/*/; do v=$(basename "$d"); mkdir -p "./__stage/lib/clang/$v"; cp -r "$d/include" "./__stage/lib/clang/$v/"; done tar -czvhf ../${{ env.ASSET_NAME }}-${{ matrix.os }}-x64.tar.gz -C ./__stage . - name: Archive Tar.GZ of Linux Asset diff --git a/scripts/install_llvm_prebuilt_release.sh b/scripts/install_llvm_prebuilt_release.sh new file mode 100755 index 000000000..8b531ed25 --- /dev/null +++ b/scripts/install_llvm_prebuilt_release.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# Installs a Release LLVM/Clang/LLD/MLIR prefix from the official LLVM Linux package +# (LLVM--Linux-X64.tar.xz, a GitHub release asset of llvm/llvm-project) - nothing is compiled. +# Unlike the Windows package it has MLIR, so it is all that is needed; it unpacks into the prefix +# config_llvm_release.sh + build_llvm_release.sh would make, so nothing that looks under +# 3rdParty/llvm/release has to change. Windows: install_llvm_prebuilt_release.ps1. +# +# Differences from the custom build: RTTI and EH are off in the LLVM libraries (config_macros.cmake +# then builds the project with -fno-rtti), and LLVMSupport needs zlib and zstd (zlib1g-dev, +# libzstd-dev) to link. There is no Debug variant - Debug still needs the custom build. +# +# The package is about 12 GB unpacked, mostly tools. Left out: flang, lldb and bolt, which the LLVM, +# Clang and MLIR CMake packages do not reference (every file those do reference must exist, or +# find_package fails). The rest stays, about 9.6 GB. +# +# Usage: install_llvm_prebuilt_release.sh [version] (PREFIX / WORK_DIR override the defaults) +# Skips everything when the prefix already has MLIR, so it never overwrites a custom build. +set -e + +VERSION=${1:-22.1.8} +ROOT=$(cd "$(dirname "$0")/.." && pwd) +PREFIX=${PREFIX:-$ROOT/3rdParty/llvm/release} +WORK_DIR=${WORK_DIR:-$ROOT/__build/llvm-prebuilt} + +if [ -f "$PREFIX/lib/cmake/mlir/MLIRConfig.cmake" ]; then + echo "MLIR is already installed in $PREFIX, nothing to do" + exit 0 +fi + +NAME=LLVM-$VERSION-Linux-X64 +PACKAGE=$WORK_DIR/$NAME.tar.xz +mkdir -p "$WORK_DIR" "$PREFIX" +if [ ! -f "$PACKAGE" ]; then + echo "Downloading $NAME.tar.xz" + curl -fsSL --retry 3 -o "$PACKAGE.part" \ + "https://github.com/llvm/llvm-project/releases/download/llvmorg-$VERSION/$NAME.tar.xz" + mv "$PACKAGE.part" "$PACKAGE" +fi + +echo "Unpacking $PACKAGE into $PREFIX" +xz -T0 -dc "$PACKAGE" | tar -x -C "$PREFIX" --strip-components=1 \ + --exclude='*/bin/flang*' --exclude='*/bin/bbc' --exclude='*/bin/tco' \ + --exclude='*/bin/fir-*' --exclude='*/bin/f18-*' --exclude='*/bin/lldb*' \ + --exclude='*/bin/llvm-bolt*' --exclude='*/bin/*bolt*' --exclude='*/bin/merge-fdata' \ + --exclude='*/lib/liblldb*' --exclude='*/lib/libFortran*' --exclude='*/lib/libflang*' \ + --exclude='*/lib/libFlang*' --exclude='*/lib/libFIR*' --exclude='*/lib/libHLFIR*' \ + --exclude='*/lib/libCUF*' --exclude='*/lib/libMIF*' --exclude='*/lib/libbolt*' \ + --exclude='*/lib/libLLVMBOLT*' --exclude='*/lib/python*' \ + --exclude='*/include/flang' --exclude='*/include/lldb' --exclude='*/include/bolt' +# The download is 1.9 GB; the prefix is what the build needs. +rm -f "$PACKAGE" + +if ! grep -q "set(LLVM_PACKAGE_VERSION $VERSION)" "$PREFIX/lib/cmake/llvm/LLVMConfig.cmake"; then + echo "$PREFIX holds an LLVM other than $VERSION" >&2 + exit 1 +fi +for required in lib/cmake/mlir/MLIRConfig.cmake lib/cmake/clang/ClangConfig.cmake bin/mlir-tblgen bin/ld.lld; do + if [ ! -e "$PREFIX/$required" ]; then + echo "The LLVM package is missing $required" >&2 + exit 1 + fi +done +echo "LLVM $VERSION with MLIR installed in $PREFIX"