diff --git a/.github/workflows/cmake-test-release-linux.yml b/.github/workflows/cmake-test-release-linux.yml index 5293e8768..836b1888e 100644 --- a/.github/workflows/cmake-test-release-linux.yml +++ b/.github/workflows/cmake-test-release-linux.yml @@ -13,9 +13,9 @@ on: env: # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.) BUILD_TYPE: Release - CACHE_VERSION: v10 - LLVM_VERSION: 22 - LLVM_TARGZFILE: "https://www.dropbox.com/scl/fi/ym6xvv8ib6yyewgf47ene/llvm_22_1_8_release.tgz?rlkey=jwkrrp82lql46qd1oysg9p736&st=p828hd39&dl=1" + # The official LLVM release package, installed where a custom LLVM build would be - see + # scripts/install_llvm_prebuilt_release.sh. + LLVM_FULL_VERSION: "22.1.8" GC_VERSION: "8.2.12" LIBATOMIC_OPS_VERSION: "7.10.0" @@ -34,27 +34,8 @@ jobs: - uses: actions/checkout@v4 - - name: Cache Build 3rd Party - id: cache-3rd-party - uses: actions/cache@v4 - with: - path: __build/llvm/ninja - key: ${{ runner.os }}-build-llvm-${{ env.LLVM_VERSION }}-${{ env.CACHE_VERSION }} - - - name: Cache 3rd Party binaries - id: cache-3rd-party-binary - uses: actions/cache@v4 - with: - path: 3rdParty - key: ${{ runner.os }}-3rd-party-${{ env.LLVM_VERSION }}-${{ env.CACHE_VERSION }} - - - name: Cache 3rd Party binaries - LLVM ${{ env.LLVM_VERSION }} Release - id: cache-3rd-party-binary-llvm-release - uses: actions/cache@v4 - with: - path: llvm_${{ env.LLVM_VERSION }}_release.tgz - key: ${{ runner.os }}-3rd-party-llvm-${{ env.LLVM_VERSION }}-release-${{ env.CACHE_VERSION }} - + # No 3rdParty cache: the LLVM prefix is about 9.6 GB, too big for the cache, and downloading it + # from GitHub each run is quick; GC is rebuilt every run anyway. - name: Delete __Build TS folders continue-on-error: true run: rm -R ./__build/tslang/ninja/release @@ -64,21 +45,11 @@ jobs: run: mkdir -p ./__build/tslang/ninja/release; mkdir -p ./__build/gc/ninja/release shell: sh - - name: Download 3rd party - LLVM - continue-on-error: true - shell: sh - run: | - if test ! -f "llvm_${{ env.LLVM_VERSION }}_release.tgz"; then - curl --insecure -L "${{ env.LLVM_TARGZFILE }}" --output llvm_${{ env.LLVM_VERSION }}_release.tgz - fi - - - name: UnZip 3rd party - LLVM + # Downloads and unpacks the official LLVM package - nothing is compiled. + - name: Install 3rd party - LLVM ${{ env.LLVM_FULL_VERSION }} continue-on-error: false shell: sh - run: | - if test -f "llvm_${{ env.LLVM_VERSION }}_release.tgz"; then - file llvm_${{ env.LLVM_VERSION }}_release.tgz; mkdir -p ./3rdParty/llvm; tar -xf llvm_${{ env.LLVM_VERSION }}_release.tgz -C ./3rdParty/llvm/ - fi + run: sh ./scripts/install_llvm_prebuilt_release.sh ${{ env.LLVM_FULL_VERSION }} && df -h . - name: Download 3rd party source - GC continue-on-error: true @@ -105,10 +76,11 @@ jobs: shell: sh run: cp -a ./3rdParty/libatomic_ops-${{ env.LIBATOMIC_OPS_VERSION }}/ ./3rdParty/gc-${{ env.GC_VERSION }}/libatomic_ops/ - - name: Install Ninja + # zlib, zstd and libxml2: the official LLVM package's CMake targets link against them. + - name: Install Ninja and LLVM's link dependencies continue-on-error: false shell: sh - run: sudo apt install ninja-build + run: sudo apt-get update && sudo apt-get install -y ninja-build zlib1g-dev libzstd-dev libxml2-dev # - name: Install Clang 12 # continue-on-error: false diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index c40971b88..3c2944a6d 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -13,11 +13,9 @@ env: # tslang-v0.0-pre-alpha83-windows-x64.zip, tslang-v0.0-pre-alpha83-ubuntu-24.04-x64.tar.gz ASSET_NAME: tslang-${{ github.ref_name }} CACHE_VERSION: v10 - LLVM_VERSION: 22 - # Windows: the official LLVM release package plus a prebuilt MLIR archive, installed where a - # custom LLVM build would be - see scripts/install_llvm_prebuilt_release.ps1. + # The official LLVM release package (plus, on Windows, a prebuilt MLIR archive), installed where + # a custom LLVM build would be - see scripts/install_llvm_prebuilt_release.{ps1,sh}. LLVM_FULL_VERSION: "22.1.8" - LLVM_TARGZFILE: "https://www.dropbox.com/scl/fi/ym6xvv8ib6yyewgf47ene/llvm_22_1_8_release.tgz?rlkey=jwkrrp82lql46qd1oysg9p736&st=p828hd39&dl=1" GC_VERSION: "8.2.12" LIBATOMIC_OPS_VERSION: "7.10.0" @@ -244,28 +242,8 @@ jobs: steps: - uses: actions/checkout@v4 - # keyed by the Ubuntu release, not just the OS: 3rdParty holds the GC built on that release - - name: Cache Build 3rd Party - id: cache-3rd-party - uses: actions/cache@v4 - with: - path: __build/llvm/ninja - key: ${{ matrix.os }}-build-llvm-${{ env.LLVM_VERSION }}-${{ env.CACHE_VERSION }} - - - name: Cache 3rd Party binaries - id: cache-3rd-party-binary - uses: actions/cache@v4 - with: - path: 3rdParty - key: ${{ matrix.os }}-3rd-party-${{ env.LLVM_VERSION }}-${{ env.CACHE_VERSION }} - - - name: Cache 3rd Party binaries - LLVM ${{ env.LLVM_VERSION }} Release - id: cache-3rd-party-binary-llvm-release - uses: actions/cache@v4 - with: - path: llvm_${{ env.LLVM_VERSION }}_release.tgz - key: ${{ matrix.os }}-3rd-party-llvm-${{ env.LLVM_VERSION }}-release-${{ env.CACHE_VERSION }} - + # No 3rdParty cache: the LLVM prefix is about 9.6 GB, too big for the cache, and downloading + # it from GitHub each run is quick; GC is rebuilt every run anyway. - name: Retrieve version run: echo "TAG_NAME=$(git describe --abbrev=0 --tags)" >> $GITHUB_OUTPUT id: version @@ -279,21 +257,11 @@ jobs: run: mkdir -p ./__build/tslang/ninja/release; mkdir -p ./__build/gc/ninja/release shell: sh - - name: Download 3rd party - LLVM - continue-on-error: true - shell: sh - run: | - if test ! -f "llvm_${{ env.LLVM_VERSION }}_release.tgz"; then - curl --insecure -L "${{ env.LLVM_TARGZFILE }}" --output llvm_${{ env.LLVM_VERSION }}_release.tgz - fi - - - name: UnZip 3rd party - LLVM + # Downloads and unpacks the official LLVM package - nothing is compiled. + - name: Install 3rd party - LLVM ${{ env.LLVM_FULL_VERSION }} continue-on-error: false shell: sh - run: | - if test -f "llvm_${{ env.LLVM_VERSION }}_release.tgz"; then - file llvm_${{ env.LLVM_VERSION }}_release.tgz; mkdir -p ./3rdParty/llvm; tar -xf llvm_${{ env.LLVM_VERSION }}_release.tgz -C ./3rdParty/llvm/ - fi + run: sh ./scripts/install_llvm_prebuilt_release.sh ${{ env.LLVM_FULL_VERSION }} && df -h . - name: Download 3rd party source - GC continue-on-error: true @@ -323,7 +291,8 @@ jobs: - name: Install Ninja continue-on-error: false shell: sh - run: sudo apt-get update && sudo apt-get install -y ninja-build libcurl4-openssl-dev + # zlib, zstd and libxml2: the official LLVM package's CMake targets link against them. + run: sudo apt-get update && sudo apt-get install -y ninja-build libcurl4-openssl-dev zlib1g-dev libzstd-dev libxml2-dev # - name: Install Clang 12 # continue-on-error: false @@ -431,9 +400,10 @@ jobs: # defaultlib/*.d.ts, defaultlib/generics/ cp -r ../TypeScriptCompilerDefaultLib/__build/defaultlib ./__stage/ # tsbindgen looks for clang's resource directory next to itself as lib/clang/ - # (findResourceDir's "release package" candidate). - mkdir -p ./__stage/lib - cp -r ../3rdParty/llvm/release/lib/clang ./__stage/lib/ + # (findResourceDir's "release package" candidate) - its headers only: the official LLVM + # package also puts compiler-rt's libraries under lib/clang//lib, which neither + # tsbindgen nor tslang uses. + for d in ../3rdParty/llvm/release/lib/clang/*/; do v=$(basename "$d"); mkdir -p "./__stage/lib/clang/$v"; cp -r "$d/include" "./__stage/lib/clang/$v/"; done tar -czvhf ../${{ env.ASSET_NAME }}-${{ matrix.os }}-x64.tar.gz -C ./__stage . - name: Archive Tar.GZ of Linux Asset diff --git a/scripts/install_llvm_prebuilt_release.sh b/scripts/install_llvm_prebuilt_release.sh new file mode 100755 index 000000000..8b531ed25 --- /dev/null +++ b/scripts/install_llvm_prebuilt_release.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# Installs a Release LLVM/Clang/LLD/MLIR prefix from the official LLVM Linux package +# (LLVM--Linux-X64.tar.xz, a GitHub release asset of llvm/llvm-project) - nothing is compiled. +# Unlike the Windows package it has MLIR, so it is all that is needed; it unpacks into the prefix +# config_llvm_release.sh + build_llvm_release.sh would make, so nothing that looks under +# 3rdParty/llvm/release has to change. Windows: install_llvm_prebuilt_release.ps1. +# +# Differences from the custom build: RTTI and EH are off in the LLVM libraries (config_macros.cmake +# then builds the project with -fno-rtti), and LLVMSupport needs zlib and zstd (zlib1g-dev, +# libzstd-dev) to link. There is no Debug variant - Debug still needs the custom build. +# +# The package is about 12 GB unpacked, mostly tools. Left out: flang, lldb and bolt, which the LLVM, +# Clang and MLIR CMake packages do not reference (every file those do reference must exist, or +# find_package fails). The rest stays, about 9.6 GB. +# +# Usage: install_llvm_prebuilt_release.sh [version] (PREFIX / WORK_DIR override the defaults) +# Skips everything when the prefix already has MLIR, so it never overwrites a custom build. +set -e + +VERSION=${1:-22.1.8} +ROOT=$(cd "$(dirname "$0")/.." && pwd) +PREFIX=${PREFIX:-$ROOT/3rdParty/llvm/release} +WORK_DIR=${WORK_DIR:-$ROOT/__build/llvm-prebuilt} + +if [ -f "$PREFIX/lib/cmake/mlir/MLIRConfig.cmake" ]; then + echo "MLIR is already installed in $PREFIX, nothing to do" + exit 0 +fi + +NAME=LLVM-$VERSION-Linux-X64 +PACKAGE=$WORK_DIR/$NAME.tar.xz +mkdir -p "$WORK_DIR" "$PREFIX" +if [ ! -f "$PACKAGE" ]; then + echo "Downloading $NAME.tar.xz" + curl -fsSL --retry 3 -o "$PACKAGE.part" \ + "https://github.com/llvm/llvm-project/releases/download/llvmorg-$VERSION/$NAME.tar.xz" + mv "$PACKAGE.part" "$PACKAGE" +fi + +echo "Unpacking $PACKAGE into $PREFIX" +xz -T0 -dc "$PACKAGE" | tar -x -C "$PREFIX" --strip-components=1 \ + --exclude='*/bin/flang*' --exclude='*/bin/bbc' --exclude='*/bin/tco' \ + --exclude='*/bin/fir-*' --exclude='*/bin/f18-*' --exclude='*/bin/lldb*' \ + --exclude='*/bin/llvm-bolt*' --exclude='*/bin/*bolt*' --exclude='*/bin/merge-fdata' \ + --exclude='*/lib/liblldb*' --exclude='*/lib/libFortran*' --exclude='*/lib/libflang*' \ + --exclude='*/lib/libFlang*' --exclude='*/lib/libFIR*' --exclude='*/lib/libHLFIR*' \ + --exclude='*/lib/libCUF*' --exclude='*/lib/libMIF*' --exclude='*/lib/libbolt*' \ + --exclude='*/lib/libLLVMBOLT*' --exclude='*/lib/python*' \ + --exclude='*/include/flang' --exclude='*/include/lldb' --exclude='*/include/bolt' +# The download is 1.9 GB; the prefix is what the build needs. +rm -f "$PACKAGE" + +if ! grep -q "set(LLVM_PACKAGE_VERSION $VERSION)" "$PREFIX/lib/cmake/llvm/LLVMConfig.cmake"; then + echo "$PREFIX holds an LLVM other than $VERSION" >&2 + exit 1 +fi +for required in lib/cmake/mlir/MLIRConfig.cmake lib/cmake/clang/ClangConfig.cmake bin/mlir-tblgen bin/ld.lld; do + if [ ! -e "$PREFIX/$required" ]; then + echo "The LLVM package is missing $required" >&2 + exit 1 + fi +done +echo "LLVM $VERSION with MLIR installed in $PREFIX"