diff --git a/.dockerignore b/.dockerignore index 09b3588..26b0a5e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,8 @@ packages packages-docker runtime.*.liblouis/runtimes LibLouis.NET.Tables/tables + +# Not needed inside the image, and excluding them means editing the build definition does not +# invalidate every COPY . /source and force a full recompile of every target. +Dockerfile +.dockerignore diff --git a/.gitignore b/.gitignore index 86c063c..cedb7b8 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ runtime.*.liblouis/runtimes/ # Rider / ReSharper per-user settings *.DotSettings.user + +# Worktrees created by spawned Claude Code sessions +.claude/worktrees/ diff --git a/Dockerfile b/Dockerfile index 916d491..a2d614b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -# The pinned platform below is deliberate, see the comment on the FROM line. +# The pinned platforms below are deliberate, see the comment on the first FROM. # check=skip=FromPlatformFlagConstDisallowed # Builds the native liblouis binaries for the Linux and Windows runtime identifiers, packs one @@ -8,85 +8,101 @@ # native-macos CI job. The managed packages are not built here either, because building them # resolves runtime.liblouis, which depends on the macOS packages this container cannot produce. # -# The gcc and llvm-mingw targets are separate stages on purpose. llvm-mingw also ships -# i686-w64-mingw32-gcc and x86_64-w64-mingw32-gcc, so having it installed alongside the Ubuntu cross -# compilers means it can displace them and silently change which toolchain builds win-x86 and -# win-x64. Keeping it out of that stage entirely makes the mistake impossible rather than merely -# documented. BuildKit also builds the independent stages concurrently, so the wall clock is the -# slowest stage rather than the sum. +# Compiling and packing are separate stages, each on an image chosen for the job. +# +# The compilers do not need a .NET SDK: dotnet appears exactly once in the native build, to pack an +# already-compiled binary into a .nupkg. Building C on a dotnet/sdk image meant apt-get installing a +# toolchain onto an image picked for something else, and pulling in packages the build never uses - +# which is how a 404 on linux-libc-dev, a dependency of build-essential, once failed CI. +# +# So: toolchain images compile and stage binaries, and the SDK image packs whatever it finds. The +# SDK stage installs nothing at all. # # --platform is pinned because the cross toolchain package names below only exist for amd64. On an # Apple Silicon machine this runs under emulation: slower, but it works. -FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:8.0-jammy AS base -LABEL org.opencontainers.image.source=https://github.com/Notalib/LibLouis.NET/ - -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - m4 \ - xz-utils \ - && rm -rf /var/lib/apt/lists/* - -ENV PACKAGE_OUTPUT_DIR=/packages -WORKDIR /source # The five targets Ubuntu has cross compilers for. -FROM base AS gcc-targets - -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - gcc-i686-linux-gnu \ - gcc-aarch64-linux-gnu \ - gcc-mingw-w64-i686 \ - gcc-mingw-w64-x86-64 \ - # The cross gcc packages only Recommend their target libc, so with - # --no-install-recommends they install a compiler that cannot link. Name them explicitly - # rather than dropping the flag, so the requirement is visible. - libc6-dev-i386-cross \ - libc6-dev-arm64-cross \ - && rm -rf /var/lib/apt/lists/* +FROM --platform=linux/amd64 ubuntu:noble AS gcc-build +# Retried, because a single apt-get run is a coin flip against archive.ubuntu.com: the index and +# the pool are not updated atomically, so a package version can be listed after it has been removed +# and the fetch 404s. Each attempt refreshes the index first, since a newer index is usually what +# resolves it. The explicit ok check matters: without it a loop that never succeeds still falls +# through and the layer builds with nothing installed. +RUN set -eu; \ + ok=0; \ + for attempt in 1 2 3; do \ + if apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + m4 \ + gcc-i686-linux-gnu \ + gcc-aarch64-linux-gnu \ + gcc-mingw-w64-i686 \ + gcc-mingw-w64-x86-64 \ + libc6-dev-i386-cross \ + libc6-dev-arm64-cross; then \ + ok=1; break; \ + fi; \ + echo "apt attempt $attempt failed, retrying" >&2; \ + sleep 10; \ + done; \ + [ "$ok" = 1 ] || exit 1; \ + rm -rf /var/lib/apt/lists/* +# The cross gcc packages above only Recommend their target libc, so with --no-install-recommends +# they would install a compiler that cannot link. libc6-dev-*-cross are named explicitly rather +# than dropping the flag, so the requirement is visible. + +ENV SKIP_PACK=1 +WORKDIR /source COPY . /source RUN sh ./build/build_runtime_packages.sh gcc -# win-arm64. Ubuntu has no aarch64 mingw-w64 cross compiler, so this stage uses the prebuilt -# llvm-mingw toolchain, pinned by digest: an unpinned toolchain would silently change what the -# published binaries were built with. Bump both values together when moving to a newer release. -FROM base AS llvm-targets - -ARG LLVM_MINGW_VERSION=20260616 -ARG LLVM_MINGW_SHA256=534b92e067b22a6b4441f48ae9240a3341b17825d04d577eab0cf85c44b4deda -RUN set -eu; \ - archive="llvm-mingw-${LLVM_MINGW_VERSION}-ucrt-ubuntu-22.04-x86_64.tar.xz"; \ - curl -fL -o "/tmp/$archive" \ - "https://github.com/mstorsjo/llvm-mingw/releases/download/${LLVM_MINGW_VERSION}/$archive"; \ - echo "${LLVM_MINGW_SHA256} /tmp/$archive" | sha256sum --check; \ - mkdir -p /opt/llvm-mingw; \ - tar xf "/tmp/$archive" -C /opt/llvm-mingw --strip-components=1; \ - rm "/tmp/$archive" - -# The build script prepends this to PATH for the targets that need it. Left off PATH here so there -# is exactly one mechanism selecting the toolchain, in the script, where it is visible. -ENV LLVM_MINGW_BIN=/opt/llvm-mingw/bin - +# win-arm64. Ubuntu has no aarch64 mingw-w64 cross compiler, so this uses llvm-mingw, taken from +# the image its own author publishes and pinned to a dated release rather than downloaded and +# checksummed by hand. The image already carries make, m4, curl and the toolchain on PATH, so this +# stage installs nothing. +# +# It is a separate stage from the gcc targets, and that separation is load bearing: llvm-mingw also +# ships i686-w64-mingw32-gcc and x86_64-w64-mingw32-gcc, so having it on PATH alongside the Ubuntu +# cross compilers silently takes over the win-x86 and win-x64 builds. That is not hypothetical - it +# happened, and it broke win-x86, because clang treats the -Wincompatible-pointer-types that gnulib +# trips on mingw as an error where gcc only warns. +FROM --platform=linux/amd64 mstorsjo/llvm-mingw:20260616 AS llvm-build + +ENV SKIP_PACK=1 +WORKDIR /source COPY . /source RUN sh ./build/build_runtime_packages.sh llvm -# The metapackage is pure metadata and needs no toolchain at all. -FROM base AS metapackage +# Packs what the toolchain stages produced, and the metapackage, which needs no native binary at +# all. Installs nothing: dotnet pack is the only thing this stage does. +# +# The SDK version barely affects the output here - these packages are netstandard2.0 metadata around +# an already-compiled binary, with IncludeBuildOutput off - but .NET 8 goes out of support in +# November 2026, and there is no reason for the build to be the thing still on it. +FROM --platform=linux/amd64 mcr.microsoft.com/dotnet/sdk:10.0-noble AS pack +LABEL org.opencontainers.image.source=https://github.com/Notalib/LibLouis.NET/ +ENV PACKAGE_OUTPUT_DIR=/packages +WORKDIR /source COPY . /source -RUN sh ./build/build_metapackage.sh + +COPY --from=gcc-build /source/runtime.linux-x86.liblouis/runtimes /source/runtime.linux-x86.liblouis/runtimes +COPY --from=gcc-build /source/runtime.linux-x64.liblouis/runtimes /source/runtime.linux-x64.liblouis/runtimes +COPY --from=gcc-build /source/runtime.linux-arm64.liblouis/runtimes /source/runtime.linux-arm64.liblouis/runtimes +COPY --from=gcc-build /source/runtime.win-x86.liblouis/runtimes /source/runtime.win-x86.liblouis/runtimes +COPY --from=gcc-build /source/runtime.win-x64.liblouis/runtimes /source/runtime.win-x64.liblouis/runtimes +COPY --from=llvm-build /source/runtime.win-arm64.liblouis/runtimes /source/runtime.win-arm64.liblouis/runtimes + +RUN sh ./build/pack_runtime_packages.sh && \ + sh ./build/build_metapackage.sh # `docker build --output=packages .` exports just the .nupkg files into ./packages. FROM scratch -COPY --from=gcc-targets /packages/* / -COPY --from=llvm-targets /packages/* / -COPY --from=metapackage /packages/* / +COPY --from=pack /packages/* / diff --git a/PACKAGING.md b/PACKAGING.md index a1ae783..6ff7250 100644 --- a/PACKAGING.md +++ b/PACKAGING.md @@ -94,13 +94,32 @@ NUGET_LOCAL_FEED=$PWD/packages sh build/build_managed_packages.sh ### Cross-compilation Linux and Windows binaries are cross-compiled in the container defined by `Dockerfile`. The -toolchain package names there only exist for amd64, so the image is pinned to `linux/amd64` and -runs under emulation on Apple Silicon. +toolchain package names there only exist for amd64, so the images are pinned to `linux/amd64` and +run under emulation on Apple Silicon. -`win-arm64` is the exception: Ubuntu has no aarch64 mingw-w64 gcc, so the image installs the -[llvm-mingw](https://github.com/mstorsjo/llvm-mingw) toolchain, pinned by SHA-256. It provides -`aarch64-w64-mingw32-gcc` driver wrappers, so the build script treats it like the other Windows -targets. Bump `LLVM_MINGW_VERSION` and `LLVM_MINGW_SHA256` together. +Compiling and packing are separate stages, on images chosen for the job: + +| Stage | Image | Installs | +| --- | --- | --- | +| `gcc-build` | `ubuntu:noble` | the cross toolchains | +| `llvm-build` | `mstorsjo/llvm-mingw` | nothing | +| `pack` | `dotnet/sdk` | nothing | + +dotnet appears exactly once in the native build, to pack an already-compiled binary. Compiling C on +a `dotnet/sdk` image meant apt-getting a toolchain onto an image chosen for something else, and +pulling in packages the build never uses — which is how a 404 on `linux-libc-dev`, a dependency of +`build-essential`, once failed CI. The toolchain stages now leave binaries staged under +`runtime..liblouis/runtimes/`, and `build/pack_runtime_packages.sh` packs whatever it finds +rather than a list that could fall out of step. Finding nothing is an error. + +`win-arm64` is the exception among the Windows targets: Ubuntu has no aarch64 mingw-w64 gcc, so it +uses [llvm-mingw](https://github.com/mstorsjo/llvm-mingw), taken from the image its own author +publishes and pinned to a dated release. That image already carries `make`, `m4`, `curl` and the +toolchain on `PATH`. + +The `Dockerfile` is excluded from the build context. It is not needed inside any image, and +excluding it means editing the build definition does not invalidate every `COPY . /source` and +force a full recompile of every target. llvm-mingw lives in its own container stage, and that separation is load bearing. It also ships `i686-w64-mingw32-gcc` and `x86_64-w64-mingw32-gcc`, so merely having it on `PATH` alongside the diff --git a/build/build_runtime_packages.sh b/build/build_runtime_packages.sh index 5d616b9..557bea9 100755 --- a/build/build_runtime_packages.sh +++ b/build/build_runtime_packages.sh @@ -89,7 +89,11 @@ build_runtime_nuget() { make distclean ) - pack_runtime_package "$rid" + # SKIP_PACK leaves the binary staged without packing it, for a container stage that has a + # toolchain but no .NET SDK. A later stage packs what this one produced. + if [ -z "${SKIP_PACK:-}" ]; then + pack_runtime_package "$rid" + fi } if [ "$group" = "gcc" ] || [ "$group" = "all" ]; then diff --git a/build/pack_runtime_packages.sh b/build/pack_runtime_packages.sh new file mode 100755 index 0000000..2c6650a --- /dev/null +++ b/build/pack_runtime_packages.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Packs a runtime package for every RID that has a native binary staged. +# +# Compiling and packing happen in different container stages: the toolchain images have no .NET SDK +# and the SDK image has no cross compilers. The build stages leave binaries staged under +# runtime..liblouis/runtimes/, and this packs whatever it finds. +# +# Packing whatever is present, rather than a list kept here, means the list cannot fall out of step +# with the RIDs the build stages actually produce. Finding nothing is an error: a stage that copied +# no binaries would otherwise produce an empty set of packages and look like a success. + +. "$(dirname "$0")/common.sh" + +packed=0 + +for project in "$REPO_ROOT"/runtime.*.liblouis; do + rid=$(basename "$project" | sed 's/^runtime\.\(.*\)\.liblouis$/\1/') + + # The metapackage has no RID of its own and no native payload. + if [ "$rid" = "liblouis" ]; then + continue + fi + + if [ -z "$(find "$project/runtimes" -type f 2>/dev/null | head -n 1)" ]; then + continue + fi + + pack_runtime_package "$rid" + packed=$((packed + 1)) +done + +if [ "$packed" -eq 0 ]; then + echo "pack_runtime_packages: no staged native binaries found under runtime.*.liblouis/runtimes/" >&2 + exit 1 +fi + +echo "==> Packed $packed runtime package(s) into $PACKAGE_OUTPUT_DIR"