check-submodules: the release blocks a build tree that is not the pinned tree, and warns when there is nothing to compare (#2129)

cmake/tests-submodules.cmake

@@ -0,0 +1,30 @@
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Submodule checkout integrity (engine#2129). Its own family file for the reason engine#1979 gives: a registration in
+# CMakeLists.txt's trailing block collides with every other test-adding PR, and this one is a new family -- nothing else
+# registers here.
+#
+# The `--self-test` row is hermetic (two throwaway local repos, no network, no submodule of this tree), so it runs in CI.
+# The `--root` mode is NOT a ctest row on purpose: it inspects a working tree's checkouts, and CI checks out at the pin by
+# construction, so a row there would test the runner rather than the tree. It is for the box, before a build:
+#
+#     scripts/check-submodules.sh --root .
+add_test(NAME submodules_self COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/check-submodules.sh --self-test)
+
+# The release preflight's use of it (engine#2129): a build tree that is not the pinned tree BLOCKS, and "no checkout to
+# compare against" only WARNs, so a fresh clone can still build. The function is cut out of scripts/weekly.sh and run
+# against stub checkers -- no submodules, no network, no build.
+add_test(NAME weekly_submodule_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_submodule_preflight_test.sh)

scripts/check-submodules.sh

@@ -0,0 +1,155 @@
+#!/usr/bin/env bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# check-submodules.sh [--root DIR] [--self-test]
+#
+# engine#2129: every submodule's CHECKOUT must be the commit the repository pins. CI checks the pins themselves ("submodule
+# pins only move forward", "hrx-system gitlink agrees with the hold file") and that is a check on the *repository*; nothing
+# checks the tree on the box, which is where a build actually reads its sources.
+#
+# Measured on this box 2026-11-11 while triaging #2129 -- 7 of 7 disagreed with their pin:
+#
+#   third_party/hrx-system   pin 7f4236924  head 51b1739ae  dirty 0
+#   third_party/llama.cpp    pin 2293f95ea  head c16424f56  dirty 301     <- the issue's "301 files short of its own HEAD"
+#   third_party/xdna-driver  pin 526c60819  head 5d302c998  dirty 32
+#   third_party/laya         pin b8a0f0991  head 593a1596f  dirty 4
+#   third_party/tokenizers   pin 301277072  head 88a4498ad  dirty 20
+#   third_party/lemonade     pin 248f818b4  head 593a1596f  dirty 4
+#   third_party/ggmlc        pin df08d1b06  head 593a1596f  dirty 4
+#
+# A build from that tree does not build the pinned code, and nothing says so: the engine reads a handful of submodule
+# files directly (CMakeLists.txt's jinja probe is one), so "the pin is healthy" and "the checkout is the pin" are two
+# different claims. This is the second one, and it is cheap enough to run before every build.
+#
+# `dirty` matters even when HEAD matches: a checkout whose worktree differs from its HEAD builds sources the pin does not
+# name, which is the same defect with a smaller diff. It is reported per submodule with its count so a reader can tell a
+# stray file from a reset (`git -C <path> checkout -- . && git -C <path> clean -fd`).
+#
+# Exit: 0 when every checkout is its pin and clean; 1 when a CHECKOUT exists and disagrees with the pin (the defect);
+# 3 when submodules are not checked out at all -- a fresh clone is not a defect and must not stop a build, but it is also
+# not a pass, which is why it is not 0; 2 when there is nothing to inspect at all (no .gitmodules, or git missing).
+set -uo pipefail
+
+root=.
+self=0
+while [ $# -gt 0 ]; do
+    case $1 in
+        --root) root=${2:?--root needs a directory}; shift 2 ;;
+        --root=*) root=${1#*=}; shift ;;
+        --self-test) self=1; shift ;;
+        -h|--help) sed -n '/^# check-submodules.sh /,/^set -uo/p' "$0" | sed '$d'; exit 0 ;;
+        *) echo "check-submodules.sh: unknown option $1" >&2; exit 2 ;;
+    esac
+done
+
+if [ "$self" = 1 ]; then
+    tmp=$(mktemp -d)
+    trap 'rm -rf "$tmp"' EXIT
+    fails=0
+    ok() { printf '  ok   %s\n' "$1"; }
+    bad() { printf '  FAIL %s\n' "$1"; fails=$((fails + 1)); }
+
+    # A real submodule, built offline: two local repos, protocol.file.allow so `submodule add` accepts a path.
+    git init -q "$tmp/sub"
+    ( cd "$tmp/sub" && echo a > a.txt && git add a.txt \
+        && git -c user.name=t -c user.email=t@t commit -qm first ) || { echo "fixture failed" >&2; exit 2; }
+    git init -q "$tmp/repo"
+    ( cd "$tmp/repo" && echo top > top.txt && git add top.txt \
+        && git -c user.name=t -c user.email=t@t commit -qm top \
+        && git -c protocol.file.allow=always -c user.name=t -c user.email=t@t \
+             submodule add -q "$tmp/sub" third_party/sub >/dev/null 2>&1 \
+        && git -c user.name=t -c user.email=t@t commit -qm pin ) || { echo "fixture failed" >&2; exit 2; }
+
+    out=$(bash "$0" --root "$tmp/repo" 2>&1)
+    [ $? = 0 ] && ok "a checkout at its pin and clean passes" || bad "a clean checkout at its pin should pass: $out"
+
+    # HEAD moves away from the pin: the engine#2129 shape.
+    ( cd "$tmp/repo/third_party/sub" && echo b > b.txt && git add b.txt \
+        && git -c user.name=t -c user.email=t@t commit -qm second ) >/dev/null 2>&1
+    out=$(bash "$0" --root "$tmp/repo" 2>&1)
+    [ $? = 1 ] && ok "a checkout ahead of its pin fails" || bad "a checkout ahead of its pin should fail"
+    printf '%s' "$out" | grep -q 'is not the pin' && ok "and the message says which submodule and both commits" \
+        || bad "the message does not name the disagreement: $out"
+
+    # HEAD back at the pin, worktree dirty: same defect, smaller diff.
+    ( cd "$tmp/repo/third_party/sub" && git reset -q --hard HEAD~1 && echo stray > stray.txt ) >/dev/null 2>&1
+    out=$(bash "$0" --root "$tmp/repo" 2>&1)
+    [ $? = 1 ] && ok "a dirty checkout fails even when HEAD is the pin" || bad "a dirty checkout should fail"
+    printf '%s' "$out" | grep -q 'clean' && ok "and it says the worktree differs from its HEAD" \
+        || bad "the dirty message does not explain itself: $out"
+
+    # A fresh clone: submodule registered in .gitmodules and in the tree, but not checked out. That is not a defect, and
+    # it must not read as one -- a build on a fresh box has to be able to proceed.
+    rm -rf "$tmp/fresh" && mkdir -p "$tmp/fresh"
+    ( cd "$tmp/repo" && git clone -q --no-local . "$tmp/fresh" ) >/dev/null 2>&1
+    ( cd "$tmp/repo/third_party/sub" && git checkout -q . && git clean -qfd ) >/dev/null 2>&1
+    out=$(bash "$0" --root "$tmp/fresh" 2>&1)
+    [ $? = 3 ] && ok "an unchecked-out submodule is exit 3 (cannot tell), not 1 and not 0" \
+        || bad "an unchecked-out submodule should exit 3: $out"
+
+    # No .gitmodules: cannot tell, not "clean".
+    mkdir -p "$tmp/plain" && git init -q "$tmp/plain"
+    bash "$0" --root "$tmp/plain" >/dev/null 2>&1
+    [ $? = 2 ] && ok "no .gitmodules is 'cannot tell' (2), not a pass" || bad "no .gitmodules should exit 2"
+
+    if [ "$fails" != 0 ]; then echo "check_submodules self-test: $fails failure(s)"; exit 1; fi
+    echo "check_submodules self-test: ok"
+    exit 0
+fi
+
+command -v git >/dev/null || { echo "check-submodules: no git on PATH" >&2; exit 2; }
+[ -f "$root/.gitmodules" ] || { echo "check-submodules: no .gitmodules under $root -- nothing to inspect, which is not a pass" >&2; exit 2; }
+
+bad=0
+missing=0
+while read -r _key path; do
+    [ -n "${path:-}" ] || continue
+    pin=$(git -C "$root" ls-tree HEAD -- "$path" 2>/dev/null | awk '{print $3}')
+    if [ -z "$pin" ]; then
+        echo "::error::check-submodules: $path is in .gitmodules but the tree pins no commit for it"
+        bad=$((bad + 1)); continue
+    fi
+    head=$(git -C "$root/$path" rev-parse HEAD 2>/dev/null)
+    # `git -C` in an unchecked-out submodule dir is not an error: it walks UP and reports the PARENT's HEAD, so a fresh
+    # clone looked like a checkout at the wrong commit. The path must be its own repository root to be a checkout at all.
+    toplevel=$(git -C "$root/$path" rev-parse --show-toplevel 2>/dev/null)
+    if [ -n "$head" ] && [ -n "$toplevel" ] && \
+       [ "$(cd "$root/$path" 2>/dev/null && pwd -P)" != "$(cd "$toplevel" 2>/dev/null && pwd -P)" ]; then
+        head=
+    fi
+    if [ -z "$head" ]; then
+        echo "::warning::check-submodules: $path is not checked out (the tree pins ${pin:0:9}); a build would not find its sources"
+        missing=$((missing + 1)); continue
+    fi
+    if [ "${pin:0:9}" != "${head:0:9}" ]; then
+        echo "::error::check-submodules: $path is not the pin: tree ${pin:0:9}, checkout ${head:0:9} -- the build would read sources the pin does not name"
+        bad=$((bad + 1)); continue
+    fi
+    n=$(git -C "$root/$path" status --porcelain 2>/dev/null | wc -l)
+    if [ "$n" != 0 ]; then
+        echo "::error::check-submodules: $path is at the pin but its worktree is not clean ($n path(s) differ from HEAD) -- a build would read sources the pin does not name"
+        bad=$((bad + 1)); continue
+    fi
+    printf 'ok   %-28s %s\n' "$path" "${pin:0:9}"
+done < <(git -C "$root" config -f .gitmodules --get-regexp '^submodule\..*\.path$' 2>/dev/null)
+
+if [ "$bad" = 0 ]; then
+    if [ "$missing" = 0 ]; then echo "check-submodules: every checkout is its pin and clean"; exit 0; fi
+    echo "check-submodules: $missing submodule(s) are not checked out, so nothing was inspected for them (a clone gets here; run git submodule update --init)"
+    exit 3
+fi
+echo "check-submodules: $bad submodule(s) are not what the tree pins"
+exit 1

scripts/weekly.sh

@@ -658,6 +658,29 @@ preflight_runner_units() {
     return 0
 }
 
+# The submodule row of the preflight (engine#2129). The bumps and build stages build the tree in front of them, and
+# "the pin is healthy" -- which CI checks, on the repository -- is not "the checkout is the pin", which is what a build
+# reads. Measured 2026-11-11 while triaging #2129: 7 of 7 submodules on this box disagreed with the commit the tree pins,
+# llama.cpp by 301 uncommitted files. Exit 1 is a checkout that disagrees and stops the run: the build would not be of the
+# pinned sources. Exit 3 is submodules not checked out at all (a fresh clone) and only WARNS -- a fresh box must still be
+# able to build, and the warning says the check inspected nothing rather than implying it passed (the exit-3 discipline of
+# engine#2094). Exit 2 (no .gitmodules) cannot happen on this repository and warning is right for it too.
+# A function of its own so tests/weekly_submodule_preflight_test.sh can run it against a stub.
+preflight_submodules() {
+    local check=$1 src=$2 out rc=0
+    [ -x "$check" ] || return 0
+    out=$("$check" --root "$src" 2>&1) || rc=$?
+    case "$rc" in
+        0) printf '%s\n' "$out" | grep -v '^ok ' | sed 's/^/preflight: /' ;;
+        2|3) printf '%s\n' "$out" | sed 's/^/preflight: /'
+             echo "preflight: WARNING, not blocking: no checkout to compare against the pin, so the build follows the tree rather than this check (scripts/check-submodules.sh exit $rc; git submodule update --init fills it)" ;;
+        *) printf '%s\n' "$out" | sed 's/^/preflight: /'
+           echo "preflight: the tree in front of this build is not the pinned tree, so the build would not be of the pinned sources (scripts/check-submodules.sh exit $rc; the lines above name the submodule and both commits)"
+           return 1 ;;
+    esac
+    return 0
+}
+
 # --- preflight ----------------------------------------------------------------------------
 # What the stages need from the machine, checked before the first one runs, so that a missing
 # tool stops the run at the start rather than after an hour of build (2026-10-19 rehearsal).
@@ -677,7 +700,9 @@ preflight() {
                              2> "$LOGS/hrx-toolchain.log"); then :; else
                            echo "preflight: cannot fetch the pinned HRX toolchain (config/hrx-toolchain.config): see $LOGS/hrx-toolchain.log"; bad=1
                        fi ;;
-                esac ;;
+                esac
+                # engine#2129: this stage builds the tree in front of it, so the tree has to be the pinned tree
+                preflight_submodules "$(cd "$(dirname "$0")" && pwd)/check-submodules.sh" "$SRC" || bad=1 ;;
             package)
                 # scripts/package-linux.sh rewrites RPATHs with patchelf
                 command -v patchelf > /dev/null ||

tests/weekly_submodule_preflight_test.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Copyright 2026 bong-water-water-bong
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# weekly_submodule_preflight_test.sh: the release preflight BLOCKS on a build tree that is not the pinned tree and only
+# WARNS when there is no checkout to compare against (engine#2129). Both halves are the real code: the function is cut out
+# of scripts/weekly.sh and run against stub checkers, so no submodules, no network and no build.
+set -u
+root=$(cd "$(dirname "$0")/.." && pwd)
+tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
+fails=0
+ok()  { printf 'ok   %s\n' "$1"; }
+bad() { printf 'FAIL %s\n' "$1"; fails=$((fails + 1)); }
+
+sed -n '/^preflight_submodules() {$/,/^}$/p' "$root/scripts/weekly.sh" > "$tmp/fn.sh"
+grep -q 'check-submodules.sh exit' "$tmp/fn.sh" || { bad "could not cut preflight_submodules() out of scripts/weekly.sh"; echo "weekly_submodule_preflight: FAIL"; exit 1; }
+grep -q 'preflight_submodules "' "$root/scripts/weekly.sh" || bad "scripts/weekly.sh's preflight no longer calls preflight_submodules"
+say() { echo "$*"; }
+# shellcheck disable=SC1091
+. "$tmp/fn.sh"
+
+# A stub checker: prints what the real one prints and exits with the code under test.
+stub() {  # <exit code> <line...>
+    local rc=$1; shift
+    printf '#!/usr/bin/env bash\nprintf "%%s\\n" %s\nexit %s\n' "$(printf '%q ' "$@")" "$rc" > "$tmp/stub-$rc"
+    chmod +x "$tmp/stub-$rc"; echo "$tmp/stub-$rc"
+}
+row() {  # <name> <want return> <want text> <rc> [lines...]
+    local name=$1 want=$2 text=$3 rc=$4; shift 4
+    local check; check=$(stub "$rc" "$@")
+    local out; out=$(preflight_submodules "$check" "$tmp" 2>&1); local got=$?
+    if [ "$got" = "$want" ] && printf '%s\n' "$out" | grep -q -- "$text"; then ok "preflight: $name -> return $got"
+    else bad "preflight: $name: want return $want with '$text', got $got: $out"; fi
+}
+
+row "a tree at its pins passes" 0 "preflight: " 0 "ok   third_party/sub 1234567890" "check-submodules: every checkout is its pin and clean"
+row "a checkout that disagrees stops the run" 1 "not the pinned tree" 1 "::error::check-submodules: third_party/x is not the pin: tree a, checkout b"
+row "an unchecked-out clone only warns" 0 "WARNING, not blocking" 3 "::warning::check-submodules: third_party/x is not checked out"
+row "no .gitmodules only warns" 0 "WARNING, not blocking" 2 "check-submodules: no .gitmodules under . -- nothing to inspect, which is not a pass"
+row "a missing checker does not stop the run" 0 "" 0
+
+# The stated asymmetry, in one assertion: the two "cannot tell" codes must not return 1.
+for rc in 2 3; do
+    check=$(stub "$rc" "::warning::check-submodules: cannot tell")
+    preflight_submodules "$check" "$tmp" >/dev/null 2>&1
+    [ $? = 0 ] && ok "exit $rc is not a block" || bad "exit $rc should not block the release"
+done
+
+if [ "$fails" != 0 ]; then echo "weekly_submodule_preflight: $fails failure(s)"; exit 1; fi
+echo "weekly_submodule_preflight: ok"