diff --git a/cmake/tests-submodules.cmake b/cmake/tests-submodules.cmake new file mode 100644 index 000000000..554f779b9 --- /dev/null +++ b/cmake/tests-submodules.cmake @@ -0,0 +1,30 @@ +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Submodule checkout integrity (engine#2129). Its own family file for the reason engine#1979 gives: a registration in +# CMakeLists.txt's trailing block collides with every other test-adding PR, and this one is a new family -- nothing else +# registers here. +# +# The `--self-test` row is hermetic (two throwaway local repos, no network, no submodule of this tree), so it runs in CI. +# The `--root` mode is NOT a ctest row on purpose: it inspects a working tree's checkouts, and CI checks out at the pin by +# construction, so a row there would test the runner rather than the tree. It is for the box, before a build: +# +# scripts/check-submodules.sh --root . +add_test(NAME submodules_self COMMAND bash ${CMAKE_SOURCE_DIR}/scripts/check-submodules.sh --self-test) + +# The release preflight's use of it (engine#2129): a build tree that is not the pinned tree BLOCKS, and "no checkout to +# compare against" only WARNs, so a fresh clone can still build. The function is cut out of scripts/weekly.sh and run +# against stub checkers -- no submodules, no network, no build. +add_test(NAME weekly_submodule_preflight COMMAND ${CMAKE_SOURCE_DIR}/tests/weekly_submodule_preflight_test.sh) diff --git a/scripts/check-submodules.sh b/scripts/check-submodules.sh new file mode 100755 index 000000000..f68893688 --- /dev/null +++ b/scripts/check-submodules.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# check-submodules.sh [--root DIR] [--self-test] +# +# engine#2129: every submodule's CHECKOUT must be the commit the repository pins. CI checks the pins themselves ("submodule +# pins only move forward", "hrx-system gitlink agrees with the hold file") and that is a check on the *repository*; nothing +# checks the tree on the box, which is where a build actually reads its sources. +# +# Measured on this box 2026-11-11 while triaging #2129 -- 7 of 7 disagreed with their pin: +# +# third_party/hrx-system pin 7f4236924 head 51b1739ae dirty 0 +# third_party/llama.cpp pin 2293f95ea head c16424f56 dirty 301 <- the issue's "301 files short of its own HEAD" +# third_party/xdna-driver pin 526c60819 head 5d302c998 dirty 32 +# third_party/laya pin b8a0f0991 head 593a1596f dirty 4 +# third_party/tokenizers pin 301277072 head 88a4498ad dirty 20 +# third_party/lemonade pin 248f818b4 head 593a1596f dirty 4 +# third_party/ggmlc pin df08d1b06 head 593a1596f dirty 4 +# +# A build from that tree does not build the pinned code, and nothing says so: the engine reads a handful of submodule +# files directly (CMakeLists.txt's jinja probe is one), so "the pin is healthy" and "the checkout is the pin" are two +# different claims. This is the second one, and it is cheap enough to run before every build. +# +# `dirty` matters even when HEAD matches: a checkout whose worktree differs from its HEAD builds sources the pin does not +# name, which is the same defect with a smaller diff. It is reported per submodule with its count so a reader can tell a +# stray file from a reset (`git -C checkout -- . && git -C clean -fd`). +# +# Exit: 0 when every checkout is its pin and clean; 1 when a CHECKOUT exists and disagrees with the pin (the defect); +# 3 when submodules are not checked out at all -- a fresh clone is not a defect and must not stop a build, but it is also +# not a pass, which is why it is not 0; 2 when there is nothing to inspect at all (no .gitmodules, or git missing). +set -uo pipefail + +root=. +self=0 +while [ $# -gt 0 ]; do + case $1 in + --root) root=${2:?--root needs a directory}; shift 2 ;; + --root=*) root=${1#*=}; shift ;; + --self-test) self=1; shift ;; + -h|--help) sed -n '/^# check-submodules.sh /,/^set -uo/p' "$0" | sed '$d'; exit 0 ;; + *) echo "check-submodules.sh: unknown option $1" >&2; exit 2 ;; + esac +done + +if [ "$self" = 1 ]; then + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' EXIT + fails=0 + ok() { printf ' ok %s\n' "$1"; } + bad() { printf ' FAIL %s\n' "$1"; fails=$((fails + 1)); } + + # A real submodule, built offline: two local repos, protocol.file.allow so `submodule add` accepts a path. + git init -q "$tmp/sub" + ( cd "$tmp/sub" && echo a > a.txt && git add a.txt \ + && git -c user.name=t -c user.email=t@t commit -qm first ) || { echo "fixture failed" >&2; exit 2; } + git init -q "$tmp/repo" + ( cd "$tmp/repo" && echo top > top.txt && git add top.txt \ + && git -c user.name=t -c user.email=t@t commit -qm top \ + && git -c protocol.file.allow=always -c user.name=t -c user.email=t@t \ + submodule add -q "$tmp/sub" third_party/sub >/dev/null 2>&1 \ + && git -c user.name=t -c user.email=t@t commit -qm pin ) || { echo "fixture failed" >&2; exit 2; } + + out=$(bash "$0" --root "$tmp/repo" 2>&1) + [ $? = 0 ] && ok "a checkout at its pin and clean passes" || bad "a clean checkout at its pin should pass: $out" + + # HEAD moves away from the pin: the engine#2129 shape. + ( cd "$tmp/repo/third_party/sub" && echo b > b.txt && git add b.txt \ + && git -c user.name=t -c user.email=t@t commit -qm second ) >/dev/null 2>&1 + out=$(bash "$0" --root "$tmp/repo" 2>&1) + [ $? = 1 ] && ok "a checkout ahead of its pin fails" || bad "a checkout ahead of its pin should fail" + printf '%s' "$out" | grep -q 'is not the pin' && ok "and the message says which submodule and both commits" \ + || bad "the message does not name the disagreement: $out" + + # HEAD back at the pin, worktree dirty: same defect, smaller diff. + ( cd "$tmp/repo/third_party/sub" && git reset -q --hard HEAD~1 && echo stray > stray.txt ) >/dev/null 2>&1 + out=$(bash "$0" --root "$tmp/repo" 2>&1) + [ $? = 1 ] && ok "a dirty checkout fails even when HEAD is the pin" || bad "a dirty checkout should fail" + printf '%s' "$out" | grep -q 'clean' && ok "and it says the worktree differs from its HEAD" \ + || bad "the dirty message does not explain itself: $out" + + # A fresh clone: submodule registered in .gitmodules and in the tree, but not checked out. That is not a defect, and + # it must not read as one -- a build on a fresh box has to be able to proceed. + rm -rf "$tmp/fresh" && mkdir -p "$tmp/fresh" + ( cd "$tmp/repo" && git clone -q --no-local . "$tmp/fresh" ) >/dev/null 2>&1 + ( cd "$tmp/repo/third_party/sub" && git checkout -q . && git clean -qfd ) >/dev/null 2>&1 + out=$(bash "$0" --root "$tmp/fresh" 2>&1) + [ $? = 3 ] && ok "an unchecked-out submodule is exit 3 (cannot tell), not 1 and not 0" \ + || bad "an unchecked-out submodule should exit 3: $out" + + # No .gitmodules: cannot tell, not "clean". + mkdir -p "$tmp/plain" && git init -q "$tmp/plain" + bash "$0" --root "$tmp/plain" >/dev/null 2>&1 + [ $? = 2 ] && ok "no .gitmodules is 'cannot tell' (2), not a pass" || bad "no .gitmodules should exit 2" + + if [ "$fails" != 0 ]; then echo "check_submodules self-test: $fails failure(s)"; exit 1; fi + echo "check_submodules self-test: ok" + exit 0 +fi + +command -v git >/dev/null || { echo "check-submodules: no git on PATH" >&2; exit 2; } +[ -f "$root/.gitmodules" ] || { echo "check-submodules: no .gitmodules under $root -- nothing to inspect, which is not a pass" >&2; exit 2; } + +bad=0 +missing=0 +while read -r _key path; do + [ -n "${path:-}" ] || continue + pin=$(git -C "$root" ls-tree HEAD -- "$path" 2>/dev/null | awk '{print $3}') + if [ -z "$pin" ]; then + echo "::error::check-submodules: $path is in .gitmodules but the tree pins no commit for it" + bad=$((bad + 1)); continue + fi + head=$(git -C "$root/$path" rev-parse HEAD 2>/dev/null) + # `git -C` in an unchecked-out submodule dir is not an error: it walks UP and reports the PARENT's HEAD, so a fresh + # clone looked like a checkout at the wrong commit. The path must be its own repository root to be a checkout at all. + toplevel=$(git -C "$root/$path" rev-parse --show-toplevel 2>/dev/null) + if [ -n "$head" ] && [ -n "$toplevel" ] && \ + [ "$(cd "$root/$path" 2>/dev/null && pwd -P)" != "$(cd "$toplevel" 2>/dev/null && pwd -P)" ]; then + head= + fi + if [ -z "$head" ]; then + echo "::warning::check-submodules: $path is not checked out (the tree pins ${pin:0:9}); a build would not find its sources" + missing=$((missing + 1)); continue + fi + if [ "${pin:0:9}" != "${head:0:9}" ]; then + echo "::error::check-submodules: $path is not the pin: tree ${pin:0:9}, checkout ${head:0:9} -- the build would read sources the pin does not name" + bad=$((bad + 1)); continue + fi + n=$(git -C "$root/$path" status --porcelain 2>/dev/null | wc -l) + if [ "$n" != 0 ]; then + echo "::error::check-submodules: $path is at the pin but its worktree is not clean ($n path(s) differ from HEAD) -- a build would read sources the pin does not name" + bad=$((bad + 1)); continue + fi + printf 'ok %-28s %s\n' "$path" "${pin:0:9}" +done < <(git -C "$root" config -f .gitmodules --get-regexp '^submodule\..*\.path$' 2>/dev/null) + +if [ "$bad" = 0 ]; then + if [ "$missing" = 0 ]; then echo "check-submodules: every checkout is its pin and clean"; exit 0; fi + echo "check-submodules: $missing submodule(s) are not checked out, so nothing was inspected for them (a clone gets here; run git submodule update --init)" + exit 3 +fi +echo "check-submodules: $bad submodule(s) are not what the tree pins" +exit 1 diff --git a/scripts/weekly.sh b/scripts/weekly.sh old mode 100755 new mode 100644 index d8ce116fc..dc2aa3f5d --- a/scripts/weekly.sh +++ b/scripts/weekly.sh @@ -658,6 +658,29 @@ preflight_runner_units() { return 0 } +# The submodule row of the preflight (engine#2129). The bumps and build stages build the tree in front of them, and +# "the pin is healthy" -- which CI checks, on the repository -- is not "the checkout is the pin", which is what a build +# reads. Measured 2026-11-11 while triaging #2129: 7 of 7 submodules on this box disagreed with the commit the tree pins, +# llama.cpp by 301 uncommitted files. Exit 1 is a checkout that disagrees and stops the run: the build would not be of the +# pinned sources. Exit 3 is submodules not checked out at all (a fresh clone) and only WARNS -- a fresh box must still be +# able to build, and the warning says the check inspected nothing rather than implying it passed (the exit-3 discipline of +# engine#2094). Exit 2 (no .gitmodules) cannot happen on this repository and warning is right for it too. +# A function of its own so tests/weekly_submodule_preflight_test.sh can run it against a stub. +preflight_submodules() { + local check=$1 src=$2 out rc=0 + [ -x "$check" ] || return 0 + out=$("$check" --root "$src" 2>&1) || rc=$? + case "$rc" in + 0) printf '%s\n' "$out" | grep -v '^ok ' | sed 's/^/preflight: /' ;; + 2|3) printf '%s\n' "$out" | sed 's/^/preflight: /' + echo "preflight: WARNING, not blocking: no checkout to compare against the pin, so the build follows the tree rather than this check (scripts/check-submodules.sh exit $rc; git submodule update --init fills it)" ;; + *) printf '%s\n' "$out" | sed 's/^/preflight: /' + echo "preflight: the tree in front of this build is not the pinned tree, so the build would not be of the pinned sources (scripts/check-submodules.sh exit $rc; the lines above name the submodule and both commits)" + return 1 ;; + esac + return 0 +} + # --- preflight ---------------------------------------------------------------------------- # What the stages need from the machine, checked before the first one runs, so that a missing # tool stops the run at the start rather than after an hour of build (2026-10-19 rehearsal). @@ -677,7 +700,9 @@ preflight() { 2> "$LOGS/hrx-toolchain.log"); then :; else echo "preflight: cannot fetch the pinned HRX toolchain (config/hrx-toolchain.config): see $LOGS/hrx-toolchain.log"; bad=1 fi ;; - esac ;; + esac + # engine#2129: this stage builds the tree in front of it, so the tree has to be the pinned tree + preflight_submodules "$(cd "$(dirname "$0")" && pwd)/check-submodules.sh" "$SRC" || bad=1 ;; package) # scripts/package-linux.sh rewrites RPATHs with patchelf command -v patchelf > /dev/null || diff --git a/tests/weekly_submodule_preflight_test.sh b/tests/weekly_submodule_preflight_test.sh new file mode 100755 index 000000000..a7b4aadf0 --- /dev/null +++ b/tests/weekly_submodule_preflight_test.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# weekly_submodule_preflight_test.sh: the release preflight BLOCKS on a build tree that is not the pinned tree and only +# WARNS when there is no checkout to compare against (engine#2129). Both halves are the real code: the function is cut out +# of scripts/weekly.sh and run against stub checkers, so no submodules, no network and no build. +set -u +root=$(cd "$(dirname "$0")/.." && pwd) +tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT +fails=0 +ok() { printf 'ok %s\n' "$1"; } +bad() { printf 'FAIL %s\n' "$1"; fails=$((fails + 1)); } + +sed -n '/^preflight_submodules() {$/,/^}$/p' "$root/scripts/weekly.sh" > "$tmp/fn.sh" +grep -q 'check-submodules.sh exit' "$tmp/fn.sh" || { bad "could not cut preflight_submodules() out of scripts/weekly.sh"; echo "weekly_submodule_preflight: FAIL"; exit 1; } +grep -q 'preflight_submodules "' "$root/scripts/weekly.sh" || bad "scripts/weekly.sh's preflight no longer calls preflight_submodules" +say() { echo "$*"; } +# shellcheck disable=SC1091 +. "$tmp/fn.sh" + +# A stub checker: prints what the real one prints and exits with the code under test. +stub() { # + local rc=$1; shift + printf '#!/usr/bin/env bash\nprintf "%%s\\n" %s\nexit %s\n' "$(printf '%q ' "$@")" "$rc" > "$tmp/stub-$rc" + chmod +x "$tmp/stub-$rc"; echo "$tmp/stub-$rc" +} +row() { # [lines...] + local name=$1 want=$2 text=$3 rc=$4; shift 4 + local check; check=$(stub "$rc" "$@") + local out; out=$(preflight_submodules "$check" "$tmp" 2>&1); local got=$? + if [ "$got" = "$want" ] && printf '%s\n' "$out" | grep -q -- "$text"; then ok "preflight: $name -> return $got" + else bad "preflight: $name: want return $want with '$text', got $got: $out"; fi +} + +row "a tree at its pins passes" 0 "preflight: " 0 "ok third_party/sub 1234567890" "check-submodules: every checkout is its pin and clean" +row "a checkout that disagrees stops the run" 1 "not the pinned tree" 1 "::error::check-submodules: third_party/x is not the pin: tree a, checkout b" +row "an unchecked-out clone only warns" 0 "WARNING, not blocking" 3 "::warning::check-submodules: third_party/x is not checked out" +row "no .gitmodules only warns" 0 "WARNING, not blocking" 2 "check-submodules: no .gitmodules under . -- nothing to inspect, which is not a pass" +row "a missing checker does not stop the run" 0 "" 0 + +# The stated asymmetry, in one assertion: the two "cannot tell" codes must not return 1. +for rc in 2 3; do + check=$(stub "$rc" "::warning::check-submodules: cannot tell") + preflight_submodules "$check" "$tmp" >/dev/null 2>&1 + [ $? = 0 ] && ok "exit $rc is not a block" || bad "exit $rc should not block the release" +done + +if [ "$fails" != 0 ]; then echo "weekly_submodule_preflight: $fails failure(s)"; exit 1; fi +echo "weekly_submodule_preflight: ok"