From d9b8d304062eea854ff4df3ee2037736e2a860b5 Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Thu, 12 Nov 2026 03:10:16 -0800 Subject: [PATCH 1/3] feat(checks): build volatile checks with lib.caisson.mkVolatileCheck `lib.caisson.mkVolatileCheck { pkgs, name, program, impureEnvVars ? [ ] }` returns the derivation of a check of a volatile category. It is a fixed-output derivation, which Nix lets reach the network. Its builder, the bash of `pkgs` with no `PATH`, runs the program and, when the program exits with status 0, writes the pass marker: a file holding the line `pass`, declared by its flat sha256 hash. The name of the derivation is `-`, with the hash part of the store path of the program, so a changed program gets a new marker path and the same program keeps it. A failure leaves nothing in the store. The derivation is a plain `derivation`, so its `drvAttrs` keep the program, and topside reads the inputs of the marker from their context. The check record of a caisson top now refuses a check of a volatile category whose derivation has `drvAttrs` that declare no `outputHash`: reading the `volatile` field of its entry fails. `topside check` reads that field for every check it runs. The `checks` output reads no property of a category, and reading `volatile` there would evaluate the declarations of the category for every `nix flake check`, so the output does not refuse. Co-Authored-By: Claude Opus 5.5 --- lib-overlays/integrations/default.nix | 35 ++++++- lib-overlays/tooling/default.nix | 2 + lib-overlays/tooling/mk-volatile-check.nix | 104 +++++++++++++++++++++ modules/generic/caisson/types.nix | 14 ++- 4 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 lib-overlays/tooling/mk-volatile-check.nix diff --git a/lib-overlays/integrations/default.nix b/lib-overlays/integrations/default.nix index 55aa934c..a6b2c76b 100644 --- a/lib-overlays/integrations/default.nix +++ b/lib-overlays/integrations/default.nix @@ -3031,6 +3031,35 @@ else [ ]; + # Whether a gathered check (`gatheredChecksOf`) is of a volatile + # category. A check of a volatile category runs in a fixed-output + # derivation (`lib.caisson.mkVolatileCheck`), and one whose + # derivation has `drvAttrs` that declare no `outputHash` is + # refused. The tools that act on volatility, such as `topside + # check`, read it here, in the check record. The `checks` output + # reads no property of a category, so it does not refuse such a + # check: reading `volatile` evaluates the declarations of the + # category, which `nix flake check` has no use for. + volatileOf = + entry: + let + derivation = entry.check.derivation; + in + if !entry.volatile then + false + else if builtins.isAttrs (derivation.drvAttrs or null) && derivation.drvAttrs ? outputHash then + true + else + throw '' + caisson: the check `${entry.checkName}` of the configuration at ${showNamePath entry.namePath}, + declared at ${showCode entry.check.code}, is of the volatile category + `${entry.check.category}`, and its derivation is not a fixed-output derivation: + its `drvAttrs` declare no `outputHash`. A check of a volatile category runs in a + fixed-output derivation, which Nix lets reach the network, and its output is a + pass marker. Build the derivation with `lib.caisson.mkVolatileCheck`, or name a + category that is not volatile. + ''; + # The check record of a caisson top, keyed as `checks` of its # outputs is, by system and then published name: for each check, # `name` and `system`, `from.namePath`, the path from the caisson @@ -3045,6 +3074,10 @@ # top as `check`, and the entry of the check record of the test # caisson top as `record`, null when the test caisson top publishes # no check record. For every other check `top` is null. + # + # The `volatile` field of a check of a volatile category is + # refused when the derivation of the check is not a fixed-output + # derivation (`volatileOf`). checkRecordOf = manifest: builtins.mapAttrs ( @@ -3059,7 +3092,7 @@ code top ; - inherit (entry) volatile; + volatile = volatileOf entry; requiredFeatures = requiredFeaturesOf entry.check.derivation; } ) diff --git a/lib-overlays/tooling/default.nix b/lib-overlays/tooling/default.nix index d29ddd96..160d6a02 100644 --- a/lib-overlays/tooling/default.nix +++ b/lib-overlays/tooling/default.nix @@ -21,6 +21,8 @@ mkMemoizedDerivationRead = import ./mk-memoized-derivation-read.nix { lib = final; }; + mkVolatileCheck = import ./mk-volatile-check.nix; + }; }; diff --git a/lib-overlays/tooling/mk-volatile-check.nix b/lib-overlays/tooling/mk-volatile-check.nix new file mode 100644 index 00000000..bc830642 --- /dev/null +++ b/lib-overlays/tooling/mk-volatile-check.nix @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: MIT +# +# This file is the function `mkVolatileCheck`. It returns the +# derivation of a check whose category is volatile: a check whose result +# depends on something outside the build, such as a service reached +# over the network. +# +# A configuration declares such a check with the derivation it returns: +# +# caisson.checks.mirror-answers = { +# category = "online"; +# derivation = lib.caisson.mkVolatileCheck { +# inherit pkgs; +# name = "mirror-answers"; +# program = pkgs.writeShellApplication { +# name = "mirror-answers"; +# runtimeInputs = [ pkgs.curl ]; +# text = '' +# curl --fail --silent --cacert ${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt \ +# https://mirror.example.org/health +# ''; +# }; +# }; +# }; +# +# The derivation is a fixed-output derivation. Nix lets the builder of +# a fixed-output derivation reach the network, and in exchange the +# derivation declares the hash of its output in advance. The builder +# runs the program. When the program exits with status 0, the builder +# writes the pass marker, a file holding the line `pass`, as its +# output. When the program fails, the build fails and leaves nothing in +# the Nix store. +# +# Nix computes the store path of the output of a fixed-output +# derivation from its name and the declared hash alone. Every pass +# marker has the same hash, so the name tells markers apart: it is +# `-`, where `` is the hash part of the store path of +# the program, the 32 characters after `/nix/store/`. A change to the +# program, or to anything its store path depends on, gives a new store +# path for the program and so a new marker. While the program stays the +# same, the marker path stays the same, and a marker already in the +# store, or in a cache, counts as a pass without the program running. +# A tool that runs the check again builds the marker with substitution +# off and `--rebuild`, as `topside check --rerun-volatile` does. +# +# Arguments: +# +# - `pkgs`: the package set for the system of the check. The builder +# is the bash of this set, and the derivation is built for its host +# platform. +# - `name`: the name of the check, the first part of the name of the +# marker. +# - `program`: the executable the check runs, with no arguments. It is +# a derivation whose output is an executable file, as +# `pkgs.writeShellScript` returns, or a string naming an executable +# inside a store path, as `lib.getExe` returns for +# `pkgs.writeShellApplication`, or a path, which Nix copies into the +# store. The builder sets no `PATH`, so the program reaches only the +# commands it names by store path or puts on its `PATH` itself, as +# `pkgs.writeShellApplication` does with its `runtimeInputs`. A +# program that reaches a server over TLS names a CA bundle, such as +# that of `pkgs.cacert`, since the build sandbox has none. +# - `impureEnvVars`: the names of the environment variables of the Nix +# process that runs the build which Nix passes on to the builder. +# With a Nix daemon, that process is the daemon. Of the machine that +# builds the check, the program reaches the network and these +# variables. +# +# The derivation is a plain `derivation`, so its `drvAttrs` hold the +# attributes above, and the store paths it depends on are the context +# of those attributes: the bash of `pkgs` and the program. +{ + pkgs, + name, + program, + impureEnvVars ? [ ], +}: +let + programPath = "${program}"; + storeHash = builtins.match "${builtins.storeDir}/([0-9a-z]{32})-.*" programPath; +in +if storeHash == null then + throw '' + caisson: mkVolatileCheck: the program of the check `${name}` is + ${builtins.toJSON (builtins.unsafeDiscardStringContext programPath)}, which is not + inside the Nix store. The name of the pass marker carries the hash of the store + path of the program. Pass a derivation, a path, or a string inside a store path, + such as `lib.getExe` of a package. + '' +else + derivation { + name = "${name}-${builtins.head storeHash}"; + inherit (pkgs.stdenv.hostPlatform) system; + builder = "${pkgs.bash}/bin/bash"; + args = [ + "-c" + ''"$program" && printf 'pass\n' > "$out"'' + ]; + program = programPath; + inherit impureEnvVars; + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = builtins.hashString "sha256" "pass\n"; + } diff --git a/modules/generic/caisson/types.nix b/modules/generic/caisson/types.nix index 0b833ab0..fa8ab5c1 100644 --- a/modules/generic/caisson/types.nix +++ b/modules/generic/caisson/types.nix @@ -97,7 +97,10 @@ in The derivation that is the check, for the system of the evaluation result that declares it: the check passes when the derivation builds. Null for an entry that annotates a check the - configuration already has, and for an entry with a `top`. + configuration already has, and for an entry with a `top`. A + check of a volatile category (`caisson.checkCategories`) has a + fixed-output derivation, such as the derivation + `lib.caisson.mkVolatileCheck` returns. ''; }; top = lib.mkOption { @@ -166,7 +169,14 @@ in description = '' Whether the result of a check of the category depends on something outside the build, such as a network, a service or a - piece of hardware. + piece of hardware. The derivation of a check of a volatile + category is a fixed-output derivation, the kind of derivation + Nix lets reach the network, and its output is a pass marker. + `lib.caisson.mkVolatileCheck` builds it from the program the + check runs. The check record of a caisson top refuses a + check of a volatile category, when its `volatile` field is + read, if the `drvAttrs` of its derivation declare no + `outputHash`. ''; }; }; From c7c5314de8a613f86b17de5107c93cfc9ab26cd1 Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Thu, 12 Nov 2026 03:10:16 -0800 Subject: [PATCH 2/3] test(checks): cover the volatile check derivation and its builds The pinned-world suite checks the name of the derivation `mkVolatileCheck` returns: it changes with the program and stays the same for the same program, whatever `impureEnvVars` hold. It checks the fixed output, the `drvAttrs` and their context, the refusal of a program outside the store, and the refusal, in the check record, of a check of a volatile category whose derivation is not a fixed-output derivation. The volatile checks of the existing probes become fixed-output derivations. tests/volatile-checks/check.sh builds a passing check and finds its marker with `nix path-info`, builds it again with `--rebuild`, then builds a failing check and finds no marker. The `volatile-checks` job of the check workflow runs it on the runner, since a check of `nix flake check` cannot run `nix build`. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/check.yml | 13 +++ tests/pinned-world/default.nix | 131 ++++++++++++++++++++++++++++-- tests/unit/lib-overlays.nix | 6 +- tests/volatile-checks/check.sh | 65 +++++++++++++++ tests/volatile-checks/default.nix | 31 +++++++ 5 files changed, 240 insertions(+), 6 deletions(-) create mode 100755 tests/volatile-checks/check.sh create mode 100644 tests/volatile-checks/default.nix diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index aa1487f2..5a48cbc2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -25,6 +25,19 @@ jobs: authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} signingKey: ${{ secrets.CACHIX_SIGNING_KEY }} - run: nix flake check -L + # The checks `lib.caisson.mkVolatileCheck` returns, built into the + # store of the runner (tests/volatile-checks/check.sh): a pass writes + # its marker and a failure leaves none. A check of `nix flake check` + # is itself a build, and its builder cannot run `nix build` against + # the store of the runner, so the script runs as a step of the job. + volatile-checks: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: DeterminateSystems/nix-installer-action@v22 + with: + determinate: false + - run: bash tests/volatile-checks/check.sh # A system-manager configuration activated in a container of each # distribution (tests/system-manager/container.nix). system-manager's # container test driver runs the container in the build sandbox, which diff --git a/tests/pinned-world/default.nix b/tests/pinned-world/default.nix index 6896de90..654b93c7 100644 --- a/tests/pinned-world/default.nix +++ b/tests/pinned-world/default.nix @@ -137,6 +137,7 @@ let "mkModules" "mkPkgOverlay" "mkPkgOverlays" + "mkVolatileCheck" "modules" "nixidy" "nixops4" @@ -984,6 +985,18 @@ let } ); checkMarker = name: pkgs.runCommand name { } "touch $out"; + # A check of a volatile category is a fixed-output derivation: this + # check writes an empty file, whose hash it declares. + fixedOutputCheckMarker = + name: attrs: + pkgs.runCommand name ( + attrs + // { + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = builtins.hashString "sha256" ""; + } + ) "touch $out"; checksNixosConfiguration = lib: module: lib.caisson.nixos.mkConfiguration { @@ -1039,7 +1052,7 @@ let description = "the NixOS configuration evaluates"; }; caisson.checks.boots = { - derivation = pkgs.runCommand "nas-1-boots" { requiredSystemFeatures = [ "kvm" ]; } "touch $out"; + derivation = fixedOutputCheckMarker "nas-1-boots" { requiredSystemFeatures = [ "kvm" ]; }; category = "hardware"; }; caisson.checks.off = { @@ -1216,7 +1229,7 @@ let specialArgs = { inherit lib; }; modules = lib.caisson.integrations.frameworkModules "nixos" { } ++ [ module ]; }; - outputs.artifact = checkMarker "stub-artifact-${manifest.system}"; + outputs.artifact = fixedOutputCheckMarker "stub-artifact-${manifest.system}" { }; }; } ); @@ -6712,6 +6725,114 @@ let && checksFail clashing.checks && !checksFail checksTop.checks; + # `mkVolatileCheck` returns a fixed-output derivation whose output + # is the file holding `pass`, named for the check and the hash part + # of the store path of the program. A changed program changes the + # name and the output path; the same program, given again or with + # other `impureEnvVars`, keeps them. The `drvAttrs` keep the + # program, so their context names its derivation. A program + # outside the store is refused. + volatileCheckNamesItsMarkerForTheProgram = + let + mk = composed.lib.caisson.mkVolatileCheck; + program = text: pkgs.writeShellScript "probe" text; + hashOf = path: builtins.head (builtins.match "${builtins.storeDir}/([0-9a-z]{32})-.*" "${path}"); + passing = mk { + inherit pkgs; + name = "probe"; + program = program "exit 0"; + }; + again = mk { + inherit pkgs; + name = "probe"; + program = program "exit 0"; + }; + withVariables = mk { + inherit pkgs; + name = "probe"; + program = program "exit 0"; + impureEnvVars = [ "PROBE_TOKEN" ]; + }; + changed = mk { + inherit pkgs; + name = "probe"; + program = program "exit 1"; + }; + application = pkgs.writeShellApplication { + name = "probe"; + runtimeInputs = [ pkgs.coreutils ]; + text = "true"; + }; + fromApplication = mk { + inherit pkgs; + name = "probe"; + program = pkgs.lib.getExe application; + }; + outside = mk { + inherit pkgs; + name = "probe"; + program = "/usr/bin/true"; + }; + in + passing.name == "probe-${hashOf (program "exit 0")}" + && fromApplication.name == "probe-${hashOf application}" + && passing.outPath == again.outPath + && passing.drvPath == again.drvPath + && passing.outPath == withVariables.outPath + && passing.drvPath != withVariables.drvPath + && passing.name != changed.name + && passing.outPath != changed.outPath + && passing.system == pkgs.stdenv.hostPlatform.system + && passing.drvAttrs.outputHashMode == "flat" + && passing.drvAttrs.outputHashAlgo == "sha256" + && passing.drvAttrs.outputHash == builtins.hashString "sha256" "pass\n" + && passing.drvAttrs.impureEnvVars == [ ] + && withVariables.drvAttrs.impureEnvVars == [ "PROBE_TOKEN" ] + && builtins.getContext (builtins.toJSON passing.drvAttrs) + ? ${builtins.unsafeDiscardStringContext (program "exit 0").drvPath} + && !(builtins.tryEval (builtins.deepSeq outside.drvPath true)).success; + + # A check of a volatile category whose derivation is not a + # fixed-output derivation is refused in the `volatile` field of + # its record entry. The other fields of the entry read, and so does + # the `checks` output. The derivation `mkVolatileCheck` returns is + # taken. + volatileChecksAreFixedOutputDerivations = + let + withProbe = + derivation: + checksTopWith ( + { lib, ... }: + { + caisson.nixos.configurations.probe = checksNixosConfiguration lib { + caisson.checks.reaches = { + derivation = derivation lib; + category = "hardware"; + }; + }; + } + ); + hermetic = withProbe (_: checkMarker "probe-reaches"); + fixedOutput = withProbe ( + lib: + lib.caisson.mkVolatileCheck { + inherit pkgs; + name = "probe-reaches"; + program = pkgs.writeShellScript "probe-reaches" "exit 0"; + } + ); + record = top: top.caisson.manifest.checkRecord.x86_64-linux.probe-reaches; + in + !(builtins.tryEval (record hermetic).volatile).success + && checksFail hermetic.caisson.manifest.checkRecord + && (record hermetic).category == "hardware" + && (record hermetic).requiredFeatures == [ ] + && !checksFail hermetic.checks + && !checksFail fixedOutput.caisson.manifest.checkRecord + && !checksFail fixedOutput.checks + && fixedOutput.checks.x86_64-linux.probe-reaches.drvAttrs ? outputHash + && (record fixedOutput).volatile; + # A stage declared inside the caisson top is refused. checkStagesDeclaredInsideTheCaissonTopAreRefused = let @@ -6849,7 +6970,7 @@ let }; off = stubChecksTopWith { caisson.checks.artifact.enable = false; }; redefined = stubChecksTopWith { - caisson.checks.artifact.derivation = checkMarker "another"; + caisson.checks.artifact.derivation = fixedOutputCheckMarker "another" { }; }; checksOf = top: top.aarch64-linux.outputs.checks; recordOf = checksLib.caisson.integrations.checkRecordOf; @@ -6857,8 +6978,8 @@ let in builtins.attrNames plain == builtins.sort builtins.lessThan checksSystems && - (checksOf plain).aarch64-linux.artifact.drvPath == (checkMarker "stub-artifact-aarch64-linux") - .drvPath + (checksOf plain).aarch64-linux.artifact.drvPath + == (fixedOutputCheckMarker "stub-artifact-aarch64-linux" { }).drvPath && artifact.category == "probe" && artifact.volatile && artifact.description == "the artifact of the configuration builds" diff --git a/tests/unit/lib-overlays.nix b/tests/unit/lib-overlays.nix index d63cdff9..d9715474 100644 --- a/tests/unit/lib-overlays.nix +++ b/tests/unit/lib-overlays.nix @@ -3694,8 +3694,12 @@ in volatile = true; }; caisson.nixos.configurations.nas-1 = nixosConfiguration lib { + # A check of a volatile category is a fixed-output + # derivation. caisson.checks.boots = { - derivation = check "boots"; + derivation = check "boots" // { + drvAttrs.outputHash = builtins.hashString "sha256" "pass\n"; + }; category = "hardware"; }; }; diff --git a/tests/volatile-checks/check.sh b/tests/volatile-checks/check.sh new file mode 100755 index 00000000..2e566534 --- /dev/null +++ b/tests/volatile-checks/check.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +# +# Builds the checks of tests/volatile-checks/default.nix into the local +# Nix store and reads the store after each build with `nix path-info`. +# A passing check writes its pass marker, at the output path its +# derivation declares, holding `pass`; building it again with +# `--rebuild` runs the program again and keeps the marker. A failing +# check fails to build, and no marker is valid at its output path. +# +# tests/volatile-checks/check.sh +# +# The pinned-world suite holds what evaluation shows of the derivation +# (its name, its fixed output and its `drvAttrs`); this script holds +# what only a build shows. + +set -euo pipefail + +repo=$(cd "$(dirname "$0")/../.." && pwd) +nonce="run-$(date +%s)-$$" +failures=0 + +fail() { + echo "check.sh: FAIL: $*" >&2 + failures=$((failures + 1)) +} + +checks=(--impure --file "$repo/tests/volatile-checks" --argstr nonce "$nonce") + +output_of() { + nix eval --raw "${checks[@]}" "$1.outPath" +} + +passing=$(output_of passing) +failing=$(output_of failing) + +for marker in "$passing" "$failing"; do + if nix path-info "$marker" >/dev/null 2>&1; then + fail "$marker is valid in the local store before any build" + fi +done + +# A pass writes the marker at the declared output path. +built=$(nix build --no-link --print-out-paths "${checks[@]}" passing) || fail "the passing check did not build" +[[ $built == "$passing" ]] || fail "the passing check built $built, not $passing" +nix path-info "$passing" >/dev/null 2>&1 || fail "$passing is not valid in the local store after a pass" +[[ $(cat "$passing") == pass ]] || fail "$passing does not hold pass" + +# The program runs again with --rebuild, and the marker stays. +nix build --no-link --rebuild "${checks[@]}" passing || fail "the passing check did not build again with --rebuild" +nix path-info "$passing" >/dev/null 2>&1 || fail "$passing is not valid in the local store after the rebuild" + +# A failure leaves no marker. +if nix build --no-link "${checks[@]}" failing; then + fail "the failing check built" +fi +if nix path-info "$failing" >/dev/null 2>&1; then + fail "$failing is valid in the local store after a failure" +fi + +if ((failures)); then + echo "check.sh: $failures expectations failed" >&2 + exit 1 +fi +echo "check.sh: every expectation holds" diff --git a/tests/volatile-checks/default.nix b/tests/volatile-checks/default.nix new file mode 100644 index 00000000..606b4bc7 --- /dev/null +++ b/tests/volatile-checks/default.nix @@ -0,0 +1,31 @@ +# SPDX-License-Identifier: MIT +# +# The checks tests/volatile-checks/check.sh builds: `passing` and +# `failing`, the derivations `lib.caisson.mkVolatileCheck` returns for a +# program that exits with status 0 and for a program that exits with +# status 1. `nonce` is part of the name of each program, so each run of +# the script builds markers no store holds yet. +# +# caisson is the flake of this checkout, and nixpkgs is the pin of +# tests/dependencies. The script evaluates this file with `--impure`, +# since it reads both flakes with `builtins.getFlake` by path. +{ nonce }: +let + caisson = builtins.getFlake (toString ../..); + nixpkgs = (builtins.getFlake (toString ../dependencies)).inputs.nixpkgs; + pkgs = nixpkgs.legacyPackages.${builtins.currentSystem}; + check = + name: status: + caisson.lib.caisson.mkVolatileCheck { + inherit pkgs; + name = "volatile-checks-${name}"; + program = pkgs.writeShellScript "volatile-checks-${name}-${nonce}" '' + echo "the program of the check ${name} exits with status ${status}" + exit ${status} + ''; + }; +in +{ + passing = check "passing" "0"; + failing = check "failing" "1"; +} From 16c32e966e3163f41262d5b6921b4e1bfee1c448 Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Thu, 12 Nov 2026 03:10:16 -0800 Subject: [PATCH 3/3] docs(testing): describe the volatile checks script Co-Authored-By: Claude Opus 5.5 --- docs/development/testing.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/development/testing.md b/docs/development/testing.md index fe8fdc23..894dc053 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -97,6 +97,25 @@ plumbing works. --impure ``` +## Volatile Checks (Builds in the Local Store) + +- **Scope:** The derivation `lib.caisson.mkVolatileCheck` returns, built: + a check whose program passes writes its pass marker at the output + path its derivation declares, building it again with `--rebuild` + keeps the marker, and a check whose program fails leaves no marker in + the store. The pinned-world suite holds what evaluation shows of the + derivation: its name, its fixed output and its `drvAttrs`. +- **Location:** `tests/volatile-checks/check.sh` and the checks it + builds in `tests/volatile-checks/default.nix`. +- **Mechanism:** the script runs `nix build` on each check and reads the + store afterwards with `nix path-info`. A check of `nix flake check` is + a build and cannot do either, so the `volatile-checks` job of the + check workflow runs the script on the runner. +- **Execution, from a checkout:** + ```bash + tests/volatile-checks/check.sh + ``` + ## Integration Testing (Test Flakes) - **Scope:** Module composition, end-to-end evaluation, and build success.