diff --git a/.checkup.yml.example b/.checkup.yml.example index c552dc5..40a520c 100644 --- a/.checkup.yml.example +++ b/.checkup.yml.example @@ -61,5 +61,14 @@ commands: # Directory globs work (a leading match covers the whole subtree): `vendor/js/*`. # exclude: [src/generated/*, "vendor/js/*", "*.pb.go"] -# ── Not yet configurable here ──────────────────────────────────────────────── -# - Per-check thresholds (complexity CCN, duplication %): planned. +# ── Thresholds ─────────────────────────────────────────────────────────────── +# Tune the warn/fail banding the complexity and duplication sections apply. Each +# is an integer; omit any to keep the default (shown). These change a check's +# STATUS only — never the health score (checkup is a localiser, not a gate). The +# complexity knobs apply to the CCN engines (ESLint, lizard); scc's heuristic +# band keeps its own scale. +thresholds: + # complexity_ccn_warn: 10 # report functions at/above this cyclomatic CCN + # complexity_ccn_fail: 30 # any function at/above this → the record fails + # duplication_warn_pct: 3 # duplication % at/above this → warn + # duplication_fail_pct: 5 # duplication % at/above this → fail diff --git a/CHANGELOG.md b/CHANGELOG.md index be785ba..864977d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,13 @@ across the whole thing, honestly*. **Focus Areas** view synthesising the forensic axes (#36–#38). - **Command profiles** + a **`.checkup.yml`** override layer (stack / checks / commands) with a documented example (#6, #69–#71, #2). +- **Per-check thresholds** in `.checkup.yml` (#72): a `thresholds:` block tunes + the complexity (`complexity_ccn_warn` / `_fail`, the CCN engines) and + duplication (`duplication_warn_pct` / `_fail_pct`) warn/fail banding — *"our + complexity budget is 15, not 10"*. Integers, validated (garbage → warn + keep + the default); each defaults to the historical literal so an absent block is + byte-identical. Tunes a check's **status only — never the health score** + (ADR-0009). - Generated/vendored exclusion for the multi-language scans + `CHECKUP_EXCLUDE` (#41, #44); evergreen reference docs, ADRs, and `AGENTS.md` (#12–#13, #19). diff --git a/README.md b/README.md index 3fc3a9a..0ac9444 100644 --- a/README.md +++ b/README.md @@ -398,6 +398,10 @@ checks: enable: [mutation] # opt in to an off-by-default check commands: test: "dotnet test" # override a check's command ("" disables it); or set CHECKUP_CMD_TEST +thresholds: # tune warn/fail banding (status only, never the score) + complexity_ccn_warn: 10 # report functions at/above this CCN (ESLint + lizard engines) + complexity_ccn_fail: 30 # any function at/above this → the complexity record fails + duplication_warn_pct: 3 # duplication % at/above this → warn; …_fail_pct → fail ``` It's a small YAML subset (inline lists `[a, b]`, `#` comments); `yq` is used if diff --git a/bin/checkup.sh b/bin/checkup.sh index 1306256..9a4ef53 100755 --- a/bin/checkup.sh +++ b/bin/checkup.sh @@ -421,6 +421,21 @@ else DETECT_ENGINE_DUPLICATION="none"; DUP_REASON="no Node target for jscpd and no lizard-parseable source" fi +# Per-check thresholds (#72): from .checkup.yml `thresholds:` (or a CHECKUP_* env +# var), each defaulting to the historical literal so a no-override run is +# byte-identical. Tunes only the warn/fail banding the complexity (CCN arms) and +# duplication sections already apply — never scoring (ADR-0009). _cfg_int guards a +# hand-set non-integer from reaching jq / `[ -lt ]`. (scc's heuristic complexity +# band is a different scale and keeps its own cutoffs.) +CPLX_CCN_WARN=$(_cfg_int "${CHECKUP_CPLX_CCN_WARN:-}" 10) +CPLX_CCN_FAIL=$(_cfg_int "${CHECKUP_CPLX_CCN_FAIL:-}" 30) +DUP_WARN_PCT=$(_cfg_int "${CHECKUP_DUP_WARN_PCT:-}" 3) +DUP_FAIL_PCT=$(_cfg_int "${CHECKUP_DUP_FAIL_PCT:-}" 5) +# The ESLint complexity warn level, built so the default (10) renders the exact +# historical --rule JSON (byte-identical). Cognitive stays 15 (not yet tunable). +CPLX_ESLINT_RULE=$(jq -nc --argjson w "$CPLX_CCN_WARN" '{complexity:["warn",$w],"sonarjs/cognitive-complexity":["warn",15]}') +CPLX_ESLINT_RULE_CYC=$(jq -nc --argjson w "$CPLX_CCN_WARN" '{complexity:["warn",$w]}') + # Primary stack + confidence (drives absence-is-signal framing, #51): the largest # scc stack that also has a manifest is a HIGH-confidence "we looked the right way # for this stack"; manifest-or-dominant-only is medium; neither is low. Empty when @@ -1443,11 +1458,11 @@ if [ "$DUP_ENGINE" = "jscpd" ]; then | .[0:10] ' reports/jscpd/jscpd-report.json) - if [ "$DUPLICATION_INT" -lt 3 ]; then + if [ "$DUPLICATION_INT" -lt "$DUP_WARN_PCT" ]; then echo -e "${GREEN}✅ Low code duplication: ${DUPLICATION_PCT}% (5/5)${NC}" HEALTH_SCORE=$((HEALTH_SCORE + 5)) JSCPD_STATUS="pass" - elif [ "$DUPLICATION_INT" -lt 5 ]; then + elif [ "$DUPLICATION_INT" -lt "$DUP_FAIL_PCT" ]; then echo -e "${YELLOW}⚠️ Moderate code duplication: ${DUPLICATION_PCT}% (3/5)${NC}" HEALTH_SCORE=$((HEALTH_SCORE + 3)) echo " Review reports/jscpd/jscpd-report.json for details" @@ -1535,11 +1550,11 @@ PY DUP_RATE_INT=$(echo "$DUP_RATE" | awk '{print int($1)}') DUP_COUNT=$(echo "$DUP_PARSED" | jq -r '.count') DUP_TOP=$(echo "$DUP_PARSED" | jq -c '.top') - if [ "$DUP_RATE_INT" -lt 3 ]; then + if [ "$DUP_RATE_INT" -lt "$DUP_WARN_PCT" ]; then echo -e "${GREEN}✅ Low code duplication: ${DUP_RATE}% (5/5)${NC}" HEALTH_SCORE=$((HEALTH_SCORE + 5)) DUP_STATUS="pass" - elif [ "$DUP_RATE_INT" -lt 5 ]; then + elif [ "$DUP_RATE_INT" -lt "$DUP_FAIL_PCT" ]; then echo -e "${YELLOW}⚠️ Moderate code duplication: ${DUP_RATE}% ($DUP_COUNT clone blocks, 3/5)${NC}" HEALTH_SCORE=$((HEALTH_SCORE + 3)) DUP_STATUS="warn" @@ -2006,7 +2021,7 @@ if [ "$DETECT_CPLX_ARM" = "merged" ]; then ESLINT_FINDINGS='[]'; ESLINT_RAN=false if [ "$RUN_ESLINT_SLICE" = true ]; then run_tool "Complexity Hotspots" "${ESLINT_INVOKE[@]}" \ - --rule '{"complexity":["warn",10],"sonarjs/cognitive-complexity":["warn",15]}' \ + --rule "$CPLX_ESLINT_RULE" \ --format json --no-warn-ignored \ "${CPLX_ESLINT_ROOTS[@]}" ESLINT_RAW="$LAST_RAW"; ESLINT_EXIT="$LAST_EXIT" @@ -2018,7 +2033,7 @@ if [ "$DETECT_CPLX_ARM" = "merged" ]; then # this run (JS/TS-only metric, best-effort). if ! is_valid_json "$ESLINT_RAW"; then run_tool "Complexity Hotspots" "${ESLINT_INVOKE[@]}" \ - --rule '{"complexity":["warn",10]}' \ + --rule "$CPLX_ESLINT_RULE_CYC" \ --format json --no-warn-ignored \ "${CPLX_ESLINT_ROOTS[@]}" ESLINT_RAW="$LAST_RAW"; ESLINT_EXIT="$LAST_EXIT" @@ -2027,7 +2042,7 @@ if [ "$DETECT_CPLX_ARM" = "merged" ]; then if is_valid_json "$ESLINT_RAW"; then # Normalise to TARGET-relative paths; filter test/build paths at the # JSON layer (tests legitimately branch more; dist/build is generated). - ESLINT_FINDINGS=$(jq --arg root "$PWD" --arg ns "$CPLX_NS" ' + ESLINT_FINDINGS=$(jq --arg root "$PWD" --arg ns "$CPLX_NS" --argjson fail "$CPLX_CCN_FAIL" ' [ .[] | select(.filePath | test("\\.test\\.ts$|\\.spec\\.ts$|/__tests__/|/dist/|/build/|/\\.svelte-kit/") | not) | .filePath as $fp @@ -2043,7 +2058,7 @@ if [ "$DETECT_CPLX_ARM" = "merged" ]; then line: .line, ccn: $ccn, code: ($kind + "-" + ($ccn | tostring)), - severity: (if $ccn >= 30 then "error" elif $ccn >= 20 then "warning" else "low" end), + severity: (if $ccn >= $fail then "error" elif $ccn >= 20 then "warning" else "low" end), message: ($fname + " — " + (if $kind == "COG" then "cognitive complexity " else "CCN " end) + ($ccn | tostring)) } ] @@ -2079,7 +2094,7 @@ if [ "$DETECT_CPLX_ARM" = "merged" ]; then if [ ! -s "$LAST_RAW" ]; then echo -e "${YELLOW}⚠️ lizard produced no output on the non-JS slice (exit $LAST_EXIT)${NC}" else - LIZARD_FINDINGS=$(jq -R -s --arg root "$PWD" --arg ns "$CPLX_NS" ' + LIZARD_FINDINGS=$(jq -R -s --arg root "$PWD" --arg ns "$CPLX_NS" --argjson warn "$CPLX_CCN_WARN" --argjson fail "$CPLX_CCN_FAIL" ' def unq: gsub("^\"|\"$"; ""); [ split("\n")[] | select(length > 0) @@ -2089,14 +2104,14 @@ if [ "$DETECT_CPLX_ARM" = "merged" ]; then | ($f[5] | unq) as $loc | ($f[6] | unq) as $file | ($f[7] | unq) as $fname - | select($ccn >= 10) + | select($ccn >= $warn) | select($file | test("\\.test\\.|\\.spec\\.|/__tests__/|/dist/|/build/|/\\.svelte-kit/") | not) | { file: (($file | sub("^" + $root + "/"; "") | sub("^\\./"; "")) | (if $ns == "" then . else $ns + "/" + . end)), line: (($loc | capture("@(?[0-9]+)-").s | tonumber) // 1), ccn: $ccn, code: ("CCN-" + ($ccn | tostring)), - severity: (if $ccn >= 30 then "error" elif $ccn >= 20 then "warning" else "low" end), + severity: (if $ccn >= $fail then "error" elif $ccn >= 20 then "warning" else "low" end), message: ($fname + " — CCN " + ($ccn | tostring)) } ] @@ -2133,7 +2148,7 @@ if [ "$DETECT_CPLX_ARM" = "merged" ]; then # non-JS findings array exceeds the 128 KB per-argv cap and jq would die # "Argument list too long" on a big polyglot like a Java monorepo (#79). ALL_FINDINGS=$(jq -s 'add' <(printf '%s' "$ESLINT_FINDINGS") <(printf '%s' "$LIZARD_FINDINGS")) - CPLX_MERGED=$(echo "$ALL_FINDINGS" | jq -f "$CHECKUP_HOME/lib/complexity-merge.jq") + CPLX_MERGED=$(echo "$ALL_FINDINGS" | jq --argjson fail "$CPLX_CCN_FAIL" -f "$CHECKUP_HOME/lib/complexity-merge.jq") TOTAL_COUNT=$(echo "$CPLX_MERGED" | jq '.count') mkdir -p "$OUT_DIR" @@ -2217,7 +2232,7 @@ elif [ "$DETECT_CPLX_ARM" = "lizard" ]; then # all precede column 9 (long_name), the only field that can contain # commas — so a plain comma split reads them reliably. The start line # comes from the location field ("name@start-end@file"), also pre-col-9. - ALL_FINDINGS=$(jq -R -s --arg root "$PWD" --arg ns "$CPLX_NS" ' + ALL_FINDINGS=$(jq -R -s --arg root "$PWD" --arg ns "$CPLX_NS" --argjson warn "$CPLX_CCN_WARN" --argjson fail "$CPLX_CCN_FAIL" ' def unq: gsub("^\"|\"$"; ""); [ split("\n")[] | select(length > 0) @@ -2227,14 +2242,14 @@ elif [ "$DETECT_CPLX_ARM" = "lizard" ]; then | ($f[5] | unq) as $loc | ($f[6] | unq) as $file | ($f[7] | unq) as $fname - | select($ccn >= 10) + | select($ccn >= $warn) | select($file | test("\\.test\\.|\\.spec\\.|/__tests__/|/dist/|/build/|/\\.svelte-kit/") | not) | { file: (($file | sub("^" + $root + "/"; "") | sub("^\\./"; "")) | (if $ns == "" then . else $ns + "/" + . end)), line: (($loc | capture("@(?[0-9]+)-").s | tonumber) // 1), ccn: $ccn, code: ("CCN-" + ($ccn | tostring)), - severity: (if $ccn >= 30 then "error" elif $ccn >= 20 then "warning" else "low" end), + severity: (if $ccn >= $fail then "error" elif $ccn >= 20 then "warning" else "low" end), message: ($fname + " — CCN " + ($ccn | tostring)) } ] @@ -2250,7 +2265,7 @@ elif [ "$DETECT_CPLX_ARM" = "lizard" ]; then HIGHEST_CCN=$(echo "$ALL_FINDINGS" | jq '[.[].ccn] | max') STATUS="warn" - [ "$HIGHEST_CCN" -ge 30 ] && STATUS="fail" + [ "$HIGHEST_CCN" -ge "$CPLX_CCN_FAIL" ] && STATUS="fail" printf "%-7s %-50s %s\n" "Score" "Function" "Location" echo "----------------------------------------------------------------------------------------" diff --git a/lib/complexity-merge.jq b/lib/complexity-merge.jq index 6ee463f..452f826 100644 --- a/lib/complexity-merge.jq +++ b/lib/complexity-merge.jq @@ -15,14 +15,15 @@ # Output: { count, highest, status, top } — # count : total findings # highest : max score (0 when empty) -# status : pass when empty; fail when any score ≥ 30; warn otherwise +# status : pass when empty; fail when any score ≥ the fail threshold (default +# 30, override with `--argjson fail N`, #72); warn otherwise # top : ranked by score desc, capped at 20, with the internal `.ccn` sort # key shed so the public top[] schema stays {file,line,code,severity,message} { count: length, highest: (([.[].ccn] | max) // 0), status: (if length == 0 then "pass" - elif (([.[].ccn] | max) // 0) >= 30 then "fail" + elif (([.[].ccn] | max) // 0) >= ($ARGS.named.fail // 30) then "fail" else "warn" end), top: (sort_by(-.ccn) | .[0:20] | map(del(.ccn))) } diff --git a/lib/config.sh b/lib/config.sh index b2a04fd..429ef32 100644 --- a/lib/config.sh +++ b/lib/config.sh @@ -23,6 +23,11 @@ # # the CHECKUP_EXCLUDE env var; reaches every engine # # (lizard inventory AND scc keep-set, #109). Top-level # # (not nested). Directory globs work: `vendor/js/*`. +# thresholds: # per-check warn/fail banding (#72) — integers, with +# complexity_ccn_warn: 10 # the historical literals as defaults so an +# complexity_ccn_fail: 30 # absent block is byte-identical. Tunes only the +# duplication_warn_pct: 3 # status the section already applies (NOT scoring, +# duplication_fail_pct: 5 # ADR-0009). Garbage → warn + keep the default. # # Grammar: `key: value` and one level of `section:`-nested ` key: value`; # inline flow lists `[a, b]`; `#` comments; quoted or bare scalars. Block-style @@ -30,7 +35,8 @@ # # Outputs (consumed by bin/checkup.sh): CHECKUP_FORCE_STACK, CHECKUP_SUPPRESS_STACKS, # CHECKUP_DISABLE, CHECKUP_ENABLE, CHECKUP_CMD_* (commands), CHECKUP_EXCLUDE -# (merged with the env var), CHECKUP_OVERRIDDEN. +# (merged with the env var), CHECKUP_CPLX_CCN_WARN/FAIL, CHECKUP_DUP_WARN_PCT/FAIL_PCT +# (thresholds), CHECKUP_OVERRIDDEN. _cfg_trim() { printf '%s' "$1" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//'; } _cfg_unquote(){ # strip one layer of matching single/double quotes @@ -53,6 +59,17 @@ _cfg_list() { # "[a, \"b c\", d]" → "a" / "b c" / "d" newline-separated # Canonical command keys accepted under `commands:` → CHECKUP_CMD_. _cfg_cmd_known=" test build typecheck lint format typeaware deps unused coverage mutation security audit outdated " +# _cfg_int — echo iff it's a non-negative +# integer, else . The use-site guard for thresholds (#72): keeps a bad +# value (from a hand-set env var, or anything the parser let through) from +# reaching jq / `[ -lt ]` — never abort, never a false pass. +_cfg_int() { + case "$1" in + ''|*[!0-9]*) printf '%s' "$2";; + *) printf '%s' "$1";; + esac +} + load_checkup_config() { # $1 = path to .checkup.yml local file="$1" CHECKUP_OVERRIDDEN="${CHECKUP_OVERRIDDEN:-false}" @@ -113,6 +130,26 @@ _cfg_apply() { # $1 = section, $2 = key, $3 = raw value else echo "⚠️ .checkup.yml: unknown command '$key' — ignoring" >&2 fi;; + thresholds) + # Per-check warn/fail banding (#72). Integers only; garbage → warn + + # keep the default (the use-site `${VAR:-literal}` supplies it). Export + # so the value also reaches jq via `$ARGS.named` / env at the use site. + local _tv; _tv="$(_cfg_unquote "$val")" + local _tvar="" + case "$key" in + complexity_ccn_warn) _tvar=CHECKUP_CPLX_CCN_WARN;; + complexity_ccn_fail) _tvar=CHECKUP_CPLX_CCN_FAIL;; + duplication_warn_pct) _tvar=CHECKUP_DUP_WARN_PCT;; + duplication_fail_pct) _tvar=CHECKUP_DUP_FAIL_PCT;; + *) echo "⚠️ .checkup.yml: unknown key 'thresholds.$key' — ignoring" >&2;; + esac + if [ -n "$_tvar" ]; then + if printf '%s' "$_tv" | grep -Eq '^[0-9]+$'; then + printf -v "$_tvar" '%s' "$_tv"; export "$_tvar"; CHECKUP_OVERRIDDEN=true + else + echo "⚠️ .checkup.yml: thresholds.$key must be a non-negative integer (got '$_tv') — ignoring" >&2 + fi + fi;; toplevel) case "$key" in exclude) @@ -140,7 +177,8 @@ _cfg_parse_yq() { # $1 = file (.checks.disable // [] | "checks disable [" + (join(", ")) + "]"), (.checks.enable // [] | "checks enable [" + (join(", ")) + "]"), (.exclude // [] | select(length > 0) | "toplevel exclude [" + (join(", ")) + "]"), - (.commands // {} | to_entries[] | "commands " + .key + " " + (.value|tostring)) + (.commands // {} | to_entries[] | "commands " + .key + " " + (.value|tostring)), + (.thresholds // {} | to_entries[] | "thresholds " + .key + " " + (.value|tostring)) ' "$file" 2>/dev/null) || return 1 local sec key val while read -r sec key val; do diff --git a/test/config.test.sh b/test/config.test.sh index 8e78768..b9a6dd2 100755 --- a/test/config.test.sh +++ b/test/config.test.sh @@ -115,6 +115,41 @@ printf '{"slug":"gitleaks","status":"pass","count":0,"summary":"clean","top":[], assert_eq "disabled+skipped → honest reason" "disabled in .checkup.yml" "$(jq -r '.summary' "$PD/unit-tests.json")" assert_eq "a check that RAN is untouched" "clean" "$(jq -r '.summary' "$PD/gitleaks.json")" +echo "" +echo "thresholds: per-check warn/fail banding (#72)" +TH=$(yml 'thresholds: + complexity_ccn_warn: 15 + complexity_ccn_fail: 25 + duplication_warn_pct: 4 + duplication_fail_pct: 8') +assert_eq "complexity_ccn_warn" "15" "$( load_checkup_config "$TH"; printf '%s' "${CHECKUP_CPLX_CCN_WARN:-}" )" +assert_eq "complexity_ccn_fail" "25" "$( load_checkup_config "$TH"; printf '%s' "${CHECKUP_CPLX_CCN_FAIL:-}" )" +assert_eq "duplication_warn_pct" "4" "$( load_checkup_config "$TH"; printf '%s' "${CHECKUP_DUP_WARN_PCT:-}" )" +assert_eq "duplication_fail_pct" "8" "$( load_checkup_config "$TH"; printf '%s' "${CHECKUP_DUP_FAIL_PCT:-}" )" +assert_eq "thresholds flip overridden" "true" "$( load_checkup_config "$TH"; printf '%s' "$CHECKUP_OVERRIDDEN" )" + +echo "" +echo "thresholds: non-integer warns + is ignored (default preserved), siblings still parse" +TG=$(yml 'thresholds: + complexity_ccn_warn: abc + duplication_fail_pct: 8') +assert_eq "garbage value → var stays unset" "unset" "$( load_checkup_config "$TG" 2>/dev/null; printf '%s' "${CHECKUP_CPLX_CCN_WARN:-unset}" )" +assert_eq "sibling valid value still set" "8" "$( load_checkup_config "$TG" 2>/dev/null; printf '%s' "${CHECKUP_DUP_FAIL_PCT:-unset}" )" +assert_eq "garbage emits a warning" "1" "$( load_checkup_config "$TG" 2>&1 >/dev/null | grep -c 'must be a non-negative integer' )" + +echo "" +echo "thresholds: unknown key warns + is ignored" +TU=$(yml 'thresholds: + complexity_ccn_budget: 12') +assert_eq "unknown threshold key warns" "1" "$( load_checkup_config "$TU" 2>&1 >/dev/null | grep -c "unknown key .thresholds.complexity_ccn_budget" )" + +echo "" +echo "_cfg_int: use-site guard (integer in, default on garbage)" +assert_eq "valid integer passes through" "15" "$(_cfg_int "15" 10)" +assert_eq "empty → default" "10" "$(_cfg_int "" 10)" +assert_eq "non-integer → default" "30" "$(_cfg_int "10x" 30)" +assert_eq "negative → default" "5" "$(_cfg_int "-2" 5)" + echo "" echo ".checkup.yml.example stays in sync with the parser (no unknown keys)" EX="$CHECKUP_HOME/.checkup.yml.example"