diff --git a/CHANGELOG.md b/CHANGELOG.md index b933140..7b8e52e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,10 +85,12 @@ across the whole thing, honestly*. paths re-prefixed so the by-file/focus join stays coherent (Phase 2). The scc-measured **codebase statistics** recover too: each sub-package's totals are re-aggregated from the *one* cached `scc --by-file` walk, sliced to its subtree - (reuse, never re-walk), labelled per package (`api/codebase-stats`) — Phase 2b, - first increment. A single package / declared workspace is unchanged - (byte-identical). Declared workspaces (npm/pnpm/yarn/nx/turbo/lerna) are healthy - and don't alarm. + (reuse, never re-walk), labelled per package (`api/codebase-stats`). **Code + duplication** recovers with its engine re-routed per sub-package — jscpd *in* + each node package (npm), lizard over the inventory file list sliced to the + subtree — labelled (`svc/duplication`). (Phase 2b.) A single package / declared + workspace is unchanged (byte-identical). Declared workspaces + (npm/pnpm/yarn/nx/turbo/lerna) are healthy and don't alarm. - **`lizard`** as a true multi-language complexity + duplication engine, and a **Focus Areas** view synthesising the forensic axes (#36–#38). - **Command profiles** + a **`.checkup.yml`** override layer (stack / checks / diff --git a/bin/checkup.sh b/bin/checkup.sh index 2d882c3..21a0496 100755 --- a/bin/checkup.sh +++ b/bin/checkup.sh @@ -1370,7 +1370,40 @@ DUP_INTENT=$(jq -n '{ fail_means: "≥5% duplication. Refactor toward shared helpers when the same pattern recurs 3+ times. NOTE: Classic ASP/VBScript has no tokeniser in either engine, so .asp duplication is not measured." }') -if [ "$DETECT_ENGINE_DUPLICATION" = "jscpd" ]; then +# Per-package recovery (#78 increment 3): duplication is engine-routed (jscpd on +# node packages, lizard elsewhere). On an undeclared fan-out, measure EACH +# assessment root with its OWN engine — jscpd runs IN the package (npm, cwd = +# the child), lizard reads the inventory file list sliced to the subtree. Single +# package / declared workspace → one iteration at "." reusing the whole-tree +# engine + probes, no cd, SLUG_NS empty → byte-identical to before (the gate). +for TOPO_ROOT in "${TOPO_ASSESSMENT_ROOTS[@]}"; do +_RECOVER_CWD="$PWD"; SLUG_NS=""; DUP_SUBTREE="." +if [ "$TOPO_ROOT" = "." ]; then + DUP_ENGINE="$DETECT_ENGINE_DUPLICATION"; DUP_REASON_R="$DUP_REASON" +else + cd "$TARGET/$TOPO_ROOT" 2>/dev/null || { cd "$_RECOVER_CWD"; continue; } + SLUG_NS="$TOPO_ROOT"; DUP_SUBTREE="$TOPO_ROOT" + echo -e "${BLUE}📦 Sub-package: $TOPO_ROOT${NC}" + # Re-route the engine for THIS child (mirrors the whole-tree routing, scoped to + # the subtree): a node package (own package.json + JS/TS-dominant source, or no + # scc read) with npm → jscpd; else lizard-parseable source under the child → + # lizard; else an honest skip. The child's stack is read from the ONE cached scc + # walk, sliced to the subtree (reuse, never re-walk). + DUP_ENGINE="none"; DUP_REASON_R="no Node target for jscpd and no lizard-parseable source in $TOPO_ROOT" + _CHILD_PRIMARY="" + if [ "${SCC_BYFILE_OK:-false}" = true ]; then + _CHILD_PRIMARY=$(scc_breakdown "$(scc_keep_for_root "$TOPO_ROOT")" < "$SCC_BYFILE" \ + | jq -c -f "$CHECKUP_HOME/lib/detect-stacks.jq" 2>/dev/null | jq -r '.[0].stack // ""' 2>/dev/null) + fi + if [ -f package.json ] && command -v npm > /dev/null 2>&1 \ + && { [ "$_CHILD_PRIMARY" = "node" ] || [ -z "$_CHILD_PRIMARY" ]; }; then + DUP_ENGINE="jscpd"; DUP_REASON_R="node package → jscpd (exact-token)" + elif [ -n "$LIZARD_BIN" ] && [ -n "$(inventory_paths_under "$TOPO_ROOT" "$INV_LIZARD_RE" | head -c1)" ]; then + DUP_ENGINE="lizard"; DUP_REASON_R="lizard-parseable source → lizard -Eduplicate (identifier-unified)" + fi +fi + +if [ "$DUP_ENGINE" = "jscpd" ]; then # ---- Tier 1: jscpd (Node best-fit; engine chosen by the detector, #7) ---- echo "Command: npm run quality:duplicates" echo "" @@ -1427,7 +1460,7 @@ if [ "$DETECT_ENGINE_DUPLICATION" = "jscpd" ]; then write_parsed "duplication" "$JSCPD_STATUS" "$DUPLICATION_LINES" "${DUPLICATION_PCT}% duplication across $DUPLICATION_LINES lines (jscpd)" "$JSCPD_TOP" "$DUP_INTENT" fi fi -elif [ "$DETECT_ENGINE_DUPLICATION" = "lizard" ]; then +elif [ "$DUP_ENGINE" = "lizard" ]; then # ---- Tier 2: lizard -Eduplicate (language-agnostic clone detection) ---- echo "Command: lizard -Eduplicate (excluding generated/vendored/repetitive paths)" echo "" @@ -1438,7 +1471,7 @@ elif [ "$DETECT_ENGINE_DUPLICATION" = "lizard" ]; then # list is non-empty. # Feed the file list via a temp file + xargs (run_tool_filelist), not a bash # array on argv — a large tree overflows ARG_MAX otherwise (exit 126). - DUP_LIST=$(mktemp); inventory_paths "$INV_LIZARD_RE" > "$DUP_LIST" + DUP_LIST=$(mktemp); inventory_paths_under "$DUP_SUBTREE" "$INV_LIZARD_RE" > "$DUP_LIST" DUP_N=$(filelist_count "$DUP_LIST") if [ "$DUP_N" -gt "$CHECKUP_LIZARD_MAX_FILES" ]; then # Single-pass clone detection holds every file's tokens in memory; on a @@ -1460,7 +1493,7 @@ elif [ "$DETECT_ENGINE_DUPLICATION" = "lizard" ]; then else # Parse lizard's text report: each "Duplicate block:" lists the cloned # locations as `file:start ~ end`; the footer gives the overall rate. - DUP_PARSED=$(python3 - "$LAST_RAW" "$TARGET" <<'PY' || echo '{"rate":0,"count":0,"top":[]}' + DUP_PARSED=$(python3 - "$LAST_RAW" "$PWD" <<'PY' || echo '{"rate":0,"count":0,"top":[]}' import sys, json, re path, target = sys.argv[1], sys.argv[2] text = open(path, errors="replace").read() @@ -1518,9 +1551,12 @@ PY fi fi # end CHECKUP_LIZARD_MAX_FILES cap guard else - echo -e "${BLUE}ℹ️ Skipped — $DUP_REASON${NC}" - write_skipped "duplication" "$DUP_REASON" "$DUP_INTENT" + echo -e "${BLUE}ℹ️ Skipped — $DUP_REASON_R${NC}" + write_skipped "duplication" "$DUP_REASON_R" "$DUP_INTENT" fi + +cd "$_RECOVER_CWD"; SLUG_NS="" +done echo "" # ─── Topology recover pass (#78) — project-built cluster C (per-package code health) ─ diff --git a/docs/architecture.md b/docs/architecture.md index 3a5e52c..5f65948 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -212,9 +212,10 @@ dependency-health (`npm-audit`, `deps-freshness`, `circular-deps`), correctness/ `unused-code` + `coverage` clusters (Phase 2/2b increment 1), plus the scc-measured **`codebase-stats`** — re-aggregated per sub-package by slicing the *one* cached `scc --by-file` walk to each root's subtree (`scc_keep_for_root`; -reuse, never re-walk), so no extra scc cost (Phase 2b increment 2). Still -root-scoped: the engine-routed `complexity` and `duplication`, which need their -engine (eslint / lizard / jscpd) re-routed per sub-package — the remaining slice. +reuse, never re-walk), so no extra scc cost — and **`duplication`**, whose engine +is re-routed per child (jscpd *in* each node package; lizard over the inventory +sliced to the subtree via `inventory_paths_under`) (Phase 2b). Still root-scoped: +the engine-routed `complexity` (eslint / lizard / scc merge), the remaining slice. Honesty survives recovery: a sub-package whose toolchain can't run still `skip`s with a reason (never green-by-default), the floor laid in #85. diff --git a/lib/source-inventory.sh b/lib/source-inventory.sh index f2b9713..d88b7c0 100644 --- a/lib/source-inventory.sh +++ b/lib/source-inventory.sh @@ -262,6 +262,29 @@ inventory_paths() { done < "$SOURCE_LST" } +# Emit (NUL-delimited) the inventory paths under , RELATIVE to (the +# "/" prefix stripped), filtered by [re] like inventory_paths. The +# per-package measurement recover pass (#78): a section cd'd into a sub-package +# needs its slice of the whole-tree inventory with CWD-relative paths (so lizard, +# run in the package, finds them). "." is the identity — the whole +# inventory, TARGET-relative — so a single package / declared workspace is +# byte-identical to inventory_paths. Consume like inventory_paths. +inventory_paths_under() { + local root="$1" re="${2:-}" p pre + [ -f "${SOURCE_LST:-}" ] || return 0 + if [ "$root" = "." ]; then + inventory_paths "$re" + return 0 + fi + pre="${root#./}/" + while IFS= read -r -d '' p; do + case "$p" in "$pre"*) ;; *) continue ;; esac + p="${p#"$pre"}" + [ -z "$re" ] || [[ "$p" =~ $re ]] || continue + printf '%s\0' "$p" + done < "$SOURCE_LST" +} + # The inventory as a JSON array of paths — for filtering a scanner's findings # back to the tracked set (e.g. ESLint, which traverses on its own). inventory_json() { diff --git a/test/source-inventory.test.sh b/test/source-inventory.test.sh index 82768eb..3b2e5c7 100755 --- a/test/source-inventory.test.sh +++ b/test/source-inventory.test.sh @@ -227,6 +227,21 @@ assert_eq "git scope → .gitignore" ".gitignore" "$(SOURCE assert_eq "override:git scope → .gitignore" ".gitignore" "$(SOURCE_SCOPE=override:git inventory_exclusion_source)" assert_eq "find scope → builtin excludes" "builtin excludes (no VCS)" "$(SOURCE_SCOPE=find inventory_exclusion_source)" +echo "" +echo "inventory_paths_under: per-subtree slice, CWD-relative (#78 increment 3)" +SOURCE_LST="$TMP/under.lst" +printf 'api/src/a.ts\0api/src/b.js\0api/README.md\0web/app.ts\0top.ts\0' > "$SOURCE_LST" +under() { inventory_paths_under "$@" | tr '\0' '\n' | sort | paste -sd',' -; } +assert_eq "root '.' is the identity (whole inventory, TARGET-relative)" \ + "api/README.md,api/src/a.ts,api/src/b.js,top.ts,web/app.ts" "$(under .)" +assert_eq "child slice strips the '/' prefix (CWD-relative)" \ + "README.md,src/a.ts,src/b.js" "$(under api)" +assert_eq "child slice + extension filter" \ + "src/a.ts" "$(under api '\.ts$')" +assert_eq "'.' identity + extension filter matches inventory_paths" \ + "$(inventory_paths '\.ts$' | tr '\0' '\n' | sort | paste -sd',' -)" "$(under . '\.ts$')" +assert_eq "no-match subtree → empty" "" "$(under nope)" + echo "" echo "eslint_flat_config_root: gates the ESLint complexity slice (#79)" CFG="$TMP/cfgprobe"; mkdir -p "$CFG/pkg"