Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,10 +85,12 @@ across the whole thing, honestly*.
paths re-prefixed so the by-file/focus join stays coherent (Phase 2). The
scc-measured **codebase statistics** recover too: each sub-package's totals are
re-aggregated from the *one* cached `scc --by-file` walk, sliced to its subtree
(reuse, never re-walk), labelled per package (`api/codebase-stats`) β€” Phase 2b,
first increment. A single package / declared workspace is unchanged
(byte-identical). Declared workspaces (npm/pnpm/yarn/nx/turbo/lerna) are healthy
and don't alarm.
(reuse, never re-walk), labelled per package (`api/codebase-stats`). **Code
duplication** recovers with its engine re-routed per sub-package β€” jscpd *in*
each node package (npm), lizard over the inventory file list sliced to the
subtree β€” labelled (`svc/duplication`). (Phase 2b.) A single package / declared
workspace is unchanged (byte-identical). Declared workspaces
(npm/pnpm/yarn/nx/turbo/lerna) are healthy and don't alarm.
- **`lizard`** as a true multi-language complexity + duplication engine, and a
**Focus Areas** view synthesising the forensic axes (#36–#38).
- **Command profiles** + a **`.checkup.yml`** override layer (stack / checks /
Expand Down
48 changes: 42 additions & 6 deletions bin/checkup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1370,7 +1370,40 @@ DUP_INTENT=$(jq -n '{
fail_means: "β‰₯5% duplication. Refactor toward shared helpers when the same pattern recurs 3+ times. NOTE: Classic ASP/VBScript has no tokeniser in either engine, so .asp duplication is not measured."
}')

if [ "$DETECT_ENGINE_DUPLICATION" = "jscpd" ]; then
# Per-package recovery (#78 increment 3): duplication is engine-routed (jscpd on
# node packages, lizard elsewhere). On an undeclared fan-out, measure EACH
# assessment root with its OWN engine β€” jscpd runs IN the package (npm, cwd =
# the child), lizard reads the inventory file list sliced to the subtree. Single
# package / declared workspace β†’ one iteration at "." reusing the whole-tree
# engine + probes, no cd, SLUG_NS empty β†’ byte-identical to before (the gate).
for TOPO_ROOT in "${TOPO_ASSESSMENT_ROOTS[@]}"; do
_RECOVER_CWD="$PWD"; SLUG_NS=""; DUP_SUBTREE="."
if [ "$TOPO_ROOT" = "." ]; then
DUP_ENGINE="$DETECT_ENGINE_DUPLICATION"; DUP_REASON_R="$DUP_REASON"
else
cd "$TARGET/$TOPO_ROOT" 2>/dev/null || { cd "$_RECOVER_CWD"; continue; }
SLUG_NS="$TOPO_ROOT"; DUP_SUBTREE="$TOPO_ROOT"
echo -e "${BLUE}πŸ“¦ Sub-package: $TOPO_ROOT${NC}"
# Re-route the engine for THIS child (mirrors the whole-tree routing, scoped to
# the subtree): a node package (own package.json + JS/TS-dominant source, or no
# scc read) with npm β†’ jscpd; else lizard-parseable source under the child β†’
# lizard; else an honest skip. The child's stack is read from the ONE cached scc
# walk, sliced to the subtree (reuse, never re-walk).
DUP_ENGINE="none"; DUP_REASON_R="no Node target for jscpd and no lizard-parseable source in $TOPO_ROOT"
_CHILD_PRIMARY=""
if [ "${SCC_BYFILE_OK:-false}" = true ]; then
_CHILD_PRIMARY=$(scc_breakdown "$(scc_keep_for_root "$TOPO_ROOT")" < "$SCC_BYFILE" \
| jq -c -f "$CHECKUP_HOME/lib/detect-stacks.jq" 2>/dev/null | jq -r '.[0].stack // ""' 2>/dev/null)
fi
if [ -f package.json ] && command -v npm > /dev/null 2>&1 \
&& { [ "$_CHILD_PRIMARY" = "node" ] || [ -z "$_CHILD_PRIMARY" ]; }; then
DUP_ENGINE="jscpd"; DUP_REASON_R="node package β†’ jscpd (exact-token)"
elif [ -n "$LIZARD_BIN" ] && [ -n "$(inventory_paths_under "$TOPO_ROOT" "$INV_LIZARD_RE" | head -c1)" ]; then
DUP_ENGINE="lizard"; DUP_REASON_R="lizard-parseable source β†’ lizard -Eduplicate (identifier-unified)"
fi
fi

if [ "$DUP_ENGINE" = "jscpd" ]; then
# ---- Tier 1: jscpd (Node best-fit; engine chosen by the detector, #7) ----
echo "Command: npm run quality:duplicates"
echo ""
Expand Down Expand Up @@ -1427,7 +1460,7 @@ if [ "$DETECT_ENGINE_DUPLICATION" = "jscpd" ]; then
write_parsed "duplication" "$JSCPD_STATUS" "$DUPLICATION_LINES" "${DUPLICATION_PCT}% duplication across $DUPLICATION_LINES lines (jscpd)" "$JSCPD_TOP" "$DUP_INTENT"
fi
fi
elif [ "$DETECT_ENGINE_DUPLICATION" = "lizard" ]; then
elif [ "$DUP_ENGINE" = "lizard" ]; then
# ---- Tier 2: lizard -Eduplicate (language-agnostic clone detection) ----
echo "Command: lizard -Eduplicate (excluding generated/vendored/repetitive paths)"
echo ""
Expand All @@ -1438,7 +1471,7 @@ elif [ "$DETECT_ENGINE_DUPLICATION" = "lizard" ]; then
# list is non-empty.
# Feed the file list via a temp file + xargs (run_tool_filelist), not a bash
# array on argv β€” a large tree overflows ARG_MAX otherwise (exit 126).
DUP_LIST=$(mktemp); inventory_paths "$INV_LIZARD_RE" > "$DUP_LIST"
DUP_LIST=$(mktemp); inventory_paths_under "$DUP_SUBTREE" "$INV_LIZARD_RE" > "$DUP_LIST"
DUP_N=$(filelist_count "$DUP_LIST")
if [ "$DUP_N" -gt "$CHECKUP_LIZARD_MAX_FILES" ]; then
# Single-pass clone detection holds every file's tokens in memory; on a
Expand All @@ -1460,7 +1493,7 @@ elif [ "$DETECT_ENGINE_DUPLICATION" = "lizard" ]; then
else
# Parse lizard's text report: each "Duplicate block:" lists the cloned
# locations as `file:start ~ end`; the footer gives the overall rate.
DUP_PARSED=$(python3 - "$LAST_RAW" "$TARGET" <<'PY' || echo '{"rate":0,"count":0,"top":[]}'
DUP_PARSED=$(python3 - "$LAST_RAW" "$PWD" <<'PY' || echo '{"rate":0,"count":0,"top":[]}'
import sys, json, re
path, target = sys.argv[1], sys.argv[2]
text = open(path, errors="replace").read()
Expand Down Expand Up @@ -1518,9 +1551,12 @@ PY
fi
fi # end CHECKUP_LIZARD_MAX_FILES cap guard
else
echo -e "${BLUE}ℹ️ Skipped β€” $DUP_REASON${NC}"
write_skipped "duplication" "$DUP_REASON" "$DUP_INTENT"
echo -e "${BLUE}ℹ️ Skipped β€” $DUP_REASON_R${NC}"
write_skipped "duplication" "$DUP_REASON_R" "$DUP_INTENT"
fi

cd "$_RECOVER_CWD"; SLUG_NS=""
done
echo ""

# ─── Topology recover pass (#78) β€” project-built cluster C (per-package code health) ─
Expand Down
7 changes: 4 additions & 3 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -212,9 +212,10 @@ dependency-health (`npm-audit`, `deps-freshness`, `circular-deps`), correctness/
`unused-code` + `coverage` clusters (Phase 2/2b increment 1), plus the
scc-measured **`codebase-stats`** β€” re-aggregated per sub-package by slicing the
*one* cached `scc --by-file` walk to each root's subtree (`scc_keep_for_root`;
reuse, never re-walk), so no extra scc cost (Phase 2b increment 2). Still
root-scoped: the engine-routed `complexity` and `duplication`, which need their
engine (eslint / lizard / jscpd) re-routed per sub-package β€” the remaining slice.
reuse, never re-walk), so no extra scc cost β€” and **`duplication`**, whose engine
is re-routed per child (jscpd *in* each node package; lizard over the inventory
sliced to the subtree via `inventory_paths_under`) (Phase 2b). Still root-scoped:
the engine-routed `complexity` (eslint / lizard / scc merge), the remaining slice.
Honesty survives recovery: a
sub-package whose toolchain can't run still `skip`s with a reason (never
green-by-default), the floor laid in #85.
Expand Down
23 changes: 23 additions & 0 deletions lib/source-inventory.sh
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,29 @@ inventory_paths() {
done < "$SOURCE_LST"
}

# Emit (NUL-delimited) the inventory paths under <root>, RELATIVE to <root> (the
# "<root>/" prefix stripped), filtered by [re] like inventory_paths. The
# per-package measurement recover pass (#78): a section cd'd into a sub-package
# needs its slice of the whole-tree inventory with CWD-relative paths (so lizard,
# run in the package, finds them). <root> "." is the identity β€” the whole
# inventory, TARGET-relative β€” so a single package / declared workspace is
# byte-identical to inventory_paths. Consume like inventory_paths.
inventory_paths_under() {
local root="$1" re="${2:-}" p pre
[ -f "${SOURCE_LST:-}" ] || return 0
if [ "$root" = "." ]; then
inventory_paths "$re"
return 0
fi
pre="${root#./}/"
while IFS= read -r -d '' p; do
case "$p" in "$pre"*) ;; *) continue ;; esac
p="${p#"$pre"}"
[ -z "$re" ] || [[ "$p" =~ $re ]] || continue
printf '%s\0' "$p"
done < "$SOURCE_LST"
}

# The inventory as a JSON array of paths β€” for filtering a scanner's findings
# back to the tracked set (e.g. ESLint, which traverses on its own).
inventory_json() {
Expand Down
15 changes: 15 additions & 0 deletions test/source-inventory.test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,21 @@ assert_eq "git scope β†’ .gitignore" ".gitignore" "$(SOURCE
assert_eq "override:git scope β†’ .gitignore" ".gitignore" "$(SOURCE_SCOPE=override:git inventory_exclusion_source)"
assert_eq "find scope β†’ builtin excludes" "builtin excludes (no VCS)" "$(SOURCE_SCOPE=find inventory_exclusion_source)"

echo ""
echo "inventory_paths_under: per-subtree slice, CWD-relative (#78 increment 3)"
SOURCE_LST="$TMP/under.lst"
printf 'api/src/a.ts\0api/src/b.js\0api/README.md\0web/app.ts\0top.ts\0' > "$SOURCE_LST"
under() { inventory_paths_under "$@" | tr '\0' '\n' | sort | paste -sd',' -; }
assert_eq "root '.' is the identity (whole inventory, TARGET-relative)" \
"api/README.md,api/src/a.ts,api/src/b.js,top.ts,web/app.ts" "$(under .)"
assert_eq "child slice strips the '<root>/' prefix (CWD-relative)" \
"README.md,src/a.ts,src/b.js" "$(under api)"
assert_eq "child slice + extension filter" \
"src/a.ts" "$(under api '\.ts$')"
assert_eq "'.' identity + extension filter matches inventory_paths" \
"$(inventory_paths '\.ts$' | tr '\0' '\n' | sort | paste -sd',' -)" "$(under . '\.ts$')"
assert_eq "no-match subtree β†’ empty" "" "$(under nope)"

echo ""
echo "eslint_flat_config_root: gates the ESLint complexity slice (#79)"
CFG="$TMP/cfgprobe"; mkdir -p "$CFG/pkg"
Expand Down