diff --git a/bench/slice/_common.py b/bench/slice/_common.py new file mode 100644 index 000000000..663b306d6 --- /dev/null +++ b/bench/slice/_common.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""_common.py — the helpers every bench/slice harness needs, defined once. + +`run_slicerecall.py` (the 2026-08-30 / 08-31 cpp rounds) and the 2026-09-20 py round's three +scripts all shell out to git and all index source by 1-based line number. Each had grown its own +copy; --quality-delta named the clone pairs, so they live here instead. +""" + +import io, subprocess, tokenize +from pathlib import Path + + +def sh( args, cwd=None, ok_fail=False ): + """run a command, capture text, raise on failure unless ok_fail. + + errors="replace": external corpora carry non-UTF-8 bytes (ugrep's own test fixtures are + deliberately latin-1/binary), and a diff that touches one must not abort a mine. Only content + bytes are ever mangled — hunk headers and funcnames are ASCII by git's own format — so + qualification is unaffected. + """ + r = subprocess.run( args, cwd=cwd, capture_output=True, text=True, errors="replace" ) + if r.returncode != 0 and not ok_fail: + raise RuntimeError( f"{args}: rc={r.returncode}\n{r.stderr[:500]}" ) + return r + + +def git( repo, *args, ok_fail=False ): + """sh() with `git -C repo` prepended.""" + return sh( [ "git", "-C", str( repo ) ] + list( args ), ok_fail=ok_fail ) + + +def line_text( lines, n ): + """the 1-based n-th source line, or "" when n is outside the file.""" + return lines[ n - 1 ] if 0 < n <= len( lines ) else "" + + +def archive_tree( repo, commit, dest ): + """materialize repo@commit READ-ONLY into dest via `git archive | tar -x` — the checkout is never + written to and nothing is cloned. True on success; dest is left present but possibly incomplete + on failure, matching probe_wholerepo_selector.py's original inline version this replaces.""" + tar = subprocess.run( [ "git", "-C", str( repo ), "archive", commit ], capture_output=True ) + if tar.returncode != 0: + return False + Path( dest ).mkdir( parents=True, exist_ok=True ) + r = subprocess.run( [ "tar", "-x", "-C", str( dest ) ], input=tar.stdout, capture_output=True ) + return r.returncode == 0 + + +def name_lines( source ): + """{identifier: {line numbers where it occurs as a NAME token}} — the strict relevance oracle. + + AMENDMENT 2026-09-20 (b) of docs/research/slice-line-recall.md, taken AFTER inspecting the + registered oracle's misses and reported BESIDE it, never instead of it: the registered oracle is + a word regex over the line text, so it counts a variable's name inside a docstring, a comment or + a string literal as an occurrence the slice ought to have rowed. Python's own tokenizer settles + which occurrences are identifiers. A source the tokenizer refuses (py2 syntax, decode trouble) + yields None, and its instance is then reported only under the registered oracle. + """ + out = {} + try: + for tok in tokenize.generate_tokens( io.StringIO( source ).readline ): + if tok.type == tokenize.NAME: + out.setdefault( tok.string, set() ).add( tok.start[ 0 ] ) + except Exception: + return None + return out diff --git a/bench/slice/inspect_slice_misses.py b/bench/slice/inspect_slice_misses.py new file mode 100644 index 000000000..eab3f4f33 --- /dev/null +++ b/bench/slice/inspect_slice_misses.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""inspect_slice_misses.py — read run_slice_linerecall.py's results json and say, per missed line, +whether the miss is the RELEVANCE ORACLE's noise or a real drop by the slicer. + +The registered oracle is a word regex over the changed line's text, so it counts a variable's name +inside a docstring, a comment, a string literal or an f-string prefix as an occurrence. The strict +oracle (AMENDMENT 2026-09-20 (b)) is Python's own tokenizer: an occurrence counts only when it is a +NAME token. This script prints the census both ways and lists every miss that survives the strict +oracle in full, because that residue is the only part that is evidence about the slicer. + +Usage: python3 bench/slice/inspect_slice_misses.py --results results.json --gold gold.json +""" + +import argparse, json +from pathlib import Path + +from _common import git, name_lines # one definition, shared across bench/slice + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( "--results", required=True ) + ap.add_argument( "--gold", required=True ) + a = ap.parse_args() + + res = json.loads( Path( a.results ).read_text() ) + gold = { g[ "instance_id" ]: g for g in json.loads( Path( a.gold ).read_text() )[ "instances" ] } + src_cache = {} + + def source_of( iid ): + if iid not in src_cache: + g = gold[ iid ] + r = git( g[ "repo_dir" ], "show", f"{g['base_commit']}:{g['path']}", ok_fail=True ) + src_cache[ iid ] = r.stdout if r.returncode == 0 else "" + return src_cache[ iid ] + + total_miss, noise, real, untokenizable = 0, 0, [], 0 + for x in res[ "var_instances" ]: + if not x[ "v1_missed" ]: + continue + names = name_lines( source_of( x[ "instance_id" ] ) ) + for m in x[ "v1_missed" ]: + total_miss += 1 + if names is None: + untokenizable += 1 + elif m[ "line" ] in names.get( x[ "var" ], set() ): + real.append( ( x[ "instance_id" ], x[ "var" ], m ) ) + else: + noise += 1 + + print( f"missed lines under the registered (word-regex) oracle : {total_miss}" ) + print( f" the name is not a NAME token on that line (oracle noise): {noise}" ) + print( f" file not tokenizable, unclassified : {untokenizable}" ) + print( f" survives the strict oracle — evidence about the slicer : {len(real)}" ) + for iid, var, m in real: + print( f" {iid} var={var} L{m['line']}: {m['text']}" ) + + +if __name__ == "__main__": + main() diff --git a/bench/slice/locbench_gold.py b/bench/slice/locbench_gold.py new file mode 100644 index 000000000..4e0509b88 --- /dev/null +++ b/bench/slice/locbench_gold.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""locbench_gold.py — build line-level gold for the --slice line-recall round from a LocBench-shaped +dataset plus local repository checkouts. Downloads nothing; reads only what is already on disk. + +Protocol: docs/research/slice-line-recall.md §2 (corpus and slice rule) and §1.1 G2 (the gold rule +for a pure-insertion hunk). This script does NOT invoke ripwire — every ripwire-dependent +qualification stage lives in run_slice_linerecall.py, so the gold is a function of the dataset and +the checkouts alone and cannot move when the binary does. + +Gold is PRE-IMAGE: the round scores the localization setting (the agent holds the pre-fix tree and +must find the lines to change), so gold line numbers are numbered in the file at base_commit. + - every '-' line of a hunk touching the target file contributes its pre-image line number; + - a run of '+' lines with no '-' line of its own contributes ONE anchor: the pre-image line + immediately preceding the insertion point, or the hunk's first pre-image line when the run + opens the hunk. + +Usage: + python3 bench/slice/locbench_gold.py --assets DIR [--dataset FILE] [--json out.json] + +--assets DIR must contain `datasets/` (the dataset json) and one or more sibling directories of +repository checkouts named `owner__repo`. Every directory directly under DIR is scanned for those. +""" + +import argparse, json, os, re, sys +from pathlib import Path + +from _common import git # one definition, shared across bench/slice + +HUNK = re.compile( r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@" ) + + +def index_checkouts( assets ): + """every `owner__repo` directory one level under any direct subdirectory of assets.""" + idx = {} + for top in sorted( os.listdir( assets ) ): + p = Path( assets ) / top + if not p.is_dir(): + continue + for name in sorted( os.listdir( p ) ): + q = p / name + if "__" in name and q.is_dir(): + idx.setdefault( name, str( q ) ) + return idx + + +def file_sections( patch ): + """split a unified diff into {post_path: [lines]} sections, keyed by the b/ path.""" + out, cur, path = {}, None, None + for line in patch.splitlines(): + if line.startswith( "diff --git " ): + cur, path = [], None + elif line.startswith( "+++ b/" ) and cur is not None: + path = line[ 6: ] + out[ path ] = cur + elif line.startswith( "+++ " ) and cur is not None: + path = None + elif cur is not None: + cur.append( line ) + return out + + +def gold_pre_lines( section ): + """pre-image gold line numbers for one file section — docs/research/slice-line-recall.md §1.1 G2.""" + deleted, anchors = set(), set() + pre_ln, hunk_start, consumed, plus_run_open = None, None, False, False + for line in section: + m = HUNK.match( line ) + if m: + hunk_start = int( m.group( 1 ) ) + pre_ln, consumed, plus_run_open = hunk_start, False, False + continue + if pre_ln is None: + continue + if line.startswith( "-" ) and not line.startswith( "---" ): + deleted.add( pre_ln ); pre_ln += 1; consumed = True; plus_run_open = False + elif line.startswith( "+" ) and not line.startswith( "+++" ): + if not plus_run_open: + # ONE anchor per '+' run: the pre-image line just before the insertion point, + # or the hunk's first pre-image line when the run opens the hunk. + anchors.add( pre_ln - 1 if consumed else hunk_start ) + plus_run_open = True + elif line.startswith( "\\" ): + continue + else: # context (leading space, or empty line) + pre_ln += 1; consumed = True; plus_run_open = False + # a line both deleted and used as an anchor is one gold line, not two + return sorted( deleted | ( anchors - deleted ) ), sorted( deleted ), sorted( anchors - deleted ) + + +def selector_for( path, fn ): + """LocBench `PATH:FN` -> a --slice selector. `Class.method` uses ripwire's scoped `::` spelling.""" + base = Path( path ).name + return f"{base}::" + "::".join( fn.split( "." ) ) if "." in fn else f"{base}:{fn}" + + +def carry_row( r, idx, census ): + """the carried instance for one dataset row, or None — every None bumps a census counter. + + Qualification stages 1, 2, 4, 6 of docs/research/slice-line-recall.md §2; stages 3 (single + function) and 5/7 (selector, inventory) are handled by the caller and by the runner, so that + nothing here needs the binary.""" + efs = r[ "edit_functions" ] + slug = r[ "repo" ].replace( "/", "__" ) + if slug not in idx: + census[ "no_checkout" ] += 1; return None + repo = idx[ slug ] + if git( repo, "cat-file", "-e", r[ "base_commit" ] + "^{commit}", ok_fail=True ).returncode != 0: + census[ "no_commit" ] += 1; return None + path, _, fn_name = efs[ 0 ].rpartition( ":" ) + if not path.endswith( ".py" ): + census[ "not_python" ] += 1; return None + if git( repo, "show", f"{r['base_commit']}:{path}", ok_fail=True ).returncode != 0: + census[ "file_missing_at_base" ] += 1; return None + secs = file_sections( r[ "patch" ] ) + if path not in secs: + census[ "no_patch_section" ] += 1; return None + gold, deleted, anchors = gold_pre_lines( secs[ path ] ) + if not gold: + census[ "no_gold_line" ] += 1; return None + return { + "instance_id": r[ "instance_id" ], "repo": r[ "repo" ], "repo_dir": repo, + "base_commit": r[ "base_commit" ], "path": path, "fn": fn_name, + "selector": selector_for( path, fn_name ), "scoped": "." in fn_name, + "gold": gold, "gold_deleted": deleted, "gold_anchor": anchors, + "patch_files": len( secs ), "category": r.get( "category" ), + } + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( "--assets", required=True, help="directory holding datasets/ and the repo checkouts" ) + ap.add_argument( "--dataset", default=None, help="dataset json (default: the single file under /datasets)" ) + ap.add_argument( "--json", default=None ) + a = ap.parse_args() + + assets = Path( a.assets ).resolve() + ds = Path( a.dataset ) if a.dataset else sorted( ( assets / "datasets" ).glob( "*.json" ) )[ 0 ] + rows = json.loads( Path( ds ).read_text() ) + idx = index_checkouts( assets ) + + census = { k: 0 for k in ( + "total", "multi_function", "zero_function", "no_checkout", "no_commit", + "not_python", "file_missing_at_base", "no_patch_section", "no_gold_line", "carried" ) } + census[ "total" ] = len( rows ) + carried, gold_lines_total, gold_lines_multi = [], 0, 0 + # dataset-level reachability, computed for ALL 560 rows independently of which repositories + # happen to be checked out locally: how much of the corpus's gold an INTRA-PROCEDURAL primitive + # can address at all. A multi-function row is out of reach by construction, not a miss. + ds_gold_single = ds_gold_multi = ds_rows_single = ds_rows_multi = 0 + for r in rows: + g = sum( len( gold_pre_lines( sec )[ 0 ] ) for sec in file_sections( r[ "patch" ] ).values() ) + if len( r[ "edit_functions" ] ) == 1: + ds_rows_single += 1; ds_gold_single += g + else: + ds_rows_multi += 1; ds_gold_multi += g + + for r in rows: + efs = r[ "edit_functions" ] + if len( efs ) != 1: + census[ "multi_function" if len( efs ) > 1 else "zero_function" ] += 1 + if len( efs ) > 1: + for sec in file_sections( r[ "patch" ] ).values(): + gold_lines_multi += len( gold_pre_lines( sec )[ 0 ] ) + continue + got = carry_row( r, idx, census ) + if got is None: + continue + census[ "carried" ] += 1 + gold_lines_total += len( got[ "gold" ] ) + carried.append( got ) + + out = { "dataset": str( ds ), "checkout_dirs": len( idx ), "census": census, + "gold_lines_carried": gold_lines_total, "gold_lines_multi_function": gold_lines_multi, + "dataset_reachability": { "rows_single_function": ds_rows_single, "rows_multi_function": ds_rows_multi, + "gold_lines_single_function": ds_gold_single, + "gold_lines_multi_function": ds_gold_multi }, + "instances": carried } + print( json.dumps( { k: v for k, v in out.items() if k != "instances" }, indent=2 ) ) + if a.json: + Path( a.json ).write_text( json.dumps( out, indent=2 ) ) + print( f"wrote {a.json} ({len(carried)} carried rows)", file=sys.stderr ) + + +if __name__ == "__main__": + main() diff --git a/bench/slice/probe_wholerepo_selector.py b/bench/slice/probe_wholerepo_selector.py new file mode 100644 index 000000000..0b0d57f88 --- /dev/null +++ b/bench/slice/probe_wholerepo_selector.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""probe_wholerepo_selector.py — price the one deviation the main run makes. + +run_slice_linerecall.py hands ripwire a ONE-FILE tree (sound, because --slice is intra-procedural by +declaration), which also removes whole-repository selector ambiguity. The selector-resolution rate it +reports is therefore an UPPER BOUND on what an agent sees on a real checkout. This probe measures the +gap on a sample: the same selector, same commit, against the WHOLE tree, materialized read-only with +`git archive` (the checkouts are never written to). + +Usage: + python3 bench/slice/probe_wholerepo_selector.py --gold gold.json --bin build/ripwire \ + --work DIR [--sample 12] [--max-mb 400] +""" + +import argparse, json, shutil, subprocess, sys, time +from pathlib import Path + +from _common import git, archive_tree # one definition, shared across bench/slice + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( "--gold", required=True ) + ap.add_argument( "--bin", default="build/ripwire" ) + ap.add_argument( "--work", required=True ) + ap.add_argument( "--sample", type=int, default=12 ) + ap.add_argument( "--json", default=None ) + a = ap.parse_args() + + binary = str( Path( a.bin ).resolve() ) + work = Path( a.work ).resolve(); work.mkdir( parents=True, exist_ok=True ) + rows = json.loads( Path( a.gold ).read_text() )[ "instances" ] + # a deterministic, spread-out sample: every k-th carried row + step = max( 1, len( rows ) // a.sample ) + sample = rows[ ::step ][ :a.sample ] + + out, resolved, ambiguous, other = [], 0, 0, 0 + for r in sample: + tree = work / r[ "instance_id" ] + shutil.rmtree( tree, ignore_errors=True ) + if not archive_tree( r[ "repo_dir" ], r[ "base_commit" ], tree ): + continue + t0 = time.perf_counter() + p = subprocess.run( [ binary, str( tree ), f"--slice={r['selector']}" ], + capture_output=True, text=True, errors="replace" ) + ms = ( time.perf_counter() - t0 ) * 1000.0 + files = sum( 1 for _ in tree.rglob( "*.py" ) ) + verdict = "resolved" if p.returncode == 0 else ( "ambiguous" if "matches" in p.stderr else "other_refusal" ) + if verdict == "resolved": resolved += 1 + elif verdict == "ambiguous": ambiguous += 1 + else: other += 1 + out.append( { "instance_id": r[ "instance_id" ], "selector": r[ "selector" ], "scoped": r[ "scoped" ], + "py_files": files, "verdict": verdict, "ms": ms, + "stderr": p.stderr.strip()[ :200 ] } ) + print( f"{r['instance_id']:<48} py={files:<6} {verdict:<14} {ms:8.0f} ms", file=sys.stderr ) + shutil.rmtree( tree, ignore_errors=True ) + + summary = { "sampled": len( out ), "resolved": resolved, "ambiguous": ambiguous, + "other_refusal": other, + "ms_mean": ( sum( x[ "ms" ] for x in out ) / len( out ) ) if out else None } + print( json.dumps( summary, indent=2 ) ) + if a.json: + Path( a.json ).write_text( json.dumps( { "summary": summary, "rows": out }, indent=2 ) ) + + +if __name__ == "__main__": + main() diff --git a/bench/slice/run_cross_fn_reach.py b/bench/slice/run_cross_fn_reach.py new file mode 100644 index 000000000..5d544e9c5 --- /dev/null +++ b/bench/slice/run_cross_fn_reach.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""run_cross_fn_reach.py — for gold lines a single-function slice cannot reach, is the enclosing +function reachable from the seed function through ripwire's own call graph, and at what hop depth? + +Protocol: docs/research/slice-line-recall.md, section "ARISE rung 3 — cross-function reach, measured +before building it". Reuses the note's existing gold build (locbench_gold.py helpers) and the SAME +instance set (the LocBench V1 test-560 dataset at the pinned checkouts), so this composes with the +already-published 70.8%-out-of-reach ceiling (docs/research/slice-line-recall.md §R3): the population +here is exactly the 208 multi-function rows / 6,809 gold lines that number counts, restricted to rows +whose repository has a local checkout (same availability constraint as the rest of the note). + +Unlike run_slice_linerecall.py's one-file trick (sound only because --slice is intra-procedural), +cross-function reach needs the WHOLE tree: a gold line's enclosing function can live in a different +file from the seed. The tree is materialized read-only with `git archive` (the checkout is never +written to, nothing is cloned) — the same technique probe_wholerepo_selector.py already uses to price +the one-file deviation, moved into _common.py so both scripts share it. + +Per carried row: + - seed = edit_functions[0] (the first-listed edited function — deterministic, and the convention + locbench_gold.py's carry_row already uses for the single-function population). The selector is + spelled with the FULL relative path, not the basename selector_for() uses for the one-file trick: + on a whole tree a bare basename is exactly the ambiguity source §R6 priced (12.5% of real-checkout + selectors), and the patch's own path is already a unique, unambiguous qualifier. + - every gold line (pre-image rule, G2, reused verbatim from locbench_gold) across EVERY file the + patch touches, not only the seed's file. + - a gold line inside the seed's own resolved span is hop0: it was already reachable by today's + single-function slice had it been pointed at this row at all — a footnote refining the ceiling, + reported apart from the extension question. + - every other gold line: --at=@FILE:LINE finds its true enclosing symbol (not the dataset's own + edit_functions naming, which the task may list imprecisely) or refuses (no indexed definition — + an import line, a decorator, a module constant, a comment) — reported as its own floor, + no_enclosing_symbol, because extending call-graph reach cannot help a line with no enclosing call + at all. + - distinct enclosing symbols are deduped per row (one --path=SEED,@FILE:LINE call per symbol, not + per gold line) and classified by hops= / reachable= into unreachable / 1 / 2 / 3+. + - for an unreachable symbol, --callers=@FILE:LINE and --callees=@FILE:LINE with count="0" on BOTH + flags it as reading like a leaf — the graph-limits disclosure the protocol requires (a dispatch + hub reached only by dynamic dispatch/callbacks/macros reads exactly the same way). + +--cache=PATH is passed on every call against one row's tree: the first call cold-parses and writes +it, every later call against the same unchanged tree reads it back instead of re-parsing. + +Usage: + python3 bench/slice/run_cross_fn_reach.py --assets DIR --bin build/ripwire --work DIR \ + [--json results.json] [--limit N] +""" + +import argparse, json, re, statistics, subprocess, sys, time +from pathlib import Path + +from _common import git, archive_tree # one definition, shared across bench/slice +from locbench_gold import index_checkouts, file_sections, gold_pre_lines + +ATTR = re.compile( r'(\w+)="([^"]*)"' ) +S_ROW = re.compile( r']*)/>' ) +BODY_H = re.compile( r"]*)>", re.S ) +HOP_BUCKETS = ( "hop1", "hop2", "hop3plus" ) + + +def attrs( s ): + return dict( ATTR.findall( s ) ) + + +def run( binary, tree, args, cache ): + t0 = time.perf_counter() + r = subprocess.run( [ binary, str( tree ) ] + args + [ f"--cache={cache}", "--legend=compact" ], + capture_output=True, text=True, errors="replace" ) + ms = ( time.perf_counter() - t0 ) * 1000.0 + return r.returncode, r.stdout, r.stderr, ms + + +def full_path_selector( path, fn ): + """PATH:FN -> a --expand/--path selector qualified by the FULL relative path (not the basename + selector_for() uses), because on a whole tree the basename is exactly the ambiguity source.""" + return f"{path}::" + "::".join( fn.split( "." ) ) if "." in fn else f"{path}:{fn}" + + +def expand_span( binary, tree, sel, cache ): + """(path, start, end) of SEL's body, or None if the selector refuses or serves no body.""" + rc, out, err, ms = run( binary, tree, [ f"--expand={sel}" ], cache ) + if rc != 0: + return None, err, ms + m = BODY_H.search( out ) + if not m: + return None, "no body in --expand output", ms + a = attrs( m.group( 1 ) ) + start = int( a[ "l" ] ) + end = start + len( m.group( 2 ).splitlines() ) - 1 + return ( a[ "p" ], start, end ), None, ms + + +def at_symbol( binary, tree, path, line, cache ): + """the innermost enclosing symbol at path:line — (path, start, end, name), or None if refused.""" + rc, out, err, ms = run( binary, tree, [ f"--at={path}:{line}" ], cache ) # --at itself takes a + # bare FILE:LINE; the @ prefix is only how ITS OWN seed composes into OTHER verbs (path/callers/…) + if rc != 0: + return None, ms + rows = [ a for a in ( attrs( m.group( 1 ) ) for m in S_ROW.finditer( out ) ) if "n" in a and "l" in a ] + if not rows: + return None, ms + innermost = rows[ -1 ] + return ( path, int( innermost[ "l" ] ), int( innermost.get( "el", innermost[ "l" ] ) ), innermost[ "n" ] ), ms + + +def hop_call( binary, tree, seed_sel, path, line, cache ): + rc, out, err, ms = run( binary, tree, [ f"--path={seed_sel},@{path}:{line}" ], cache ) + if rc != 0: + return None, ms + a = attrs( re.search( r"]*)>", out ).group( 1 ) ) + return a, ms + + +def edge_count( binary, tree, path, line, flag, cache ): + rc, out, err, ms = run( binary, tree, [ f"--{flag}=@{path}:{line}" ], cache ) + if rc != 0: + return None + m = re.search( rf"<{flag}\b([^>]*)>", out ) + return int( attrs( m.group( 1 ) )[ "count" ] ) if m else None + + +def hop_bucket( a ): + if a[ "reachable" ] == "0": + return "unreachable" + h = int( a[ "hops" ] ) + return "hop1" if h == 1 else "hop2" if h == 2 else "hop3plus" + + +def measure_row( binary, tree, r, cache, tag ): + """one carried multi-function row -> its result dict, or None with a census bucket bumped.""" + efs = r[ "edit_functions" ] + path0, _, fn0 = efs[ 0 ].rpartition( ":" ) + seed_sel = full_path_selector( path0, fn0 ) + seed_span, err, _ = expand_span( binary, tree, seed_sel, cache ) + if seed_span is None: + return None, "seed_selector_refused" + seed_path, seed_start, seed_end = seed_span + + gold_by_file = {} + for path, sec in file_sections( r[ "patch" ] ).items(): + if not path.endswith( ".py" ): + continue + lines = gold_pre_lines( sec )[ 0 ] + if lines: + gold_by_file[ path ] = lines + total_gold = sum( len( v ) for v in gold_by_file.values() ) + if total_gold == 0: + return None, "no_gold_line" + + hop0, outside = [], [] + for path, lines in gold_by_file.items(): + for ln in lines: + if path == seed_path and seed_start <= ln <= seed_end: + hop0.append( ( path, ln ) ) + else: + outside.append( ( path, ln ) ) + + # ---- resolve each outside line's true enclosing symbol, dedup by symbol identity ----------- + sym_of, no_enclosing, group = {}, [], {} + for path, ln in outside: + sym, _ = at_symbol( binary, tree, path, ln, cache ) + if sym is None: + no_enclosing.append( ( path, ln ) ) + continue + sym_of[ ( path, ln ) ] = sym + group.setdefault( sym, [] ).append( ( path, ln ) ) + + # ---- one --path call per DISTINCT enclosing symbol, not per gold line ----------------------- + sym_result, ambiguous_seen, unresolved_seen = {}, [], [] + for ( spath, sstart, send, sname ) in group: + a, _ = hop_call( binary, tree, seed_sel, spath, sstart, cache ) + if a is None: + sym_result[ ( spath, sstart, send, sname ) ] = ( "path_refused", None ) + continue + ambiguous_seen.append( int( a.get( "graph_ambiguous", 0 ) ) ) + unresolved_seen.append( int( a.get( "graph_unresolved", 0 ) ) ) + bucket = hop_bucket( a ) + leaf = None + if bucket == "unreachable": + c_in = edge_count( binary, tree, spath, sstart, "callers", cache ) + c_out = edge_count( binary, tree, spath, sstart, "callees", cache ) + leaf = ( c_in == 0 and c_out == 0 ) + sym_result[ ( spath, sstart, send, sname ) ] = ( bucket, leaf ) + + per_line = {} + for sym, pairs in group.items(): + bucket, leaf = sym_result[ sym ] + for pl in pairs: + per_line[ pl ] = ( bucket, leaf, sym ) + + counts = { "hop0": len( hop0 ), "no_enclosing_symbol": len( no_enclosing ), + "unreachable": 0, "hop1": 0, "hop2": 0, "hop3plus": 0, "path_refused": 0 } + leaf_unreachable, total_unreachable = 0, 0 + for pl, ( bucket, leaf, sym ) in per_line.items(): + counts[ bucket ] = counts.get( bucket, 0 ) + 1 + if bucket == "unreachable": + total_unreachable += 1 + if leaf: + leaf_unreachable += 1 + + max_needed = 0 + for pl, ( bucket, leaf, sym ) in per_line.items(): + if bucket in ( "unreachable", "path_refused" ): + max_needed = 10 ** 6 + else: + max_needed = max( max_needed, { "hop1": 1, "hop2": 2, "hop3plus": 3 }[ bucket ] ) + if no_enclosing: + max_needed = 10 ** 6 + + row = { "instance_id": r[ "instance_id" ], "repo": r[ "repo" ], "n_edit_functions": len( efs ), + "total_gold": total_gold, "counts": counts, + "distinct_enclosing_symbols": len( group ), + "leaf_unreachable": leaf_unreachable, "total_unreachable": total_unreachable, + "graph_ambiguous_max": max( ambiguous_seen ) if ambiguous_seen else None, + "graph_unresolved_max": max( unresolved_seen ) if unresolved_seen else None, + "fully_covered_at": { str( n ): ( max_needed <= n ) for n in ( 1, 2, 3 ) } } + print( f"[{tag}] {r['instance_id']} n_fn={len(efs)} gold={total_gold} " + f"hop0={counts['hop0']} unreach={counts['unreachable']} " + f"h1={counts['hop1']} h2={counts['hop2']} h3+={counts['hop3plus']} " + f"no_sym={counts['no_enclosing_symbol']}", file=sys.stderr ) + return row, None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( "--assets", required=True ) + ap.add_argument( "--dataset", default=None ) + ap.add_argument( "--bin", default="build/ripwire" ) + ap.add_argument( "--work", required=True ) + ap.add_argument( "--json", default=None ) + ap.add_argument( "--limit", type=int, default=0 ) + a = ap.parse_args() + + assets = Path( a.assets ).resolve() + binary = str( Path( a.bin ).resolve() ) + work = Path( a.work ).resolve(); work.mkdir( parents=True, exist_ok=True ) + ds = Path( a.dataset ) if a.dataset else sorted( ( assets / "datasets" ).glob( "*.json" ) )[ 0 ] + rows = json.loads( Path( ds ).read_text() ) + idx = index_checkouts( assets ) + + multi = [ r for r in rows if len( r[ "edit_functions" ] ) > 1 ] + ds_gold_multi = sum( len( gold_pre_lines( sec )[ 0 ] ) + for r in multi for sec in file_sections( r[ "patch" ] ).values() ) + + census = { "dataset_multi_function_rows": len( multi ), "dataset_multi_function_gold_lines": ds_gold_multi, + "no_checkout": 0, "no_commit": 0, "seed_selector_refused": 0, "no_gold_line": 0, + "archive_failed": 0, "carried": 0 } + carried_gold = 0 + candidates = [] + for r in multi: + slug = r[ "repo" ].replace( "/", "__" ) + if slug not in idx: + census[ "no_checkout" ] += 1; continue + repo = idx[ slug ] + if git( repo, "cat-file", "-e", r[ "base_commit" ] + "^{commit}", ok_fail=True ).returncode != 0: + census[ "no_commit" ] += 1; continue + candidates.append( ( r, repo ) ) + if a.limit: + candidates = candidates[ :a.limit ] + + results = [] + for i, ( r, repo ) in enumerate( candidates ): + tree = work / f"x{i:04d}" + cache = work / f"x{i:04d}.ripwirecache" + tree.mkdir( parents=True, exist_ok=True ) + ok = archive_tree( repo, r[ "base_commit" ], tree ) + if not ok: + census[ "archive_failed" ] += 1; continue + row, why = measure_row( binary, tree, r, cache, f"{i+1}/{len(candidates)}" ) + if row is None: + census[ why ] += 1 + continue + census[ "carried" ] += 1 + carried_gold += row[ "total_gold" ] + results.append( row ) + + # ---- summary: counts against BOTH the carried subset and the full 6,809-gold-line ceiling --- + agg = { k: sum( x[ "counts" ].get( k, 0 ) for x in results ) + for k in ( "hop0", "unreachable", "hop1", "hop2", "hop3plus", "no_enclosing_symbol", "path_refused" ) } + outside_seed = sum( agg[ k ] for k in ( "unreachable", "hop1", "hop2", "hop3plus", "no_enclosing_symbol", "path_refused" ) ) + not_measured_of_ceiling = census[ "dataset_multi_function_gold_lines" ] - carried_gold + total_unreach = sum( x[ "total_unreachable" ] for x in results ) + total_leaf = sum( x[ "leaf_unreachable" ] for x in results ) + + def cov_share( n ): + rows_n = [ x for x in results if x[ "fully_covered_at" ][ str( n ) ] ] + return len( rows_n ) / len( results ) if results else None + + summary = { + "binary": binary, "assets": str( assets ), "dataset": str( ds ), + "census": census, "carried_rows": census[ "carried" ], "carried_gold_lines": carried_gold, + "gold_line_distribution_of_carried": agg, + "gold_line_distribution_share_of_6809_ceiling": { + k: agg[ k ] / census[ "dataset_multi_function_gold_lines" ] for k in agg }, + "not_measured_share_of_6809_ceiling": not_measured_of_ceiling / census[ "dataset_multi_function_gold_lines" ], + "outside_seed_gold_lines": outside_seed, + "instance_fully_covered_share": { str( n ): cov_share( n ) for n in ( 1, 2, 3 ) }, + "unreachable_symbols_total": total_unreach, "unreachable_symbols_leaf_looking": total_leaf, + "unreachable_leaf_share": ( total_leaf / total_unreach ) if total_unreach else None, + "graph_ambiguous_max_over_rows": max( ( x[ "graph_ambiguous_max" ] for x in results if x[ "graph_ambiguous_max" ] is not None ), default=None ), + "graph_unresolved_max_over_rows": max( ( x[ "graph_unresolved_max" ] for x in results if x[ "graph_unresolved_max" ] is not None ), default=None ), + } + print( json.dumps( summary, indent=2 ) ) + if a.json: + Path( a.json ).write_text( json.dumps( { "summary": summary, "rows": results }, indent=2 ) ) + print( f"wrote {a.json}", file=sys.stderr ) + + +if __name__ == "__main__": + main() diff --git a/bench/slice/run_slice_linerecall.py b/bench/slice/run_slice_linerecall.py new file mode 100644 index 000000000..1eca177f5 --- /dev/null +++ b/bench/slice/run_slice_linerecall.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""run_slice_linerecall.py — the measurement arms of the --slice line-recall round. + +Protocol, fixed before this ran: docs/research/slice-line-recall.md §§2-5. Input is the gold file +produced by bench/slice/locbench_gold.py (which never invokes ripwire, so gold cannot move when the +binary does). Output is one json with every per-instance row plus the summary tables. + +What it does per carried row: + - materialize the ONE target file at base_commit into a scratch tree (`git show`, read-only on the + checkout) — sound because --slice is intra-procedural by declaration; + - resolve the selector: --slice=SEL (inventory, gives the definition's start line) and + --expand=SEL (gives the body, hence the span); + - restrict gold to the span; everything outside is reachability loss, not a slicer miss; + - arms v1 (--slice=SEL:VAR) and v2 (--slice=SEL:VAR --slice-flow=both) for EVERY inventory + variable (seed-free), so the ranking rules never look at the gold; + - metrics (a) set recall, (b) Recall@k for R1 / R2 / R2-oracle / random control, + (c) cost in bytes and wall-clock ms, (d) the §5 fixed-budget granularity comparison. + +Usage: + python3 bench/slice/run_slice_linerecall.py --gold gold.json --bin build/ripwire \ + --work DIR [--json results.json] [--limit N] +""" + +import argparse, json, random, re, statistics, subprocess, sys, time +from pathlib import Path + +from _common import git, line_text, name_lines # one definition, shared across bench/slice + +S_ROW = re.compile( r']*)>' ) +ATTR = re.compile( r'(\w+)="([^"]*)"' ) +V_ROW = re.compile( r']*)/>' ) +SLICE_H = re.compile( r']*)>' ) +WORD = re.compile( r"[A-Za-z_]\w*" ) +BUDGETS = ( 512, 1024, 2048, 4096 ) +KS = ( 1, 3, 5, 10, 20 ) +SEED = 20260920 +SHUFFLES = 200 + + +def attrs( s ): + return dict( ATTR.findall( s ) ) + + +def run( binary, tree, args ): + t0 = time.perf_counter() + r = subprocess.run( [ binary, str( tree ) ] + args, capture_output=True, text=True, errors="replace" ) + ms = ( time.perf_counter() - t0 ) * 1000.0 + return r.returncode, r.stdout, len( r.stdout.encode() ), ms + + +def slice_rows( out ): + """[(line, role, depth)] for every row; a row without d= is a v1 (depth 0) row.""" + rows = [] + for a in ( attrs( m.group( 1 ) ) for m in S_ROW.finditer( out ) ): + if "l" not in a: + continue + rows.append( ( int( a[ "l" ] ), a.get( "t", "" ), int( a.get( "d", "0" ) ) ) ) + return rows + + +def recall_at_k( order, gold, k ): + if not gold: + return None + return len( set( order[ :k ] ) & gold ) / len( gold ) + + +def mrr( order, gold ): + for i, ln in enumerate( order, 1 ): + if ln in gold: + return 1.0 / i + return 0.0 + + +def control_curve( pool, gold, rng ): + """mean Recall@k and MRR of a uniform random permutation of the same candidate pool.""" + acc = { k: 0.0 for k in KS }; acc_mrr = 0.0 + pool = list( pool ) + for _ in range( SHUFFLES ): + rng.shuffle( pool ) + for k in KS: + acc[ k ] += recall_at_k( pool, gold, k ) + acc_mrr += mrr( pool, gold ) + return { k: acc[ k ] / SHUFFLES for k in KS }, acc_mrr / SHUFFLES + + +def pack( numbered, budget ): + """line numbers delivered when `numbered` [(n, text)] is packed until budget bytes.""" + used, got = 0, [] + for n, text in numbered: + piece = f"{n}: {text}\n".encode() + if used + len( piece ) > budget: + break + used += len( piece ); got.append( n ) + return got + + +def rank_scores( span, gold, cover, depth, depth_o, rng ): + """§4(b): Recall@k and MRR for R0/R1/R2/R2-oracle and the random control, plus the R2 order. + + R0 is plain source order, R1 orders by how many inventory variables' flat slices cover the line, + R2 by the flow depth that reached it, R2-oracle by the same depth restricted to the gold-touching + seeds, and CTL is the mean over SHUFFLES permutations of the same candidate pool.""" + orders = { "r0": list( span ), + "r1": sorted( span, key=lambda n: ( -cover[ n ], n ) ), + "r2": sorted( span, key=lambda n: ( depth[ n ], -cover[ n ], n ) ), + "oracle": sorted( span, key=lambda n: ( depth_o[ n ], -cover[ n ], n ) ) } + ctl, ctl_mrr = control_curve( span, gold, rng ) + out = { f"mrr_{t}": mrr( o, gold ) for t, o in orders.items() } + out[ "mrr_ctl" ] = ctl_mrr + for k in KS: + for t, o in orders.items(): + out[ f"{t}@{k}" ] = recall_at_k( o, gold, k ) + out[ f"ctl@{k}" ] = ctl[ k ] + return orders[ "r2" ], out + + +def budget_scores( span, gold, lines, r2_order, cover, span_bounds ): + """§5: the share of gold lines delivered under each byte budget at each granularity. + + Every payload is line-numbered `N: text`, so the numbering costs the same in each arm and the + score is an exact line-number match. `file_window` is the strongest fair file-level arm: it packs + outward from the function's centre instead of from the file's first line.""" + start, span_end = span_bounds + mid = ( start + span_end ) // 2 + txt = lambda n: line_text( lines, n ) + file_head = [ ( n, txt( n ) ) for n in range( 1, len( lines ) + 1 ) ] + file_window = sorted( file_head, key=lambda t: ( abs( t[ 0 ] - mid ), t[ 0 ] ) ) + symbol = [ ( n, txt( n ) ) for n in span ] + line_level = [ ( n, txt( n ) ) for n in r2_order ] + line_filt = [ ( n, txt( n ) ) for n in + [ n for n in span if cover[ n ] > 0 ] + [ n for n in span if cover[ n ] == 0 ] ] + arms = ( ( "file_head", file_head ), ( "file_window", file_window ), ( "symbol", symbol ), + ( "line_filtered", line_filt ), ( "line", line_level ) ) + out = {} + for b in BUDGETS: + for name, payload in arms: + out[ f"budget{b}_{name}" ] = len( set( pack( payload, b ) ) & gold ) / len( gold ) + out[ f"budget{b}_symbol_fits" ] = sum( len( f"{n}: {t}\n".encode() ) for n, t in symbol ) <= b + return out + + +def measure_instance( binary, tree, r, sink, tag ): + """measure ONE carried row; return its instance row, or None when a stage disqualifies it. + + `sink` carries everything that outlives one row: `skips` and `acc` are the disclosure counters + (why a row dropped out, and the per-gold-line reachability cascade), `varinst` and `timings` + collect the per-variable rows and the wall clock, and `rng` is the control's seeded generator. + Every early return is a counted skip, never a silent one.""" + skips, varinst, timings, acc, rng = ( sink[ "skips" ], sink[ "varinst" ], sink[ "timings" ], + sink[ "acc" ], sink[ "rng" ] ) + src = tree / Path( r[ "path" ] ).name + show = git( r[ "repo_dir" ], "show", f"{r['base_commit']}:{r['path']}", ok_fail=True ) + if show.returncode != 0: + skips[ "no_body" ] += 1; return None + src.write_text( show.stdout ) + lines = show.stdout.splitlines() + + sel = r[ "selector" ] + rc_inv, inv_out, inv_bytes, inv_ms = run( binary, tree, [ f"--slice={sel}" ] ) + if rc_inv != 0: + skips[ "selector_scoped_refused" if r[ "scoped" ] else "selector_refused" ] += 1 + return None + head = attrs( SLICE_H.search( inv_out ).group( 1 ) ) + start = int( head[ "p" ].rsplit( ":", 1 )[ 1 ] ) + invent = [ attrs( m.group( 1 ) )[ "n" ] for m in V_ROW.finditer( inv_out ) ] + + rc_exp, exp_out, exp_bytes, exp_ms = run( binary, tree, [ f"--expand={sel}" ] ) + # the body CDATA is followed by only when the definition has no callees; with callees a + # element sits between, so anchor on the CDATA close, never on . + body = re.search( r"]*>", exp_out, re.S ) + if rc_exp != 0 or not body: + skips[ "no_body" ] += 1; return None + span_end = start + len( body.group( 1 ).splitlines() ) - 1 + span = list( range( start, span_end + 1 ) ) + gold_all = set( r[ "gold" ] ) + acc[ "resolved" ] += len( gold_all ) + gold = { n for n in gold_all if start <= n <= span_end } + acc[ "in_span" ] += len( gold ) + if not gold: + skips[ "gold_all_outside_span" ] += 1; return None + if not invent: + skips[ "empty_inventory" ] += 1 + # still counted in the reachability cascade above; no v1/v2 arm exists for it + return { "instance_id": r[ "instance_id" ], "empty_inventory": True, + "gold_total": len( gold_all ), "gold_in_span": len( gold ), + "span_lines": len( span ) } + + # ---- arms, seed-free: every inventory variable, v1 and v2 ------------------------------ + v1_by_var, v2_by_var, v1_bytes, v2_bytes = {}, {}, [], [] + for var in invent: + rc1, o1, b1, ms1 = run( binary, tree, [ f"--slice={sel}:{var}" ] ) + rc2, o2, b2, ms2 = run( binary, tree, [ f"--slice={sel}:{var}", "--slice-flow=both" ] ) + timings.append( ( "v1", ms1 ) ); timings.append( ( "v2", ms2 ) ) + if rc1 == 0: + v1_by_var[ var ] = slice_rows( o1 ); v1_bytes.append( b1 ) + if rc2 == 0: + v2_by_var[ var ] = slice_rows( o2 ); v2_bytes.append( b2 ) + timings.append( ( "inv", inv_ms ) ); timings.append( ( "expand", exp_ms ) ) + + def text_of( n ): return line_text( lines, n ) + names = name_lines( show.stdout ) + touched = sorted( { v for n in gold for v in WORD.findall( text_of( n ) ) if v in v1_by_var } ) + if touched: + acc[ "naming_local" ] += len( { n for n in gold + if any( re.search( r"\b%s\b" % re.escape( v ), text_of( n ) ) for v in touched ) } ) + + # (a) set recall, per (instance, variable) + for var in touched: + rel = [ n for n in sorted( gold ) if re.search( r"\b%s\b" % re.escape( var ), text_of( n ) ) ] + if not rel: + continue # this variable contributes no (instance, var) pair + l1 = { ln for ln, _, _ in v1_by_var[ var ] } + l2 = { ln for ln, _, _ in v2_by_var.get( var, [] ) } + hit1 = sum( 1 for n in rel if n in l1 ) + hit2 = sum( 1 for n in rel if n in l2 ) + rel_s = [ n for n in rel if n in names.get( var, () ) ] if names is not None else None + varinst.append( { + "instance_id": r[ "instance_id" ], "var": var, "relevant": len( rel ), + "v1_line_recall": hit1 / len( rel ), "v1_hit_all": hit1 == len( rel ), + "v2_line_recall": hit2 / len( rel ), + "v1_overinclusion": len( l1 ) / len( rel ), "v2_overinclusion": len( l2 ) / len( rel ), + "v1_missed": [ { "line": n, "text": text_of( n ).strip()[ :160 ] } for n in rel if n not in l1 ], + "relevant_strict": ( len( rel_s ) if rel_s is not None else None ), + "v1_line_recall_strict": ( ( sum( 1 for n in rel_s if n in l1 ) / len( rel_s ) ) if rel_s else None ), + "v1_hit_all_strict": ( all( n in l1 for n in rel_s ) if rel_s else None ), + } ) + + # (b) rank: R1 coverage, R2 flow depth, R2-oracle, random control + cover = { n: 0 for n in span } + for var, rr in v1_by_var.items(): + for ln in { x[ 0 ] for x in rr }: + if ln in cover: + cover[ ln ] += 1 + depth = { n: 10 ** 6 for n in span } + for var, rr in v2_by_var.items(): + for ln, _, d in rr: + if ln in depth: + depth[ ln ] = min( depth[ ln ], d ) + depth_o = { n: 10 ** 6 for n in span } + for var in touched: + for ln, _, d in v2_by_var.get( var, [] ): + if ln in depth_o: + depth_o[ ln ] = min( depth_o[ ln ], d ) + + r2_order, rank = rank_scores( span, gold, cover, depth, depth_o, rng ) + row = { "instance_id": r[ "instance_id" ], "repo": r[ "repo" ], "scoped": r[ "scoped" ], + "span_lines": len( span ), "gold_total": len( gold_all ), "gold_in_span": len( gold ), + "inventory": len( invent ), "touched_vars": len( touched ), + "inv_bytes": inv_bytes, "expand_bytes": exp_bytes, + "v1_bytes_mean": statistics.mean( v1_bytes ) if v1_bytes else None, + "v2_bytes_mean": statistics.mean( v2_bytes ) if v2_bytes else None, + "covered_lines": sum( 1 for n in span if cover[ n ] > 0 ), + "flow_lines": sum( 1 for n in span if depth[ n ] < 10 ** 6 ), + **rank, + **budget_scores( span, gold, lines, r2_order, cover, ( start, span_end ) ) } + + print( f"[{tag}] {r['instance_id']} span={len(span)} gold={len(gold)}/{len(gold_all)} " + f"inv={len(invent)} touched={len(touched)}", file=sys.stderr ) + return row + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument( "--gold", required=True ) + ap.add_argument( "--bin", default="build/ripwire" ) + ap.add_argument( "--work", required=True, help="scratch directory for the one-file trees" ) + ap.add_argument( "--json", default=None ) + ap.add_argument( "--limit", type=int, default=0 ) + a = ap.parse_args() + + binary = str( Path( a.bin ).resolve() ) + work = Path( a.work ).resolve(); work.mkdir( parents=True, exist_ok=True ) + gold_doc = json.loads( Path( a.gold ).read_text() ) + rows = gold_doc[ "instances" ] + if a.limit: + rows = rows[ :a.limit ] + + skips = { "selector_refused": 0, "selector_scoped_refused": 0, "no_body": 0, + "gold_all_outside_span": 0, "empty_inventory": 0 } + inst, varinst, timings = [], [], [] + acc = { "resolved": 0, "in_span": 0, "naming_local": 0 } + sink = { "skips": skips, "varinst": varinst, "timings": timings, "acc": acc, + "rng": random.Random( SEED ) } + gold_carried = sum( len( x[ "gold" ] ) for x in rows ) + + for i, r in enumerate( rows ): + tree = work / f"i{i:04d}" + tree.mkdir( exist_ok=True ) + row = measure_instance( binary, tree, r, sink, f"{i+1}/{len(rows)}" ) + if row is not None: + inst.append( row ) + # ---- summary ------------------------------------------------------------------------------- + scored = [ x for x in inst if not x.get( "empty_inventory" ) ] + def m( key, src=None ): + vals = [ x[ key ] for x in ( src if src is not None else scored ) if x.get( key ) is not None ] + return statistics.mean( vals ) if vals else None + tl = lambda tag: [ ms for t, ms in timings if t == tag ] + summary = { + "binary": binary, "gold_file": a.gold, + "rows_in": len( rows ), "scored_instances": len( scored ), "skips": skips, + "var_instances": len( varinst ), + "gold_lines_carried": gold_carried, "gold_lines_after_resolve": acc[ "resolved" ], + "gold_lines_in_span": acc[ "in_span" ], + "gold_lines_naming_a_sliceable_local": acc[ "naming_local" ], + "v1_line_recall_mean": statistics.mean( [ x[ "v1_line_recall" ] for x in varinst ] ) if varinst else None, + "v1_hit_all_rate": ( sum( 1 for x in varinst if x[ "v1_hit_all" ] ) / len( varinst ) ) if varinst else None, + "v1_line_recall_strict_mean": ( statistics.mean( [ x[ "v1_line_recall_strict" ] for x in varinst if x[ "v1_line_recall_strict" ] is not None ] ) + if any( x[ "v1_line_recall_strict" ] is not None for x in varinst ) else None ), + "v1_hit_all_strict_rate": ( ( sum( 1 for x in varinst if x[ "v1_hit_all_strict" ] ) / + sum( 1 for x in varinst if x[ "v1_hit_all_strict" ] is not None ) ) + if any( x[ "v1_hit_all_strict" ] is not None for x in varinst ) else None ), + "var_instances_strict": sum( 1 for x in varinst if x[ "v1_line_recall_strict" ] is not None ), + "v2_line_recall_mean": statistics.mean( [ x[ "v2_line_recall" ] for x in varinst ] ) if varinst else None, + "v1_overinclusion_mean": statistics.mean( [ x[ "v1_overinclusion" ] for x in varinst ] ) if varinst else None, + "v2_overinclusion_mean": statistics.mean( [ x[ "v2_overinclusion" ] for x in varinst ] ) if varinst else None, + "span_lines_mean": m( "span_lines" ), "inventory_mean": m( "inventory" ), + "covered_frac_mean": statistics.mean( [ x[ "covered_lines" ] / x[ "span_lines" ] for x in scored ] ) if scored else None, + "flow_frac_mean": statistics.mean( [ x[ "flow_lines" ] / x[ "span_lines" ] for x in scored ] ) if scored else None, + "mrr": { t: m( f"mrr_{t}" ) for t in ( "r0", "r1", "r2", "oracle", "ctl" ) }, + "bytes": { "inv": m( "inv_bytes" ), "v1": m( "v1_bytes_mean" ), "v2": m( "v2_bytes_mean" ), "expand": m( "expand_bytes" ) }, + "ms": { t: { "mean": statistics.mean( tl( t ) ), "median": statistics.median( tl( t ) ), "n": len( tl( t ) ) } + for t in ( "inv", "v1", "v2", "expand" ) if tl( t ) }, + } + summary[ "recall_at_k" ] = { t: { k: m( f"{t}@{k}" ) for k in KS } for t in ( "r0", "r1", "r2", "oracle", "ctl" ) } + summary[ "budget" ] = { b: { name: m( f"budget{b}_{name}" ) for name in ( "file_head", "file_window", "symbol", "line_filtered", "line" ) } + for b in BUDGETS } + fits = { b: [ x for x in scored if x.get( f"budget{b}_symbol_fits" ) ] for b in BUDGETS } + summary[ "budget_symbol_does_not_fit" ] = { + b: { "n": len( scored ) - len( fits[ b ] ), + **{ name: ( statistics.mean( [ x[ f"budget{b}_{name}" ] for x in scored if not x.get( f"budget{b}_symbol_fits" ) ] ) + if len( scored ) > len( fits[ b ] ) else None ) + for name in ( "file_head", "file_window", "symbol", "line_filtered", "line" ) } } + for b in BUDGETS } + print( json.dumps( summary, indent=2 ) ) + if a.json: + Path( a.json ).write_text( json.dumps( { "summary": summary, "instances": inst, "var_instances": varinst }, indent=2 ) ) + print( f"wrote {a.json}", file=sys.stderr ) + + +if __name__ == "__main__": + main() diff --git a/bench/slice/run_slicerecall.py b/bench/slice/run_slicerecall.py index 95b3bcba0..2bfc777ce 100644 --- a/bench/slice/run_slicerecall.py +++ b/bench/slice/run_slicerecall.py @@ -40,21 +40,13 @@ import argparse, json, re, subprocess, sys, tempfile, shutil, os from pathlib import Path +from _common import sh, line_text # one definition, shared with the py-round harnesses + CPP_EXT = { ".h", ".hpp", ".cpp", ".cc", ".cxx" } WORD = re.compile( r"[A-Za-z_]\w*" ) ROW_L = re.compile( r' INDEXING or OUTPUT, the taxonomy `LIMITS.md` renders in its `class` column: does this cap bound what can EVER be found, or only what is shown from what was found. A sidecar with a known expiry — the tag belongs on the declaration in `src/` — kept honest by `limitstablecheck.sh`, which fails if a row names a cap that no longer exists. | | **[`lineage-paper-dates.tsv`](lineage-paper-dates.tsv)** | Maintainers | arXiv id -> publication date for every 2026 paper in `LINEAGE.md`. The ID stem does not track the date (`2607.09691` was published 2026-06-19), so the README's recency claim is re-derived from this file by `readmedriftcheck.sh` arm (H2) rather than from the ids. Adding a 2026 paper without a date row fails that arm. | +| **[`research/`](research/)** | Anyone working on an open question here, or offering to help with one | Investigation notes: a question this tool has not answered, the measurement run against it, and the pre-registration that says what would license a change. Not a claim surface — a number here is local evidence for a decision not yet taken, and nothing in it is quoted on `README.md`. | | **[`assets/`](assets/)** | The front page | The README banner and tagline artwork (SVG, self-contained). | | **[`captures/`](captures/)** | Maintainers, and the curious | One recorded run of every verb against a real repository — the source of `COMMANDS.md`'s sample output, and the harvest source for the differential argv harness. | diff --git a/docs/research/slice-line-recall.md b/docs/research/slice-line-recall.md new file mode 100644 index 000000000..c50e02438 --- /dev/null +++ b/docs/research/slice-line-recall.md @@ -0,0 +1,698 @@ +# `--slice` line recall on an issue-derived Python corpus — ARISE rung 1, measured + +**Status: RUN, 2026-09-20.** The protocol in sections 1 to 6 was committed in `3d994cda` with the +results section empty — that commit contains no number produced by the harness, which is the proof +of ordering. Amendments are dated inline and labelled **AMENDMENT**, and each says whether it was +made before or after a result was seen. + +**Scope.** This is the retrieval-quality question ARISE (arXiv:2605.03117) raises, asked of +ripwire's `--slice` primitive on a corpus ripwire has not measured it on: issue-derived Python fix +patches, addressed at the **pre-fix** tree. It is a research note, not a product claim. No number +here is published anywhere else until an owner pass. + +--- + +## 1. What is already registered, and what this round adds + +`docs/EVALS.md` § *"The `--slice` def-use primitive (2026-08-28)"* registered a line-recall shape. +§ *"`--slice-flow` — ARISE rung 2"* executed it (2026-08-30, ripwire's own history, 38 instances) +and extended it (2026-08-31, three D4-pinned C/C++ trees). **So the registered shape has been run — +twice — and this round does not "finally run it".** What those runs did *not* cover, and what this +round adds: + +| already measured | not measured before this round | +| --- | --- | +| cpp family only (4 corpora, all C/C++) | **py family** — the one ARISE itself measured on | +| corpus mined from git history by subject regex | **issue-derived fix patches** (LocBench), the shape ARISE evaluates | +| gold = ADDED lines at the POST-fix tree | **gold = PRE-image lines at the PRE-fix tree** — the actual localization setting | +| set recall: are the gold lines among the rows | **rank**: do the rows put gold lines *first*, vs a random-order control | +| misses pooled | **reachability split**: gold out of reach *by construction* reported apart from gold the slicer dropped | +| bytes | **bytes and wall time** | +| — | **granularity vs presentation**: file-level / symbol-level / line-level answers under one byte budget | + +Everything in the right-hand column is new registration and is fixed below before the harness runs. + +### 1.1 Gap in the existing registration, closed here before measuring + +The 2026-08-28 registration says "take the variables named on the changed lines and ask whether +`--slice=fn:var` surfaces those changed lines among its rows". It is silent on three things that +decide the number: + +- **G1 — which tree.** A fix commit has a before and an after. The 2026-08-30/31 runs sliced the + POST-fix tree and scored ADDED lines. That measures *can the slicer see lines that already exist*. + The localization task ARISE scores is the other one: the agent holds the **pre-fix** tree and must + find the lines to change. This round slices the **base commit** and scores **pre-image** lines. + Both are defensible; they are different questions, and the earlier numbers are not comparable to + these. Stated, not reconciled. +- **G2 — which lines are gold when a hunk only inserts.** A pure insertion has no pre-image line of + its own. Rule fixed here: the gold line for an insertion run is the pre-image line **immediately + preceding** the insertion point (`pre_ln - 1`), or the hunk's first pre-image line when the run + opens the hunk. One anchor per run, never both neighbours. +- **G3 — what "surfaces" means when the row set is unordered.** `` rows are emitted in source + order, which is not a relevance ranking. The registration's metric is therefore a *set* metric and + cannot answer "does the primitive rank". The two ranking rules used here (§4) are defined below, + before any result, and both are computed from attributes the primitive already emits — no new + scorer, nothing tuned. + +**AMENDMENT 2026-09-20 (a):** G1, G2, G3 are amendments to the 2026-08-28 registration, written and +committed before the harness ran. They do not alter the 2026-08-30 or 2026-08-31 numbers, which +stand under the original reading. + +--- + +## 2. Corpus and slice rule — fixed before looking at any result + +**Source, already on disk, nothing downloaded.** `czlll/Loc-Bench_V1` test split, 560 rows, at +`/datasets/rows_czlll__Loc-Bench_V1_test_560.json`; 165 distinct repositories, of which +**90 have a local checkout**. Every `edit_functions` entry in all 560 rows is a `.py` path, so this +corpus is py-family in its entirety and is reported as such, never averaged with the cpp corpora. + +**A row is USABLE iff all of:** + +1. its `repo` has a local checkout; +2. `base_commit` resolves as a commit in that checkout; +3. `edit_functions_length == 1` — exactly one edited function. **Rows failing only this are the + by-construction-unreachable population** (§4c) and are counted, never scored as misses; +4. the single `edit_functions` entry is `PATH:FN` with `PATH` ending `.py`, and `PATH` materializes + at `base_commit`; +5. the selector resolves **uniquely**: `FILE:FN` for a plain name, `FILE::CLASS::METHOD` for a + dotted `Class.method`. An ambiguous or unresolved selector disqualifies the row and is counted by + reason; +6. at least one gold line (§G2 above, restricted to the resolved function's line span) exists; +7. at least one gold line names a variable in the function's own `--slice=SEL` inventory (otherwise + there is no v1 instance to score — counted as `no_touched_var`). + +**Cap: none.** Every usable row is measured. The corpus is small enough that a cap would only add a +choice to defend. + +**The tree handed to ripwire is a one-file tree**: the target file alone, materialized with +`git show BASE:PATH` into a scratch directory. The checkouts are never written to, never checked +out, never `git worktree add`-ed. This is sound *because the primitive is intra-procedural by +declaration* — no row of `--slice` can depend on a file it does not read — and it removes the +basename-ambiguity that thinned 34/63 candidates in the 2026-08-31 cpp run. It is stated as a +deviation because it also removes a real-world failure mode (whole-repo selector ambiguity), so the +selector-resolution rate reported here is an **upper bound** on what an agent would see on a whole +repo. + +**Determinism.** Given the dataset file, the checkouts and the binary, the instance list and every +number are a pure function of the inputs; row order is the dataset's own. The random-order control +uses a fixed seed (`20260920`) and a fixed shuffle count (200). + +--- + +## 3. Arms + +All arms run on the same instances, at the base commit's own file. + +| arm | command | what it is | +| --- | --- | --- | +| **v1** | `--slice=SEL:VAR` | the registered flat slice | +| **v2** | `--slice=SEL:VAR --slice-flow=both` | rung 2, bounded def-use BFS | +| **inv** | `--slice=SEL` | the sliceable-local inventory (addressing cost) | +| **expand** | `--expand=SEL` | whole-body baseline, recall 1.0 by construction, priced in bytes | +| **file** | the raw file | file-level baseline for §5 | + +--- + +## 4. Metrics — all defined before measuring + +**(a) Set recall (the registered shape, at the pre-fix tree).** Per (row, var) instance with +`G_var` = gold lines naming `var`: +`v1_line_recall = |G_var ∩ rows| / |G_var|`; `hit_all` = that ratio is 1.0; +`over_inclusion = |rows| / |G_var|`. Same for v2. + +**(b) Rank — the new question.** Candidate pool = **every line in the resolved function's span**. +Two ranking rules, both from attributes already emitted, neither tuned, plus a control: + +- **R1 (coverage).** Score a line by the number of *distinct* inventory variables whose v1 slice + contains it. Ties broken by line number ascending. Rationale: a line participating in several + tracked locals is the more central statement. Seed-free — it unions over the whole inventory and + never looks at the gold. +- **R2 (flow depth).** Score a line by `-min(d)` over the `--slice-flow=both` rows of every + inventory variable (v1 rows count as `d = 0`). Ties broken by R1, then line number. This is the + primitive's own relevance signal. +- **CTL (random).** A uniform random permutation of the same candidate pool, averaged over 200 + shuffles at seed `20260920`. This is the honest floor: a slice that merely *presents* lines will + score like CTL. + +Metric: `Recall@k = |G ∩ top-k| / |G|`, k ∈ {1, 3, 5, 10, 20}, reported as the instance mean, for +R1, R2 and CTL. Also `MRR` of the first gold line. + +**A seeded upper bound is reported separately** (`R2-oracle`): the same R2 ranking computed from the +gold-touched variables only. It is an oracle and is labelled one; it bounds what a perfect seed +choice could buy. + +**(c) Reachability.** A cascade, each stage counted, reported as fractions of the 560 rows and of +the gold lines: +`multi-function row` → `no checkout / no commit` → `selector unresolved` → `gold line outside the +resolved span` → `gold line names no sliceable local` → `scoreable`. +A gold line lost at any stage but the last is **out of reach by construction**, not a slicer miss, +and is reported on its own line. + +**(d) Cost.** Per instance: output bytes of inv / v1 / v2 / expand / file, and wall-clock +milliseconds of each ripwire invocation (single process, warm cache, median and mean). + +--- + +## 5. Granularity vs presentation — the paper's actual question + +ARISE's claim is that the *granularity floor* binds, not the ranking. The test that separates +granularity from presentation: hold the **answer** fixed (the correct function is given) and the +**budget** fixed, and vary only the granularity of what is delivered. + +Budgets `B ∈ {512, 1024, 2048, 4096}` bytes of delivered payload. Three deliveries: + +- **file-level** — the file's source, from its first line, truncated at `B` bytes; +- **symbol-level** — `--expand=SEL`'s body, truncated at `B` bytes; +- **line-level** — slice rows in **R2** order, each as `line-number: source text`, packed until `B` + bytes is reached. + +Score: the fraction of gold lines whose exact source text appears in the delivered payload. +Reported split by whether the function body **fits** in `B` (where symbol-level is 1.0 by +construction and the comparison is uninformative) and where it does not (where the question bites). +This is a *presentation-controlled* comparison: same correct function, same bytes, different +granularity. It cannot speak to ranking a whole repository — see §8. + +--- + +## 6. What we are NOT claiming + +- Nothing here is a Function Recall or Line Recall@1 number comparable to ARISE's. ARISE ranks over + a whole repository from a natural-language issue; this measures a primitive **given** the correct + function. The two numbers are not on the same axis and are never put in the same table. +- The one-file tree is an upper bound on selector resolution (§2). +- py-family only. The cpp numbers in `docs/EVALS.md` are a different population. + +--- + +## Results + +**Binary** `ripwire 0.6.1 (dev, built_from=755f9026f)` — plain dev build, no `-DCMAKE_BUILD_TYPE`. +**Re-runs:** the full harness was run end to end twice; the summary, every per-instance row and +every per-variable row compared **identical** (wall-clock timings excluded, as they must be). + +**AMENDMENT 2026-09-20 (b), made AFTER seeing the first misses and reported BESIDE the registered +metric, never instead of it.** The registered relevance oracle is a word regex over the changed +line's text. It counts a variable's name inside a docstring, a comment, a string literal, and even +the `f` of an f-string prefix, as an occurrence the slice "ought" to have rowed. A *strict* oracle is +added: an occurrence counts only when Python's own tokenizer calls it a `NAME` token on that line. +Both numbers are reported. The registered number is the headline; the strict number is what the +misses turn out to be made of. + +**AMENDMENT 2026-09-20 (c), made after a 6-row smoke run and before the corpus ran.** Two arms were +added: **R0**, plain source order over the function's lines — "just read the function top-down", +the baseline an agent actually has — and **line-filtered**, the def-use-covered lines in *source* +order, which separates the granularity FILTER from the RANKING in §5. + +### R1. Corpus — what was usable, and why the rest was not + +| stage | rows | note | +| --- | ---: | --- | +| dataset | **560** | LocBench V1 test; every `edit_functions` path is `.py` | +| multi-function (`edit_functions_length > 1`) | **208** | out of reach by construction — §R3, not a miss | +| single-function | 352 | | +| …no local checkout | 169 | 90 of the dataset's 165 repositories are on disk | +| …`base_commit` absent from the checkout | 1 | | +| **carried to the harness** | **182** | 1 040 gold lines | +| …selector refused, plain `FILE:FN` | 3 | | +| …selector refused, scoped `FILE::CLASS::METHOD` | 2 | | +| …`--expand` served no body | 2 | | +| …every gold line outside the resolved span | 2 | | +| **scored instances** | **173** | 2 453 `--slice` calls per arm; **498** (instance, variable) pairs | + +Selector resolution on the one-file tree is **177/182 = 97.3 %**. That is an upper bound (§2), and +§R6 prices the gap. + +### R2. Set recall — the registered shape, at the pre-fix tree + +498 (instance, variable) pairs; 480 of them also scoreable under the strict oracle. + +| metric | registered oracle | strict oracle | +| --- | ---: | ---: | +| v1 per-variable line-recall (mean) | **0.932** | **0.995** | +| v1 hit-all rate | **0.902** | **0.994** | +| v2 (`--slice-flow=both`) per-variable line-recall | 0.935 | — | +| v1 over-inclusion, rows / relevant lines | 5.12× | — | +| v2 over-inclusion | 12.46× | — | + +**Every miss was inspected, not sampled** (`bench/slice/inspect_slice_misses.py`). 100 gold lines +miss under the registered oracle. **97 of the 100** are lines where the variable's name is not a +`NAME` token at all — docstring prose, a trailing comment, a string literal, and in one instance the +`f` of `f"Incompatible safetensors file…"` matching a local called `f`. The **3** that survive the +strict oracle are all the same construct, a **keyword-argument name that collides with a local**: + +``` +pydantic-10789 var=schema L1918: lambda x, h: h(x), schema=core_schema.any_schema() +dask-11539 var=store L3759: z = zarr.open_array(store=url, read_only=True, path=component, **kwargs) +feast-4727 var=actions L235 : assert_permissions(resource=feature_view, actions=[AuthzedAction.WRITE_ONLINE]) +``` + +In all three the `store=` / `schema=` / `actions=` token is the **callee's** parameter name, not a +use of the local — so the classifier is right and the gold line is one the localization task wants +but the def-use relation genuinely does not contain. **On this corpus the slicer drops no real +identifier occurrence.** This replicates, on a different family and a different corpus shape, the +2026-08-30 cpp reading that the misses belong to the oracle rather than to the slice; that reading +was a per-instance inspection then and is an exhaustive, tokenizer-decided classification now. + +### R3. Reachability — what is out of reach by construction + +The primitive is name-based and intra-procedural. Two cascades, kept apart on purpose. + +**Dataset-level, over all 560 rows and all 9 615 gold lines** (computed without reference to which +repositories happen to be on disk): + +| population | rows | gold lines | share of gold | +| --- | ---: | ---: | ---: | +| multi-function fixes — **out of reach by construction** | 208 | **6 809** | **70.8 %** | +| single-function fixes — addressable in principle | 352 | 2 806 | 29.2 % | + +A fix spanning functions cannot be served by an intra-procedural slice at all. **Seven in ten gold +lines in this corpus live in such a fix.** That is the single largest number in this document and it +is a statement about the primitive's ceiling, not about its accuracy. + +**Within the 173 scored instances**, per gold line: + +| stage | gold lines | share of resolved | +| --- | ---: | ---: | +| carried into the harness | 1 040 | — | +| in an instance whose selector resolved and whose body was served | 945 | 100 % | +| **inside the resolved function's span** | **809** | **85.6 %** | +| …and naming a variable in that function's own sliceable inventory | **536** | **56.7 %** | + +The 136 lines inside a single-function row but outside the function's span are import lines, +decorators and module-level constants the patch also touched — the row's `edit_functions` names one +function, the patch is not confined to it. The 273 further lines are inside the function but name no +local: `self.x` attribute writes, bare `return`, `raise`, blank/comment anchors, and calls whose +arguments are all literals. + +**So: 56.7 % of the gold of the reachable population is addressable by a per-variable slice, and of +that 56.7 %, the slice recovers 99.5 % (strict) / 93.2 % (registered).** Those two numbers multiply; +neither on its own is the primitive's line recall. + +### R4. Rank — does the primitive order, or only present? + +Candidate pool = every line of the resolved function (mean span **86.9** lines; mean inventory +**14.2** sliceable locals). Instance mean of Recall@k, n = 173. + +| order | @1 | @3 | @5 | @10 | @20 | MRR | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| **R0** source order (read the function top-down) | 0.029 | 0.099 | 0.156 | 0.263 | 0.428 | 0.154 | +| **CTL** random permutation (200 shuffles, seed 20260920) | 0.042 | 0.123 | 0.191 | 0.321 | 0.498 | 0.234 | +| **R1** def-use coverage | **0.048** | **0.206** | **0.311** | **0.455** | **0.595** | **0.289** | +| **R2** flow depth, `--slice-flow=both` | 0.048 | 0.206 | 0.311 | 0.455 | 0.595 | 0.289 | +| *R2-oracle* seeded with the gold-touching variables | *0.053* | *0.292* | *0.415* | *0.570* | *0.706* | *0.349* | + +Three readings, in descending order of how much they matter. + +1. **The ordering carries real signal, and it is modest.** R1 beats the random control at every + depth — +8.3 pp @3, +12.0 pp @5, +13.4 pp @10 — and beats it on MRR 0.289 vs 0.234. It is not + close to pinpointing: **@1 is 0.048 against a 0.042 chance rate**, which is no effect worth + naming. The primitive re-ranks a shortlist; it does not name the line. +2. **Reading the function top-down is worse than random** (MRR 0.154 vs 0.234). Fixes cluster away + from the function head, so the default presentation order an agent gets is an actively bad + ranking, and *anything* is an improvement on it. Half of what looks like "the slice ranks well" + is really "source order ranks badly". +3. **`--slice-flow=both` adds exactly nothing here, and the reason is structural.** R1 and R2 are + identical to every digit, and the fraction of the function's lines the flow rows reach equals the + fraction the flat rows reach, to 16 decimal places (0.5126 both). This is not a coincidence and + not a bug: a flow row at depth ≥ 1 is a line where *another* variable `w` occurs, so that line is + already in `w`'s own flat slice. **Unioned over the whole inventory, rung 2 is provably + redundant.** Flow's value is confined to the *seeded* case — you know which variable you care + about and you do not want the other thirteen slices — which is exactly what the R2-oracle row + measures, and there it is worth +8.6 pp @3 over unseeded R1. + +The def-use filter keeps **51.3 %** of the function's lines. That is the granularity floor moving: +half the body is dropped before any ranking happens. + +### R5. Granularity versus presentation — the paper's question + +Same correct function, same byte budget, different granularity. All four payloads are delivered as +`line-number: source text`, so the numbering costs the same in every arm and the score is an exact +line-number match. Instance mean over the 173 instances. + +**All instances:** + +| budget | file (from line 1) | file (window on the fn) | symbol (`--expand`) | line-filtered | line-ranked | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 512 B | 0.004 | 0.329 | 0.286 | 0.373 | **0.411** | +| 1 024 B | 0.015 | 0.500 | 0.457 | 0.505 | **0.560** | +| 2 048 B | 0.057 | 0.693 | 0.682 | **0.744** | 0.738 | +| 4 096 B | 0.116 | 0.840 | 0.857 | 0.868 | **0.872** | + +Symbol-level is 1.0 by construction whenever the body fits the budget, so the comparison only bites +where it does not. **Restricted to instances whose body does NOT fit:** + +| budget | n | file (window) | symbol | line-filtered | line-ranked | filter gain | ranking gain | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 512 B | 150 | 0.226 | 0.176 | 0.276 | **0.321** | **+10.0 pp** | +4.4 pp | +| 1 024 B | 129 | 0.330 | 0.272 | 0.336 | **0.410** | **+6.4 pp** | +7.4 pp | +| 2 048 B | 94 | 0.435 | 0.414 | **0.530** | 0.517 | **+11.6 pp** | −1.3 pp | +| 4 096 B | 51 | 0.458 | 0.517 | 0.552 | **0.567** | +3.6 pp | +1.5 pp | + +*filter gain* = line-filtered − symbol (granularity alone, source order preserved). +*ranking gain* = line-ranked − line-filtered (ordering alone, same lines). + +**Answer: finer granularity changes retrieval quality, not only presentation — and on this corpus +the granularity is worth more than the ordering.** The filter is worth +3.6 to +11.6 pp at equal +bytes and is positive at every budget; the ranking adds +1.5 to +7.4 pp and at 2 048 B it is +*negative*. Both effects shrink as the budget grows, which is what "granularity floor" predicts: the +floor only binds when the budget is below the symbol. + +The honest size of the claim: at a 512-byte budget, line-level delivery gets 32 % of the gold lines +in front of the agent where whole-function delivery gets 18 %. That is a real lift and it is +nowhere near ARISE's +17 pp on Function Recall@1, because it is not the same measurement — this one +is handed the correct function and ARISE's is not (§6). + +### R6. Cost + +| arm | mean output bytes | share of `--expand` | wall clock, median | mean | +| --- | ---: | ---: | ---: | ---: | +| `--slice=SEL` (inventory) | 1 297 | 21 % | 43.9 ms | 51.4 ms | +| `--slice=SEL:VAR` (v1) | 1 152 | **18 %** | 9.3 ms | 11.5 ms | +| `--slice=SEL:VAR --slice-flow=both` (v2) | 2 087 | 34 % | 9.7 ms | 12.0 ms | +| `--expand=SEL` | 6 230 | 100 % | 7.1 ms | 8.1 ms | + +The inventory call is the first invocation against each tree and pays the index build; v1/v2/expand +are warm. n = 2 453 for v1 and v2, 173 for inv and expand. + +**The one-file-tree deviation, priced** (`bench/slice/probe_wholerepo_selector.py`, a deterministic +16-instance sample materialized read-only with `git archive`): + +| | one-file tree | whole tree at `base_commit` | +| --- | ---: | ---: | +| selector resolves uniquely | 97.3 % (177/182) | **87.5 % (14/16)** | +| cold `--slice=SEL` wall clock | 43.9 ms median | **435 ms mean** (68 – 979 ms, 65 – 2 794 py files) | + +Both failures on the whole tree are ambiguity refusals — a bare method name matching definitions in +several classes across the repository. So roughly **one selector in eight needs qualifying** on a +real checkout, and the refusal names the qualifying spellings rather than guessing, which is the +behaviour we want but is also a round trip the agent pays for. + +--- + +## 7. Where our construct is weaker than ARISE's + +Named so the gaps can be argued with, each with what we expect it to cost. + +| gap | ours | ARISE | expected cost | +| --- | --- | --- | --- | +| **def-use relation** | name-based: an occurrence of the identifier, classified by role | true def-use by the reaching-definition rule over an AST | Over-inclusion, not under-inclusion: 5.12× rows per relevant line. R2's strict recall of 0.995 says we lose almost nothing; the price is precision, and precision is exactly what a ranking needs. We expect a true def-use relation to help @1 far more than @10. | +| **aliasing** | none | none stated for the slicer either | Unknown, probably similar. Python's `a = b` on a mutable object makes both names live; we row neither for the other. | +| **scope** | scope-insensitive; shadowing may over-include | explicit global/nonlocal handling | Python comprehension and `except … as` bindings shadow constantly; we suspect a share of the 5.12× over-inclusion is this, and we have not separated it. | +| **statement granularity** | one row per source LINE | AST statement nodes | A multi-statement line merges; a statement continued across lines splits. Python's style makes the second the common case, and it will *understate* recall wherever a gold line is a continuation line of a statement we rowed at its first line. We have not measured how often. | +| **inter-procedural** | none — refuses to leave the definition | also stops at function boundaries in the slicer; expansion lives in its call-graph tier | **This is the big one.** 70.8 % of this corpus's gold lines are in multi-function fixes. ARISE has a tier that covers them; rung 1 and rung 2 do not, and the number above is what that costs. | +| **seed** | `(symbol, variable)`, or `@FILE:LINE` | `(file, line, variable)` | Equivalent in reach. Ours forces a name the caller may not have; the inventory call that supplies it costs 1 297 B and a round trip. | +| **addressing** | no `Class.method` spelling — `FILE::CLASS::METHOD` only | n/a | 12.5 % of real-checkout selectors refuse as ambiguous. Costs a round trip each; never a wrong answer. | + +## 8. What we would like help with + +Written as questions, because we expect several of these to have obvious answers we have missed. + +1. **Is the +17 pp attributable to the relation, or to the ceiling?** Our name-based relation loses + essentially nothing in recall (0.995 strict) and pays in precision (5.12× over-inclusion). If the + paper's gain came from *precision* rather than *coverage*, name-based def-use is a dead end for + ranking and we should build the real thing. Does the ablation separate these? +2. **Is our intra-procedural reading of the paper's slicer right?** We read §"stops at function + boundaries" as the slicer proper, with cross-function expansion in the call-graph tier. If the + slicer itself crosses boundaries, our 70.8 %-out-of-reach figure is a self-inflicted ceiling and + we would change the design rather than report the number. +3. **What is the right relevance oracle for a line-level gold?** Ours went from 0.932 to 0.995 by + switching from a word regex to a tokenizer. Both are defensible and the difference is 6 points. + How is Line Recall's gold decided in the paper — post-image added lines, pre-image deleted lines, + or a tokenized identifier match? +4. **How should a keyword-argument name that shadows a local be scored?** The only three misses that + survive our strict oracle are exactly this. The def-use relation says "not an occurrence"; the + localization task says "a line the fix touched". We currently count it as a miss and think that + is wrong. +5. **Does statement-node granularity beat line granularity enough to be worth the rebuild?** We + expect it matters most for continuation lines, which Python produces constantly, and we have not + measured it. + +## 9. Reproducing + +```bash +python3 bench/slice/locbench_gold.py --assets --json gold.json +python3 bench/slice/run_slice_linerecall.py --gold gold.json --bin build/ripwire --work --json results.json +python3 bench/slice/inspect_slice_misses.py --results results.json --gold gold.json +python3 bench/slice/probe_wholerepo_selector.py --gold gold.json --bin build/ripwire --work --sample 16 +python3 bench/slice/run_cross_fn_reach.py --assets --bin build/ripwire --work --json crossfn.json +``` + +`` is a directory holding `datasets/.json` and one or more directories of +`owner__repo` checkouts. Nothing is downloaded and no checkout is written to. Gold is built without +invoking ripwire, so it cannot move when the binary does. + +--- + +## 10. ARISE rung 3 — is cross-function reach worth building? (protocol pre-registered 2026-09-22) + +**Status: RUN, 2026-09-22.** §§10.1–10.3 (the protocol and the decision bands) were committed in +`21737125` with §10.4–§10.6 as headings only — that commit contains no number the harness could have +produced, the same ordering proof the 2026-09-20 round's own pre-registration commit (`3d994cda`) used. +A second commit (`f18af585`) filled in §10.4–§10.6 from `bench/slice/run_cross_fn_reach.py`'s output. +**AMENDMENT 2026-09-22 (b):** made after both of those, on review — `extend_2` was reported only +against the full 6,809-line ceiling, of which 57.2 % was never measured; §10.4 and §10.5 now also +report `extend_2` against the 2,913-line **measured** subset alone (14.35 %), so the verdict is checked +against both denominators rather than resting on the one that could be challenged. Both stay under the +pre-registered 20 % kill band. §10.5 also names the specific follow-up measurement (`--connect`-style +undirected reach over the 83.8 % of unreachable lines that have graph edges but no directed path) that +would need to change before this verdict does, and states the verdict's scope (LocBench Python +multi-function fixes, `edit_functions[0]`-seeded, directed reach) explicitly. §10.1–§10.3 are +unchanged. + +### 10.1 The question + +§R3 measured that **70.8 % of this corpus's gold lines (6,809 of 9,615) sit in fixes touching more +than one function**, out of reach of an intra-procedural primitive *by construction* — the single +largest number in this document. ripwire already builds a call graph for every other verb in the +catalog. Before spending any engineering on extending `--slice` across call boundaries, this round +asks the cheap question first: **using the call graph the tool already has, how much of that 6,809 +would a cross-function extension actually recover, and at what hop depth?** + +### 10.2 Method + +**Same population, same pin.** Reuses `bench/slice/locbench_gold.py`'s dataset and gold rule (§2, §1.1 +G2) — the LocBench V1 test-560 rows at each row's own `base_commit` — restricted to the **208 +multi-function rows already counted in §R3's 6,809**, at the *same* checkouts as the rest of this note +(one directory containing all `owner__repo` checkouts and `datasets/rows_czlll__Loc-Bench_V1_test_560.json`, +passed as `--assets`; not committed, not named here — see §2's own `` convention). This +composes with the published ceiling: it is the same denominator, not a resample. + +**Whole tree, not the one-file trick.** `run_slice_linerecall.py`'s one-file tree is sound only because +`--slice` is declared intra-procedural — a cross-function question cannot reuse it, because the +enclosing function of a gold line outside the seed can live in a different file from the seed +entirely. The tree is materialized **read-only** with `git archive | tar -x` into scratch +(the checkout is never written to, nothing is cloned) — `probe_wholerepo_selector.py`'s existing +technique, factored into `_common.archive_tree()` so both scripts share one definition. + +**Seed = `edit_functions[0]`.** The dataset gives a patch's edited functions as a bare list, not a +ranked one. The first-listed function is used as the seed — deterministic, stated before measuring, +and the same convention `locbench_gold.py`'s `carry_row` already uses for the single-function +population (`efs[0]`). The selector is spelled with the **full relative path**, not the basename +`selector_for()` uses for the one-file trick: on a whole tree a bare basename is exactly the ambiguity +source §R6 priced (12.5 % of real-checkout selectors refuse); the patch's own path is already a unique +qualifier, so this removes a chunk of that refusal rate by construction rather than by luck. + +**Reachability, per row:** + +1. Resolve the seed via `--expand=SEED` against the whole tree; refusal drops the row (counted). +2. Compute every gold line of the row (§1.1 G2, unchanged) across **every file the patch touches**, + not only the seed's file. +3. A gold line inside the seed's own resolved span is **hop0** — reported apart from the rest, + because it was already reachable by today's single-function slice had one been pointed at this row + at all; the existing methodology never attempts a multi-function row, so this is a footnote on the + ceiling, not new reach. +4. Every other gold line: `--at=FILE:LINE` names its **true** enclosing symbol — not the dataset's own + `edit_functions` naming, which the task can list imprecisely — or refuses (no indexed definition: + an import line, a decorator, a module constant, a comment). A refusal is its own bucket, + `no_enclosing_symbol`, disclosed apart from `unreachable`: extending call-graph reach cannot help a + line that names no enclosing call at all, so folding the two together would overstate what a + cross-function extension could ever buy. +5. Distinct enclosing symbols are **deduped per row** — one `--path=SEED,@FILE:LINE` call per symbol, + not per gold line, and every gold line under that symbol inherits its verdict. `--path` is a + directed shortest call-path with its own `hops=`/`reachable=`, which is a closer fit to "how many + hops would a call-graph-extended slice need to walk" than reconstructing depth from `--impact`'s + transitive-but-undated reach set or from repeated 1-hop `--callers=`/`--callees=` BFS — both of + which this round could have used instead and neither of which reports a per-target hop count + directly. **Direction matters and is deliberate**: `--path=SEED,TARGET` asks whether *walking + outward from the seed's own calls* reaches TARGET, which is exactly what a call-graph-extended + slice would do; it does *not* find a shared-caller sibling relationship (two functions invoked + by a common third function but not by each other) — `--connect` would, and §10.6 says what that + means for the reading. +6. For a symbol `--path` calls **unreachable**, `--callers=@FILE:LINE` and `--callees=@FILE:LINE` are + both checked: `count="0"` on both is the honesty disclosure the protocol requires — it reads + exactly like a symbol reached only by dynamic dispatch, a callback, or a macro (the same blind spots + named on every graph verb in `docs/COMMANDS.md`), and this round cannot tell the two apart from the + outside. +7. `--cache=PATH` is passed on every call against one row's tree — the first call cold-parses and + writes it, later calls against the same unchanged tree read it back. + +**Reported, per gold line of the 6,809:** `hop0` / `hop1` / `hop2` / `hop3plus` / `unreachable` / +`no_enclosing_symbol` / not measured (no checkout, no commit, seed selector refused) — every bucket a +share of 6,809, summing to it exactly, because a count that cannot be a total is a floor and a +floor is disclosed, never silently dropped from the denominator. + +**Reported, per instance (row):** "fully covered at reach = N" — every one of the row's gold lines is +either `hop0` or at or under N hops — for N ∈ {1, 2, 3}, as a share of the **measured** rows (the +denominator here is rows this round could actually resolve a seed for, not all 208, because an +unmeasured row has no "fully covered" verdict to report, and reporting one against the full 208 would +manufacture a number this round never produced). + +### 10.3 Pre-registered decision, before any number exists + +The question this buys an answer to is **marginal**: of the gold lines a single-function slice cannot +reach today, how many would a call-graph extension **newly** reach? `hop0` lines are already reachable +in principle (§10.2 step 3) and are not the extension's credit to claim; `no_enclosing_symbol` lines +cannot be reached by any amount of call-graph walking. So the decision metric is: + +**`extend_2 = (hop1 + hop2) / 6809`** — the share of the published ceiling a reach-2 extension would +newly recover. + +- **`extend_2 ≥ 50 %`** — strong case: build it. +- **`extend_2 < 20 %`** — kills it: the ceiling barely moves for the engineering cost. +- **`20 % ≤ extend_2 < 50 %`** — inconclusive: report it, do not build from this number alone; it + needs a cost estimate (§7's `--slice-flow=both`'s own redundancy finding is the cautionary + precedent — a rung that sounded obviously useful and measured provably redundant). + +`hop0`, `hop3plus`, `unreachable`, `no_enclosing_symbol` and the not-measured share are reported beside +`extend_2`, every one of them as its own share of 6,809, so the bands are graded against a number nothing +else in this section can quietly inflate. + +### 10.4 Results + +**Binary** `ripwire 0.6.1 (dev, built_from=81b7322ce)` — plain dev build, no `-DCMAKE_BUILD_TYPE`, same +assets as the rest of this note (§2). **Population.** 208 dataset-level multi-function rows / 6,809 +gold lines — recomputed independently by this round's own harness and it matches §R3's published ceiling +exactly, which is the composability check §10.2 promised. + +**Availability cascade** (rows, out of the 208): + +| stage | rows | note | +| --- | ---: | --- | +| multi-function rows | **208** | the §R3 ceiling population | +| …no local checkout | 80 | same 90/165-repository availability ceiling as §R1 | +| …`base_commit` absent from the checkout | 2 | | +| …seed selector (`edit_functions[0]`) refused on the whole tree | 4 | **3.2 %** of the 126 real-checkout candidates — well under §R6's 12.5 % basename-refusal rate; the full-path qualifier (§10.2) is doing the work that number predicted | +| **carried** | **122** | 2,913 gold lines, 42.8 % of the 6,809 ceiling | + +**Gold-line distribution, every bucket a share of the full 6,809 ceiling — they sum to it exactly:** + +| bucket | gold lines | share of 6,809 | meaning | +| --- | ---: | ---: | --- | +| not measured | 3,896 | 57.2 % | no checkout / no commit / seed refused (cascade above) | +| `hop0` | 572 | 8.4 % | inside the seed's own span — already reachable today, a footnote on the ceiling, not new reach | +| **`hop1`** | **291** | **4.3 %** | one call hop from the seed | +| **`hop2`** | **127** | **1.9 %** | two call hops | +| `hop3plus` | 58 | 0.9 % | three or more hops | +| `unreachable` | 1,559 | 22.9 % | `--path` found no directed call path at all | +| `no_enclosing_symbol` | 306 | 4.5 % | outside the seed, and `--at` found no indexed definition there either — an import line, a decorator, a module constant, a comment; no amount of call-graph walking reaches these | + +2,035 of the 2,341 outside-seed gold lines resolved to a real enclosing symbol via `--at` (the rest are +the 306 `no_enclosing_symbol` lines above); those 2,035 lines deduped to **513 distinct enclosing +symbols**, one `--path` call each, and every line inherited its symbol's hop verdict. + +**The decision metric:** + +**`extend_2 = (hop1 + hop2) / 6809 = 418 / 6809 = 6.14 %`** + +(`extend_1 = 4.27 %`, `extend_3 = 6.99 %` — depth 3 buys less than one more point over depth 2, which +is itself the point: whatever is reachable at all is mostly reachable in one hop or not in three.) + +**Denominator sensitivity, stated before this is read as a verdict.** `extend_2` above is computed +against the full 6,809-line ceiling, and 57.2 % of that ceiling was never measured (the availability +cascade above). Scoring the unmeasured 57.2 % as non-extendable is the conservative choice — it favours +killing the feature — so it is worth asking what `extend_2` is over the **measured subset alone**, +where every line actually got a `--path` answer: + +**`extend_2 (measured) = (hop1 + hop2) / 2913 = 418 / 2913 = 14.35 %`** + +Both denominators are reported because a reader who does not trust the availability cascade should not +have to take the ceiling-scoped number on faith. **14.35 % is still under the pre-registered 20 % kill +band**, so the verdict does not depend on which denominator is used — it holds on the conservative +figure (6.14 %) and on the more forgiving one (14.35 %) alike. Had the measured-subset figure landed at +or above 20 %, §10.5 would have downgraded to inconclusive-pending-wider-measurement rather than kill, +because the pre-registered bands decide this, not a preferred outcome. + +**Instance-level: share of the 122 measured rows fully covered if reach extended to N** (every gold +line in the row is `hop0` or at/under N hops — a `no_enclosing_symbol` line anywhere in the row makes +it uncoverable at any N): + +| reach | rows fully covered | share of 122 | +| ---: | ---: | ---: | +| 1 | 2 | 1.6 % | +| 2 | 3 | 2.5 % | +| 3 | 6 | 4.9 % | + +**Graph-limits disclosure, as required by §10.2 step 6 and the protocol's own honesty rule.** Of the +1,559 `unreachable` gold lines, 252 (**16.2 %, gold-line-weighted** — a line inherits its symbol's +verdict, and a symbol with several gold lines is counted once per line, not once) sit behind a symbol +with `count="0"` on BOTH `--callers` and `--callees`. Read literally: about one unreachable gold line in +six sits in a function with no recorded edge into or out of it at all — indistinguishable, from the +outside, between "this function truly stands alone" and "the only calls into or out of it go through +dynamic dispatch, a callback, or a macro the name-based graph does not see" (the same blind spot named +on every graph verb in `docs/COMMANDS.md`). **The other 83.8 % of unreachable gold lines sit behind a +symbol that DOES have edges elsewhere in the graph** — not a leaf — and is still unreached by a +*directed* path from the seed; the most likely structural reading is the one §10.2 step 5 named before +measuring: sibling functions a patch edits together because a common caller uses both, not because +either calls the other. A directed +`--path` cannot see that relationship by design; `--connect` could, and did not run here (§10.6). +`graph_ambiguous=` ranged 0–34,114 across the 122 trees (mean 3,176, median 2,392) and +`graph_unresolved=` 0–11,208 (mean 1,040, median 146) — both scale with repository size, and both are +resolver gauges over the WHOLE tree, not specific to any one call queried. + +### 10.5 Verdict, against the bands fixed in §10.3 before any of this was measured + +**Scope: this verdict is about LocBench's Python multi-function fixes, seeded at `edit_functions[0]`, +under DIRECTED reach from that one seed — not a general claim about slicing, and not about Python +patches in general.** It says what extending `--slice` across call boundaries would buy *this specific +primitive on this specific corpus*, nothing wider. + +**`extend_2 = 6.14 %` over the full 6,809-line ceiling, `14.35 %` over the 2,913-line measured +subset — both under the pre-registered 20 % kill line, so the verdict does not turn on which +denominator is used.** Even the most generous read (`extend_3 = 6.99 %` ceiling-scoped / `16.3 %` +measured-scoped, or crediting every `hop0` line as if the extension bought it too, which it did not) +stays under the band. At the instance level the picture agrees: extending reach to depth 3 still +leaves 95 % of measured multi-function rows with at least one gold line the extension cannot touch. +**Kill it** — cross-function reach via the existing directed call graph would recover a small, +single-digit-to-low-teens slice of the 70.8 % ceiling on either denominator, not the "close most of the +gap" outcome that would justify the engineering. This is a **measured** answer to the question §10.1 +asked cheaply before building anything, and it came back negative on both readings of the denominator. + +**The specific measurement that would change this answer, named rather than left vague:** §10.4's +graph-limits paragraph found that 83.8 % of unreachable gold lines sit behind a symbol that has *some* +call-graph edge, just not on a directed path from the seed — the structural signature of a sibling +function a patch touches via a shared caller, not via a call between the two. `--path` cannot see that +relationship by design. **The next measurement, if this is revisited, is `--connect=SEED,TARGET` (or an +equivalent undirected/bidirectional reach) over exactly that 83.8 % population**, asking how much of it +joins through a shared caller. If that share turns out to be large and cheaply reachable, this verdict +does not transfer — a directed-reach kill says nothing about an undirected join, and the two are +different features with different costs. This round did not run that measurement; it is future work, +not a caveat folded into the kill above. + +### 10.6 What this does not tell us + +- **Directed reach only.** `--path=SEED,TARGET` asks whether walking outward from the seed's own calls + reaches the target. It cannot find a shared-caller sibling (two functions a common third function + calls, never calling each other) — `--connect` could, and this round did not run it over the + unreachable population. §10.5's one open caveat is exactly this gap; the 6.14 % verdict is a verdict + on directed extension, not on every shape a cross-function join could take. +- **57.2 % of the ceiling was never measured**, for the same repository-availability reason as the + rest of this note (§R1): only 90 of 165 repositories have a local checkout. The `extend_2` figure is + computed against the FULL 6,809 denominator specifically so this gap reads as an honest floor rather + than vanishing into a smaller, rosier-looking base — but a measurement over the other 57.2 % could + still move the number, in either direction. +- **`edit_functions[0]` is one seed choice, not the best one.** A different, better-informed pick + (the function with the most gold lines, or the one PageRank ranks highest) could reach more; this + round deliberately used the cheapest, most defensible rule and did not search over seed choices. +- **A leaf-looking unreachable symbol (16.2 % of them) is not proof of isolation** — §10.4 already + says this cannot be told apart from a dynamic-dispatch/callback/macro blind spot from the outside; + reading it as "the code really has no callers" would overstate what the graph knows. +- **This is still the given-the-correct-seed question**, the same limit §6 already states for the rest + of the note: an agent doing real cross-repository localization does not start from a known-correct + `edit_functions[0]`, so even a favorable `extend_2` would not transfer directly to end-to-end + localization accuracy — it would only bound what the *primitive* could contribute once a seed is + already in hand. +- **Cost was not priced.** The bands in §10.3 were deliberately about the ceiling only; even had + `extend_2` cleared 50 %, this round says nothing about the byte or latency cost of a cross-function + slice, which would need its own measurement before a build decision — the same discipline §R6 already + applied to the one-file-tree deviation.