docs(research): measure --slice line recall on an issue-derived Python corpus

bench/slice/_common.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+"""_common.py — the helpers every bench/slice harness needs, defined once.
+
+`run_slicerecall.py` (the 2026-08-30 / 08-31 cpp rounds) and the 2026-09-20 py round's three
+scripts all shell out to git and all index source by 1-based line number. Each had grown its own
+copy; --quality-delta named the clone pairs, so they live here instead.
+"""
+
+import io, subprocess, tokenize
+from pathlib import Path
+
+
+def sh( args, cwd=None, ok_fail=False ):
+    """run a command, capture text, raise on failure unless ok_fail.
+
+    errors="replace": external corpora carry non-UTF-8 bytes (ugrep's own test fixtures are
+    deliberately latin-1/binary), and a diff that touches one must not abort a mine. Only content
+    bytes are ever mangled — hunk headers and funcnames are ASCII by git's own format — so
+    qualification is unaffected.
+    """
+    r = subprocess.run( args, cwd=cwd, capture_output=True, text=True, errors="replace" )
+    if r.returncode != 0 and not ok_fail:
+        raise RuntimeError( f"{args}: rc={r.returncode}\n{r.stderr[:500]}" )
+    return r
+
+
+def git( repo, *args, ok_fail=False ):
+    """sh() with `git -C repo` prepended."""
+    return sh( [ "git", "-C", str( repo ) ] + list( args ), ok_fail=ok_fail )
+
+
+def line_text( lines, n ):
+    """the 1-based n-th source line, or "" when n is outside the file."""
+    return lines[ n - 1 ] if 0 < n <= len( lines ) else ""
+
+
+def archive_tree( repo, commit, dest ):
+    """materialize repo@commit READ-ONLY into dest via `git archive | tar -x` — the checkout is never
+    written to and nothing is cloned. True on success; dest is left present but possibly incomplete
+    on failure, matching probe_wholerepo_selector.py's original inline version this replaces."""
+    tar = subprocess.run( [ "git", "-C", str( repo ), "archive", commit ], capture_output=True )
+    if tar.returncode != 0:
+        return False
+    Path( dest ).mkdir( parents=True, exist_ok=True )
+    r = subprocess.run( [ "tar", "-x", "-C", str( dest ) ], input=tar.stdout, capture_output=True )
+    return r.returncode == 0
+
+
+def name_lines( source ):
+    """{identifier: {line numbers where it occurs as a NAME token}} — the strict relevance oracle.
+
+    AMENDMENT 2026-09-20 (b) of docs/research/slice-line-recall.md, taken AFTER inspecting the
+    registered oracle's misses and reported BESIDE it, never instead of it: the registered oracle is
+    a word regex over the line text, so it counts a variable's name inside a docstring, a comment or
+    a string literal as an occurrence the slice ought to have rowed. Python's own tokenizer settles
+    which occurrences are identifiers. A source the tokenizer refuses (py2 syntax, decode trouble)
+    yields None, and its instance is then reported only under the registered oracle.
+    """
+    out = {}
+    try:
+        for tok in tokenize.generate_tokens( io.StringIO( source ).readline ):
+            if tok.type == tokenize.NAME:
+                out.setdefault( tok.string, set() ).add( tok.start[ 0 ] )
+    except Exception:
+        return None
+    return out

bench/slice/inspect_slice_misses.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""inspect_slice_misses.py — read run_slice_linerecall.py's results json and say, per missed line,
+whether the miss is the RELEVANCE ORACLE's noise or a real drop by the slicer.
+
+The registered oracle is a word regex over the changed line's text, so it counts a variable's name
+inside a docstring, a comment, a string literal or an f-string prefix as an occurrence. The strict
+oracle (AMENDMENT 2026-09-20 (b)) is Python's own tokenizer: an occurrence counts only when it is a
+NAME token. This script prints the census both ways and lists every miss that survives the strict
+oracle in full, because that residue is the only part that is evidence about the slicer.
+
+Usage: python3 bench/slice/inspect_slice_misses.py --results results.json --gold gold.json
+"""
+
+import argparse, json
+from pathlib import Path
+
+from _common import git, name_lines                    # one definition, shared across bench/slice
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument( "--results", required=True )
+    ap.add_argument( "--gold", required=True )
+    a = ap.parse_args()
+
+    res = json.loads( Path( a.results ).read_text() )
+    gold = { g[ "instance_id" ]: g for g in json.loads( Path( a.gold ).read_text() )[ "instances" ] }
+    src_cache = {}
+
+    def source_of( iid ):
+        if iid not in src_cache:
+            g = gold[ iid ]
+            r = git( g[ "repo_dir" ], "show", f"{g['base_commit']}:{g['path']}", ok_fail=True )
+            src_cache[ iid ] = r.stdout if r.returncode == 0 else ""
+        return src_cache[ iid ]
+
+    total_miss, noise, real, untokenizable = 0, 0, [], 0
+    for x in res[ "var_instances" ]:
+        if not x[ "v1_missed" ]:
+            continue
+        names = name_lines( source_of( x[ "instance_id" ] ) )
+        for m in x[ "v1_missed" ]:
+            total_miss += 1
+            if names is None:
+                untokenizable += 1
+            elif m[ "line" ] in names.get( x[ "var" ], set() ):
+                real.append( ( x[ "instance_id" ], x[ "var" ], m ) )
+            else:
+                noise += 1
+
+    print( f"missed lines under the registered (word-regex) oracle : {total_miss}" )
+    print( f"  the name is not a NAME token on that line (oracle noise): {noise}" )
+    print( f"  file not tokenizable, unclassified                      : {untokenizable}" )
+    print( f"  survives the strict oracle — evidence about the slicer  : {len(real)}" )
+    for iid, var, m in real:
+        print( f"    {iid} var={var} L{m['line']}: {m['text']}" )
+
+
+if __name__ == "__main__":
+    main()

bench/slice/locbench_gold.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""locbench_gold.py — build line-level gold for the --slice line-recall round from a LocBench-shaped
+dataset plus local repository checkouts. Downloads nothing; reads only what is already on disk.
+
+Protocol: docs/research/slice-line-recall.md §2 (corpus and slice rule) and §1.1 G2 (the gold rule
+for a pure-insertion hunk). This script does NOT invoke ripwire — every ripwire-dependent
+qualification stage lives in run_slice_linerecall.py, so the gold is a function of the dataset and
+the checkouts alone and cannot move when the binary does.
+
+Gold is PRE-IMAGE: the round scores the localization setting (the agent holds the pre-fix tree and
+must find the lines to change), so gold line numbers are numbered in the file at base_commit.
+  - every '-' line of a hunk touching the target file contributes its pre-image line number;
+  - a run of '+' lines with no '-' line of its own contributes ONE anchor: the pre-image line
+    immediately preceding the insertion point, or the hunk's first pre-image line when the run
+    opens the hunk.
+
+Usage:
+  python3 bench/slice/locbench_gold.py --assets DIR [--dataset FILE] [--json out.json]
+
+--assets DIR must contain `datasets/` (the dataset json) and one or more sibling directories of
+repository checkouts named `owner__repo`. Every directory directly under DIR is scanned for those.
+"""
+
+import argparse, json, os, re, sys
+from pathlib import Path
+
+from _common import git                                # one definition, shared across bench/slice
+
+HUNK = re.compile( r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@" )
+
+
+def index_checkouts( assets ):
+    """every `owner__repo` directory one level under any direct subdirectory of assets."""
+    idx = {}
+    for top in sorted( os.listdir( assets ) ):
+        p = Path( assets ) / top
+        if not p.is_dir():
+            continue
+        for name in sorted( os.listdir( p ) ):
+            q = p / name
+            if "__" in name and q.is_dir():
+                idx.setdefault( name, str( q ) )
+    return idx
+
+
+def file_sections( patch ):
+    """split a unified diff into {post_path: [lines]} sections, keyed by the b/ path."""
+    out, cur, path = {}, None, None
+    for line in patch.splitlines():
+        if line.startswith( "diff --git " ):
+            cur, path = [], None
+        elif line.startswith( "+++ b/" ) and cur is not None:
+            path = line[ 6: ]
+            out[ path ] = cur
+        elif line.startswith( "+++ " ) and cur is not None:
+            path = None
+        elif cur is not None:
+            cur.append( line )
+    return out
+
+
+def gold_pre_lines( section ):
+    """pre-image gold line numbers for one file section — docs/research/slice-line-recall.md §1.1 G2."""
+    deleted, anchors = set(), set()
+    pre_ln, hunk_start, consumed, plus_run_open = None, None, False, False
+    for line in section:
+        m = HUNK.match( line )
+        if m:
+            hunk_start = int( m.group( 1 ) )
+            pre_ln, consumed, plus_run_open = hunk_start, False, False
+            continue
+        if pre_ln is None:
+            continue
+        if line.startswith( "-" ) and not line.startswith( "---" ):
+            deleted.add( pre_ln ); pre_ln += 1; consumed = True; plus_run_open = False
+        elif line.startswith( "+" ) and not line.startswith( "+++" ):
+            if not plus_run_open:
+                # ONE anchor per '+' run: the pre-image line just before the insertion point,
+                # or the hunk's first pre-image line when the run opens the hunk.
+                anchors.add( pre_ln - 1 if consumed else hunk_start )
+                plus_run_open = True
+        elif line.startswith( "\\" ):
+            continue
+        else:                                              # context (leading space, or empty line)
+            pre_ln += 1; consumed = True; plus_run_open = False
+    # a line both deleted and used as an anchor is one gold line, not two
+    return sorted( deleted | ( anchors - deleted ) ), sorted( deleted ), sorted( anchors - deleted )
+
+
+def selector_for( path, fn ):
+    """LocBench `PATH:FN` -> a --slice selector. `Class.method` uses ripwire's scoped `::` spelling."""
+    base = Path( path ).name
+    return f"{base}::" + "::".join( fn.split( "." ) ) if "." in fn else f"{base}:{fn}"
+
+
+def carry_row( r, idx, census ):
+    """the carried instance for one dataset row, or None — every None bumps a census counter.
+
+    Qualification stages 1, 2, 4, 6 of docs/research/slice-line-recall.md §2; stages 3 (single
+    function) and 5/7 (selector, inventory) are handled by the caller and by the runner, so that
+    nothing here needs the binary."""
+    efs = r[ "edit_functions" ]
+    slug = r[ "repo" ].replace( "/", "__" )
+    if slug not in idx:
+        census[ "no_checkout" ] += 1; return None
+    repo = idx[ slug ]
+    if git( repo, "cat-file", "-e", r[ "base_commit" ] + "^{commit}", ok_fail=True ).returncode != 0:
+        census[ "no_commit" ] += 1; return None
+    path, _, fn_name = efs[ 0 ].rpartition( ":" )
+    if not path.endswith( ".py" ):
+        census[ "not_python" ] += 1; return None
+    if git( repo, "show", f"{r['base_commit']}:{path}", ok_fail=True ).returncode != 0:
+        census[ "file_missing_at_base" ] += 1; return None
+    secs = file_sections( r[ "patch" ] )
+    if path not in secs:
+        census[ "no_patch_section" ] += 1; return None
+    gold, deleted, anchors = gold_pre_lines( secs[ path ] )
+    if not gold:
+        census[ "no_gold_line" ] += 1; return None
+    return {
+        "instance_id": r[ "instance_id" ], "repo": r[ "repo" ], "repo_dir": repo,
+        "base_commit": r[ "base_commit" ], "path": path, "fn": fn_name,
+        "selector": selector_for( path, fn_name ), "scoped": "." in fn_name,
+        "gold": gold, "gold_deleted": deleted, "gold_anchor": anchors,
+        "patch_files": len( secs ), "category": r.get( "category" ),
+    }
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument( "--assets", required=True, help="directory holding datasets/ and the repo checkouts" )
+    ap.add_argument( "--dataset", default=None, help="dataset json (default: the single file under <assets>/datasets)" )
+    ap.add_argument( "--json", default=None )
+    a = ap.parse_args()
+
+    assets = Path( a.assets ).resolve()
+    ds = Path( a.dataset ) if a.dataset else sorted( ( assets / "datasets" ).glob( "*.json" ) )[ 0 ]
+    rows = json.loads( Path( ds ).read_text() )
+    idx = index_checkouts( assets )
+
+    census = { k: 0 for k in (
+        "total", "multi_function", "zero_function", "no_checkout", "no_commit",
+        "not_python", "file_missing_at_base", "no_patch_section", "no_gold_line", "carried" ) }
+    census[ "total" ] = len( rows )
+    carried, gold_lines_total, gold_lines_multi = [], 0, 0
+    # dataset-level reachability, computed for ALL 560 rows independently of which repositories
+    # happen to be checked out locally: how much of the corpus's gold an INTRA-PROCEDURAL primitive
+    # can address at all. A multi-function row is out of reach by construction, not a miss.
+    ds_gold_single = ds_gold_multi = ds_rows_single = ds_rows_multi = 0
+    for r in rows:
+        g = sum( len( gold_pre_lines( sec )[ 0 ] ) for sec in file_sections( r[ "patch" ] ).values() )
+        if len( r[ "edit_functions" ] ) == 1:
+            ds_rows_single += 1; ds_gold_single += g
+        else:
+            ds_rows_multi += 1; ds_gold_multi += g
+
+    for r in rows:
+        efs = r[ "edit_functions" ]
+        if len( efs ) != 1:
+            census[ "multi_function" if len( efs ) > 1 else "zero_function" ] += 1
+            if len( efs ) > 1:
+                for sec in file_sections( r[ "patch" ] ).values():
+                    gold_lines_multi += len( gold_pre_lines( sec )[ 0 ] )
+            continue
+        got = carry_row( r, idx, census )
+        if got is None:
+            continue
+        census[ "carried" ] += 1
+        gold_lines_total += len( got[ "gold" ] )
+        carried.append( got )
+
+    out = { "dataset": str( ds ), "checkout_dirs": len( idx ), "census": census,
+            "gold_lines_carried": gold_lines_total, "gold_lines_multi_function": gold_lines_multi,
+            "dataset_reachability": { "rows_single_function": ds_rows_single, "rows_multi_function": ds_rows_multi,
+                                      "gold_lines_single_function": ds_gold_single,
+                                      "gold_lines_multi_function": ds_gold_multi },
+            "instances": carried }
+    print( json.dumps( { k: v for k, v in out.items() if k != "instances" }, indent=2 ) )
+    if a.json:
+        Path( a.json ).write_text( json.dumps( out, indent=2 ) )
+        print( f"wrote {a.json} ({len(carried)} carried rows)", file=sys.stderr )
+
+
+if __name__ == "__main__":
+    main()

bench/slice/probe_wholerepo_selector.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""probe_wholerepo_selector.py — price the one deviation the main run makes.
+
+run_slice_linerecall.py hands ripwire a ONE-FILE tree (sound, because --slice is intra-procedural by
+declaration), which also removes whole-repository selector ambiguity. The selector-resolution rate it
+reports is therefore an UPPER BOUND on what an agent sees on a real checkout. This probe measures the
+gap on a sample: the same selector, same commit, against the WHOLE tree, materialized read-only with
+`git archive` (the checkouts are never written to).
+
+Usage:
+  python3 bench/slice/probe_wholerepo_selector.py --gold gold.json --bin build/ripwire \
+      --work DIR [--sample 12] [--max-mb 400]
+"""
+
+import argparse, json, shutil, subprocess, sys, time
+from pathlib import Path
+
+from _common import git, archive_tree                  # one definition, shared across bench/slice
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument( "--gold", required=True )
+    ap.add_argument( "--bin", default="build/ripwire" )
+    ap.add_argument( "--work", required=True )
+    ap.add_argument( "--sample", type=int, default=12 )
+    ap.add_argument( "--json", default=None )
+    a = ap.parse_args()
+
+    binary = str( Path( a.bin ).resolve() )
+    work = Path( a.work ).resolve(); work.mkdir( parents=True, exist_ok=True )
+    rows = json.loads( Path( a.gold ).read_text() )[ "instances" ]
+    # a deterministic, spread-out sample: every k-th carried row
+    step = max( 1, len( rows ) // a.sample )
+    sample = rows[ ::step ][ :a.sample ]
+
+    out, resolved, ambiguous, other = [], 0, 0, 0
+    for r in sample:
+        tree = work / r[ "instance_id" ]
+        shutil.rmtree( tree, ignore_errors=True )
+        if not archive_tree( r[ "repo_dir" ], r[ "base_commit" ], tree ):
+            continue
+        t0 = time.perf_counter()
+        p = subprocess.run( [ binary, str( tree ), f"--slice={r['selector']}" ],
+                            capture_output=True, text=True, errors="replace" )
+        ms = ( time.perf_counter() - t0 ) * 1000.0
+        files = sum( 1 for _ in tree.rglob( "*.py" ) )
+        verdict = "resolved" if p.returncode == 0 else ( "ambiguous" if "matches" in p.stderr else "other_refusal" )
+        if verdict == "resolved": resolved += 1
+        elif verdict == "ambiguous": ambiguous += 1
+        else: other += 1
+        out.append( { "instance_id": r[ "instance_id" ], "selector": r[ "selector" ], "scoped": r[ "scoped" ],
+                      "py_files": files, "verdict": verdict, "ms": ms,
+                      "stderr": p.stderr.strip()[ :200 ] } )
+        print( f"{r['instance_id']:<48} py={files:<6} {verdict:<14} {ms:8.0f} ms", file=sys.stderr )
+        shutil.rmtree( tree, ignore_errors=True )
+
+    summary = { "sampled": len( out ), "resolved": resolved, "ambiguous": ambiguous,
+                "other_refusal": other,
+                "ms_mean": ( sum( x[ "ms" ] for x in out ) / len( out ) ) if out else None }
+    print( json.dumps( summary, indent=2 ) )
+    if a.json:
+        Path( a.json ).write_text( json.dumps( { "summary": summary, "rows": out }, indent=2 ) )
+
+
+if __name__ == "__main__":
+    main()