Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 12 additions & 12 deletions claude-context-kernel/bench/ephemeral_dividend.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,9 +122,9 @@ def main() -> int:
ap.add_argument("--transcripts",
default=os.path.expanduser("~/.claude/projects"))
ap.add_argument("--scale", type=float, default=0.5,
help="CK_EPHEMERAL_SCALE del braccio B")
help="CK_EPHEMERAL_SCALE for arm B")
ap.add_argument("--adaptive", type=float, default=0.75,
help="scala adattiva live comune ai due bracci")
help="live adaptive scale shared by both arms")
ap.add_argument("--json", action="store_true")
args = ap.parse_args()

Expand Down Expand Up @@ -170,7 +170,7 @@ def main() -> int:
tot[arm]["lost_by_tool"].get(key, 0) + lost)

if n == 0:
print("corpus vuoto: nessun output effimero trovato", file=sys.stderr)
print("empty corpus: no ephemeral output found", file=sys.stderr)
return 1
res = {
"outputs": n, "per_tool": per_tool, "token_before": before_tot,
Expand All @@ -190,15 +190,15 @@ def main() -> int:
print(json.dumps(res, indent=2))
return 0
a, b = res["arms"]["A"], res["arms"]["B"]
print(f"corpus: {n} output effimeri reali ({per_tool}), "
f"~{before_tot:,} token grezzi")
print(f"A baseline (scala {a['scale']}): ~{a['token_after']:,} token "
f"(-{a['saving_pct']}%), {a['elisions']} elisioni, "
f"{a['signal_lines_lost']} righe di segnale perse")
print(f"B dividendo (scala {b['scale']}): ~{b['token_after']:,} token "
f"(-{b['saving_pct']}%), {b['elisions']} elisioni, "
f"{b['signal_lines_lost']} righe di segnale perse")
print(f"extra risparmio di B: ~{res['extra_saving_tokens']:,} token")
print(f"corpus: {n} real ephemeral outputs ({per_tool}), "
f"~{before_tot:,} raw tokens")
print(f"A baseline (scale {a['scale']}): ~{a['token_after']:,} tokens "
f"(-{a['saving_pct']}%), {a['elisions']} elisions, "
f"{a['signal_lines_lost']} signal lines lost")
print(f"B dividend (scale {b['scale']}): ~{b['token_after']:,} tokens "
f"(-{b['saving_pct']}%), {b['elisions']} elisions, "
f"{b['signal_lines_lost']} signal lines lost")
print(f"extra savings from B: ~{res['extra_saving_tokens']:,} tokens")
return 0


Expand Down
30 changes: 15 additions & 15 deletions claude-context-kernel/bench/exploration_ab.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,21 +69,21 @@
"CK_DELTA": "0",
"CK_PRETOOL": "0"}

PROMPT = """Lavori nel repository nella directory corrente. Un utente riporta \
questo errore, in forma parziale come capita nella realta':
PROMPT = """You are working in the repository in the current directory. A user \
reports this error, in the partial form you get in the real world:

{symptom}

Trova il punto ESATTO del repository che SOLLEVA questo errore. Esplora coi \
tool a disposizione (read, grep, find, bash in sola lettura). Non modificare \
nulla. Quando sei sicuro, l'ULTIMA riga della risposta deve essere SOLO:
percorso/relativo/file.py::nome_funzione"""
Find the EXACT point in the repository that RAISES this error. Explore with \
the tools available (read, grep, find, read-only bash). Do not modify \
anything. When you are sure, the LAST line of your answer must be ONLY:
relative/path/file.py::function_name"""

INJECT = """

[context-kernel] Un tool ha gia' calcolato il working set rilevante per \
questo sintomo (slicer deterministico sul grafo degli import). Usalo come \
prior, non come divieto:
[context-kernel] A tool has already computed the working set relevant to \
this symptom (deterministic slicer over the import graph). Use it as a \
prior, not as a prohibition:

{manifest}"""

Expand All @@ -96,13 +96,13 @@ def degraded_symptom(err: str, msg: str, caller: str,
l'esplorazione deve navigare il grafo: e' il regime dove l'ipotesi
prevede che la correttezza possa divergere tra i bracci."""
if difficulty == "hard":
return (f"{err} da qualche parte durante l'esecuzione.\n"
f"(nessun messaggio disponibile, traceback perso; il modulo "
f"coinvolto lato applicazione dovrebbe essere {caller})")
return (f"{err} somewhere during execution.\n"
f"(no message available, traceback lost; the module involved "
f"on the application side should be {caller})")
vague = " ".join(msg.split()[:3])
return (f"{err}: {vague} ...\n"
f"(il traceback e' andato perso; il modulo coinvolto lato "
f"applicazione dovrebbe essere {caller})")
f"(the traceback was lost; the module involved on the application "
f"side should be {caller})")


def slice_manifest(root: str, symptom: str) -> str:
Expand Down Expand Up @@ -188,7 +188,7 @@ def main() -> int:
reverse.setdefault(d, set()).add(f)
cases = sb.find_cases(rs, root, files, reverse, args.cases)
if not cases:
print("nessun caso", file=sys.stderr)
print("no case", file=sys.stderr)
return 2

rows = []
Expand Down
6 changes: 3 additions & 3 deletions claude-context-kernel/bench/sufficiency_bench.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ def run(root: str, n_cases: int, as_json: bool) -> int:
root = os.path.abspath(root)
files = [f.replace(os.sep, "/") for f in rs.collect_files(root)]
if not files:
print("nessun sorgente", file=sys.stderr)
print("no source", file=sys.stderr)
return 2
graph = rs.build_graph(root, files)
reverse: dict[str, set[str]] = {}
Expand All @@ -85,7 +85,7 @@ def run(root: str, n_cases: int, as_json: bool) -> int:

cases = find_cases(rs, root, files, reverse, n_cases)
if not cases:
print("nessun raise-site candidato (con caller e messaggio letterale)",
print("no candidate raise site (with caller and literal message)",
file=sys.stderr)
return 2

Expand Down Expand Up @@ -123,7 +123,7 @@ def run(root: str, n_cases: int, as_json: bool) -> int:
return 0
print(f"# sufficiency bench — {root}")
print(f"sorgenti: {len(files)} | casi: {len(cases)} "
f"(raise-site reali, sintomo parziale: frame del caller + messaggio)")
f"(real raise sites, partial symptom: caller frame + message)")
print(f"{'deps':>5} {'imp':>4} {'suff.':>7} {'rate':>7} {'slice medio':>12}")
for r in results:
print(f"{r['deps_depth']:>5} {r['importers_depth']:>4} "
Expand Down
20 changes: 10 additions & 10 deletions claude-context-kernel/docs/ck_demo_gif.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def _seg_color(line: str):
("[--]", DIM)):
if stripped.startswith(token):
return [(indent + token, col), (stripped[len(token):], FG)]
if stripped.startswith("VERDETTO"):
if stripped.startswith("VERDICT"):
return [(line, GREEN)]
if stripped.startswith("→"): # freccia routing
return [(line, CYAN)]
Expand Down Expand Up @@ -100,19 +100,19 @@ def _frame(lines, cursor=True):
"context-kernel — doctor",
"========================================",
" [ok] Python 3.10.7",
" [ok] hook registrati (hooks.json)",
" [ok] script core presenti (6)",
" [ok] comandi /ck-* presenti (8)",
" [ok] linguaggio naturale (kernel-ops)",
" [ok] canary verde",
" [ok] A/B: coda vuota",
" VERDETTO: tutto a posto ✓",
" [ok] hooks registered (hooks.json)",
" [ok] core scripts present (6)",
" [ok] /ck-* commands present (8)",
" [ok] natural language (kernel-ops)",
" [ok] canary green",
" [ok] A/B: queue empty",
" VERDICT: all clear ✓",
]
DOOR2 = [
" → kernel-ops: savings + canary + A/B",
" saved 1,214,963 tokens (−61%)",
" 1358 compressioni · canary verde",
" coda A/B vuota",
" 1358 compressions · canary green",
" A/B queue empty",
]

frames: list[Image.Image] = []
Expand Down
98 changes: 49 additions & 49 deletions claude-context-kernel/hooks/ab_verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@
CK_AB_TIMEOUT timeout per giudizio in secondi (default 180)

Contratto col giudice: l'ULTIMA riga della sua risposta deve essere
VERDETTO: INVARIANTE
VERDICT: INVARIANT
oppure
VERDETTO: DEGRADATO — <i segnali persi>
VERDICT: DEGRADED — <i segnali persi>
Risposte non parsabili lasciano il campione in attesa (max 3 tentativi).
"""
from __future__ import annotations
Expand Down Expand Up @@ -56,30 +56,30 @@
TIMEOUT_S = int(os.environ.get("CK_AB_TIMEOUT", "180"))
MAX_ATTEMPTS = 3

VERDICT_OK = "VERDETTO: INVARIANTE"
VERDICT_BAD = "VERDETTO: DEGRADATO"
VERDICT_OK = "VERDICT: INVARIANT"
VERDICT_BAD = "VERDICT: DEGRADED"

PROMPT = """\
Sei un giudice di answer-invariance (T4 della pipeline context-kernel).
Sotto trovi l'ORIGINALE di un output di tool ({tool}) e la versione COMPRESSA
che e' entrata al suo posto nel contesto di un agente di coding.

Giudica UNA cosa sola: la compressa conserva tutti i segnali su cui l'agente
potrebbe dover agire? (errori, warning, esiti di test, path, versioni,
conteggi, nomi di simboli). I marcatori [context-kernel: ...] e [x N] sono
attesi e legittimi: non sono perdita di segnale. La perdita di righe di puro
rumore (progress, log ripetitivi, corpo di codice riassunto in firme) e'
il comportamento voluto: e' DEGRADATO solo se manca un segnale AZIONABILE.

Rispondi in massimo 6 righe. L'ULTIMA riga deve essere esattamente:
VERDETTO: INVARIANTE
oppure:
VERDETTO: DEGRADATO — <i segnali persi, brevi>

=== ORIGINALE ===
You are an answer-invariance judge (T4 of the context-kernel pipeline).
Below is the ORIGINAL of a tool output ({tool}) and the COMPRESSED version
that entered a coding agent's context in its place.

Judge ONE thing only: does the compressed version preserve every signal the
agent might have to act on? (errors, warnings, test outcomes, paths, versions,
counts, symbol names). The [context-kernel: ...] and [x N] markers are
expected and legitimate: they are not signal loss. Losing lines of pure noise
(progress, repetitive logs, code bodies summarised into signatures) is the
intended behaviour: it is DEGRADED only if an ACTIONABLE signal is missing.

Answer in at most 6 lines. The LAST line must be exactly:
VERDICT: INVARIANT
or:
VERDICT: DEGRADED — <the lost signals, briefly>

=== ORIGINAL ===
{original}

=== COMPRESSA ===
=== COMPRESSED ===
{compressed}
"""

Expand Down Expand Up @@ -120,7 +120,7 @@ def _sample_label(s: dict) -> str:
ts = s.get("ts")
when = (datetime.datetime.fromtimestamp(ts).isoformat(timespec="seconds")
if ts else "?")
return f"{s.get('tool', '?')} {where} ({when}, sessione {s.get('session', '?')})"
return f"{s.get('tool', '?')} {where} ({when}, session {s.get('session', '?')})"


def _judge(prompt: str) -> str | None:
Expand All @@ -139,15 +139,15 @@ def _judge(prompt: str) -> str | None:
encoding="utf-8", errors="replace",
env={**os.environ, "PYTHONIOENCODING": "utf-8"})
except FileNotFoundError:
print(f"Giudice non trovato: `{CLAUDE_BIN}`. Imposta CK_AB_CLAUDE "
"o installa Claude Code.", file=sys.stderr)
print(f"Judge not found: `{CLAUDE_BIN}`. Set CK_AB_CLAUDE "
"or install Claude Code.", file=sys.stderr)
return None
except subprocess.TimeoutExpired:
print(f"Giudizio scaduto dopo {TIMEOUT_S}s.", file=sys.stderr)
print(f"Judging timed out after {TIMEOUT_S}s.", file=sys.stderr)
return None
if proc.returncode != 0:
err = (proc.stderr or "").strip().split("\n")[-1:]
print(f"Giudice uscito con {proc.returncode}: {' '.join(err)}",
print(f"Judge exited with {proc.returncode}: {' '.join(err)}",
file=sys.stderr)
return None
return proc.stdout
Expand Down Expand Up @@ -176,21 +176,21 @@ def print_cron() -> int:
+ (f' CK_AB_MODEL="{MODEL}"' if MODEL else "")
+ f' "{sys.executable}" "{me}" --limit 5'
f' >> "$HOME/.context-kernel-ab-cron.log" 2>&1')
print("Riga per `crontab -e` (giudizio A/B quotidiano alle 09:30, "
"max 5 campioni, log in ~/.context-kernel-ab-cron.log):\n")
print("Line for `crontab -e` (daily A/B judging at 09:30, "
"max 5 samples, log in ~/.context-kernel-ab-cron.log):\n")
print(line)
print("\nRicorda: ab_verify.py e' l'unico comando del plugin che MANDA "
"contenuti a un modello (README §11). Installa il cron solo se "
"questo ti sta bene; in alternativa il brief di SessionStart "
"ricorda i campioni in attesa a ogni nuova sessione.")
print("\nReminder: ab_verify.py is the only command in the plugin that SENDS "
"content to a model (README §11). Install the cron job only if "
"you are comfortable with that; otherwise the SessionStart brief "
"reminds you of pending samples at the start of every session.")
return 0


def status(st: dict) -> str:
pend = len(st.get("pending", []))
return (f"A/B invariance: {st.get('ok', 0)} invarianti, "
f"{st.get('degraded', 0)} degradate, {pend} campioni in attesa "
f"(campionate {st.get('counter', 0)} elisioni)")
return (f"A/B invariance: {st.get('ok', 0)} invariant, "
f"{st.get('degraded', 0)} degraded, {pend} samples pending "
f"({st.get('counter', 0)} elisions sampled)")


def main() -> int:
Expand All @@ -203,19 +203,19 @@ def main() -> int:
try:
limit = int(argv[argv.index("--limit") + 1])
except (IndexError, ValueError):
print("--limit vuole un intero.", file=sys.stderr)
print("--limit expects an integer.", file=sys.stderr)
return 2

st = _load()
if "--status" in argv:
print(status(st))
return 0
if not st["pending"]:
print(f"Nessun campione in attesa. {status(st)}")
print(f"No samples pending. {status(st)}")
return 0

todo = st["pending"] if limit is None else st["pending"][:limit]
print(f"{len(todo)} campioni da giudicare (giudice: {CLAUDE_BIN}"
print(f"{len(todo)} samples to judge (judge: {CLAUDE_BIN}"
f"{' --model ' + MODEL if MODEL else ''})\n")

judged_any = False
Expand All @@ -227,12 +227,12 @@ def main() -> int:
original = _unpack(s.get("orig_z", ""))
compressed = _unpack(s.get("comp_z", ""))
if not original or not compressed:
print(f" scartato (campione illeggibile): {_sample_label(s)}")
print(f" discarded (unreadable sample): {_sample_label(s)}")
continue
prompt = PROMPT.format(tool=s.get("tool", "?"),
original=original, compressed=compressed)
if dry:
print(f"--- prompt per {_sample_label(s)} ---")
print(f"--- prompt for {_sample_label(s)} ---")
print(prompt)
still.append(s)
continue
Expand All @@ -241,38 +241,38 @@ def main() -> int:
if verdict is None:
s["attempts"] = s.get("attempts", 0) + 1
if s["attempts"] >= MAX_ATTEMPTS:
print(f" scartato dopo {MAX_ATTEMPTS} tentativi: "
print(f" discarded after {MAX_ATTEMPTS} attempts: "
f"{_sample_label(s)}")
else:
still.append(s)
print(f" rimandato (risposta non parsabile, tentativo "
print(f" deferred (unparsable answer, attempt "
f"{s['attempts']}): {_sample_label(s)}")
continue
judged_any = True
kind, detail = verdict
iso = datetime.datetime.now().isoformat(timespec="seconds")
if kind == "ok":
st["ok"] = st.get("ok", 0) + 1
print(f" ✓ INVARIANTE {_sample_label(s)}")
print(f" ✓ INVARIANT {_sample_label(s)}")
else:
st["degraded"] = st.get("degraded", 0) + 1
st["degradations"] = (st.get("degradations", []) + [{
"ts": iso, "tool": s.get("tool"), "file": s.get("file"),
"session": s.get("session"), "missing": detail,
}])[-20:]
print(f" ⚠ DEGRADATO {_sample_label(s)}")
print(f" ⚠ DEGRADED {_sample_label(s)}")
if detail:
print(f" persi: {detail}")
print(f" lost: {detail}")
st["last_run"] = iso

st["pending"] = still
if not dry:
_save(st)
print(f"\n{status(st)}")
if judged_any and st.get("degraded"):
print("Le degradazioni (ultime 20) sono in "
f"{AB_STATE} -> 'degradations': usale per tarare "
"le euristiche di compress.py.")
print("The degradations (last 20) are in "
f"{AB_STATE} -> 'degradations': use them to tune "
"the heuristics in compress.py.")
return 0


Expand Down
Loading
Loading