diff --git a/README.md b/README.md
index b392d62..952b950 100644
--- a/README.md
+++ b/README.md
@@ -105,6 +105,9 @@ is overhead.
+Watch all three in the [nine-second overview](https://simplejev.org/JevAny/assets/media/docs/jev-agent-harness.mp4),
+or open the [agent harness demos](https://simplejev.org/JevAny/docs/agent-harness.html).
+
Broader paired evaluations show that gains vary by task. The [full results](https://simplejev.org/JevAny/docs/reports-JevAny_Tech_Report_Agent_Harness_Appendix.html),
[delegation protocol](https://simplejev.org/JevAny/docs/experiments-AGENT_HARNESS_FRONTIER_PROTOCOL.html), and
[technical report](https://simplejev.org/JevAny/files/reports/JevAny_Tech_Report.pdf) describe
diff --git a/README.zh-CN.md b/README.zh-CN.md
index 868eac1..4edf585 100644
--- a/README.zh-CN.md
+++ b/README.zh-CN.md
@@ -96,6 +96,9 @@
+三个案例也可在[九秒总览](https://simplejev.org/JevAny/assets/media/docs/jev-agent-harness.mp4)中连续观看，或打开
+[Agent harness 演示页](https://simplejev.org/JevAny/docs/agent-harness.html)查看单独回放。
+
下表展示更多配对评测结果，收益随任务而变化。[完整结果](https://simplejev.org/JevAny/docs/reports-JevAny_Tech_Report_Agent_Harness_Appendix.html)、
[委托协议](https://simplejev.org/JevAny/docs/experiments-AGENT_HARNESS_FRONTIER_PROTOCOL.html)和
[技术报告](https://simplejev.org/JevAny/files/reports/JevAny_Tech_Report.pdf)说明了 Jev 适合处理哪些选择，以及何时交回 LLM。
diff --git a/docs/demos/jev-agent-harness.gif b/docs/demos/jev-agent-harness.gif
index bbd666a..c7560a6 100644
Binary files a/docs/demos/jev-agent-harness.gif and b/docs/demos/jev-agent-harness.gif differ
diff --git a/docs/demos/jev-decision-frozen-lake-v2.gif b/docs/demos/jev-decision-frozen-lake-v2.gif
index 501b808..c1f28d6 100644
Binary files a/docs/demos/jev-decision-frozen-lake-v2.gif and b/docs/demos/jev-decision-frozen-lake-v2.gif differ
diff --git a/docs/demos/jev-decision-terminal-v2.gif b/docs/demos/jev-decision-terminal-v2.gif
index 511801f..30c1f01 100644
Binary files a/docs/demos/jev-decision-terminal-v2.gif and b/docs/demos/jev-decision-terminal-v2.gif differ
diff --git a/docs/demos/jev-decision-webshop-v2.gif b/docs/demos/jev-decision-webshop-v2.gif
index 26d09b5..d912dba 100644
Binary files a/docs/demos/jev-decision-webshop-v2.gif and b/docs/demos/jev-decision-webshop-v2.gif differ
diff --git a/scripts/render_agent_decision_demos_v2.py b/scripts/render_agent_decision_demos_v2.py
index 6bbe99b..09655d8 100644
--- a/scripts/render_agent_decision_demos_v2.py
+++ b/scripts/render_agent_decision_demos_v2.py
@@ -12,10 +12,13 @@
from functools import lru_cache
import json
from pathlib import Path
+import shutil
+import subprocess
-from PIL import Image, ImageDraw, ImageFont
+from PIL import Image, ImageChops, ImageColor, ImageDraw, ImageFont
-from render_agent_harness_demo import ACCENT, BLUE, BORDER, MUTED, PANEL, PANEL_2, TEXT
+from render_agent_harness_demo import ACCENT, AMBER, BLUE, BORDER, MUTED, PANEL, PANEL_2, TEXT
+from render_agent_robot_icons import ROBOT_COLORS, actor_icon
WIDTH, HEIGHT = 1080, 700
@@ -24,6 +27,13 @@
END_HOLD_MS = 600
BACKGROUND = "#070b14"
GREEN_PANEL = "#0b2426"
+BLUE_PANEL = "#14223a"
+AMBER_PANEL = "#292313"
+TASK_CONTEXT = {
+ "webshop": "z-dark green · small · under $80",
+ "frozen_lake": "Start [1, 0] → goal [0, 3]",
+ "sqlite": "Target: 10 verified rows",
+}
@lru_cache
@@ -83,7 +93,125 @@ def text(draw: ImageDraw.ImageDraw, x: float, y: float, value: str,
draw.text((x, y), value, font=font(size, bold, mono), fill=color)
-def webshop_scene(draw: ImageDraw.ImageDraw, x: int, state: int, color: str) -> None:
+def right_text(draw: ImageDraw.ImageDraw, right: float, y: float, value: str,
+ size: int = 18, color: str = MUTED, bold: bool = False) -> None:
+ text(draw, right - draw.textlength(value, font=font(size, bold)), y,
+ value, size, color, bold)
+
+
+def actor_style(actor: str) -> tuple[str, str]:
+ return (ACCENT, GREEN_PANEL) if actor == "Jev" else (
+ (BLUE, BLUE_PANEL) if actor == "LLM" else (AMBER, AMBER_PANEL)
+ )
+
+
+def wrapped_text(draw: ImageDraw.ImageDraw, x: int, y: int, value: str,
+ width: int, size: int = 17, color: str = MUTED) -> None:
+ lines = [""]
+ for word in value.split():
+ candidate_line = f"{lines[-1]} {word}".strip()
+ if draw.textlength(candidate_line, font=font(size)) > width and lines[-1]:
+ lines.append(word)
+ else:
+ lines[-1] = candidate_line
+ for index, line in enumerate(lines):
+ text(draw, x, y + index * 20, line, size, color)
+
+
+def phase_panel(image: Image.Image, draw: ImageDraw.ImageDraw, x: int, key: str,
+ step: Step, index: int, count: int, finished: bool, tick: int) -> None:
+ actor = ("Verifier" if key == "sqlite" else "Env") if finished else step.actor
+ color, fill = actor_style(actor)
+ draw.rounded_rectangle((x + 20, 414, x + 484, 533), 12, fill=PANEL_2)
+ draw.rounded_rectangle((x + 29, 423, x + 123, 524), 10,
+ fill=fill, outline=color, width=2)
+ portrait = actor_icon(actor, 78, phase=0 if finished else tick // 100 % 6)
+ image.paste(portrait, (x + 37, 424), portrait)
+ label = "CHECK" if actor == "Verifier" else "RESULT" if actor == "Env" else actor.upper()
+ center = x + 76
+ text(draw, center - draw.textlength(label, font=font(13, True)) / 2,
+ 503, label, 13, color, True)
+ text(draw, x + 141, 429,
+ "VERIFIED RESULT" if finished else f"PHASE {index + 1} / {count}", 13, MUTED)
+ title = "✓ Task complete" if finished else step.title
+ text(draw, x + 141, 452, title, 24, ACCENT if finished else TEXT, True)
+ outcome = {"webshop": "Purchased, reward 1", "frozen_lake": "Goal reached, reward 1",
+ "sqlite": "10 rows recovered, reward 1"}[key]
+ wrapped_text(draw, x + 141, 491, outcome if finished else step.detail, 322)
+
+
+def actor_rail(image: Image.Image, draw: ImageDraw.ImageDraw, x: int, steps: list[Step],
+ current: int, finished: bool, tick: int) -> None:
+ # Each controller keeps one position, including when it regains control.
+ actors = list(dict.fromkeys(step.actor for step in steps))
+ gap = 8
+ width = (460 - gap * (len(actors) - 1)) / len(actors)
+ for index, actor in enumerate(actors):
+ left = x + 22 + index * (width + gap)
+ color, fill = actor_style(actor)
+ active = not finished and steps[current].actor == actor
+ draw.rounded_rectangle((left, 546, left + width, 589), 9,
+ fill=fill if active else PANEL,
+ outline=color if active else BORDER, width=2 if active else 1)
+ portrait = actor_icon(actor, 36, active, tick // 100 % 6 if active else 0)
+ image.paste(portrait, (round(left + 10), 548), portrait)
+ text(draw, left + 55, 557, actor.upper(), 15, color if active else MUTED, True)
+ if active:
+ draw.ellipse((left + width - 18, 564, left + width - 12, 570), fill=color)
+ text(draw, left + width - 69, 561, "ACTIVE", 9, color, True)
+ text(draw, x + 22, 594, "TASK COMPLETE" if finished else "ACTIVE CONTROLLER", 10, MUTED)
+
+
+def savings_strip(draw: ImageDraw.ImageDraw, baseline: dict, assisted: dict) -> None:
+ # Static totals from this recorded pair; percentages use the baseline.
+ tokens_saved = baseline["tokens"] - assisted["tokens"]
+ time_saved = baseline["wall_time_seconds"] - assisted["wall_time_seconds"]
+ rewards = baseline["reward"], assisted["reward"]
+ both_succeeded = all(reward == 1 for reward in rewards)
+ outcome = "Both succeeded" if both_succeeded else (
+ "Same reward" if rewards[0] == rewards[1] else "Reward changed"
+ )
+ cards = [
+ ("LLM TOKENS SAVED", f"{tokens_saved:,}",
+ f"{tokens_saved / baseline['tokens']:.1%}",
+ f"{baseline['tokens']:,} → {assisted['tokens']:,}", ACCENT),
+ ("TIME SAVED", f"{time_saved:g} s",
+ f"{time_saved / baseline['wall_time_seconds']:.1%}",
+ f"{baseline['wall_time_seconds']:g} s → {assisted['wall_time_seconds']:g} s", ACCENT),
+ ("TASK RESULT", outcome, "✓" if both_succeeded else "",
+ f"Reward {rewards[0]:g} → {rewards[1]:g}", AMBER),
+ ]
+ for index, (label, value, badge, detail, color) in enumerate(cards):
+ x = 24 + index * 348
+ metric = index < 2
+ draw.rounded_rectangle((x, 616, x + 336, 676), 10,
+ fill=GREEN_PANEL if metric else PANEL,
+ outline=ACCENT if metric else BORDER)
+ text(draw, x + 14, 619, label, 14, MUTED, True)
+ if metric:
+ text(draw, x + 14, 631, badge, 42, color, True)
+ right_text(draw, x + 322, 620, value, 28, TEXT, True)
+ right_text(draw, x + 322, 651, detail, 15)
+ else:
+ text(draw, x + 14, 632, value, 26, TEXT, True)
+ right_text(draw, x + 322, 637, badge, 24, color, True)
+ text(draw, x + 14, 657, detail, 15, MUTED)
+
+
+def candidate(draw: ImageDraw.ImageDraw, x: int, y: int, width: int,
+ label: str, selected: bool, color: str = ACCENT) -> None:
+ fill = BLUE_PANEL if color == BLUE else GREEN_PANEL
+ draw.rounded_rectangle((x, y, x + width, y + 41), 8,
+ fill=fill if selected else PANEL_2,
+ outline=color if selected else BORDER, width=2)
+ text(draw, x + 13, y + 7, label, 21, color if selected else MUTED, selected)
+ if selected:
+ text(draw, x + width - 32, y + 5, "✓", 23, color, True)
+
+
+def webshop_scene(draw: ImageDraw.ImageDraw, x: int, trace: dict, step: Step,
+ assisted: bool, finished: bool, color: str) -> None:
+ state = step.state
if state == 0:
text(draw, x + 22, 220, "SEARCH", 22, MUTED, True)
draw.rounded_rectangle((x + 20, 264, x + 444, 324), 12, fill=PANEL_2, outline=BORDER)
@@ -99,10 +227,20 @@ def webshop_scene(draw: ImageDraw.ImageDraw, x: int, state: int, color: str) ->
]
draw.polygon(jacket, fill="#277464" if state >= 2 else "#3d4e67")
draw.line((x + 93, 222, x + 91, 357), fill="#9fbab5", width=3)
+ if assisted and step.actor == "Jev" and not finished:
+ decision = trace["candidate_groups"][state - 2]
+ kind = "color" if state == 2 else "size"
+ text(draw, x + 179, 216, f"LLM {kind} menu", 24, BLUE, True)
+ for index, action in enumerate(decision["actions"]):
+ candidate(draw, x + 176, 256 + index * 47, 265,
+ action.removeprefix("click[").removesuffix("]"),
+ action == decision["selected"])
+ text(draw, x + 29, 377, "3 alternatives", 17, MUTED)
+ return
text(draw, x + 179, 216, "Women's blazer", 26, bold=True)
for y, label, selected in ((265, "z-dark green", state >= 2), (327, "small", state >= 3)):
draw.rounded_rectangle((x + 176, y, x + 441, y + 48), 10,
- fill=GREEN_PANEL if selected else PANEL_2,
+ fill=(GREEN_PANEL if assisted else BLUE_PANEL) if selected else PANEL_2,
outline=color if selected else BORDER, width=2)
text(draw, x + 192, y + 9, label, 24, color if selected else MUTED)
if selected:
@@ -110,7 +248,7 @@ def webshop_scene(draw: ImageDraw.ImageDraw, x: int, state: int, color: str) ->
def lake_scene(draw: ImageDraw.ImageDraw, x: int, trace: dict, step: Step,
- fraction: float, finished: bool, color: str) -> None:
+ fraction: float, finished: bool, color: str, assisted: bool) -> None:
positions = [tuple(s["player"]) for s in trace["steps"]] + [(0, 3)]
state = step.state
player = positions[state]
@@ -129,28 +267,31 @@ def lake_scene(draw: ImageDraw.ImageDraw, x: int, trace: dict, step: Step,
text(draw, left + 10, top + 4, "G" if goal else "×", 23, ACCENT if goal else "#fb7185", True)
cy, cx = gy + player[0] * cell + 19, gx + player[1] * cell + 19
draw.ellipse((cx - 15, cy - 15, cx + 15, cy + 15), fill=color)
- text(draw, x + 241, 216, "4 directions", 24, MUTED)
+ text(draw, x + 241, 216, "1 LLM plan" if assisted else "4 directions",
+ 24, BLUE if assisted else MUTED)
for i, action in enumerate(trace["action_menu"]):
left, top = x + 238 + (i % 2) * 105, 262 + (i // 2) * 57
selected = state > 0 and action == trace["steps"][state - 1]["action"]
draw.rounded_rectangle((left, top, left + 96, top + 44), 9,
- fill=GREEN_PANEL if selected else PANEL_2,
+ fill=(GREEN_PANEL if assisted else BLUE_PANEL) if selected else PANEL_2,
outline=color if selected else BORDER, width=2)
text(draw, left + 12, top + 8, action, 23, color if selected else MUTED, selected)
-def terminal_scene(draw: ImageDraw.ImageDraw, x: int, state: int,
+def terminal_scene(draw: ImageDraw.ImageDraw, x: int, trace: dict, state: int,
assisted: bool, finished: bool) -> None:
draw.rounded_rectangle((x + 20, 207, x + 444, 393), 12, fill="#080e1b", outline=BORDER)
+ if assisted and state < 2:
+ text(draw, x + 38, 220, "trunc.db · 3 inspections", 19, MUTED, mono=True)
+ for index, option in enumerate(trace["options"]):
+ candidate(draw, x + 32, 250 + index * 46, 398, option["name"],
+ state == 1 and option["name"] == trace["jev_choice"])
+ return
text(draw, x + 38, 220, "trunc.db", 23, MUTED, mono=True)
if state == 0:
- lines = ("raw page | schema | tools", "Jev chooses the inspection") if assisted else (
- "damaged SQLite page", "LLM chooses inspections",
- )
+ lines = ("damaged SQLite page", "LLM chooses inspections")
elif state == 1:
- lines = ("$ od ... trunc.db", "0d 00 00 00 0a 0f 49 ...") if assisted else (
- "inspect page bytes", "interpret page structure",
- )
+ lines = ("inspect page bytes", "interpret page structure")
elif state == 2:
lines = ("cell pointers → records", "parse recoverable rows")
else:
@@ -184,6 +325,7 @@ def render_frame(key: str, trace: dict, tick: int) -> Image.Image:
image = Image.new("RGB", (WIDTH, HEIGHT), BACKGROUND)
draw = ImageDraw.Draw(image)
text(draw, 28, 21, titles[key], 28, bold=True)
+ right_text(draw, WIDTH - 28, 30, TASK_CONTEXT[key], 17)
for lane, run in enumerate((baseline, assisted)):
x = 24 + lane * 528
color = ACCENT if lane else BLUE
@@ -198,43 +340,36 @@ def render_frame(key: str, trace: dict, tick: int) -> Image.Image:
draw.rounded_rectangle((x, 73, x + 504, 608), 18, fill=PANEL, outline=color, width=2)
text(draw, x + 22, 90, "LLM + Jev" if lane else "LLM only", 30, color, True)
- text(draw, x + 22, 131, f"{elapsed:.2f} s" if key == "webshop" else f"{elapsed:.1f} s", 40, bold=True)
- text(draw, x + 280, 148, "Completed" if finished else "Running…", 28, ACCENT if finished else MUTED, True)
+ calls = str(run["llm_calls"])
+ text(draw, x + 22, 124, calls, 44, color, True)
+ label_x = x + 34 + draw.textlength(calls, font=font(44, True))
+ text(draw, label_x, 131, "RECORDED TOTAL", 10, MUTED)
+ text(draw, label_x, 149, "LLM call" if run["llm_calls"] == 1 else "LLM calls", 21, TEXT)
+ text(draw, x + 300, 105, "COMPLETED" if finished else "ELAPSED TIME",
+ 11, ACCENT if finished else MUTED, True)
+ draw.rounded_rectangle((x + 300, 125, x + 484, 188), 10,
+ fill=BACKGROUND, outline=BORDER)
+ text(draw, x + 314, 133,
+ f"{elapsed:.2f} s" if key == "webshop" else f"{elapsed:.1f} s", 29, TEXT, True)
+ draw.rounded_rectangle((x + 314, 178, x + 470, 182), 2, fill=BORDER)
+ length = round(156 * progress)
+ if length:
+ draw.rounded_rectangle((x + 314, 178, x + 314 + length, 182), 2, fill=color)
inner = x + 20
if key == "webshop":
- webshop_scene(draw, inner, step.state, color)
+ webshop_scene(draw, inner, trace, step, bool(lane), finished, color)
elif key == "frozen_lake":
- lake_scene(draw, inner, trace, step, fraction, finished, color)
+ lake_scene(draw, inner, trace, step, fraction, finished, color, bool(lane))
else:
- terminal_scene(draw, inner, step.state, bool(lane), finished)
-
- draw.rounded_rectangle((x + 20, 414, x + 484, 533), 12,
- fill=GREEN_PANEL if finished else PANEL_2)
- if finished:
- text(draw, x + 36, 427, "✓ Completed", 32, ACCENT, True)
- outcome = {"webshop": "Purchased, reward 1", "frozen_lake": "Goal reached, reward 1",
- "sqlite": "10 rows recovered, reward 1"}[key]
- text(draw, x + 36, 480, outcome, 25)
- else:
- actor_color = ACCENT if step.actor == "Jev" else BLUE
- text(draw, x + 36, 425, f"{step.actor}: {step.title}", 26, actor_color, True)
- text(draw, x + 36, 477, step.detail, 24, MUTED)
- draw.rounded_rectangle((x + 22, 551, x + 482, 560), 4, fill=BORDER)
- length = round(460 * progress)
- if length:
- draw.rounded_rectangle((x + 22, 551, x + 22 + length, 560), 4, fill=color)
- text(draw, x + 22, 577,
- f"{run['steps']} {run['step_label']} · {run['llm_calls']} LLM calls · {run['tokens']:,} tokens",
- 20, MUTED)
- saved = baseline["wall_time_seconds"] - assisted["wall_time_seconds"]
- step_saved = baseline["steps"] - assisted["steps"]
- call_saved = baseline["llm_calls"] - assisted["llm_calls"]
- token_saved = baseline["tokens"] - assisted["tokens"]
- text(draw, 28, 626,
- f"Saved: {step_saved} {baseline['step_label']} · {call_saved} LLM calls · "
- f"{token_saved:,} tokens · {saved:.2f} s",
- 25, ACCENT, True)
- text(draw, 28, 668, f"Illustrated steps; measured totals at {longest * 1000 / PLAY_MS:.2f}× playback", 23, MUTED)
+ terminal_scene(draw, inner, trace, step.state, bool(lane), finished)
+
+ phase_panel(image, draw, x, key, step, index, len(steps), finished, tick)
+ actor_rail(image, draw, x, steps, index, finished, tick)
+ savings_strip(draw, baseline, assisted)
+ text(draw, 28, 681,
+ f"One recorded pair · illustrated steps · measured totals at {longest * 1000 / PLAY_MS:.2f}× playback",
+ 13, MUTED)
+ right_text(draw, WIDTH - 28, 681, "LLM only → LLM + Jev", 13)
return image
@@ -254,10 +389,29 @@ def save(path: str, key: str, trace: dict) -> None:
for i, sample in enumerate(samples):
sheet.paste(sample, (0, 175 * i))
palette = sheet.quantize(colors=128)
- frames = [
- render_frame(key, trace, tick).quantize(palette=palette, dither=Image.Dither.NONE)
- for tick in ticks
- ]
+ # Keep controller and portrait colors exact even when small art is undersampled.
+ # Remaining entries retain the sampled colors for antialiased text and art.
+ theme = (BACKGROUND, PANEL, PANEL_2, BORDER, TEXT, MUTED, ACCENT, BLUE,
+ AMBER, GREEN_PANEL, BLUE_PANEL, AMBER_PANEL) + ROBOT_COLORS
+ colors = palette.getpalette()[:128 * 3]
+ colors += [channel for color in theme for channel in ImageColor.getrgb(color)]
+ colors += list(ImageColor.getrgb(BACKGROUND)) * ((768 - len(colors)) // 3)
+ palette.putpalette(colors)
+ frames = []
+ for tick in ticks:
+ original = render_frame(key, trace, tick)
+ frame = original.quantize(palette=palette, dither=Image.Dither.NONE)
+ # Quantization can reuse a nearby swatch even for an exact palette color.
+ # Pin flat theme pixels; retain sampled colors for antialiased edges.
+ channels = original.split()
+ for index, color in enumerate(theme, start=128):
+ masks = [
+ channel.point([255 if value == target else 0 for value in range(256)])
+ for channel, target in zip(channels, ImageColor.getrgb(color))
+ ]
+ mask = ImageChops.multiply(ImageChops.multiply(masks[0], masks[1]), masks[2])
+ frame.paste(index, mask=mask)
+ frames.append(frame)
output = Path(path)
output.parent.mkdir(parents=True, exist_ok=True)
frames[0].save(output, save_all=True, append_images=frames[1:], duration=durations,
@@ -267,17 +421,80 @@ def save(path: str, key: str, trace: dict) -> None:
f"{output.stat().st_size / 1024:.0f} KiB")
+def save_overview(path: str, sources: list[str]) -> None:
+ """Join the three approved loops without changing their pixels or timing."""
+ frames = []
+ durations = []
+ for source in sources:
+ with Image.open(source) as clip:
+ if clip.size != (WIDTH, HEIGHT):
+ raise ValueError(f"{source}: expected a {WIDTH}×{HEIGHT} replay")
+ for index in range(clip.n_frames):
+ clip.seek(index)
+ original = clip.convert("RGB")
+ frame = original.quantize(colors=256, dither=Image.Dither.NONE)
+ if ImageChops.difference(original, frame.convert("RGB")).getbbox():
+ raise ValueError(f"{source}: frame {index} cannot be preserved exactly")
+ frames.append(frame)
+ durations.append(clip.info["duration"])
+ output = Path(path)
+ output.parent.mkdir(parents=True, exist_ok=True)
+ frames[0].save(output, save_all=True, append_images=frames[1:], duration=durations,
+ loop=0, optimize=False, disposal=1)
+ print(f"{output}: {len(frames)} frames, {sum(durations) / 1000:.1f}s, "
+ f"{output.stat().st_size / 1024:.0f} KiB")
+
+
+def save_site_media(sources: list[str]) -> None:
+ """Export only this presentation's videos and posters at native resolution."""
+ executable = shutil.which("ffmpeg")
+ if not executable:
+ try:
+ from imageio_ffmpeg import get_ffmpeg_exe
+ except ImportError as error:
+ raise SystemExit("Install ffmpeg or imageio-ffmpeg to export site media.") from error
+ executable = get_ffmpeg_exe()
+ directory = Path(__file__).resolve().parents[1] / "site/assets/media/docs"
+ directory.mkdir(parents=True, exist_ok=True)
+ for source in sources:
+ name = Path(source).stem
+ common = [
+ executable, "-nostdin", "-hide_banner", "-loglevel", "error", "-y",
+ "-ignore_loop", "1", "-i", source, "-an", "-vf", "setsar=1,fps=25",
+ "-pix_fmt", "yuv420p", "-threads", "2",
+ ]
+ subprocess.run(common + [
+ "-c:v", "libx264", "-crf", "26", "-preset", "fast",
+ "-movflags", "+faststart", str(directory / f"{name}.mp4"),
+ ], check=True)
+ subprocess.run(common + [
+ "-c:v", "libvpx-vp9", "-crf", "36", "-b:v", "0", "-cpu-used", "3",
+ "-row-mt", "1", str(directory / f"{name}.webm"),
+ ], check=True)
+ with Image.open(source) as image:
+ image.seek(min(12, image.n_frames - 1))
+ image.convert("RGB").save(directory / f"{name}.webp", quality=88, method=6)
+ print(f"Prepared site media: {name}")
+
+
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--trace", default="docs/demos/jev-agent-harness-traces.json")
- parser.add_argument("--terminal-out", default="docs/demos/jev-decision-terminal-v2.gif")
+ parser.add_argument("--terminal-out", "--sqlite-out", default="docs/demos/jev-decision-terminal-v2.gif")
parser.add_argument("--frozen-lake-out", default="docs/demos/jev-decision-frozen-lake-v2.gif")
parser.add_argument("--webshop-out", default="docs/demos/jev-decision-webshop-v2.gif")
+ parser.add_argument("--overview-out", "--out", default="docs/demos/jev-agent-harness.gif")
+ parser.add_argument("--site-media", action="store_true",
+ help="Also export MP4, WebM, and WebP for these four GIFs")
args = parser.parse_args()
trace = json.loads(Path(args.trace).read_text(encoding="utf-8"))
for key, output in (("webshop", args.webshop_out), ("frozen_lake", args.frozen_lake_out),
("sqlite", args.terminal_out)):
save(output, key, trace[key])
+ clips = [args.webshop_out, args.frozen_lake_out, args.terminal_out]
+ save_overview(args.overview_out, clips)
+ if args.site_media:
+ save_site_media([args.overview_out, *clips])
if __name__ == "__main__":
diff --git a/scripts/render_agent_harness_demo.py b/scripts/render_agent_harness_demo.py
index 6c76a5d..17e02f3 100644
--- a/scripts/render_agent_harness_demo.py
+++ b/scripts/render_agent_harness_demo.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Render recorded Terminal-Bench, WebShop, and FrozenLake harness traces."""
+"""Shared theme and legacy helpers; the CLI exports the current paired demos."""
from __future__ import annotations
@@ -444,7 +444,7 @@ def summary_frame() -> Image.Image:
return image
-def main() -> None:
+def legacy_main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--out", default="docs/demos/jev-agent-harness.gif")
parser.add_argument("--sqlite-out", default="docs/demos/jev-agent-harness-sqlite.gif")
@@ -563,5 +563,11 @@ def save(output_name: str, frames: list[Image.Image], durations: list[int]) -> N
)
+def main() -> None:
+ from render_agent_decision_demos_v2 import main as render_current_demos
+
+ render_current_demos()
+
+
if __name__ == "__main__":
main()
diff --git a/scripts/render_agent_robot_icons.py b/scripts/render_agent_robot_icons.py
new file mode 100644
index 0000000..76c811e
--- /dev/null
+++ b/scripts/render_agent_robot_icons.py
@@ -0,0 +1,127 @@
+"""Small, deterministic robot portraits for the recorded agent demos."""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from PIL import Image, ImageColor, ImageDraw
+
+from render_agent_harness_demo import ACCENT, AMBER, BLUE
+
+
+PORTRAIT_STYLES = {
+ "LLM": ("#c3dcff", "#4674a9", "#10233f"),
+ "Jev": ("#a9f0df", "#237768", "#092d2b"),
+ "idle": ("#43516a", "#334158", "#121b2c"),
+}
+IDLE_EDGE, EYE_LIGHT, IDLE_LIGHT = "#52617a", "#f1fbff", "#8290a6"
+ROBOT_COLORS = tuple(color for style in PORTRAIT_STYLES.values() for color in style) + (
+ IDLE_EDGE, EYE_LIGHT, IDLE_LIGHT,
+)
+
+
+@lru_cache(maxsize=128)
+def actor_icon(actor: str, size: int, active: bool = True, phase: int = 0) -> Image.Image:
+ """Return an antialiased portrait; motion is decorative, not measured time."""
+ image = Image.new("RGBA", (size * 3, size * 3))
+ draw = ImageDraw.Draw(image)
+ scale = size * 3 / 100
+ accent = BLUE if actor == "LLM" else ACCENT if actor == "Jev" else AMBER
+ edge = accent if active else IDLE_EDGE
+ shell, joint, face = PORTRAIT_STYLES[
+ "idle" if not active else "LLM" if actor == "LLM" else "Jev"
+ ]
+ light = EYE_LIGHT if active else IDLE_LIGHT
+ phase = phase % 6 if active else 0
+
+ def coords(values: tuple | list) -> tuple:
+ return tuple(round(value * scale) for value in values)
+
+ def rgba(color: str, alpha: int) -> tuple[int, int, int, int]:
+ return (*ImageColor.getrgb(color), alpha)
+
+ def line(points: tuple | list, fill: str | tuple, width: float = 2) -> None:
+ draw.line(coords(points), fill=fill, width=max(1, round(width * scale)),
+ joint="curve")
+
+ def rect(box: tuple, radius: float, fill: str | tuple,
+ outline: str | tuple | None = None, width: float = 1.5) -> None:
+ draw.rounded_rectangle(coords(box), round(radius * scale), fill=fill,
+ outline=outline, width=max(1, round(width * scale)))
+
+ def ellipse(box: tuple, fill: str | tuple, outline: str | tuple | None = None,
+ width: float = 1) -> None:
+ draw.ellipse(coords(box), fill=fill, outline=outline,
+ width=max(1, round(width * scale)))
+
+ def polygon(points: list[tuple], fill: str | tuple) -> None:
+ draw.polygon([coords(point) for point in points], fill=fill)
+
+ if active:
+ ellipse((7, 13, 95, 98), rgba(accent, 13))
+ ellipse((15, 22, 89, 94), rgba(accent, 14))
+ ellipse((24, 88, 80, 95), rgba(accent if active else edge, 35))
+
+ if actor == "LLM":
+ # Broad shell, stable stance, and three small planning lights.
+ rect((30, 68, 73, 85), 8, joint, edge)
+ rect((32, 82, 45, 90), 4, shell, edge)
+ rect((59, 82, 72, 90), 4, shell, edge)
+ rect((20, 69, 30, 80), 4, joint, edge)
+ rect((74, 69, 84, 80), 4, joint, edge)
+ rect((11, 42, 21, 59), 4, joint, edge)
+ rect((82, 42, 92, 59), 4, joint, edge)
+ line((51, 18, 51, 28), edge, 3)
+ ellipse((47, 11, 55, 19), shell, edge)
+ rect((19, 26, 84, 73), 15, shell, edge, 2)
+ rect((26, 36, 77, 65), 10, face)
+ line((36, 31, 65, 31), rgba(light, 190), 2)
+ rect((36, 44, 42, 55), 3, edge)
+ rect((61, 44, 67, 55), 3, edge)
+ ellipse((37, 45, 40, 48), light)
+ ellipse((62, 45, 65, 48), light)
+ rect((46, 59, 57, 61), 1, rgba(edge, 150))
+ for index in range(3):
+ left = 42 + index * 8
+ ellipse((left, 76, left + 4, 80), edge if active else joint)
+ if active:
+ line((66, 20, 75, 11, 87, 17), rgba(accent, 100), 1)
+ for index, (cx, cy) in enumerate(((66, 20), (75, 11), (87, 17))):
+ focused = index == phase // 2
+ radius = 3 if focused else 2
+ ellipse((cx - radius, cy - radius, cx + radius, cy + radius),
+ light if focused else accent)
+ elif actor == "Jev":
+ # Compact shell, forward stance, and short moving trails.
+ if active:
+ for index, y in enumerate((43, 55, 67)):
+ shift = (phase * 3 + index * 4) % 10
+ line((3 + shift, y, 23 + shift // 2, y),
+ rgba(accent, 115 + index * 45), 2)
+ rect((42, 66, 77, 80), 7, joint, edge)
+ polygon([(43, 77), (53, 79), (44, 88), (31, 88)], shell)
+ polygon([(66, 78), (75, 75), (87, 86), (77, 89)], shell)
+ line((34, 89, 44, 89), edge, 2)
+ line((77, 89, 87, 86), edge, 2)
+ rect((27, 59, 39, 68), 4, joint, edge)
+ rect((82, 56, 92, 64), 4, joint, edge)
+ rect((32, 31, 87, 69), 12, shell, edge, 2)
+ line((45, 35, 74, 32), rgba(light, 190), 2)
+ rect((40, 42, 83, 62), 8, face)
+ rect((48, 47, 54, 55), 3, edge)
+ rect((68, 46, 74, 54), 3, edge)
+ ellipse((49, 47, 52, 49), light)
+ ellipse((69, 46, 72, 48), light)
+ polygon([(65, 12), (54, 29), (63, 28), (59, 39),
+ (77, 20), (67, 21), (73, 12)], edge)
+ polygon([(60, 69), (54, 76), (59, 76), (56, 81),
+ (66, 73), (61, 73)], light)
+ else:
+ # Environment feedback and verification share the amber result symbol.
+ polygon([(50, 17), (82, 29), (78, 60), (67, 78), (50, 90),
+ (31, 78), (21, 60), (18, 29)], rgba(accent, 40))
+ line((50, 17, 82, 29, 78, 60, 67, 78, 50, 90,
+ 31, 78, 21, 60, 18, 29, 50, 17), edge, 2)
+ line((33, 51, 46, 64, 69, 38), light, 6)
+
+ return image.resize((size, size), Image.Resampling.LANCZOS)
diff --git a/site/README.md b/site/README.md
index 3c4b6be..18c6685 100644
--- a/site/README.md
+++ b/site/README.md
@@ -25,6 +25,61 @@ released checkpoints and the supported-model catalog. Guides, the technical
report and linked reference files are hosted on the site. GitHub remains an
explicit source-code link; checkpoint download links point to Hugging Face.
+The README and `docs/agent-harness.html` use the same paired comparisons from
+`scripts/render_agent_decision_demos_v2.py`. Three individual GIFs show WebShop,
+FrozenLake, and SQLite recovery. `docs/demos/jev-agent-harness.gif` joins those
+replays in that order into a nine-second overview, preserving every frame and
+its timing. The gallery uses matching MP4/WebM videos and WebP posters.
+
+Both lanes retain the measured completion-time ratio within each three-second loop.
+Recorded LLM call totals are prominent alongside each lane's clock. Task
+constraints stay visible; large actor panels identify the current phase,
+and recorded menus highlight the selected option. The footer shows LLM tokens
+saved, wall time saved, and the task result for this recorded pair. Large
+highlighted percentages lead the two savings cards, with absolute savings and
+original totals alongside them. Percentages use the LLM-only baseline. These
+are full-run measurements for one recorded pair.
+A fixed role bar shows each controller once and highlights only
+the current controller. When control returns to the LLM, the same LLM position
+lights up again. Both roles dim when the task completes, and completed lanes
+remain frozen.
+
+Shared robot portraits in `scripts/render_agent_robot_icons.py` distinguish the
+blue LLM from the smaller teal Jev. Planning lights and short motion trails
+animate only on the active controller; inactive portraits stay gray and still.
+These decorative animations do not represent measured inference time. Amber
+result symbols mark completion. Icons are drawn and antialiased locally with
+Pillow, so all three cases use the same artwork without external image assets.
+
+`docs/agent-harness.html` leads with the overview, followed by the three individual
+replays and a short explanation of each delegation: choosing within LLM-provided
+menus, retaining one plan across local moves, and choosing useful evidence.
+It uses the same navy, blue, teal, and amber palette as the GIFs.
+Playback controls provide pause, seek, and fullscreen beneath the frame,
+keeping the savings visible even when paused. Native controls remain available
+when JavaScript is disabled. Only the overview
+starts automatically; videos pause offscreen and when the tab is hidden, and
+only one plays at a time. Reduced-motion preferences disable automatic playback.
+A manual pause or selection takes precedence over automatic playback.
+
+The gallery and its CSS/JS live in `docs/`, which the existing Pages artifact includes.
+The gallery links to the recorded traces, delegation protocol, and full results.
+WebShop uses seed 3107 and Jev 4B from the D4 supplement; SQLite and FrozenLake
+keep their recorded cases. Savings describe one recorded pair, with the broader
+D2 matrix, separate D4 supplement, fallback cases, and uncertainty documented
+in the report. Rendering uses existing recordings and runs no model or environment.
+
+Regenerate all four GIFs and their site media from the repository root:
+
+```bash
+python3 -m pip install pillow imageio-ffmpeg
+python3 scripts/render_agent_decision_demos_v2.py --site-media
+```
+
+Omit `--site-media` to export only GIFs. Video conversion uses ffmpeg;
+imageio-ffmpeg is optional when ffmpeg is already on PATH. The legacy command
+`python3 scripts/render_agent_harness_demo.py` also exports this presentation.
+
Regenerate the homepage's data sections and documentation from the repository:
```bash
diff --git a/site/assets/i18n/zh-CN.js b/site/assets/i18n/zh-CN.js
index 022b9d4..a9b967a 100644
--- a/site/assets/i18n/zh-CN.js
+++ b/site/assets/i18n/zh-CN.js
@@ -10,6 +10,7 @@ window.jevanyMessages = {
"Main navigation": "主导航",
"Demos": "演示",
"Benchmarks": "评测",
+ "Agent harness": "Agent 协作演示",
"Models": "模型",
"Docs": "文档",
"Language": "语言",
diff --git a/site/assets/media/docs/jev-agent-harness.mp4 b/site/assets/media/docs/jev-agent-harness.mp4
index 311fa91..434d4c6 100644
Binary files a/site/assets/media/docs/jev-agent-harness.mp4 and b/site/assets/media/docs/jev-agent-harness.mp4 differ
diff --git a/site/assets/media/docs/jev-agent-harness.webm b/site/assets/media/docs/jev-agent-harness.webm
index a224daf..1b53946 100644
Binary files a/site/assets/media/docs/jev-agent-harness.webm and b/site/assets/media/docs/jev-agent-harness.webm differ
diff --git a/site/assets/media/docs/jev-agent-harness.webp b/site/assets/media/docs/jev-agent-harness.webp
index a5b1384..233c1de 100644
Binary files a/site/assets/media/docs/jev-agent-harness.webp and b/site/assets/media/docs/jev-agent-harness.webp differ
diff --git a/site/assets/media/docs/jev-decision-frozen-lake-v2.mp4 b/site/assets/media/docs/jev-decision-frozen-lake-v2.mp4
index 413f066..e5b68b1 100644
Binary files a/site/assets/media/docs/jev-decision-frozen-lake-v2.mp4 and b/site/assets/media/docs/jev-decision-frozen-lake-v2.mp4 differ
diff --git a/site/assets/media/docs/jev-decision-frozen-lake-v2.webm b/site/assets/media/docs/jev-decision-frozen-lake-v2.webm
index 3ddd448..4b20a5e 100644
Binary files a/site/assets/media/docs/jev-decision-frozen-lake-v2.webm and b/site/assets/media/docs/jev-decision-frozen-lake-v2.webm differ
diff --git a/site/assets/media/docs/jev-decision-frozen-lake-v2.webp b/site/assets/media/docs/jev-decision-frozen-lake-v2.webp
index 0a5582d..e7ba168 100644
Binary files a/site/assets/media/docs/jev-decision-frozen-lake-v2.webp and b/site/assets/media/docs/jev-decision-frozen-lake-v2.webp differ
diff --git a/site/assets/media/docs/jev-decision-terminal-v2.mp4 b/site/assets/media/docs/jev-decision-terminal-v2.mp4
index ef565f6..faffd92 100644
Binary files a/site/assets/media/docs/jev-decision-terminal-v2.mp4 and b/site/assets/media/docs/jev-decision-terminal-v2.mp4 differ
diff --git a/site/assets/media/docs/jev-decision-terminal-v2.webm b/site/assets/media/docs/jev-decision-terminal-v2.webm
index c6fd208..dd3b005 100644
Binary files a/site/assets/media/docs/jev-decision-terminal-v2.webm and b/site/assets/media/docs/jev-decision-terminal-v2.webm differ
diff --git a/site/assets/media/docs/jev-decision-terminal-v2.webp b/site/assets/media/docs/jev-decision-terminal-v2.webp
index cffdcbe..f7e9776 100644
Binary files a/site/assets/media/docs/jev-decision-terminal-v2.webp and b/site/assets/media/docs/jev-decision-terminal-v2.webp differ
diff --git a/site/assets/media/docs/jev-decision-webshop-v2.mp4 b/site/assets/media/docs/jev-decision-webshop-v2.mp4
index ed56e56..1dd5d29 100644
Binary files a/site/assets/media/docs/jev-decision-webshop-v2.mp4 and b/site/assets/media/docs/jev-decision-webshop-v2.mp4 differ
diff --git a/site/assets/media/docs/jev-decision-webshop-v2.webm b/site/assets/media/docs/jev-decision-webshop-v2.webm
index dc6cea4..a4d8bad 100644
Binary files a/site/assets/media/docs/jev-decision-webshop-v2.webm and b/site/assets/media/docs/jev-decision-webshop-v2.webm differ
diff --git a/site/assets/media/docs/jev-decision-webshop-v2.webp b/site/assets/media/docs/jev-decision-webshop-v2.webp
index 9733bb2..233c1de 100644
Binary files a/site/assets/media/docs/jev-decision-webshop-v2.webp and b/site/assets/media/docs/jev-decision-webshop-v2.webp differ
diff --git a/site/docs/ATTRIBUTIONBENCH_EVALUATION.html b/site/docs/ATTRIBUTIONBENCH_EVALUATION.html
new file mode 100644
index 0000000..34282f2
--- /dev/null
+++ b/site/docs/ATTRIBUTIONBENCH_EVALUATION.html
@@ -0,0 +1,117 @@
+
+
+
+AttributionBench Evaluation — JevAny
+Skip to content
+JevAny.
+
+
+
JEVANY / DOCUMENTATION
AttributionBench Evaluation
This standalone converter makes AttributionBench
+usable with JevAny's existing evaluation interfaces. It does not train a model,
+change default benchmarks, or run inference during conversion.
+
Task
+
Each original row becomes one noul question: is the entire claim supported
+by its complete references bundle? The model receives those two fields and
+an instruction to judge only the supplied evidence.
+
The original labels map exactly as follows:
+
+
attributable becomes true.
+
not attributable becomes false.
+
+
Claim and reference strings, reference order, and repeated references are
+preserved. Empty reference lists retain their original annotations; the
+converter does not repair labels. Original responses, labels, raw claims, and
+citation links are not added to model evidence. The task is not per-reference
+ranking, full-response correctness, or end-to-end retrieval.
+
Source And License
+
The documented source is osunlp/AttributionBench, pinned to revision
+62569e644f4186606f54f742178a4517431b42e1. Its dataset card declares
+Apache-2.0. Review and retain the notices and terms of constituent sources;
+this converter does not grant additional rights to those texts.
+
Download original exports separately. Conversion itself needs no network,
+Hugging Face libraries, tokenizer, or model. Do not commit downloaded corpora
+or generated suites. For example, after creating a local export directory:
All balancing configurations share the same evaluation files. They are not
+independent evaluation panels and must not be counted multiple times.
+
Convert A Local Export
+
Use a base JevAny installation with Python 3.12 or newer. Run the module from
+the checkout root; paths below are placeholders for local absolute paths:
The required original fields are id, src_dataset, claim, references,
+and attribution_label. The first three must be non-empty strings;
+references must be a list of strings. An optional question must be a
+string or null. Unknown labels, malformed rows, duplicate IDs, and empty
+exports are rejected with physical input line numbers. All rows are validated
+before creating output. Existing output directories are never overwritten.
+
Original dev maps to development. Original test and test_ood each map
+to the locked test partition in a separate output directory. Supply the
+matching original export and --split value; the converter does not infer or
+verify an export's upstream split from its contents. It does not accept
+training splits or resplit, shuffle, truncate, or relabel records.
+
The output is one JSONL partition and a manifest, for example
+/path/to/suites/attributionbench-dev/development.jsonl and
+/path/to/suites/attributionbench-dev/manifest.json. The manifest records the
+input revision/checksum, original split, output checksum/counts, and an
+evaluation-only source policy. Record metadata retains original IDs, source
+datasets, original splits, revision, and physical input lines. Model requests
+exclude labels and provenance.
+
Validate And Evaluate
+
Development data can be checked and scored with the existing commands:
Evaluation output directories must also be new. Freeze the checkpoint,
+readout, prompt, source revision, and partition before comparing models.
+The task names retain each original src_dataset for per-source reporting.
+
Limits
+
Related records are grouped by source dataset and original question when it
+is nonblank. Without a question, the grouping uses that source dataset and
+an exact reference-bundle hash. Group IDs do not depend on the output split.
+This is a conservative grouping convention, not a complete semantic
+contamination audit. It does not detect paraphrases or every shared document.
+
check-suite compares only partitions in the same manifest. It does not
+compare separate conversion directories. Creating independent development,
+test, or OOD suites does not establish that they are disjoint. Audit overlap
+before using any data for selection or training, and keep test data locked.
+The evaluation-only manifest policy is not a security boundary against
+someone deliberately loading raw records into a separate training workflow.
+
Conversion validates schema and provenance, not model token limits. Frozen
+suite scoring aborts on overlong inputs rather than silently dropping them.
+For custom-data scoring that records context rejections and coverage, use
+--data /path/to/suites/attributionbench-dev/development.jsonl instead of
+--suite; report requested/evaluated counts and count rejected questions as
+wrong in any all-request headline accuracy. Do not claim complete coverage
+from answered-only accuracy.
+
The public release training corpus is not fully disclosed. These results
+would measure external attribution performance, not prove contamination-free
+generalization or improved end-to-end RAG quality.
With max_batch_size above 1, concurrent text requests that arrive while a
+forward pass is running share the next one. An idle server still scores each
+request alone, with no delay to collect a batch. Combining requests is limited
+to max_packed_tokens padded token slots; a single request keeps the existing
+context limits, including repeated state tokens in row mode. Multimodal
+checkpoints and requests served from the prefix cache run alone. FP32
+batches match unbatched scores within floating-point tolerance; BF16/FP16 GPU
+kernels round differently by batch shape, so keep the default of 1 when
+reproducing published metrics. On one H200 with 32 concurrent requests to a
+Qwen3.5-0.8B checkpoint, batching raised throughput 2.3x in FP32 (largest
+probability change 1.5e-5, no decision changed) and 4.6x in BF16, where 2 of 32
+requests changed at least one decision. GET /v1/models reports batch counts.
A branch includes its shared state. State and branch limits are capped at the
backbone's context window; the packed limit covers the complete request, including
all branches. Oversized inputs are rejected without truncation. Set the cache size
@@ -341,6 +359,13 @@
Optional CUDA acceleration
2048
Longer rows stay on eager inference instead of being padded into a slower graph
+
+
JEVANY_FUSED_KERNELS / --fused-kernels
+
0, 1
+
1 with merged BF16 LoRA and direct graphs
+
0 until measured
+
Fused Qwen3.5 kernels: one-kernel norms, merged projections and transposed weights
+
Qwen3.5 and Qwen3.8 mix full-attention layers with Gated DeltaNet layers.
@@ -415,6 +440,16 @@
Optional CUDA acceleration
describe() reports the lengths, capture time and how many calls replayed or ran
eagerly under acceleration.cuda_graphs. Direct graphs cannot be combined with
JEVANY_COMPILE.
+
Transformers counts CUDA stream capture as tracing. Left alone, it materialises
+the causal mask inside every captured graph, so SDPA runs its memory-efficient
+kernel there instead of the FlashAttention kernel that eager inference uses.
+During capture JevAny therefore makes the eager mask decisions for an unpadded
+single row; rows whose positions do not rise by one run eagerly. On an
+A100-40GB with BF16 merged JevAny-Qwen3.5-4B, this made captured public-JevBench
+requests of 513 to 2,048 tokens 3–5% faster and left shorter ones unchanged.
+With a 4,096-token limit, requests of 2,049 to 4,096 tokens became 6–10% faster
+in graphs, but they were still no faster than eager inference, so the default
+limit stays at 2,048 tokens.
Padded shapes select different kernels, so probabilities are close to, not
identical with, the eager path. On an A100-40GB with FLA and causal-conv1d
installed and SDPA, over every request of the development suites, median
@@ -432,10 +467,33 @@
Optional CUDA acceleration
changes, while 27B stayed at 207 with no argmax changes. Maximum probability
changes were 0.0160 and 0.0146 respectively. These serving measurements do not
replace the exact benchmark scores reported by the model cards.
+
--fused-kernels (JEVANY_FUSED_KERNELS=1, LoadOptions(fused_kernels=True))
+rewrites a merged Qwen3.5-architecture backbone (the Qwen3.5 and Qwen3.8 bases)
+for batch-1 inference; see jevany/fused_kernels.py. Each RMSNorm runs as one
+FLA kernel; the MLP gate and up projections, the four Gated DeltaNet input
+projections and the attention query, key and value projections each run as one
+GEMM; FLA's chunked kernel applies the DeltaNet gate and beta sigmoid itself;
+and decoder weights are stored transposed so that cuBLAS reads both GEMM
+operands in their natural layout. The original modules keep views of the fused
+weights, so memory does not grow. The option needs the LoRA merged into the
+backbone (JEVANY_MERGE_BF16=1 for BF16 checkpoints; direct-token checkpoints
+stay unmerged), one CUDA device and flash-linear-attention. It cannot be
+combined with JEVANY_COMPILE, a device map or choice readout, and
+describe() reports what was fused under acceleration.fused_kernels.
+
On an A100-40GB with BF16 merged JevAny-Qwen3.5-4B, SDPA, FLA, causal-conv1d
+and direct graphs, fused kernels changed public-JevBench forward latency (231
+requests × 3, scripts/benchmark_latency.py --serving-kernels) from 16.43 to
+11.25 ms at the median, from 41.79 to 34.10 ms on average and from 130.14 to
+112.30 ms at p90. Requests above the graph limit run eagerly and became 12%
+faster as well. On the Transfer-v9 development suite with the same serving
+settings, clean accuracy moved from 78.68% to 78.78% and NLL from 0.587 to
+0.588; 5 of 1,264 argmax decisions changed and the largest probability change
+was 0.044. The option has not been measured on 27B.
scripts/benchmark_latency.py accepts --cuda-graphs,
---cuda-graph-max-tokens, --max-packed and --serving-kernels. The last
-option keeps the fused SDPA kernels used by jevany serve instead of selecting
-the math kernel used for fp32-exact evaluation.
+--cuda-graph-max-tokens, --fused-kernels, --max-packed and
+--serving-kernels. The last option keeps the fused SDPA kernels used by
+jevany serve instead of selecting the math kernel used for fp32-exact
+evaluation.
For a latency-oriented 27B deployment, keep compilation off and cap direct
graphs at 2,048 tokens:
GET /v1/models for the active limits and capabilities. The default packed limit is 8,192 tokens;
evaluation covers the 2,048-token training window.
Confidence thresholds may need recalibration on your domain.
+
Video decoding goes through the Transformers video processor. torchvision 0.26 and
+later no longer ship read_video, so with those releases install torchcodec
+and a system FFmpeg that it supports. Image requests do not need either.
+
python -m pip install torchcodec # also requires FFmpeg libraries on the host
+
Extend backbone support
Serving uses the same backbone_adapter = "my_package.adapters:MyAdapter" saved
by training. Install that trusted package in the serving environment; no HTTP
diff --git a/site/docs/VSR_EVALUATION.html b/site/docs/VSR_EVALUATION.html
new file mode 100644
index 0000000..71186ad
--- /dev/null
+++ b/site/docs/VSR_EVALUATION.html
@@ -0,0 +1,144 @@
+
+
This standalone offline converter prepares a Visual Spatial Reasoning (VSR)
+subset using images whose COCO metadata identifies CC BY 2.0. It does not
+train a model, change default benchmarks, download images, or run inference.
+
Call the output a COCO-metadata CC BY 2.0-filtered VSR subset, not the full
+VSR benchmark or a legally cleared image collection.
+
Task and Source
+
An original VSR caption becomes one noul question about the supplied image.
+The converter preserves the caption exactly and maps integer 0 to false,
+integer 1 to true. Labels, relation categories, votes, object metadata,
+source identifiers and licenses do not enter model evidence. It does not
+rewrite captions, generate negative examples, or repair annotations.
+
Use the author repository,
+starting with revision b27a0af0ee1462d2b6b92c8c83e869d9254a241a.
+Its original configurations are random and zeroshot, with train, dev
+and test partitions. This converter accepts only dev and test and keeps
+those partitions separate. The author provides public test labels.
+
The Hugging Face loading script and older dataset-card statistics need not
+match this author revision. Pin the original export rather than mixing
+versions. Record the revision actually used; declaring it is not proof that
+a local file was obtained from that revision.
+
Licensing and Attribution
+
These are three different licensing layers:
+
+
The VSR project and dataset card declare Apache-2.0. Retain applicable
+ upstream notices, cite the source, and describe the conversion when
+ distributing VSR annotations or derived records.
Individual photographs retain their authors' rights and conditions.
+ The filter accepts only COCO image records whose resolved license URL is
+ http://creativecommons.org/licenses/by/2.0/ or its HTTPS equivalent.
+ It does not infer permission from a license ID or display name. Other
+ licenses are excluded from this particular policy, not declared invalid.
+
+
CC BY 2.0 has attribution and other conditions. Review the original source
+and retain required author, source and license information for the intended
+use. The converter preserves available Flickr URLs and COCO license records,
+but it does not retrieve photographer names, verify a license grant, check
+current Flickr pages, or grant rights. Follow your organization's data-use
+requirements where applicable. Private evaluation is not automatically exempt
+from image terms.
+
Do not commit real photographs, downloaded annotations or generated suites.
+Committed tests use original synthetic images and annotation fixtures. If
+sharing a generated suite or results containing photographs, separately
+satisfy the applicable attribution and redistribution conditions. Do not
+relabel all source material as Apache-2.0.
+
Prepare Local Inputs
+
Obtain a fixed author JSONL export and COCO image/license tables separately.
+The official COCO download page links the
+2017 train/validation annotation archive. Caption or instance annotation JSON
+contains images and licenses tables. Supply both train and validation
+metadata when the VSR export references both COCO partitions; consistent
+repeated tables across files are supported.
+
For example, prepare these absolute local paths outside the checkout:
+
+
/tmp/vsr-input/random-dev.jsonl
+
/tmp/vsr-input/captions_train2017.json
+
/tmp/vsr-input/captions_val2017.json
+
/tmp/vsr-input/images/train2017/ and /tmp/vsr-input/images/val2017/
+
+
Place only needed, appropriately usable images under their original twelve-digit
+COCO filenames. Excluded images need not be present. Missing selected images,
+unknown license mappings and contradictory metadata are errors, not silent
+exclusions. image_link, COCO filename, image ID and original COCO partition
+must agree. Symlinks resolving outside the image root are rejected.
+
Conversion needs Pillow, already included in JevAny's dev and multimodal
+extras. Run commands from the repository checkout with JevAny installed:
The output must not already exist. All annotations, associations and selected
+images are validated before output creation. Images are copied byte-for-byte
+into content-addressed media paths, without resizing or re-encoding. An I/O
+failure during writing may leave an incomplete output; inspect it and use a
+new output path rather than overwriting it.
+
The manifest records input SHA-256 values, the exact license selection rule,
+retained/excluded counts and reasons, and before/after label and relation
+counts. Record metadata retains source configuration, original partition and
+line number, COCO ID, Flickr/COCO URLs, image license and media hash. Relative
+media references make the suite portable. Hashes identify supplied bytes;
+they do not establish the authenticity or legal status of the input metadata.
+
Optional Evaluation
+
Conversion and synthetic tests require no model. Scoring requires a checkpoint
+or endpoint that supports image inputs and Noul decisions; a text-only model
+is not sufficient.
+
For a development suite, the existing evaluator can be used:
The existing multimodal evaluation script also supports development media
+controls (full, blank and shuffled). Its CLI loads development only. These
+controls diagnose image sensitivity; replacing the image does not create
+new ground-truth annotations.
+
For an original test export, convert with --split test into another fresh
+directory, then explicitly validate the locked partition:
+
python -m jevany data check-suite /tmp/vsr-random-test-cc-by --allow-test
+
+
For test scoring, the generic evaluator selects the locked test partition when
+--allow-test is supplied with --suite:
Without that flag, --suite selects development. Do not rename test to
+development or treat raw JSONL loading as a suite lock.
+
Interpretation Limits
+
Freeze the data revision, configuration, partition, filtering policy, prompt,
+checkpoint and readout before scoring. Report the retained size, exclusions,
+label/relation counts, rejected predictions and coverage. Do not select records
+according to model performance or compare a filtered result as if it were the
+full benchmark.
+
Multiple captions for one image remain distinct records but share a media-hash
+group. Original random splits were not designed to guarantee image-disjoint
+partitions. Zeroshot describes VSR's object-concept split, not proof that a
+modern model has never seen those concepts or COCO pictures. The configurations
+are not necessarily independent evaluation sets.
+
Each output contains one original partition. Suite validation does not compare
+separate directories or audit undisclosed pretraining corpora. Compare media
+hashes/groups across available prior data when making exposure claims; no such
+claim follows from license filtering. This contribution supplies evaluation
+infrastructure, not model scores or evidence of capability improvement.
Calls, tokens, total time, and rewards come from the recorded runs. Savings use the LLM-only baseline. Playback preserves each pair’s measured completion-time ratio; individual step durations are illustrated.
+
Broader results vary by task. The report includes the original D2 matrix, the separate D4 WebShop supplement, fallbacks, and uncertainty.
Broader paired evaluations show that gains vary by task. The full results,
delegation protocol, and
technical report describe
diff --git a/site/docs/zh.html b/site/docs/zh.html
index c3795af..b9a929c 100644
--- a/site/docs/zh.html
+++ b/site/docs/zh.html
@@ -107,6 +107,8 @@
