From 46e46850d9f2d45a7d151a73733f0d7862b9c8ff Mon Sep 17 00:00:00 2001 From: Juan Sanchez RC <316677333+JuanRC7@users.noreply.github.com> Date: Wed, 30 Sep 2026 18:40:14 -0700 Subject: [PATCH] Rename benchmark to XEbench and emphasize cross-embodiment scores --- .env.example | 6 +-- CHANGELOG.md | 6 +++ CITATION.cff | 4 +- NOTICE | 2 +- README.md | 66 ++++++++++++++++++-------- SPEC.md | 30 ++++++++++-- docs/add_an_adapter.md | 12 ++--- docs/write_a_task.md | 2 +- examples/keypress-ldr/README.md | 2 +- pyproject.toml | 11 +++-- results/README.md | 2 +- results/submission.schema.json | 2 +- src/dexbench/__init__.py | 14 +++++- src/xebench/__init__.py | 3 ++ src/{dexbench => xebench}/adapters.py | 4 +- src/{dexbench => xebench}/cli.py | 22 ++++----- src/{dexbench => xebench}/models.py | 4 +- src/{dexbench => xebench}/policies.py | 21 ++++---- src/{dexbench => xebench}/recording.py | 4 +- src/{dexbench => xebench}/tasks.py | 4 +- 20 files changed, 145 insertions(+), 76 deletions(-) create mode 100644 src/xebench/__init__.py rename src/{dexbench => xebench}/adapters.py (96%) rename src/{dexbench => xebench}/cli.py (96%) rename src/{dexbench => xebench}/models.py (94%) rename src/{dexbench => xebench}/policies.py (91%) rename src/{dexbench => xebench}/recording.py (96%) rename src/{dexbench => xebench}/tasks.py (99%) diff --git a/.env.example b/.env.example index 8bcc9cb..a8765b6 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ # Optional OpenAI-compatible policy endpoint. Keep real API keys out of source control. -DEXBENCH_LLM_BASE_URL=https://api.openai.com/v1 -DEXBENCH_LLM_API_KEY= -DEXBENCH_LLM_MODEL=gpt-4.1-mini +XEBENCH_LLM_BASE_URL=https://api.openai.com/v1 +XEBENCH_LLM_API_KEY= +XEBENCH_LLM_MODEL=gpt-4.1-mini diff --git a/CHANGELOG.md b/CHANGELOG.md index 45b1a5d..4e205d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- Rename the toolkit to XEbench (Cross-Embodiment Benchmark), with the `rc-xebench` distribution, `xebench` Python package and command, and `XEBENCH_*` configuration. +- Lead the README with the cross-embodiment σ, mean success, and embodiment coverage, and document the controlled-cohort aggregation rules. +- Keep legacy imports, command, environment variables, recording topics, and submission identifiers compatible. + ## 0.1.0 — 2026-09-27 - Define the three benchmark tasks: `keypress-ldr`, `piano-seq`, and `pick-place-ab`. diff --git a/CITATION.cff b/CITATION.cff index f217a4c..a20bc35 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,6 +1,6 @@ cff-version: 1.2.0 -message: "If you use RC DexBench, please cite this software." -title: "RC DexBench: An Open Benchmark for Dexterous Robot Evaluation" +message: "If you use RC XEbench, please cite this software." +title: "RC XEbench: An Open Cross-Embodiment Benchmark for Robotic Manipulation" type: software authors: - name: Robotics Center diff --git a/NOTICE b/NOTICE index c315be0..14e7dc9 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,2 @@ -RC DexBench +RC XEbench Copyright 2026 Robotics Center diff --git a/README.md b/README.md index fd30000..6bce9a5 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Robotics Center

-# RC DexBench +# RC XEbench — Cross-Embodiment Benchmark

CI status @@ -12,9 +12,33 @@ Apache 2.0 license

-RC DexBench is an open benchmark and evaluation toolkit for dexterous robotic hands. It defines tasks, hardware and session metadata, event-based scoring, and a common adapter interface so teams can compare teleoperation, replay, and learned policies with the same protocol. +RC XEbench (Cross-Embodiment Benchmark) evaluates how one robotic manipulation policy performs across different robot bodies. Its headline cross-embodiment statistic is **σ: the spread in a policy's task success rate across eligible embodiments**, reported alongside **mean success and embodiment coverage**. Evaluate teleoperation, replay, and learned policies under the same protocol on different hands and arm-and-hand systems. -DexBench measures finger-level contact and force control, plus fixed-scene arm-and-hand pick-and-place. The benchmark records MIDI velocity, keyboard events, tactile measurements, command-to-contact latency, and clock alignment. +XEbench measures finger-level contact and force control, plus fixed-scene arm-and-hand pick-and-place. The benchmark records MIDI velocity, keyboard events, tactile measurements, command-to-contact latency, and clock alignment. + +The Python package and command are `xebench`. Existing `dexbench` imports and commands, `DEXBENCH_*` environment variables, MCAP event topics, and submission schema identifiers remain supported for compatibility with recorded runs and existing integrations. + +## Cross-embodiment score + +**Same policy. Same task. Different bodies.** Measure task success on each embodiment, then report how much performance changes with the body. For pick and place, an embodiment is the complete arm + hand system; keyboard and piano tasks use the mounted hand and its fixture. + +| Measure | What it tells you | Read it as | +| --- | --- | --- | +| **Cross-embodiment σ ↓** | Population standard deviation of per-body success rates | Lower σ means less variation between bodies; report in percentage points | +| **Mean success ↑** | Arithmetic mean of the same per-body success rates | Each eligible embodiment has equal weight | +| **Embodiment coverage** | Number and identity of eligible bodies | Compare policies on the same body set | + +```text +SR_body (%) = 100 × successes / valid trials +mean_SR = mean(SR_body) +σ_body = √ mean((SR_body − mean_SR)²) +``` + +**Low σ alone does not establish a good policy:** consistent failure also produces low σ. Read consistency together with mean success, coverage, per-body uncertainty, and recordings. Human-relative throughput is reported separately against human teleoperation on each identical embodiment. + +Use a controlled cohort: hold the task, protocol, scene distribution, scoring rules, and checkpoint fixed; disclose every embodiment adapter. For the RC cross-embodiment summary, the proposed publication floor is at least **three eligible embodiments** and **40 valid trials per body × policy × task cell**. Compare policy columns over their common eligible body set. Different checkpoints or fine-tuning per body must be labeled as an adapted-policy comparison, rather than zero-shot transfer. + +The CLI records and scores individual runs; it does not compute a fleet-level σ from a single run. See the [cross-embodiment evaluation page](https://dexterity.roboticscenter.ai/benchmark) for the matrix, cohort rules, human baseline, and inspectable results. Its preview and XE-Table demonstration values are illustrative, not measured benchmark results. The detailed aggregation contract is in [SPEC.md](SPEC.md#cross-embodiment-evaluation). ## Tasks @@ -47,19 +71,19 @@ Each square GIF shows its matching task. The photo shows the robot hand and arm Install from a fresh checkout with Python 3.10 or later: ```bash -git clone https://github.com/RoboticsCenter/dexbench.git -cd dexbench +git clone https://github.com/RoboticsCenter/dexbench.git xebench +cd xebench python -m venv .venv source .venv/bin/activate python -m pip install -e . -dexbench tasks +xebench tasks ``` Run the bundled pipeline example and inspect its per-trial MCAP and JSON summary: ```bash -dexbench run --task keypress-ldr --policy scripted --adapter mock --trials 2 -dexbench score --task keypress-ldr --mcap runs//trial-001.mcap +xebench run --task keypress-ldr --policy scripted --adapter mock --trials 2 +xebench score --task keypress-ldr --mcap runs//trial-001.mcap ``` The scripted policy and mock adapter generate synthetic events to demonstrate the recording and scoring workflow. @@ -71,11 +95,11 @@ Real-robot runs use an adapter for the hand, sensors, and arm in the configured The built-in GPT-6 Astra policy is an example for `keypress-ldr`. It uses the OpenAI Chat Completions API and returns one bounded `press` action at a time for `index`, `middle`, or `ring`; the adapter maps that action to the robot's configured finger motion. Use a [custom policy factory](docs/add_an_adapter.md#add-a-task-policy) for `piano-seq`, `pick-place-ab`, or another policy. See the [GPT-6 Astra API documentation](https://developers.openai.com/api/docs/models/gpt-6-astra). ```bash -export DEXBENCH_LLM_BASE_URL="https://api.openai.com/v1" +export XEBENCH_LLM_BASE_URL="https://api.openai.com/v1" export OPENAI_API_KEY="" -export DEXBENCH_LLM_MODEL="gpt-6-astra" -export DEXBENCH_LLM_REASONING_EFFORT="low" -dexbench run --task keypress-ldr --policy llm --adapter mock --trials 1 +export XEBENCH_LLM_MODEL="gpt-6-astra" +export XEBENCH_LLM_REASONING_EFFORT="low" +xebench run --task keypress-ldr --policy llm --adapter mock --trials 1 ``` The request and response are recorded on `/dexbench/policy_trace`; credentials are omitted. The policy accepts only the listed finger actions or `stop`. Replace `mock` with your robot adapter to connect the policy to a hand. See [the adapter guide](docs/add_an_adapter.md). @@ -85,12 +109,12 @@ The request and response are recorded on `/dexbench/policy_trace`; credentials a A robot adapter can expose `teleop_action(observation)` to collect an operator baseline. The replay policy reads the command stream from one successful teleop trial and preserves its original relative timing. To select the successful teleop trial closest to the median completion time and replay it on the same rig: ```bash -dexbench run --task keypress-ldr --policy teleop \ - --adapter my_robot.dexbench_adapter:create_adapter --trials 20 \ +xebench run --task keypress-ldr --policy teleop \ + --adapter my_robot.xebench_adapter:create_adapter --trials 20 \ --calib-id session-2026-09-27-a --session-metadata session.json -dexbench run --task keypress-ldr --policy replay \ - --adapter my_robot.dexbench_adapter:create_adapter --trials 20 \ +xebench run --task keypress-ldr --policy replay \ + --adapter my_robot.xebench_adapter:create_adapter --trials 20 \ --calib-id session-2026-09-27-a --session-metadata session.json \ --replay-results runs//results.json ``` @@ -106,13 +130,13 @@ Keep every trial in the run's `results.json`, including failures and hardware-in | Episode 4 | Success | [View recording](examples/keypress-ldr/episode_4_success.mcap) | | Episode 5 | Failure | [View recording](examples/keypress-ldr/episode_5_failure.mcap) | -These recordings are examples, not DexBench-scored results or a complete benchmark round. The source GUI logger may report repeated key events while a key is held; see the [example notes](examples/keypress-ldr/) and the key-repeat event contract in [SPEC.md](SPEC.md#event-format). +These recordings are examples, not XEbench-scored results or a complete benchmark round. The source GUI logger may report repeated key events while a key is held; see the [example notes](examples/keypress-ldr/) and the key-repeat event contract in [SPEC.md](SPEC.md#event-format). ## Baselines and protocol Each policy run writes one JSON summary and one MCAP per trial. Benchmark runs keep failures. The reference baselines are: -- **Teleop:** a human operator using the same robot and task setup. Its run can be generated by a hardware adapter that maps the operator's controls into DexBench actions. +- **Teleop:** a human operator using the same robot and task setup. Its run can be generated by a hardware adapter that maps the operator's controls into XEbench actions. - **Replay:** open-loop playback of the `/hand/command` stream, plus `/arm/command` for pick-and-place, from the successful teleop episode with the median completion time. Keep hardware and calibration fixed between source and replay runs. - **Learned policy:** report the model, weights, endpoint/provider, prompt or policy version, action limits, and all run settings. @@ -127,16 +151,16 @@ For piano-seq, report each note's MIDI velocity, within-trial velocity standard - [Recordings and scoring](docs/recordings-and-scoring.md) - [Hardware reference sheets](hardware/) -The command-line workflow is native to DexBench. Its packaging and benchmark workflow take inspiration from [Inspect Robots](https://github.com/robocurve/inspect-robots), [WorldEvals](https://github.com/robocurve/worldevals), and [StationeryBench](https://github.com/robocurve/stationerybench), while remaining independent so tactile and contact-event channels are part of its own event and scoring contract. +The command-line workflow is native to XEbench. Its packaging and benchmark workflow take inspiration from [Inspect Robots](https://github.com/robocurve/inspect-robots), [WorldEvals](https://github.com/robocurve/worldevals), and [StationeryBench](https://github.com/robocurve/stationerybench), while remaining independent so tactile and contact-event channels are part of its own event and scoring contract. ## License and citation Released under the Apache License 2.0. See [LICENSE](LICENSE). ```bibtex -@software{rc_dexbench, +@software{rc_xebench, author = {{Robotics Center}}, - title = {RC DexBench: An Open Benchmark for Dexterous Robot Evaluation}, + title = {RC XEbench: An Open Cross-Embodiment Benchmark for Robotic Manipulation}, year = {2026}, url = {https://github.com/RoboticsCenter/dexbench}, version = {0.1.0}, diff --git a/SPEC.md b/SPEC.md index 661fbe0..b7663ba 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,12 +1,32 @@ -# RC DexBench Interface and Scoring Specification +# RC XEbench Interface and Scoring Specification Version 0.1.0 -This repository implements the benchmark workflow and reference scorers from the RC DexBench operating specification. Physical task definitions are fixed before a baseline run. Record actual dimensions, model numbers, settings, calibration, and photos in each hardware/session record. Do not tune the task after inspecting results. +This repository implements the benchmark workflow and reference scorers from the RC XEbench operating specification. Physical task definitions are fixed before a baseline run. Record actual dimensions, model numbers, settings, calibration, and photos in each hardware/session record. Do not tune the task after inspecting results. + +## Cross-embodiment evaluation + +XEbench's cross-embodiment statistic is the population standard deviation of a single policy's success rates over eligible robot bodies, in percentage points: + +```text +SR_body (%) = 100 × successes_body / valid_trials_body +mean_SR = (1 / B) × sum(SR_body) +σ_body = sqrt((1 / B) × sum((SR_body − mean_SR)²)) +``` + +Here `B` is the number of eligible embodiments in one controlled cohort. Report `σ_body`, `mean_SR`, `B`, and the included embodiment IDs together. Each body has equal weight; pooling all trials across bodies gives a different statistic. Lower σ means less variation, and cannot establish good performance without high mean success. + +The RC publication rules proposed on the [evaluation page](https://dexterity.roboticscenter.ai/benchmark) require at least three eligible bodies and 40 valid trials per body × policy × task cell. A cell may combine repeated complete protocol rounds within the same cohort. Standard community rounds remain 20 trials for keyboard/piano or 30 for pick and place; a single complete round is insufficient for this summary. + +For a comparison across policies, use the intersection of eligible bodies across every policy column in the selected controller group. Never aggregate different tasks, protocol revisions, scene sets, or cohorts. Record the fixed checkpoint, task settings, time cap, scoring rules, calibration, observation setup, exclusions, adapter, and session/episode structure in the cohort manifest. Disclose any body-specific fine-tuning or checkpoint change as an adapted-policy comparison. Pick-and-place embodiments include both the arm and hand; hand-only tasks include the mounted hand and fixture. + +Policy failures and timeouts stay in the valid-trial denominator. Justified hardware-invalid trials are excluded and reported separately. Publish per-body counts, uncertainty, and full-run recordings so the summary can be reproduced. Meeting the trial floor does not establish statistical significance. Mean reported successful-trial completion time cannot substitute for human-relative throughput, which requires matched full-run timing and a human reference on the same embodiment. + +The current command-line toolkit scores individual runs. The evaluation page computes cohort summaries; a run's `success_rate` is its per-body outcome, not a cross-embodiment σ. Preview and XE-Table demonstration values do not establish measured cross-embodiment results. ## Core interfaces -An adapter connects DexBench to a robot or simulator. A policy reads an `Observation` and returns one bounded `Action` or `None` to stop. Adapter event timestamps and action timestamps use the same monotonic clock. +An adapter connects XEbench to a robot or simulator. A policy reads an `Observation` and returns one bounded `Action` or `None` to stop. Adapter event timestamps and action timestamps use the same monotonic clock. ```python class Adapter: @@ -25,7 +45,7 @@ class Policy: def act(self, observation: Observation) -> Action | None: ... ``` -The Python versions of these contracts live in `dexbench.models`. A hardware adapter factory is loaded with `--adapter package.module:factory` and is called with the session metadata dictionary. `execute` must enforce that policy actions remain within the rig's configured limits. Adapters may expose only the observations/actions supported by the hardware; they should not fabricate sensor events. +The Python versions of these contracts live in `xebench.models`. A hardware adapter factory is loaded with `--adapter package.module:factory` and is called with the session metadata dictionary. `execute` must enforce that policy actions remain within the rig's configured limits. Adapters may expose only the observations/actions supported by the hardware; they should not fabricate sensor events. ## Event format @@ -54,7 +74,7 @@ Common topics: | `/glove/*` | glove skeleton, tactile, EMF, IMU | Operator glove | | `/camera/*` | image data and camera timestamps | Camera adapter | | `/scorer/tray_state` | `object_inside`, `stable_duration_s` | Depth + color scorer | -| `/dexbench/actions` | policy action and step | DexBench runner | +| `/dexbench/actions` | policy action and step | XEbench runner | | `/dexbench/policy_trace` | model, request, response | Example LLM policy | Scorers require these event fields: keyboard `key` values are uppercase `LEFT`, `DOWN`, or `RIGHT`; MIDI `note` and `velocity` are integers; `/hand/command` uses `finger` and `pressed`; `/hand/state` uses `open` as a boolean; `/scorer/tray_state` uses boolean `object_inside` and numeric `stable_duration_s`. Put fields inside the event's `data` object. Emit `/dexbench/invalid` with a `reason` string for hardware faults. diff --git a/docs/add_an_adapter.md b/docs/add_an_adapter.md index 0dc7df5..c615178 100644 --- a/docs/add_an_adapter.md +++ b/docs/add_an_adapter.md @@ -1,12 +1,12 @@ # Connect a hand, simulator, or rig -An adapter maps the DexBench action space to one configured embodiment, reads sensors, and emits timestamped `Event` records. Install DexBench in the adapter's environment, then pass its factory to the runner: +An adapter maps the XEbench action space to one configured embodiment, reads sensors, and emits timestamped `Event` records. Install XEbench in the adapter's environment, then pass its factory to the runner: ```bash -dexbench run \ +xebench run \ --task keypress-ldr \ --policy llm \ - --adapter my_robot.dexbench_adapter:create_adapter \ + --adapter my_robot.xebench_adapter:create_adapter \ --trials 20 \ --calib-id session-2026-09-27-a \ --session-metadata session.json @@ -27,16 +27,16 @@ For Wuji Hand 2, record `/hand/command` and `/hand/state` separately. Read the d The built-in GPT-6 Astra and scripted policies support `keypress-ldr`. Supply a custom policy factory to run a policy for `piano-seq` or `pick-place-ab`, or to connect another learned policy: ```bash -dexbench run \ +xebench run \ --task piano-seq \ --policy custom \ --policy-factory my_project.policies:create_policy \ - --adapter my_robot.dexbench_adapter:create_adapter \ + --adapter my_robot.xebench_adapter:create_adapter \ --trials 20 \ --calib-id session-2026-09-27-a \ --session-metadata session.json ``` -The policy factory receives the session metadata dictionary and returns an object with `name`, `reset(task, trial_id)`, and `act(observation)` methods. `act` returns an `Action` or `None`. The observation contains the task instruction, recent timestamped events, and run metadata; for `pick-place-ab`, `observation.metadata` includes the selected `object_id` and `grasp_type`. Robot-specific action kinds are handled by the adapter. Implement the policy and factory in an importable Python module in the environment running DexBench. +The policy factory receives the session metadata dictionary and returns an object with `name`, `reset(task, trial_id)`, and `act(observation)` methods. `act` returns an `Action` or `None`. The observation contains the task instruction, recent timestamped events, and run metadata; for `pick-place-ab`, `observation.metadata` includes the selected `object_id` and `grasp_type`. Robot-specific action kinds are handled by the adapter. Implement the policy and factory in an importable Python module in the environment running XEbench. See [SPEC.md](../SPEC.md) for the shared task event fields and outcome rules. diff --git a/docs/write_a_task.md b/docs/write_a_task.md index d5fbba2..601f149 100644 --- a/docs/write_a_task.md +++ b/docs/write_a_task.md @@ -3,7 +3,7 @@ 1. Give the task a stable kebab-case ID and a fixed start state. 2. Write success, failure, invalid-trial, timeout, reset, and trial-count rules before collecting results. 3. Prefer a hardware event or calibrated sensor as the scorer. Store scorer inputs on named MCAP topics so a run can be rescored offline. -4. Implement a deterministic scorer from `list[Event]` and `t0_ns` in `src/dexbench/tasks.py`. +4. Implement a deterministic scorer from `list[Event]` and `t0_ns` in `src/xebench/tasks.py`. 5. Add the public task definition and hardware/session fields to `SPEC.md` and `hardware/`. 6. Include a scripted or simulator example only when it is clearly labeled as a software-path demonstration. Never publish it as a hardware baseline. diff --git a/examples/keypress-ldr/README.md b/examples/keypress-ldr/README.md index 0522a31..b738ea4 100644 --- a/examples/keypress-ldr/README.md +++ b/examples/keypress-ldr/README.md @@ -7,6 +7,6 @@ These two MCAP files contain real recorded camera, hand, glove, and keyboard str | Episode 4 | Success | [`episode_4_success.mcap`](episode_4_success.mcap) | | Episode 5 | Failure | [`episode_5_failure.mcap`](episode_5_failure.mcap) | -The source application logged GUI key press and release callbacks. Operating-system key repeat can generate repeated press records while a key is held, so the raw press count is not necessarily the number of physical presses. These recordings are examples for inspecting event streams; their source labels are not DexBench scores. +The source application logged GUI key press and release callbacks. Operating-system key repeat can generate repeated press records while a key is held, so the raw press count is not necessarily the number of physical presses. These recordings are examples for inspecting event streams; their source labels are not XEbench scores. The example MCAP metadata has been cleaned of machine addresses and device serial numbers. Browse MCAP recordings with [DexData](https://dexdata.roboticscenter.ai). See the [keypress-ldr protocol](../../SPEC.md#keypress-ldr) for the canonical event contract. diff --git a/pyproject.toml b/pyproject.toml index 6d56da6..c5de855 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,26 +3,27 @@ requires = ["hatchling>=1.24"] build-backend = "hatchling.build" [project] -name = "rc-dexbench" +name = "rc-xebench" version = "0.1.0" -description = "An open benchmark and evaluation toolkit for dexterous robotic hands." +description = "An open cross-embodiment benchmark and evaluation toolkit for robotic manipulation." readme = "README.md" requires-python = ">=3.10" license = "Apache-2.0" license-files = ["LICENSE"] authors = [{ name = "Robotics Center" }] -keywords = ["robotics", "dexterous-manipulation", "benchmark", "tactile", "evaluation"] +keywords = ["robotics", "cross-embodiment", "dexterous-manipulation", "benchmark", "tactile", "evaluation"] dependencies = ["httpx>=0.27", "mcap>=1.2"] [project.scripts] -dexbench = "dexbench.cli:main" +xebench = "xebench.cli:main" +dexbench = "xebench.cli:main" [project.urls] Homepage = "https://github.com/RoboticsCenter/dexbench" Issues = "https://github.com/RoboticsCenter/dexbench/issues" [tool.hatch.build.targets.wheel] -packages = ["src/dexbench"] +packages = ["src/xebench", "src/dexbench"] [tool.ruff] line-length = 100 diff --git a/results/README.md b/results/README.md index cb37f8d..250042e 100644 --- a/results/README.md +++ b/results/README.md @@ -1,6 +1,6 @@ # Publish a result -The Dexterity leaderboard accepts complete, non-mock DexBench runs. Upload the `results.json` file written by `dexbench run`, provide the hardware and policy metadata below, and link public MCAP trial recordings in DexData. The public site validates the fixed protocol and recalculates the score from the per-trial outcomes. +The Dexterity leaderboard accepts complete, non-mock XEbench runs. Upload the `results.json` file written by `xebench run`, provide the hardware and policy metadata below, and link public MCAP trial recordings in DexData. The public site validates the fixed protocol and recalculates the score from the per-trial outcomes. ## Submit on the website diff --git a/results/submission.schema.json b/results/submission.schema.json index 86c19a6..b8fbcd0 100644 --- a/results/submission.schema.json +++ b/results/submission.schema.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/RoboticsCenter/dexbench/blob/main/results/submission.schema.json", - "title": "DexBench public leaderboard submission", + "title": "XEbench public leaderboard submission", "description": "Public, community-reported result envelope accepted by the Robotics Center Dexterity leaderboard.", "type": "object", "additionalProperties": false, diff --git a/src/dexbench/__init__.py b/src/dexbench/__init__.py index dccf286..5136c0d 100644 --- a/src/dexbench/__init__.py +++ b/src/dexbench/__init__.py @@ -1,3 +1,13 @@ -"""DexBench: an open benchmark toolkit for dexterous robot evaluation.""" +"""Compatibility imports for the toolkit now named XEbench.""" -__version__ = "0.1.0" +import sys +from importlib import import_module + +from xebench import __version__ + +__all__ = ["__version__"] + +for _name in ("models", "tasks", "recording", "adapters", "policies", "cli"): + _module = import_module(f"xebench.{_name}") + sys.modules[f"{__name__}.{_name}"] = _module + globals()[_name] = _module diff --git a/src/xebench/__init__.py b/src/xebench/__init__.py new file mode 100644 index 0000000..cab2e0f --- /dev/null +++ b/src/xebench/__init__.py @@ -0,0 +1,3 @@ +"""XEbench: an open cross-embodiment benchmark for robotic manipulation.""" + +__version__ = "0.1.0" diff --git a/src/dexbench/adapters.py b/src/xebench/adapters.py similarity index 96% rename from src/dexbench/adapters.py rename to src/xebench/adapters.py index 283bd03..e956e19 100644 --- a/src/dexbench/adapters.py +++ b/src/xebench/adapters.py @@ -6,8 +6,8 @@ import time from typing import Any -from dexbench.models import Action, Event, Observation -from dexbench.tasks import FINGER_FOR_KEY, INSTRUCTIONS +from xebench.models import Action, Event, Observation +from xebench.tasks import FINGER_FOR_KEY, INSTRUCTIONS class MockKeyboardAdapter: diff --git a/src/dexbench/cli.py b/src/xebench/cli.py similarity index 96% rename from src/dexbench/cli.py rename to src/xebench/cli.py index aac1251..86cca08 100644 --- a/src/dexbench/cli.py +++ b/src/xebench/cli.py @@ -1,4 +1,4 @@ -"""Command-line interface for running and scoring DexBench trials.""" +"""Command-line interface for running and scoring XEbench trials.""" from __future__ import annotations @@ -14,16 +14,16 @@ import httpx -from dexbench.adapters import MockKeyboardAdapter, create_observation, load_adapter -from dexbench.models import Event, Policy -from dexbench.policies import ( +from xebench.adapters import MockKeyboardAdapter, create_observation, load_adapter +from xebench.models import Event, Policy +from xebench.policies import ( OpenAICompatiblePolicy, ReplayPolicy, ScriptedKeyboardPolicy, TeleopPolicy, ) -from dexbench.recording import read_mcap_events, write_json, write_mcap -from dexbench.tasks import OBJECT_GRASP_TYPE, TASKS, score_trial +from xebench.recording import read_mcap_events, write_json, write_mcap +from xebench.tasks import OBJECT_GRASP_TYPE, TASKS, score_trial def _policy(name: str, args: argparse.Namespace, session_metadata: dict[str, Any]) -> Policy: @@ -162,7 +162,7 @@ def _trial( def _run(args: argparse.Namespace) -> int: """Run benchmark trials and save per-trial MCAP plus a run summary.""" if args.task not in TASKS: - raise ValueError(f"Unknown task {args.task!r}; use 'dexbench tasks' to see available tasks") + raise ValueError(f"Unknown task {args.task!r}; use 'xebench tasks' to see available tasks") trial_count = args.trials if args.trials is not None else _protocol_trials(args.task) if trial_count < 1 or args.max_actions < 1: raise ValueError("--trials and --max-actions must be positive") @@ -395,7 +395,7 @@ def _score(args: argparse.Namespace) -> int: None, ) if t0_event is None: - raise ValueError(f"{path} does not contain a DexBench trial start record") + raise ValueError(f"{path} does not contain a XEbench trial start record") score = score_trial(args.task, events, t0_event.timestamp_ns) print( json.dumps( @@ -419,9 +419,9 @@ def _tasks(_args: argparse.Namespace) -> int: def build_parser() -> argparse.ArgumentParser: - """Build the DexBench argument parser.""" + """Build the XEbench argument parser.""" parser = argparse.ArgumentParser( - prog="dexbench", description="Run and score dexterous hand tasks" + prog="xebench", description="Run and score cross-embodiment robotic manipulation tasks" ) commands = parser.add_subparsers(dest="command", required=True) tasks_parser = commands.add_parser("tasks", help="list included benchmark tasks") @@ -467,6 +467,6 @@ def main() -> None: try: status = args.func(args) except (ValueError, OSError, httpx.HTTPError, KeyError, ImportError) as error: - print(f"dexbench: error: {error}", file=sys.stderr) + print(f"xebench: error: {error}", file=sys.stderr) status = 2 raise SystemExit(status) diff --git a/src/dexbench/models.py b/src/xebench/models.py similarity index 94% rename from src/dexbench/models.py rename to src/xebench/models.py index bb4dead..09e98a7 100644 --- a/src/dexbench/models.py +++ b/src/xebench/models.py @@ -1,4 +1,4 @@ -"""Shared data structures and adapter contracts for DexBench.""" +"""Shared data structures and adapter contracts for XEbench.""" from __future__ import annotations @@ -66,7 +66,7 @@ def act(self, observation: Observation) -> Action | None: class Adapter(Protocol): - """Bridge between DexBench and a simulator or physical robot.""" + """Bridge between XEbench and a simulator or physical robot.""" name: str diff --git a/src/dexbench/policies.py b/src/xebench/policies.py similarity index 91% rename from src/dexbench/policies.py rename to src/xebench/policies.py index f753c57..6def8fa 100644 --- a/src/dexbench/policies.py +++ b/src/xebench/policies.py @@ -10,9 +10,14 @@ import httpx -from dexbench.models import Action, Adapter, Observation -from dexbench.recording import read_mcap_events -from dexbench.tasks import FINGER_FOR_KEY, TASKS +from xebench.models import Action, Adapter, Observation +from xebench.recording import read_mcap_events +from xebench.tasks import FINGER_FOR_KEY, TASKS + + +def _llm_setting(name: str, default: str = "") -> str: + """Prefer XEbench configuration, falling back to legacy deployment settings.""" + return os.environ.get(f"XEBENCH_LLM_{name}", os.environ.get(f"DEXBENCH_LLM_{name}", default)) class ScriptedKeyboardPolicy: @@ -49,14 +54,14 @@ class OpenAICompatiblePolicy: def __init__(self) -> None: """Read endpoint, credential, and model from environment variables.""" - self.base_url = os.environ.get("DEXBENCH_LLM_BASE_URL", "https://api.openai.com/v1") - self.api_key = os.environ.get("DEXBENCH_LLM_API_KEY") or os.environ.get("OPENAI_API_KEY") - self.model = os.environ.get("DEXBENCH_LLM_MODEL", "gpt-6-astra") - self.reasoning_effort = os.environ.get("DEXBENCH_LLM_REASONING_EFFORT", "low") + self.base_url = _llm_setting("BASE_URL", "https://api.openai.com/v1") + self.api_key = _llm_setting("API_KEY") or os.environ.get("OPENAI_API_KEY") + self.model = _llm_setting("MODEL", "gpt-6-astra") + self.reasoning_effort = _llm_setting("REASONING_EFFORT", "low") self.last_trace: dict[str, Any] = {} if not self.api_key: raise ValueError( - "Set DEXBENCH_LLM_API_KEY (or OPENAI_API_KEY) to use the example LLM policy" + "Set XEBENCH_LLM_API_KEY (or OPENAI_API_KEY) to use the example LLM policy" ) def reset(self, task: str, trial_id: str) -> None: diff --git a/src/dexbench/recording.py b/src/xebench/recording.py similarity index 96% rename from src/dexbench/recording.py rename to src/xebench/recording.py index b1b55a1..2e8fae4 100644 --- a/src/dexbench/recording.py +++ b/src/xebench/recording.py @@ -10,7 +10,7 @@ from mcap.reader import make_reader from mcap.writer import Writer -from dexbench.models import Event +from xebench.models import Event JSON_SCHEMA = { "type": "object", @@ -57,7 +57,7 @@ def write_mcap(path: Path, events: list[Event], metadata: dict[str, Any]) -> Non def read_mcap_events(path: Path) -> list[Event]: - """Read DexBench JSON events from an MCAP file.""" + """Read XEbench JSON events from an MCAP file.""" events = [] with path.open("rb") as stream: reader = make_reader(stream) diff --git a/src/dexbench/tasks.py b/src/xebench/tasks.py similarity index 99% rename from src/dexbench/tasks.py rename to src/xebench/tasks.py index ce3da4c..e4c4efa 100644 --- a/src/dexbench/tasks.py +++ b/src/xebench/tasks.py @@ -5,7 +5,7 @@ import math from typing import Any -from dexbench.models import Event, Score +from xebench.models import Event, Score TASKS: dict[str, dict[str, Any]] = { "keypress-ldr": { @@ -107,7 +107,7 @@ def _pearson(xs: list[float], ys: list[float]) -> float | None: def score_trial(task: str, events: list[Event], t0_ns: int) -> Score: """Score a trial using only its recorded event stream.""" if task not in TASKS: - raise ValueError(f"Unknown DexBench task: {task}") + raise ValueError(f"Unknown XEbench task: {task}") invalid = next((event for event in events if event.topic == "/dexbench/invalid"), None) if invalid is not None: return Score(False, str(invalid.data.get("reason", "hardware_fault")), {}, True)