From bcf1fac963d996142d2aa0815342d28a8f3c3d2e Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Fri, 4 Sep 2026 23:23:24 -0700 Subject: [PATCH] Move Edge exporter to tools/hf and add compile-time parity. --- docsrc/user_guide/edge_exporter.rst | 111 ++++++++++++------ py/torch_tensorrt/hf/__init__.py | 4 - py/torch_tensorrt/hf/exporters/__init__.py | 26 ---- .../hf/exporters/models/__init__.py | 5 - pyproject.toml | 10 -- setup.py | 16 --- tools/hf/__init__.py | 4 + tools/hf/exporters/__init__.py | 11 ++ .../hf/exporters/compile.py | 28 ++++- .../hf/exporters/config.py | 2 +- .../hf/exporters/data.py | 0 .../hf/exporters/exporter.py | 14 ++- tools/hf/exporters/measure.py | 62 ++++++++++ tools/hf/exporters/models/__init__.py | 5 + .../hf/exporters/models/common/__init__.py | 0 .../hf/exporters/models/common/helpers.py | 2 +- .../hf/exporters/models/common/patches.py | 0 .../hf/exporters/models/groot/__init__.py | 0 .../hf/exporters/models/groot/helpers.py | 0 .../hf/exporters/models/groot/patches.py | 30 ++++- .../hf/exporters/models/groot/spec.py | 29 +++-- .../hf/exporters/models/nemotron/__init__.py | 0 .../hf/exporters/models/nemotron/helpers.py | 0 .../exporters/models/nemotron/mamba_stub.py | 0 .../hf/exporters/models/nemotron/patches.py | 8 +- .../hf/exporters/models/nemotron/spec.py | 21 ++-- .../hf/exporters/models/pi05/__init__.py | 0 .../hf/exporters/models/pi05/helpers.py | 0 .../hf/exporters/models/pi05/patches.py | 8 +- .../hf/exporters/models/pi05/spec.py | 28 +++-- .../hf/exporters/ops.py | 0 .../hf/exporters/plugin/__init__.py | 0 .../hf/exporters/plugin/attention.py | 0 .../hf/exporters/plugin/attn_patches.py | 3 +- .../hf/exporters/plugin/mamba.py | 0 .../hf/exporters/plugin/moe.py | 0 .../hf/exporters/plugin/plugin_converter.py | 0 .../hf/exporters/plugin/plugin_utils.py | 0 .../hf/exporters/prefix_cache.py | 0 .../hf/exporters/rope.py | 0 .../hf/exporters/runtime.py | 3 +- .../hf/exporters/spec.py | 0 .../hf/exporters/tests}/test_edge_exporter.py | 95 +++++++++++---- .../hf/exporters/utils.py | 11 -- .../dynamo => tools/hf}/run_groot_export.py | 26 ++-- .../hf}/run_nemotron_export.py | 25 ++-- .../dynamo => tools/hf}/run_pi05_export.py | 31 +++-- 47 files changed, 376 insertions(+), 242 deletions(-) delete mode 100644 py/torch_tensorrt/hf/__init__.py delete mode 100644 py/torch_tensorrt/hf/exporters/__init__.py delete mode 100644 py/torch_tensorrt/hf/exporters/models/__init__.py create mode 100644 tools/hf/__init__.py create mode 100644 tools/hf/exporters/__init__.py rename {py/torch_tensorrt => tools}/hf/exporters/compile.py (81%) rename {py/torch_tensorrt => tools}/hf/exporters/config.py (92%) rename {py/torch_tensorrt => tools}/hf/exporters/data.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/exporter.py (94%) create mode 100644 tools/hf/exporters/measure.py create mode 100644 tools/hf/exporters/models/__init__.py rename {py/torch_tensorrt => tools}/hf/exporters/models/common/__init__.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/models/common/helpers.py (96%) rename {py/torch_tensorrt => tools}/hf/exporters/models/common/patches.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/models/groot/__init__.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/models/groot/helpers.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/models/groot/patches.py (85%) rename {py/torch_tensorrt => tools}/hf/exporters/models/groot/spec.py (94%) rename {py/torch_tensorrt => tools}/hf/exporters/models/nemotron/__init__.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/models/nemotron/helpers.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/models/nemotron/mamba_stub.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/models/nemotron/patches.py (91%) rename {py/torch_tensorrt => tools}/hf/exporters/models/nemotron/spec.py (91%) rename {py/torch_tensorrt => tools}/hf/exporters/models/pi05/__init__.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/models/pi05/helpers.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/models/pi05/patches.py (95%) rename {py/torch_tensorrt => tools}/hf/exporters/models/pi05/spec.py (93%) rename {py/torch_tensorrt => tools}/hf/exporters/ops.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/plugin/__init__.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/plugin/attention.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/plugin/attn_patches.py (99%) rename {py/torch_tensorrt => tools}/hf/exporters/plugin/mamba.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/plugin/moe.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/plugin/plugin_converter.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/plugin/plugin_utils.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/prefix_cache.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/rope.py (100%) rename {py/torch_tensorrt => tools}/hf/exporters/runtime.py (90%) rename {py/torch_tensorrt => tools}/hf/exporters/spec.py (100%) rename {tests/py/dynamo/hf => tools/hf/exporters/tests}/test_edge_exporter.py (82%) rename {py/torch_tensorrt => tools}/hf/exporters/utils.py (75%) rename {examples/dynamo => tools/hf}/run_groot_export.py (81%) rename {examples/dynamo => tools/hf}/run_nemotron_export.py (80%) rename {examples/dynamo => tools/hf}/run_pi05_export.py (79%) diff --git a/docsrc/user_guide/edge_exporter.rst b/docsrc/user_guide/edge_exporter.rst index 2ab1564bc00..7263ebeb39a 100644 --- a/docsrc/user_guide/edge_exporter.rst +++ b/docsrc/user_guide/edge_exporter.rst @@ -8,7 +8,7 @@ one ``torch.export`` graph that calls those engines. .. code-block:: python - from torch_tensorrt.hf.exporters import EdgeExporter, EdgeConfig + from hf.exporters import EdgeExporter, EdgeConfig exporter = EdgeExporter() config = EdgeConfig(dryrun=True, engine_dir="/tmp/pi05_edge") @@ -20,6 +20,9 @@ difference is what happens inside. Instead of tracing the whole policy as one graph, Edge compiles one TensorRT engine per component, then records a small outer graph that only *calls* those engines. +The code is in ``tools/hf``. The entry points are ``tools/hf/run_pi05_export.py``, +``run_groot_export.py``, and ``run_nemotron_export.py``. + .. note:: The Edge exporter is experimental. Family patches target a specific modeling @@ -67,8 +70,8 @@ Before ``export()``, load the Edge-LLM plugins and force HuggingFace attention t .. code-block:: python - from torch_tensorrt.hf.exporters.plugin.plugin_utils import load_plugins_for_trt - from torch_tensorrt.hf.exporters.utils import force_hf_attention + from hf.exporters.plugin.plugin_utils import load_plugins_for_trt + from hf.exporters.utils import force_hf_attention load_plugins_for_trt() force_hf_attention(policy.model.paligemma_with_expert.paligemma.model.vision_tower, "eager") @@ -82,8 +85,8 @@ plus sample inputs, and call ``export()``. .. code-block:: python - from torch_tensorrt.hf.exporters import EdgeExporter, EdgeConfig - from torch_tensorrt.hf.exporters.plugin.plugin_utils import load_plugins_for_trt + from hf.exporters import EdgeExporter, EdgeConfig + from hf.exporters.plugin.plugin_utils import load_plugins_for_trt load_plugins_for_trt() @@ -166,7 +169,7 @@ and no ``context_projection`` engine. Nemotron is a single ``spec.run()`` calls ``call_engine(...)``, so ``torch.export`` records **one node per engine**. Matching ``register_fake`` kernels give Dynamo the output shapes. Two packing ops live in the same file -(``py/torch_tensorrt/hf/exporters/ops.py``): +(``tools/hf/exporters/ops.py``): * ``edge_llm::fuse_prefix`` — PI05: concat vision tokens with language embeddings and gather the compact prefix. @@ -178,20 +181,26 @@ These appear in the **outer** ExportedProgram. They are not TensorRT plugins. Patches ------- -HuggingFace ``DynamoExporter`` uses ``@register_patch`` plus a temporary class -``setattr``. Edge uses the same contract. +Edge does not wrap the policy in a new module. It temporarily replaces +``Class.forward`` on the original HuggingFace / LeRobot class, compiles that +submodule, then restores the method (dryrun leaves the replacement in place). + +``@register_patch`` does not install anything. It records a factory and a dotted +class path on a backend (``"pi05"``, ``"groot"``, ``"nemotron"``). +``apply_patches(backend)`` imports that class and does +``setattr(Cls, "forward", factory(original))`` for the duration of +``export()``. -Each family has a ``patches.py`` that registers factories on a backend name -(``"pi05"``, ``"groot"``, ``"nemotron"``). A factory receives the **original** -``Class.forward`` and returns a replacement. ``apply_patches(backend)`` resolves -the dotted class path and does ``setattr(Cls, "forward", factory(original))`` -for the duration of ``export()``. After a real compile the original methods are -restored. Dryrun leaves the replacements installed so ``execute_engine`` still -hits the patched Python modules. +HuggingFace ``DynamoExporter`` uses the same two steps. The purpose is +different. HF patches make the original modeling ``forward`` traceable. Edge +patches change ``forward`` first so TensorRT traces plugin ops +(``torch.ops.trt.*``), not HuggingFace attention. After compile, the outer +graph is ``spec.run()`` → ``execute_engine``. There is no HF attention left +to patch, so the HuggingFace ``"dynamo"`` registry does not apply. .. code-block:: python - from torch_tensorrt.hf.exporters.plugin.attn_patches import register_patch + from exporters.plugin.attn_patches import register_patch PI05 = "pi05" @@ -207,15 +216,15 @@ hits the patched Python modules. return forward -The replacement is the thing TensorRT traces. You compile the **original -submodule** (``PaliGemmaModel``, ``PiGemmaModel``, ``FlowmatchingActionHead``, …), -not a wrapper ``nn.Module``. The patched ``forward`` is what makes that submodule -look like an Edge engine: a tensor in, a tensor out, plugin attention inside. +You compile the original submodule (``PaliGemmaModel``, ``PiGemmaModel``, +``FlowmatchingActionHead``, …), not a wrapper ``nn.Module``. The patched +``forward`` is what TensorRT traces: a tensor in, a tensor out, plugin +attention inside. -When the same class is used in two roles, the patched ``forward`` dispatches. -PI05 language is ``PiGemmaModel`` for both the language tower and the action -expert. Edge prefill passes ``rope_rotary_cos_sin``; the action expert does not. -If that argument is missing, the original HuggingFace forward runs: +When the same class is used twice (PI05 language vs action expert), the +patched ``forward`` checks for Edge arguments such as +``rope_rotary_cos_sin``. If they are missing, the original HuggingFace +``forward`` runs: .. code-block:: python @@ -228,18 +237,52 @@ Attention patches follow the same rule: ``GemmaAttention.forward`` uses the language plugin when ``rope_rotary_cos_sin`` is present, otherwise eager HF attention. -The spec installs the whole family backend once around the component loop: +The spec installs the family once around the component loop: .. code-block:: python class Pi05Spec(EdgeSpec): def apply_patches(self, model=None): - from torch_tensorrt.hf.exporters.plugin.attn_patches import apply_patches + from exporters.plugin.attn_patches import apply_patches return apply_patches("pi05") -Nemotron also wraps hybrid mixers on the live model inside ``apply_patches`` -(MoE packing needs the instance). That is still not a separate export wrapper; -the compiled module is the original ``NemotronHForCausalLM``. +When the decorator is enough +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If ``type(module)`` is the class named in the dotted path, +``@register_patch`` plus ``apply_patches`` is all you need. That is Siglip, +Qwen3, Llama, ``GR00TN15``, the action head, and so on. + +When it is not +^^^^^^^^^^^^^^ + +The path must be the **same class object** as the live module. A look-alike +file under another import is a different class. ``setattr`` on one does not +change the other. + +``trust_remote_code=True`` downloads Hub ``.py`` files into HuggingFace's +module cache and imports them. ``AutoModel.from_config`` then builds an +instance in memory. That class's module path looks like +``transformers_modules...modeling_...``. It is not stable and +does not exist until load, so the decorator cannot name it. The cache stores +source, not the ``nn.Module``. + +GR00T's Eagle is this case. The LeRobot path +``lerobot.policies.groot.eagle2_hg_model....Eagle25VLForConditionalGeneration`` +is a different class from the HuggingFace cache copy that ``from_config`` +actually constructs. + +Live-object patches +^^^^^^^^^^^^^^^^^^^ + +``apply_groot_patches(model)`` is not a second decorator. It is the place +that has the instance, so it can patch ``type(eagle_model)``. It still runs +``apply_patches("groot")`` for every class that has a stable path. + +Nemotron's ``apply_nemotron_patches(model)`` is the same idea for mixers: +the registry is string paths; anything that only exists on the live object +needs ``model``. The compiled module is still the original +``NemotronHForCausalLM``, not a wrapper. Add a new model --------------- @@ -248,7 +291,7 @@ Add a new model and loops ``spec.components``. A new architecture is a new spec plus a patch backend. -Create ``py/torch_tensorrt/hf/exporters/models//``: +Create ``tools/hf/exporters/models//``: .. code-block:: text @@ -262,21 +305,21 @@ exporter package loads: .. code-block:: python - from torch_tensorrt.hf.exporters.models.my_vla import spec as _my_vla # noqa: F401 + from hf.exporters.models.my_vla import spec as _my_vla # noqa: F401 1. Register the spec ^^^^^^^^^^^^^^^^^^^^ .. code-block:: python - from torch_tensorrt.hf.exporters.spec import EdgeSpec, register_edge_spec + from hf.exporters.spec import EdgeSpec, register_edge_spec @register_edge_spec("my_vla") class MyVlaSpec(EdgeSpec): components = ("vision", "language", "action") def apply_patches(self, model=None): - from torch_tensorrt.hf.exporters.plugin.attn_patches import apply_patches + from hf.exporters.plugin.attn_patches import apply_patches from .patches import MY_VLA return apply_patches(MY_VLA) @@ -452,7 +495,7 @@ either graph-breaks or fails. With a converter, the op becomes one TensorRT plugin layer. Converters live in -``py/torch_tensorrt/hf/exporters/plugin/plugin_converter.py`` and are +``tools/hf/exporters/plugin/plugin_converter.py`` and are registered with ``@dynamo_tensorrt_converter``. Example for ViT attention: .. code-block:: python diff --git a/py/torch_tensorrt/hf/__init__.py b/py/torch_tensorrt/hf/__init__.py deleted file mode 100644 index 1a52009e770..00000000000 --- a/py/torch_tensorrt/hf/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -"""HuggingFace-facing export helpers for Torch-TensorRT. - -Use ``from torch_tensorrt.hf.exporters import EdgeExporter, EdgeConfig``. -""" diff --git a/py/torch_tensorrt/hf/exporters/__init__.py b/py/torch_tensorrt/hf/exporters/__init__.py deleted file mode 100644 index 01dd627defa..00000000000 --- a/py/torch_tensorrt/hf/exporters/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -from torch_tensorrt.hf.exporters.config import EdgeConfig -from torch_tensorrt.hf.exporters.exporter import EdgeExporter -from torch_tensorrt.hf.exporters.models.groot.spec import ( # noqa: F401 - GrootSpec as _GrootSpec, -) -from torch_tensorrt.hf.exporters.models.nemotron.spec import ( # noqa: F401 - NemotronSpec as _NemotronSpec, -) -from torch_tensorrt.hf.exporters.models.pi05.spec import ( # noqa: F401 - Pi05Spec as _Pi05Spec, -) -from torch_tensorrt.hf.exporters.spec import ( - ComponentBundle, - EdgeSpec, - get_edge_spec, - register_edge_spec, -) - -__all__ = [ - "ComponentBundle", - "EdgeConfig", - "EdgeExporter", - "EdgeSpec", - "get_edge_spec", - "register_edge_spec", -] diff --git a/py/torch_tensorrt/hf/exporters/models/__init__.py b/py/torch_tensorrt/hf/exporters/models/__init__.py deleted file mode 100644 index bfb17db20d4..00000000000 --- a/py/torch_tensorrt/hf/exporters/models/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Model families. Importing this package registers EdgeSpecs.""" - -from torch_tensorrt.hf.exporters.models.groot import spec as _groot # noqa: F401 -from torch_tensorrt.hf.exporters.models.nemotron import spec as _nemotron # noqa: F401 -from torch_tensorrt.hf.exporters.models.pi05 import spec as _pi05 # noqa: F401 diff --git a/pyproject.toml b/pyproject.toml index 9dff51f61e2..f48d7abdc4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -451,16 +451,6 @@ module = "torch_tensorrt.fx.*" ignore_errors = true follow_imports = "skip" -[[tool.mypy.overrides]] -module = [ - "torch_tensorrt.hf.exporters.plugin.*", - "torch_tensorrt.hf.exporters.models.*", - "torch_tensorrt.hf.exporters.data", - "torch_tensorrt.hf.exporters.rope", - "torch_tensorrt.hf.exporters.prefix_cache", -] -ignore_errors = true - [tool.typos] files.extend-exclude = [ "docs/**/*", diff --git a/setup.py b/setup.py index 64a039f61fa..1c994f6f39b 100644 --- a/setup.py +++ b/setup.py @@ -616,14 +616,6 @@ def run(self): "torch_tensorrt.dynamo.runtime", "torch_tensorrt.dynamo.tools", "torch_tensorrt.executorch", - "torch_tensorrt.hf", - "torch_tensorrt.hf.exporters", - "torch_tensorrt.hf.exporters.models", - "torch_tensorrt.hf.exporters.models.common", - "torch_tensorrt.hf.exporters.models.groot", - "torch_tensorrt.hf.exporters.models.nemotron", - "torch_tensorrt.hf.exporters.models.pi05", - "torch_tensorrt.hf.exporters.plugin", "torch_tensorrt.runtime", ] @@ -663,14 +655,6 @@ def run(self): "torch_tensorrt.dynamo.runtime": "py/torch_tensorrt/dynamo/runtime", "torch_tensorrt.dynamo.tools": "py/torch_tensorrt/dynamo/tools", "torch_tensorrt.executorch": "py/torch_tensorrt/executorch", - "torch_tensorrt.hf": "py/torch_tensorrt/hf", - "torch_tensorrt.hf.exporters": "py/torch_tensorrt/hf/exporters", - "torch_tensorrt.hf.exporters.models": "py/torch_tensorrt/hf/exporters/models", - "torch_tensorrt.hf.exporters.models.common": "py/torch_tensorrt/hf/exporters/models/common", - "torch_tensorrt.hf.exporters.models.groot": "py/torch_tensorrt/hf/exporters/models/groot", - "torch_tensorrt.hf.exporters.models.nemotron": "py/torch_tensorrt/hf/exporters/models/nemotron", - "torch_tensorrt.hf.exporters.models.pi05": "py/torch_tensorrt/hf/exporters/models/pi05", - "torch_tensorrt.hf.exporters.plugin": "py/torch_tensorrt/hf/exporters/plugin", "torch_tensorrt.runtime": "py/torch_tensorrt/runtime", } diff --git a/tools/hf/__init__.py b/tools/hf/__init__.py new file mode 100644 index 00000000000..a58922e44e3 --- /dev/null +++ b/tools/hf/__init__.py @@ -0,0 +1,4 @@ +"""HuggingFace-facing export helpers for Torch-TensorRT. + +Use ``from exporters import EdgeExporter, EdgeConfig``. +""" diff --git a/tools/hf/exporters/__init__.py b/tools/hf/exporters/__init__.py new file mode 100644 index 00000000000..2ed7d30ed14 --- /dev/null +++ b/tools/hf/exporters/__init__.py @@ -0,0 +1,11 @@ +from .config import EdgeConfig +from .exporter import EdgeExporter +from .models.groot.spec import GrootSpec as _GrootSpec # noqa: F401 +from .models.nemotron.spec import NemotronSpec as _NemotronSpec # noqa: F401 +from .models.pi05.spec import Pi05Spec as _Pi05Spec # noqa: F401 +from .spec import ( + ComponentBundle, + EdgeSpec, + get_edge_spec, + register_edge_spec, +) diff --git a/py/torch_tensorrt/hf/exporters/compile.py b/tools/hf/exporters/compile.py similarity index 81% rename from py/torch_tensorrt/hf/exporters/compile.py rename to tools/hf/exporters/compile.py index a8b9dee20d6..81777e5c66c 100644 --- a/py/torch_tensorrt/hf/exporters/compile.py +++ b/tools/hf/exporters/compile.py @@ -6,14 +6,17 @@ import torch import torch_tensorrt -from torch_tensorrt.hf.exporters.ops import _as_tuple, record_engine -from torch_tensorrt.hf.exporters.spec import ComponentBundle + +from .measure import cuda_ms, parity +from .ops import _as_tuple, record_engine +from .spec import ComponentBundle DEFAULT_TRT_SETTINGS: dict[str, Any] = { "min_block_size": 1, "require_full_compilation": True, "immutable_weights": True, "disable_tf32": True, + "truncate_double": True, } _TRT_COMPILE_KEYS = frozenset(DEFAULT_TRT_SETTINGS) | { @@ -33,6 +36,7 @@ def compile_component( engine_dir: Path, dryrun: bool = False, trt_settings: dict[str, Any] | None = None, + bench: dict[str, tuple[float, float]] | None = None, ) -> tuple[str, tuple[torch.Tensor, ...]]: """Export one component, compile it, write ``engine_dir//``. @@ -42,7 +46,7 @@ def compile_component( Family setattr is owned by ``EdgeSpec.apply_patches``, not this helper. ``dryrun`` records the patched eager module for ``execute_engine``. """ - from torch_tensorrt.hf.exporters.plugin.attn_patches import ( + from .plugin.attn_patches import ( set_language_mask_type, ) @@ -89,6 +93,22 @@ def compile_component( arg_inputs=trace_args, **settings, ) + + with torch.no_grad(): + trt_out = _as_tuple(compiled(*execute_args)) + for i, (eager_t, trt_t) in enumerate(zip(outputs, trt_out)): + if not isinstance(eager_t, torch.Tensor) or not isinstance( + trt_t, torch.Tensor + ): + continue + label = name if i == 0 else f"{name}[{i}]" + parity(f"{label} A vs C (TRT)", eager_t, trt_t) + + eager_ms = cuda_ms(lambda: module(*execute_args)) + trt_ms = cuda_ms(lambda: compiled(*execute_args)) + if bench is not None: + bench[name] = (eager_ms, trt_ms) + record_engine( engine_path, component=name, @@ -111,7 +131,7 @@ def compile_component( return engine_path, outputs finally: if not dryrun and patched is not None: - from torch_tensorrt.hf.exporters.plugin.plugin_utils import ( + from .plugin.plugin_utils import ( restore_attention, ) diff --git a/py/torch_tensorrt/hf/exporters/config.py b/tools/hf/exporters/config.py similarity index 92% rename from py/torch_tensorrt/hf/exporters/config.py rename to tools/hf/exporters/config.py index ae9f140366e..d81055589b8 100644 --- a/py/torch_tensorrt/hf/exporters/config.py +++ b/tools/hf/exporters/config.py @@ -7,7 +7,7 @@ @dataclass class EdgeConfig: - """Knobs for :class:`~torch_tensorrt.hf.exporters.EdgeExporter`. + """Knobs for :class:`~exporters.EdgeExporter`. ``strict`` / ``dynamic`` / ``dynamic_shapes`` match HuggingFace ``DynamoConfig`` so this can subclass it later without an API break. diff --git a/py/torch_tensorrt/hf/exporters/data.py b/tools/hf/exporters/data.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/data.py rename to tools/hf/exporters/data.py diff --git a/py/torch_tensorrt/hf/exporters/exporter.py b/tools/hf/exporters/exporter.py similarity index 94% rename from py/torch_tensorrt/hf/exporters/exporter.py rename to tools/hf/exporters/exporter.py index f619ac01b23..f93ca57bc08 100644 --- a/py/torch_tensorrt/hf/exporters/exporter.py +++ b/tools/hf/exporters/exporter.py @@ -11,13 +11,14 @@ import torch import torch.nn as nn from torch.export import ExportedProgram -from torch_tensorrt.hf.exporters import ops as _ops # noqa: F401 -from torch_tensorrt.hf.exporters.compile import compile_component -from torch_tensorrt.hf.exporters.config import EdgeConfig -from torch_tensorrt.hf.exporters.runtime import EdgeRuntimeModule -from torch_tensorrt.hf.exporters.spec import get_edge_spec from transformers.exporters.exporter_dynamo import DynamoExporter +from . import ops as _ops # noqa: F401 +from .compile import compile_component +from .config import EdgeConfig +from .runtime import EdgeRuntimeModule +from .spec import get_edge_spec + logger = logging.getLogger(__name__) @@ -42,6 +43,7 @@ def __init__(self) -> None: self.engines: dict[str, str] = {} self.runtime: EdgeRuntimeModule | None = None self.sample: dict[str, Any] = {} + self.bench: dict[str, tuple[float, float]] = {} self._dryrun_patches: contextlib.ExitStack | None = None def export( @@ -71,6 +73,7 @@ def export( engines: dict[str, str] = {} upstream: dict[str, Any] = {} + self.bench = {} def _compile_components() -> None: for name in names: @@ -81,6 +84,7 @@ def _compile_components() -> None: engine_dir=engine_dir, dryrun=config.dryrun, trt_settings=config.trt_settings, + bench=self.bench, ) upstream.update(spec.capture_upstream(name, outs, sample, bundle)) diff --git a/tools/hf/exporters/measure.py b/tools/hf/exporters/measure.py new file mode 100644 index 00000000000..cdd43b3f878 --- /dev/null +++ b/tools/hf/exporters/measure.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import time +from collections.abc import Callable, Mapping + +import torch + + +def parity(name: str, eager: torch.Tensor, trt: torch.Tensor) -> None: + """Print eager-vs-TRT numeric agreement for one tensor.""" + a, b = eager.float(), trt.float() + delta = a - b + diff = delta.abs() + rel_l2 = delta.norm() / b.norm().clamp_min(1e-8) + close = torch.isclose(a, b, rtol=1e-2, atol=1e-2).float().mean() * 100 + print( + f"{name:<36} mean_abs={float(diff.mean()):.6f} " + f"max_abs={float(diff.max()):.6f} rel_l2={float(rel_l2):.4f} " + f"close%={float(close):.1f}" + ) + + +def cuda_ms(fn: Callable[[], object], *, warmup: int = 10, iters: int = 100) -> float: + """Average runtime of ``fn`` in milliseconds (CUDA events, else wall time).""" + with torch.no_grad(): + for _ in range(warmup): + fn() + if torch.cuda.is_available(): + torch.cuda.synchronize() + start, end = torch.cuda.Event(True), torch.cuda.Event(True) + start.record() + for _ in range(iters): + fn() + end.record() + torch.cuda.synchronize() + return start.elapsed_time(end) / iters + t0 = time.perf_counter() + for _ in range(iters): + fn() + return (time.perf_counter() - t0) * 1000.0 / iters + + +def speedup(eager_ms: float, trt_ms: float) -> str: + if eager_ms <= 0.0 or trt_ms <= 0.0: + return "n/a" + return f"{eager_ms / trt_ms:.3f}x" + + +def print_bench(bench: Mapping[str, tuple[float, float]]) -> None: + """Print per-component CUDA timings collected during ``export()``.""" + if not bench: + return + eager_total = trt_total = 0.0 + for name, (eager_ms, trt_ms) in bench.items(): + print(f"{name} eager execute: {eager_ms:.3f} ms") + print(f"{name} trt execute: {trt_ms:.3f} ms") + print(f"{name} speedup: {speedup(eager_ms, trt_ms)}") + eager_total += eager_ms + trt_total += trt_ms + print(f"total eager execute: {eager_total:.3f} ms") + print(f"total trt execute: {trt_total:.3f} ms") + print(f"total speedup: {speedup(eager_total, trt_total)}") diff --git a/tools/hf/exporters/models/__init__.py b/tools/hf/exporters/models/__init__.py new file mode 100644 index 00000000000..eaebd1daeb8 --- /dev/null +++ b/tools/hf/exporters/models/__init__.py @@ -0,0 +1,5 @@ +"""Model families. Importing this package registers EdgeSpecs.""" + +from .groot import spec as _groot # noqa: F401 +from .nemotron import spec as _nemotron # noqa: F401 +from .pi05 import spec as _pi05 # noqa: F401 diff --git a/py/torch_tensorrt/hf/exporters/models/common/__init__.py b/tools/hf/exporters/models/common/__init__.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/models/common/__init__.py rename to tools/hf/exporters/models/common/__init__.py diff --git a/py/torch_tensorrt/hf/exporters/models/common/helpers.py b/tools/hf/exporters/models/common/helpers.py similarity index 96% rename from py/torch_tensorrt/hf/exporters/models/common/helpers.py rename to tools/hf/exporters/models/common/helpers.py index a50c6bf0a6b..5f4951e07c7 100644 --- a/py/torch_tensorrt/hf/exporters/models/common/helpers.py +++ b/tools/hf/exporters/models/common/helpers.py @@ -25,7 +25,7 @@ def causal_lm_flat( num_kv = int(cfg.num_key_value_heads) head_dim = int(getattr(cfg, "head_dim", cfg.hidden_size // cfg.num_attention_heads)) try: - from torch_tensorrt.hf.exporters.rope import make_rope_rotary_cos_sin + from ...rope import make_rope_rotary_cos_sin rope = make_rope_rotary_cos_sin( cfg, int(max_seq_len), device, language_model=language diff --git a/py/torch_tensorrt/hf/exporters/models/common/patches.py b/tools/hf/exporters/models/common/patches.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/models/common/patches.py rename to tools/hf/exporters/models/common/patches.py diff --git a/py/torch_tensorrt/hf/exporters/models/groot/__init__.py b/tools/hf/exporters/models/groot/__init__.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/models/groot/__init__.py rename to tools/hf/exporters/models/groot/__init__.py diff --git a/py/torch_tensorrt/hf/exporters/models/groot/helpers.py b/tools/hf/exporters/models/groot/helpers.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/models/groot/helpers.py rename to tools/hf/exporters/models/groot/helpers.py diff --git a/py/torch_tensorrt/hf/exporters/models/groot/patches.py b/tools/hf/exporters/models/groot/patches.py similarity index 85% rename from py/torch_tensorrt/hf/exporters/models/groot/patches.py rename to tools/hf/exporters/models/groot/patches.py index b0eb154d095..4dbc9feb1e2 100644 --- a/py/torch_tensorrt/hf/exporters/models/groot/patches.py +++ b/tools/hf/exporters/models/groot/patches.py @@ -2,15 +2,17 @@ from __future__ import annotations -from typing import Any, Callable +from contextlib import contextmanager +from typing import Any, Callable, Iterator import torch -from torch_tensorrt.hf.exporters.models.common.patches import causal_lm_plugin_forward -from torch_tensorrt.hf.exporters.plugin.attn_patches import ( + +from ...plugin.attn_patches import ( _patch_language_attention, _patch_vision_attention, register_patch, ) +from ..common.patches import causal_lm_plugin_forward GROOT = "groot" @@ -45,6 +47,28 @@ def forward(self, pixel_values, input_ids=None, **kwargs: Any): return forward +@contextmanager +def apply_groot_patches(model: Any | None = None) -> Iterator[None]: + """Family setattr, plus the live Eagle class. + + LeRobot builds Eagle with ``AutoModel.from_config(..., trust_remote_code=True)``, + so the running class is HuggingFace ``transformers_modules`` code, not + ``lerobot.policies.groot.eagle2_hg_model``. The dotted path still covers the + in-tree copy; this patches ``type(eagle_model)`` so vision + ``eagle(pixel_values)`` hits ``extract_feature``. + """ + from ...plugin.attn_patches import apply_patches, patch_attribute + from .helpers import _groot + + with apply_patches(GROOT): + if model is None: + yield + return + eagle_cls = type(_groot(model).backbone.eagle_model) + with patch_attribute(eagle_cls, "forward", _patch_eagle_image_features): + yield + + @register_patch( GROOT, "transformers.models.llama.modeling_llama.LlamaForCausalLM.forward", diff --git a/py/torch_tensorrt/hf/exporters/models/groot/spec.py b/tools/hf/exporters/models/groot/spec.py similarity index 94% rename from py/torch_tensorrt/hf/exporters/models/groot/spec.py rename to tools/hf/exporters/models/groot/spec.py index 59b8a97e139..82a542b57b7 100644 --- a/py/torch_tensorrt/hf/exporters/models/groot/spec.py +++ b/tools/hf/exporters/models/groot/spec.py @@ -5,22 +5,23 @@ import torch import torch.nn as nn -from torch_tensorrt.hf.exporters.models.common.helpers import ( + +from ...ops import call_engine, scatter_image_tokens +from ...spec import ( + ComponentBundle, + EdgeSpec, + register_edge_spec, +) +from ..common.helpers import ( causal_lm_flat, kv_kwargs, split_flat_to_kwargs, ) -from torch_tensorrt.hf.exporters.models.groot.helpers import ( +from .helpers import ( _groot, make_embodiment_id, ) -from torch_tensorrt.hf.exporters.models.groot.patches import GROOT -from torch_tensorrt.hf.exporters.ops import call_engine, scatter_image_tokens -from torch_tensorrt.hf.exporters.spec import ( - ComponentBundle, - EdgeSpec, - register_edge_spec, -) +from .patches import apply_groot_patches def _export_module(module: nn.Module, sample: Mapping[str, Any]) -> nn.Module: @@ -44,10 +45,7 @@ class GrootSpec(EdgeSpec): # type: ignore[misc] components = ("vision", "language", "context_projection", "action") def apply_patches(self, model=None): - del model - from torch_tensorrt.hf.exporters.plugin.attn_patches import apply_patches - - return apply_patches(GROOT) + return apply_groot_patches(model) def prepare_sample_inputs( self, model: nn.Module, raw: Mapping[str, Any], config: Any @@ -57,7 +55,8 @@ def prepare_sample_inputs( from lerobot.policies.factory import make_pre_post_processors from lerobot.policies.groot.processor_groot import GrootEagleEncodeStep - from torch_tensorrt.hf.exporters.data import ( + + from ...data import ( create_pil_messages, load_test_data, pack_state, @@ -127,7 +126,7 @@ def prepare( upstream: Mapping[str, Any], config: Any, ) -> ComponentBundle: - from torch_tensorrt.hf.exporters.plugin.attention import ( + from ...plugin.attention import ( ContextAttentionMaskType, ) diff --git a/py/torch_tensorrt/hf/exporters/models/nemotron/__init__.py b/tools/hf/exporters/models/nemotron/__init__.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/models/nemotron/__init__.py rename to tools/hf/exporters/models/nemotron/__init__.py diff --git a/py/torch_tensorrt/hf/exporters/models/nemotron/helpers.py b/tools/hf/exporters/models/nemotron/helpers.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/models/nemotron/helpers.py rename to tools/hf/exporters/models/nemotron/helpers.py diff --git a/py/torch_tensorrt/hf/exporters/models/nemotron/mamba_stub.py b/tools/hf/exporters/models/nemotron/mamba_stub.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/models/nemotron/mamba_stub.py rename to tools/hf/exporters/models/nemotron/mamba_stub.py diff --git a/py/torch_tensorrt/hf/exporters/models/nemotron/patches.py b/tools/hf/exporters/models/nemotron/patches.py similarity index 91% rename from py/torch_tensorrt/hf/exporters/models/nemotron/patches.py rename to tools/hf/exporters/models/nemotron/patches.py index e9856dc285e..7b772500d9c 100644 --- a/py/torch_tensorrt/hf/exporters/models/nemotron/patches.py +++ b/tools/hf/exporters/models/nemotron/patches.py @@ -5,12 +5,12 @@ from contextlib import contextmanager from typing import Any, Callable, Iterator -from torch_tensorrt.hf.exporters.models.common.patches import gather_last_token_hidden -from torch_tensorrt.hf.exporters.models.nemotron.helpers import _decoder, _kind -from torch_tensorrt.hf.exporters.plugin.attn_patches import ( +from ...plugin.attn_patches import ( apply_patches, register_patch, ) +from ..common.patches import gather_last_token_hidden +from .helpers import _decoder, _kind NEMOTRON = "nemotron" @@ -78,7 +78,7 @@ def forward( @contextmanager def apply_nemotron_patches(model: Any | None = None) -> Iterator[None]: """Class setattr plus mixer plugin wrappers (MoE packing needs the instance).""" - from torch_tensorrt.hf.exporters.plugin.plugin_utils import ( + from ...plugin.plugin_utils import ( patch_nemotron_mixers, restore_attention, ) diff --git a/py/torch_tensorrt/hf/exporters/models/nemotron/spec.py b/tools/hf/exporters/models/nemotron/spec.py similarity index 91% rename from py/torch_tensorrt/hf/exporters/models/nemotron/spec.py rename to tools/hf/exporters/models/nemotron/spec.py index 413a5b21e85..b4a283f410b 100644 --- a/py/torch_tensorrt/hf/exporters/models/nemotron/spec.py +++ b/tools/hf/exporters/models/nemotron/spec.py @@ -5,24 +5,25 @@ import torch import torch.nn as nn -from torch_tensorrt.hf.exporters.models.common.helpers import ( + +from ...ops import call_engine +from ...spec import ( + ComponentBundle, + EdgeSpec, + register_edge_spec, +) +from ..common.helpers import ( kv_kwargs, split_flat_to_kwargs, ) -from torch_tensorrt.hf.exporters.models.nemotron.helpers import ( +from .helpers import ( _decoder, _kind, allocate_plugin_states, ) -from torch_tensorrt.hf.exporters.models.nemotron.patches import ( +from .patches import ( apply_nemotron_patches, ) -from torch_tensorrt.hf.exporters.ops import call_engine -from torch_tensorrt.hf.exporters.spec import ( - ComponentBundle, - EdgeSpec, - register_edge_spec, -) @register_edge_spec("nemotron_h", "nemotron") @@ -72,7 +73,7 @@ def prepare( upstream: Mapping[str, Any], config: Any, ) -> ComponentBundle: - from torch_tensorrt.hf.exporters.rope import make_rope_rotary_cos_sin + from ...rope import make_rope_rotary_cos_sin del name, upstream embeds = sample["inputs_embeds"] diff --git a/py/torch_tensorrt/hf/exporters/models/pi05/__init__.py b/tools/hf/exporters/models/pi05/__init__.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/models/pi05/__init__.py rename to tools/hf/exporters/models/pi05/__init__.py diff --git a/py/torch_tensorrt/hf/exporters/models/pi05/helpers.py b/tools/hf/exporters/models/pi05/helpers.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/models/pi05/helpers.py rename to tools/hf/exporters/models/pi05/helpers.py diff --git a/py/torch_tensorrt/hf/exporters/models/pi05/patches.py b/tools/hf/exporters/models/pi05/patches.py similarity index 95% rename from py/torch_tensorrt/hf/exporters/models/pi05/patches.py rename to tools/hf/exporters/models/pi05/patches.py index 1ef966509a2..5291c96ebe3 100644 --- a/py/torch_tensorrt/hf/exporters/models/pi05/patches.py +++ b/tools/hf/exporters/models/pi05/patches.py @@ -6,12 +6,13 @@ import torch import torch.nn.functional as F -from torch_tensorrt.hf.exporters.models.common.patches import causal_lm_plugin_forward -from torch_tensorrt.hf.exporters.plugin.attn_patches import ( + +from ...plugin.attn_patches import ( _patch_language_attention, _patch_vision_attention, register_patch, ) +from ..common.patches import causal_lm_plugin_forward PI05 = "pi05" @@ -97,7 +98,8 @@ def forward( **kwargs, ) from lerobot.policies.pi05.modeling_pi05 import create_sinusoidal_pos_embedding - from torch_tensorrt.hf.exporters.prefix_cache import PrefixKVCache + + from ...prefix_cache import PrefixKVCache suffix_embs = self.action_in_proj(x_t) time_emb = create_sinusoidal_pos_embedding( diff --git a/py/torch_tensorrt/hf/exporters/models/pi05/spec.py b/tools/hf/exporters/models/pi05/spec.py similarity index 93% rename from py/torch_tensorrt/hf/exporters/models/pi05/spec.py rename to tools/hf/exporters/models/pi05/spec.py index 7fb9381f449..d53dd05dcc7 100644 --- a/py/torch_tensorrt/hf/exporters/models/pi05/spec.py +++ b/tools/hf/exporters/models/pi05/spec.py @@ -5,25 +5,26 @@ import torch import torch.nn as nn -from torch_tensorrt.hf.exporters.models.common.helpers import ( + +from ...ops import call_engine, fuse_prefix +from ...spec import ( + ComponentBundle, + EdgeSpec, + register_edge_spec, +) +from ..common.helpers import ( causal_lm_flat, kv_kwargs, split_flat_to_kwargs, ) -from torch_tensorrt.hf.exporters.models.common.patches import language_decoder -from torch_tensorrt.hf.exporters.models.pi05.helpers import ( +from ..common.patches import language_decoder +from .helpers import ( _core, build_pi05_prefix_embs, make_pi05_suffix_position_and_mask, pi05_compact_index, ) -from torch_tensorrt.hf.exporters.models.pi05.patches import PI05 -from torch_tensorrt.hf.exporters.ops import call_engine, fuse_prefix -from torch_tensorrt.hf.exporters.spec import ( - ComponentBundle, - EdgeSpec, - register_edge_spec, -) +from .patches import PI05 @register_edge_spec("pi05") @@ -33,7 +34,7 @@ class Pi05Spec(EdgeSpec): # type: ignore[misc] def apply_patches(self, model=None): """Install vision, language, and action setattr replacements.""" del model - from torch_tensorrt.hf.exporters.plugin.attn_patches import apply_patches + from ...plugin.attn_patches import apply_patches return apply_patches(PI05) @@ -48,7 +49,8 @@ def prepare_sample_inputs( OBS_LANGUAGE_ATTENTION_MASK, OBS_LANGUAGE_TOKENS, ) - from torch_tensorrt.hf.exporters.data import ( + + from ...data import ( frame_from_test_data, load_test_data, ) @@ -96,7 +98,7 @@ def prepare( upstream: Mapping[str, Any], config: Any, ) -> ComponentBundle: - from torch_tensorrt.hf.exporters.plugin.attention import ( + from ...plugin.attention import ( ContextAttentionMaskType, ) diff --git a/py/torch_tensorrt/hf/exporters/ops.py b/tools/hf/exporters/ops.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/ops.py rename to tools/hf/exporters/ops.py diff --git a/py/torch_tensorrt/hf/exporters/plugin/__init__.py b/tools/hf/exporters/plugin/__init__.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/plugin/__init__.py rename to tools/hf/exporters/plugin/__init__.py diff --git a/py/torch_tensorrt/hf/exporters/plugin/attention.py b/tools/hf/exporters/plugin/attention.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/plugin/attention.py rename to tools/hf/exporters/plugin/attention.py diff --git a/py/torch_tensorrt/hf/exporters/plugin/attn_patches.py b/tools/hf/exporters/plugin/attn_patches.py similarity index 99% rename from py/torch_tensorrt/hf/exporters/plugin/attn_patches.py rename to tools/hf/exporters/plugin/attn_patches.py index cea2ea5b6c5..4b30576132b 100644 --- a/py/torch_tensorrt/hf/exporters/plugin/attn_patches.py +++ b/tools/hf/exporters/plugin/attn_patches.py @@ -20,7 +20,8 @@ import torch import torch.nn as nn -from torch_tensorrt.hf.exporters.plugin.attention import ContextAttentionMaskType + +from .attention import ContextAttentionMaskType _PATCHES: dict[str, list[tuple[str, Callable]]] = {} _LANGUAGE_MASK_TYPE = int(ContextAttentionMaskType.PADDING) diff --git a/py/torch_tensorrt/hf/exporters/plugin/mamba.py b/tools/hf/exporters/plugin/mamba.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/plugin/mamba.py rename to tools/hf/exporters/plugin/mamba.py diff --git a/py/torch_tensorrt/hf/exporters/plugin/moe.py b/tools/hf/exporters/plugin/moe.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/plugin/moe.py rename to tools/hf/exporters/plugin/moe.py diff --git a/py/torch_tensorrt/hf/exporters/plugin/plugin_converter.py b/tools/hf/exporters/plugin/plugin_converter.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/plugin/plugin_converter.py rename to tools/hf/exporters/plugin/plugin_converter.py diff --git a/py/torch_tensorrt/hf/exporters/plugin/plugin_utils.py b/tools/hf/exporters/plugin/plugin_utils.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/plugin/plugin_utils.py rename to tools/hf/exporters/plugin/plugin_utils.py diff --git a/py/torch_tensorrt/hf/exporters/prefix_cache.py b/tools/hf/exporters/prefix_cache.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/prefix_cache.py rename to tools/hf/exporters/prefix_cache.py diff --git a/py/torch_tensorrt/hf/exporters/rope.py b/tools/hf/exporters/rope.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/rope.py rename to tools/hf/exporters/rope.py diff --git a/py/torch_tensorrt/hf/exporters/runtime.py b/tools/hf/exporters/runtime.py similarity index 90% rename from py/torch_tensorrt/hf/exporters/runtime.py rename to tools/hf/exporters/runtime.py index 8e98947140a..e55547ea222 100644 --- a/py/torch_tensorrt/hf/exporters/runtime.py +++ b/tools/hf/exporters/runtime.py @@ -4,7 +4,8 @@ from typing import Any import torch.nn as nn -from torch_tensorrt.hf.exporters.spec import EdgeSpec + +from .spec import EdgeSpec class EdgeRuntimeModule(nn.Module): # type: ignore[misc] diff --git a/py/torch_tensorrt/hf/exporters/spec.py b/tools/hf/exporters/spec.py similarity index 100% rename from py/torch_tensorrt/hf/exporters/spec.py rename to tools/hf/exporters/spec.py diff --git a/tests/py/dynamo/hf/test_edge_exporter.py b/tools/hf/exporters/tests/test_edge_exporter.py similarity index 82% rename from tests/py/dynamo/hf/test_edge_exporter.py rename to tools/hf/exporters/tests/test_edge_exporter.py index 43bd6fec71f..23437408fd9 100644 --- a/tests/py/dynamo/hf/test_edge_exporter.py +++ b/tools/hf/exporters/tests/test_edge_exporter.py @@ -6,9 +6,9 @@ import pytest import torch import torch.nn as nn -from torch_tensorrt.hf.exporters import EdgeConfig, EdgeExporter, register_edge_spec -from torch_tensorrt.hf.exporters.ops import call_engine -from torch_tensorrt.hf.exporters.spec import ComponentBundle, EdgeSpec, registered_specs +from exporters import EdgeConfig, EdgeExporter, register_edge_spec +from exporters.ops import call_engine +from exporters.spec import ComponentBundle, EdgeSpec, registered_specs @register_edge_spec("dummy_edge") @@ -177,7 +177,7 @@ def test_edge_exporter_dryrun_keeps_attention_patch(tmp_path): @pytest.mark.unit def test_attn_patch_attribute_restores(): - from torch_tensorrt.hf.exporters.plugin.attn_patches import patch_attribute + from exporters.plugin.attn_patches import patch_attribute class Owner: def go(self): @@ -196,7 +196,7 @@ def go(self): @pytest.mark.unit def test_language_attn_keeps_hf_forward_without_rope(): - from torch_tensorrt.hf.exporters.plugin.attn_patches import ( + from exporters.plugin.attn_patches import ( _patch_language_attention, ) @@ -214,8 +214,8 @@ def forward(self, hidden_states, past_key_values=None, **kwargs): @pytest.mark.unit def test_pi05_backend_registers_vision_and_language(): - from torch_tensorrt.hf.exporters.models.pi05.patches import PI05 - from torch_tensorrt.hf.exporters.plugin.attn_patches import _PATCHES + from exporters.models.pi05.patches import PI05 + from exporters.plugin.attn_patches import _PATCHES paths = [p for p, _ in _PATCHES[PI05]] assert any("SiglipAttention.forward" in p for p in paths) @@ -227,7 +227,7 @@ def test_pi05_backend_registers_vision_and_language(): @pytest.mark.unit def test_paligemma_image_features_patch_returns_tensor(): - from torch_tensorrt.hf.exporters.models.pi05.patches import ( + from exporters.models.pi05.patches import ( _patch_paligemma_image_features, ) @@ -262,7 +262,7 @@ def forward(self, *args, **kwargs): @pytest.mark.unit def test_pi05_language_model_keeps_hf_forward_without_rope(): - from torch_tensorrt.hf.exporters.models.pi05.patches import ( + from exporters.models.pi05.patches import ( _patch_pi05_language_model, ) @@ -279,7 +279,7 @@ def forward(self, inputs_embeds=None, past_key_values=None, **kwargs): @pytest.mark.unit def test_pi05_action_keeps_training_forward_without_prefix_kv(): - from torch_tensorrt.hf.exporters.models.pi05.patches import ( + from exporters.models.pi05.patches import ( _patch_pi05_action_step_forward, ) @@ -294,10 +294,10 @@ def forward(self, images, img_masks, tokens, masks, actions, noise, time): @pytest.mark.unit def test_language_attn_plugin_when_rope_present(): - from torch_tensorrt.hf.exporters.plugin.attn_patches import ( + from exporters.plugin.attn_patches import ( _patch_language_attention, ) - from torch_tensorrt.hf.exporters.plugin.plugin_utils import ( + from exporters.plugin.plugin_utils import ( _register_attention_plugin_op, ) @@ -336,8 +336,8 @@ def forward(self, hidden_states, **kwargs): @pytest.mark.unit def test_groot_backend_registers_components(): - from torch_tensorrt.hf.exporters.models.groot.patches import GROOT - from torch_tensorrt.hf.exporters.plugin.attn_patches import _PATCHES + from exporters.models.groot.patches import GROOT + from exporters.plugin.attn_patches import _PATCHES paths = [p for p, _ in _PATCHES[GROOT]] assert any("SiglipAttention.forward" in p for p in paths) @@ -351,8 +351,8 @@ def test_groot_backend_registers_components(): @pytest.mark.unit def test_nemotron_backend_registers_causal_lm(): - from torch_tensorrt.hf.exporters.models.nemotron.patches import NEMOTRON - from torch_tensorrt.hf.exporters.plugin.attn_patches import _PATCHES + from exporters.models.nemotron.patches import NEMOTRON + from exporters.plugin.attn_patches import _PATCHES paths = [p for p, _ in _PATCHES[NEMOTRON]] assert any("NemotronHForCausalLM.forward" in p for p in paths) @@ -360,7 +360,7 @@ def test_nemotron_backend_registers_causal_lm(): @pytest.mark.unit def test_eagle_vision_patch_extracts_features(): - from torch_tensorrt.hf.exporters.models.groot.patches import ( + from exporters.models.groot.patches import ( _patch_eagle_image_features, ) @@ -376,9 +376,36 @@ def forward(self, *args, **kwargs): torch.testing.assert_close(Dummy()(pixel_values), pixel_values + 1) +@pytest.mark.unit +def test_groot_patches_live_eagle_class(): + from exporters.models.groot.patches import apply_groot_patches + + class Eagle: + def extract_feature(self, pixel_values): + return pixel_values + 1 + + def forward(self, pixel_values, input_ids=None, **kwargs): + raise AssertionError("unpatched Eagle.forward should not run") + + class Groot: + def __init__(self): + self.backbone = type("Backbone", (), {})() + self.backbone.eagle_model = Eagle() + + class Policy: + def __init__(self): + self._groot_model = Groot() + + policy = Policy() + eagle = policy._groot_model.backbone.eagle_model + pixel_values = torch.zeros(1, 3, 4, 4) + with apply_groot_patches(policy): + torch.testing.assert_close(eagle(pixel_values), pixel_values + 1) + + @pytest.mark.unit def test_eagle_vision_keeps_vlm_forward_with_input_ids(): - from torch_tensorrt.hf.exporters.models.groot.patches import ( + from exporters.models.groot.patches import ( _patch_eagle_image_features, ) @@ -399,7 +426,7 @@ def forward(self, pixel_values, input_ids=None, **kwargs): @pytest.mark.unit def test_groot_action_keeps_training_forward_without_context(): - from torch_tensorrt.hf.exporters.models.groot.patches import ( + from exporters.models.groot.patches import ( _patch_groot_action_step_forward, ) @@ -413,7 +440,7 @@ def forward(self, backbone_output, action_input): @pytest.mark.unit def test_groot_context_keeps_training_forward_without_hidden(): - from torch_tensorrt.hf.exporters.models.groot.patches import ( + from exporters.models.groot.patches import ( _patch_groot_context_projection, ) @@ -427,7 +454,7 @@ def forward(self, backbone_inputs, action_inputs): @pytest.mark.unit def test_nemotron_keeps_hf_forward_without_rope(): - from torch_tensorrt.hf.exporters.models.nemotron.patches import ( + from exporters.models.nemotron.patches import ( _patch_nemotron_causal_lm, ) @@ -444,7 +471,7 @@ def forward(self, input_ids=None, inputs_embeds=None, **kwargs): @pytest.mark.unit def test_category_specific_linear_uses_index_select(): - from torch_tensorrt.hf.exporters.models.groot.patches import ( + from exporters.models.groot.patches import ( _patch_category_specific_linear, ) @@ -470,3 +497,27 @@ def forward(self, x, cat_ids): out = layer(x, cat_ids) expected = torch.bmm(x, layer.W[cat_ids]) + layer.b[cat_ids].unsqueeze(1) torch.testing.assert_close(out, expected) + + +@pytest.mark.unit +def test_measure_parity_and_bench(capsys): + from exporters.measure import cuda_ms, parity, print_bench, speedup + + a = torch.ones(2, 2) + parity("dummy A vs C (TRT)", a, a) + log = capsys.readouterr().out + assert "dummy A vs C (TRT)" in log + assert "close%=100.0" in log + assert speedup(10.0, 5.0) == "2.000x" + assert speedup(0.0, 5.0) == "n/a" + + elapsed = cuda_ms(lambda: torch.ones(2, 2).sum(), warmup=1, iters=3) + assert elapsed >= 0.0 + + print_bench({"vision": (10.0, 5.0), "language": (4.0, 2.0)}) + log = capsys.readouterr().out + assert "vision eager execute: 10.000 ms" in log + assert "vision trt execute: 5.000 ms" in log + assert "total speedup: 2.000x" in log + print_bench({}) + assert capsys.readouterr().out == "" diff --git a/py/torch_tensorrt/hf/exporters/utils.py b/tools/hf/exporters/utils.py similarity index 75% rename from py/torch_tensorrt/hf/exporters/utils.py rename to tools/hf/exporters/utils.py index fb6c04b1779..b8d21d3b40b 100644 --- a/py/torch_tensorrt/hf/exporters/utils.py +++ b/tools/hf/exporters/utils.py @@ -8,17 +8,6 @@ import torch -_THOR_CUDA_LIB = Path("/usr/local/cuda-13.0/thor/targets/aarch64-linux/lib") - - -def configure_thor_pytorch() -> None: - """Use PyTorch fallbacks for ops whose pip CUDA wheels mismatch DriveOS Thor.""" - on_thor = os.environ.get("TRT_VLA_THOR", "auto") - if on_thor == "auto": - on_thor = "1" if _THOR_CUDA_LIB.is_dir() else "0" - if on_thor == "1": - torch.backends.cudnn.enabled = False - def force_hf_attention(module: Any, attn: str, use_cache: bool | None = False) -> None: """Force HuggingFace attention implementation on a module tree.""" diff --git a/examples/dynamo/run_groot_export.py b/tools/hf/run_groot_export.py similarity index 81% rename from examples/dynamo/run_groot_export.py rename to tools/hf/run_groot_export.py index 19baf8558f9..bf918c9add6 100644 --- a/examples/dynamo/run_groot_export.py +++ b/tools/hf/run_groot_export.py @@ -8,25 +8,19 @@ from __future__ import annotations import argparse +import sys from pathlib import Path -_REPO_ROOT = Path(__file__).resolve().parents[2] # TensorRT/ -_TRT_PY = _REPO_ROOT / "py" - import torch # noqa: E402 import torch_tensorrt # noqa: E402 - -_src_pkg = str(_TRT_PY / "torch_tensorrt") -if _src_pkg not in list(torch_tensorrt.__path__): - torch_tensorrt.__path__.append(_src_pkg) - +from exporters import EdgeConfig, EdgeExporter +from exporters.measure import print_bench +from exporters.plugin.plugin_utils import load_plugins_for_trt +from exporters.utils import force_hf_attention from lerobot.configs import FeatureType, PolicyFeature from lerobot.policies.groot import GrootPolicy from lerobot.policies.groot.configuration_groot import GrootConfig from lerobot.utils.constants import ACTION, OBS_STATE -from torch_tensorrt.hf.exporters import EdgeConfig, EdgeExporter -from torch_tensorrt.hf.exporters.plugin.plugin_utils import load_plugins_for_trt -from torch_tensorrt.hf.exporters.utils import configure_thor_pytorch, force_hf_attention def load_groot(device: torch.device) -> GrootPolicy: @@ -62,7 +56,6 @@ def main() -> None: parser.add_argument("--engine-dir", default="/tmp/groot_edge_exporter") args = parser.parse_args() - configure_thor_pytorch() load_plugins_for_trt() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -75,13 +68,7 @@ def main() -> None: force_hf_attention(eagle.language_model, "eager") exporter = EdgeExporter() - config = EdgeConfig( - model_type="groot", - engine_dir=args.engine_dir, - max_seq_len=968, - dryrun=not args.compile, - skip_runtime_export=False, - ) + config = EdgeConfig(model_type="groot", engine_dir=args.engine_dir, max_seq_len=968) # Spec tokenizes libero via Eagle chat template because we pass the policy. sample_inputs = {"device": device, "dtype": dtype} @@ -98,6 +85,7 @@ def main() -> None: out = velocity[0] if isinstance(velocity, (tuple, list)) else velocity print("velocity", tuple(out.shape), "mean", float(out.float().mean())) + print_bench(exporter.bench) if __name__ == "__main__": diff --git a/examples/dynamo/run_nemotron_export.py b/tools/hf/run_nemotron_export.py similarity index 80% rename from examples/dynamo/run_nemotron_export.py rename to tools/hf/run_nemotron_export.py index 2df1af5139f..977cfe0056b 100644 --- a/examples/dynamo/run_nemotron_export.py +++ b/tools/hf/run_nemotron_export.py @@ -8,24 +8,15 @@ from __future__ import annotations import argparse +import sys from pathlib import Path -_REPO_ROOT = Path(__file__).resolve().parents[2] # TensorRT/ -_TRT_PY = _REPO_ROOT / "py" - -import torch # noqa: E402 -import torch_tensorrt # noqa: E402 - -_src_pkg = str(_TRT_PY / "torch_tensorrt") -if _src_pkg not in list(torch_tensorrt.__path__): - torch_tensorrt.__path__.append(_src_pkg) - -from torch_tensorrt.hf.exporters import EdgeConfig, EdgeExporter -from torch_tensorrt.hf.exporters.models.nemotron.mamba_stub import ( - apply as apply_mamba_stub, -) -from torch_tensorrt.hf.exporters.plugin.plugin_utils import load_plugins_for_trt -from torch_tensorrt.hf.exporters.utils import configure_thor_pytorch +import torch +import torch_tensorrt +from exporters import EdgeConfig, EdgeExporter +from exporters.measure import print_bench +from exporters.models.nemotron.mamba_stub import apply as apply_mamba_stub +from exporters.plugin.plugin_utils import load_plugins_for_trt from transformers import AutoModelForCausalLM, AutoTokenizer @@ -60,7 +51,6 @@ def main() -> None: parser.add_argument("--max-seq-len", type=int, default=128) args = parser.parse_args() - configure_thor_pytorch() load_plugins_for_trt() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -94,6 +84,7 @@ def main() -> None: logits = out[0] if isinstance(out, (tuple, list)) else out print("logits", tuple(logits.shape), "mean", float(logits.float().mean())) + print_bench(exporter.bench) if __name__ == "__main__": diff --git a/examples/dynamo/run_pi05_export.py b/tools/hf/run_pi05_export.py similarity index 79% rename from examples/dynamo/run_pi05_export.py rename to tools/hf/run_pi05_export.py index fc84c8ce07f..2c2017c0e06 100644 --- a/examples/dynamo/run_pi05_export.py +++ b/tools/hf/run_pi05_export.py @@ -1,25 +1,25 @@ #!/usr/bin/env python3 +"""Smoke EdgeExporter on pi05 model. + +Pass the LeRobot PI05Policy, not policy.model — prepare_sample_inputs +needs the preprocessor on the policy wrapper. +""" + from __future__ import annotations import argparse +import sys from pathlib import Path -_REPO_ROOT = Path(__file__).resolve().parents[2] # TensorRT/ -_TRT_PY = _REPO_ROOT / "py" - -import torch # noqa: E402 -import torch_tensorrt # noqa: E402 - -_src_pkg = str(_TRT_PY / "torch_tensorrt") -if _src_pkg not in list(torch_tensorrt.__path__): - torch_tensorrt.__path__.append(_src_pkg) - +import torch +import torch_tensorrt +from exporters import EdgeConfig, EdgeExporter +from exporters.measure import print_bench +from exporters.plugin.plugin_utils import load_plugins_for_trt +from exporters.utils import force_hf_attention from lerobot.configs import FeatureType, PolicyFeature from lerobot.policies.pi05 import PI05Policy from lerobot.utils.constants import ACTION, OBS_IMAGES, OBS_STATE -from torch_tensorrt.hf.exporters import EdgeConfig, EdgeExporter -from torch_tensorrt.hf.exporters.plugin.plugin_utils import load_plugins_for_trt -from torch_tensorrt.hf.exporters.utils import configure_thor_pytorch, force_hf_attention def load_pi05(device: torch.device) -> PI05Policy: @@ -59,7 +59,6 @@ def main() -> None: parser.add_argument("--engine-dir", default="/tmp/pi05_edge_exporter") args = parser.parse_args() - configure_thor_pytorch() load_plugins_for_trt() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") @@ -78,9 +77,6 @@ def main() -> None: model_type="pi05", # optional; inferred from paligemma_with_expert engine_dir=args.engine_dir, max_seq_len=968, - dryrun=not args.compile, # True = no TRT, still writes config.json + runtime graph - skip_runtime_export=False, # False = also torch.export the stitched execute_engine graph - # components=("vision",), # uncomment to export only vision ) # Spec loads libero + preprocessor because we pass the policy, not a tensor dict. @@ -102,6 +98,7 @@ def main() -> None: out = velocity[0] if isinstance(velocity, (tuple, list)) else velocity print("velocity", tuple(out.shape), "mean", float(out.float().mean())) + print_bench(exporter.bench) if __name__ == "__main__":