diff --git a/benchmarks/single_node/fixed_seq_len/qwen3-0.6b_bf16_h100_trt.sh b/benchmarks/single_node/fixed_seq_len/qwen3-0.6b_bf16_h100_trt.sh new file mode 100755 index 0000000000..f764bf5900 --- /dev/null +++ b/benchmarks/single_node/fixed_seq_len/qwen3-0.6b_bf16_h100_trt.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +source "$(dirname "$0")/../../benchmark_lib.sh" + +check_env_vars \ + MODEL \ + TP \ + CONC \ + ISL \ + OSL \ + MAX_MODEL_LEN \ + RANDOM_RANGE_RATIO \ + RESULT_FILENAME \ + EVAL_ONLY \ + RUN_EVAL \ + PORT \ + HF_HUB_CACHE + +if [[ -n "$SLURM_JOB_ID" ]]; then + echo "JOB $SLURM_JOB_ID running on $SLURMD_NODENAME" +fi + +python3 - <<'PY' +from importlib.metadata import version + +expected = "1.3.0rc27" +actual = version("tensorrt_llm") +if actual != expected: + raise SystemExit( + f"Expected TensorRT-LLM {expected} for the pinned NGC image, got {actual}" + ) +PY + +python3 -m pip install --quiet --disable-pip-version-check \ + "modelscope==1.40.1" "modelscope-hub==0.4.3" +python3 "$(dirname "$0")/../../../runners/patch_trtllm_modelscope.py" + +export TRTLLM_USE_MODELSCOPE=true +export MODELSCOPE_CACHE="$HF_HUB_CACHE/modelscope" + +# Resolve through TensorRT-LLM's patched hub boundary on the H100 node. Keep +# serving the remote model ID below so model loading, config, and tokenizer +# paths all exercise the ModelScope integration. +MODEL_PATH_FILE=$(mktemp) +python3 - "$MODEL" "$MODEL_PATH_FILE" <<'PY' +import sys +from pathlib import Path + +from tensorrt_llm.llmapi.utils import download_hf_model + +model_path = download_hf_model(sys.argv[1]) +Path(sys.argv[2]).write_text(str(model_path), encoding="utf-8") +PY +MODEL_PATH=$(<"$MODEL_PATH_FILE") +rm -f "$MODEL_PATH_FILE" +export MODEL_PATH + +if [[ ! -f "$MODEL_PATH/config.json" ]]; then + echo "ModelScope snapshot is missing config.json: $MODEL_PATH" >&2 + exit 1 +fi + +echo "ModelScope snapshot: $MODEL_PATH" +echo "TP: $TP, CONC: $CONC, ISL: $ISL, OSL: $OSL" +nvidia-smi + +SERVER_LOG=/workspace/server.log +EXTRA_CONFIG_FILE=$(mktemp --suffix=.yaml) +MAX_BATCH_SIZE=$((CONC > 16 ? CONC : 16)) +MAX_NUM_TOKENS=$((((ISL + CONC + 127) / 128) * 128)) +MAX_NUM_TOKENS=$((MAX_NUM_TOKENS > 8192 ? MAX_NUM_TOKENS : 8192)) + +cat > "$EXTRA_CONFIG_FILE" < "$SERVER_LOG" 2>&1 & + +SERVER_PID=$! + +wait_for_server_ready --port "$PORT" --server-log "$SERVER_LOG" --server-pid "$SERVER_PID" + +run_benchmark_serving \ + --model "$MODEL" \ + --tokenizer "$MODEL_PATH" \ + --port "$PORT" \ + --backend openai \ + --input-len "$ISL" \ + --output-len "$OSL" \ + --random-range-ratio "$RANDOM_RANGE_RATIO" \ + --num-prompts "$((CONC * 10))" \ + --max-concurrency "$CONC" \ + --result-filename "$RESULT_FILENAME" \ + --result-dir /workspace/ + +if [[ "$RUN_EVAL" == "true" ]]; then + run_eval --framework lm-eval --port "$PORT" + append_lm_eval_summary +fi + +stop_gpu_monitor +rm -f "$EXTRA_CONFIG_FILE" +set +x diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index c5d5edac68..fdc754285d 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -3953,6 +3953,23 @@ qwen3.5-fp8-h100-sglang: - { tp: 8, ep: 1, conc-start: 1, conc-end: 8 } - { tp: 8, ep: 8, conc-start: 16, conc-end: 256 } +# ModelScope integration coverage for TensorRT-LLM. The image version maps to +# NVIDIA/TensorRT-LLM tag v1.3.0rc27 at commit 6e1cc953c071b8a9055b03ef2ae4ee0bc4c645c4. +qwen3-0.6b-bf16-h100-trt-modelscope: + image: nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc27 + model: Qwen/Qwen3-0.6B + model-prefix: qwen3-0.6b + runner: cluster:h100-dgxc + precision: bf16 + framework: trt + multinode: false + scenarios: + fixed-seq-len: + - isl: 8192 + osl: 1024 + search-space: + - { tp: 1, conc-list: [1, 4, 16, 32, 64] } + qwen3.5-fp8-h100-sglang-mtp: image: lmsysorg/sglang:v0.5.19-cu130 model: Qwen/Qwen3.5-397B-A17B-FP8 diff --git a/docs/eval-agentx-procedures.md b/docs/eval-agentx-procedures.md index 06c9ef6cf9..4db916d0e1 100644 --- a/docs/eval-agentx-procedures.md +++ b/docs/eval-agentx-procedures.md @@ -162,6 +162,40 @@ python3 -m infx.evals.validate_scores \ Validation resolves the threshold in this order: `models..`, `default.`, then `--min-score` (default `0.85`). By default it checks numeric, non-stderr metrics beginning with `exact_match,`. It fails when a score is below threshold, no metric matches, a requested concurrency is absent, metadata has duplicates/invalid values, any point is marked failed, or result suffixes do not match the manifest. Current floors are authoritative in [`thresholds.yaml`](../infx/evals/thresholds.yaml). See [threshold resolution](../infx/evals/validate_scores.py#L61-L69) and the [validation flow](../infx/evals/validate_scores.py#L174-L302). +### Qwen3-0.6B GSM8K floor + +`models.qwen3-0.6b.gsm8k` is **0.60** for both strict-match and flexible-extract. +This is a conservative integration regression floor for the 0.6B checkpoint, not +an expected leaderboard score. The global 0.90 floor remains unchanged. + +Keep the standard five-shot chat evaluation, full 1,319-question test split, +`temperature=0`, `top_p=1`, and 5,376 generated-token limit. The +[initial H100 BF16 run](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35534330261) +at concurrency 64 scored 886/1,319 (0.6717) strict and 895/1,319 (0.6785) +flexible. All responses were nonempty; 86 lacked the required numeric `####` +answer marker, 87 lacked a closing ``, and 61 repeated an identical +nonempty line of at least 30 characters five or more times. These categories +overlap. The nine-answer extraction gain does not explain most errors; inspected +wrong answers also contained arithmetic and reasoning mistakes. + +The [Qwen3 technical report, Table 8 and Section 3.3](https://arxiv.org/html/2505.09388v1) +reports 59.59% GSM8K for **Qwen3-0.6B-Base**, using four-shot chain of thought. +That is scale context only: the checkpoint and prompt differ, and it must not be +presented as a comparable five-shot chat baseline. No directly comparable +published baseline was established. The 0.60 floor is an explicit conservative +policy choice supported by that context and the inspected full-split result; +it leaves 7.17 percentage points below the observed strict score (reported +standard error 1.29 points), rather than rounding the observed score into a gate. +Validate the same fixed floor independently at concurrency 32 and 64. + +[Qwen's model guidance](https://huggingface.co/Qwen/Qwen3-0.6B#best-practices) +recommends sampling for thinking mode and warns that greedy decoding can repeat. +This integration keeps InferenceX's deterministic protocol for comparability; +the floor does not establish optimal Qwen quality or excuse request failures. +Changing thinking mode, sampling, prompts, or token budget requires a separately +documented policy and fresh full-split evidence. Preserve failed and passing +artifacts, and do not lower this floor in response to a later regression. + A manual combined throughput+eval recipe uploads eval output but the template's automatic score gate is specific to eval-only jobs. Run the validator explicitly for manual or combined runs. ## 6. Collect and inspect eval artifacts diff --git a/docs/eval-agentx-procedures_zh.md b/docs/eval-agentx-procedures_zh.md index 03c0580439..aadab4db20 100644 --- a/docs/eval-agentx-procedures_zh.md +++ b/docs/eval-agentx-procedures_zh.md @@ -162,6 +162,32 @@ python3 -m infx.evals.validate_scores \ 手动的吞吐量+eval 组合 recipe 会上传 eval 输出，但模板的自动分数 gate 专用于 eval-only 作业。对手动或组合运行必须显式执行 validator。 +### Qwen3-0.6B 的 GSM8K 下限 + +`models.qwen3-0.6b.gsm8k` 对 strict-match 和 flexible-extract 均使用 **0.60**。 +这是针对 0.6B 检查点的保守集成回归下限，不是排行榜预期分数；全局 0.90 下限保持不变。 + +保留标准五样本聊天评测、完整的 1,319 道测试题、`temperature=0`、`top_p=1` +和 5,376 个生成 token 的上限。 +[首次 H100 BF16 运行](https://github.com/SemiAnalysisAI/InferenceX/actions/runs/35534330261) +在并发 64 下取得 strict 886/1,319（0.6717）、flexible 895/1,319（0.6785）。 +所有响应均非空；86 个响应缺少所需的数字 `####` 答案标记，87 个缺少 `` +结束标记，61 个将同一条至少 30 个字符的非空行重复了五次或更多。这些类别存在重叠。 +宽松提取仅多判对九题，无法解释大多数错误；抽查的错误答案也包含算术和推理错误。 + +[Qwen3 技术报告表 8 和第 3.3 节](https://arxiv.org/html/2505.09388v1) +报告 **Qwen3-0.6B-Base** 在四样本思维链设置下的 GSM8K 分数为 59.59%。 +该结果仅用于说明模型规模背景：检查点和提示不同，不能视为可比的五样本聊天基线。 +目前未找到直接可比的已发表基线。0.60 是结合该背景和完整测试集响应检查作出的保守策略选择； +它比已观察到的 strict 分数低 7.17 个百分点（报告的标准误为 1.29 个百分点）， +并非将单次分数取整后作为门槛。应在并发 32 和 64 下独立验证同一个固定下限。 + +[Qwen 模型指南](https://huggingface.co/Qwen/Qwen3-0.6B#best-practices) +建议思考模式使用采样，并警告贪心解码可能产生重复。为保持可比性，本集成沿用 +InferenceX 的确定性协议；该下限不代表 Qwen 的最佳质量，也不豁免请求失败。 +改变思考模式、采样、提示或 token 预算，需要另行记录策略并提供新的完整测试集证据。 +保留失败和成功的产物，不应因为后续回归而继续降低此下限。 + ## 6. 收集并检查 eval artifact 收集工作流会下载 `eval_*`，用 `infx/results/collect_eval_results.py` 聚合原始集合，上传 `eval_results_all/agg_eval_all.json`，并将表格写入 step summary（[`collect-evals.yml`](../.github/workflows/collect-evals.yml)）。 diff --git a/docs/waiver/3323.md b/docs/waiver/3323.md new file mode 100644 index 0000000000..e65e1f9829 --- /dev/null +++ b/docs/waiver/3323.md @@ -0,0 +1,49 @@ +# Inference-engine patch waiver — PR #3323 + +Filed per [`docs/PR_REVIEW_CHECKLIST.md`](../PR_REVIEW_CHECKLIST.md): this PR patches the pinned +TensorRT-LLM image before serving because the released image predates ModelScope model loading. + +## Config covered + +- **Master config entry:** `qwen3-0.6b-bf16-h100-trt-modelscope` in + [`configs/nvidia-master.yaml`](../../configs/nvidia-master.yaml) +- **Pinned image:** `nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc27` +- **Image source:** NVIDIA/TensorRT-LLM tag `v1.3.0rc27`, commit + `6e1cc953c071b8a9055b03ef2ae4ee0bc4c645c4` +- **Patch entrypoint:** + [`runners/patch_trtllm_modelscope.py`](../../runners/patch_trtllm_modelscope.py), invoked by + [`benchmarks/single_node/fixed_seq_len/qwen3-0.6b_bf16_h100_trt.sh`](../../benchmarks/single_node/fixed_seq_len/qwen3-0.6b_bf16_h100_trt.sh) + +## What is patched + +The patcher backports the ModelScope integration from +[SemiAnalysisAI/TensorRT-LLM#2](https://github.com/SemiAnalysisAI/TensorRT-LLM/pull/2) to the two +installed Python modules that participate in this benchmark: + +- `tensorrt_llm/llmapi/utils.py` routes full and partial snapshot downloads through ModelScope when + `TRTLLM_USE_MODELSCOPE=true`, while preserving Hugging Face as the default. +- `tensorrt_llm/llmapi/llm.py` loads the tokenizer, generation config, and model config from the + resolved local snapshot rather than retrying the remote Hugging Face model ID. + +The backport is source-matched to `v1.3.0rc27`, exact-anchor gated, and idempotent. It refuses an +unknown or partially patched installed source tree. `modelscope==1.40.1` and +`modelscope-hub==0.4.3` are installed in the H100 container before the patch is applied. + +## Why the unmodified upstream image cannot run this benchmark + +TensorRT-LLM `1.3.0rc27` resolves remote model IDs exclusively with `huggingface_hub`. It has no +ModelScope switch or downloader and subsequently loads tokenizer and configuration files from the +original remote ID. Therefore the stock image cannot validate TensorRT-LLM model loading from +ModelScope for `Qwen/Qwen3-0.6B`; installing the optional ModelScope dependency alone does not change +that behavior. + +## Upstream PR + +- https://github.com/SemiAnalysisAI/TensorRT-LLM/pull/2 + +## Removal plan + +Once an NGC TensorRT-LLM release includes the ModelScope integration, update +`qwen3-0.6b-bf16-h100-trt-modelscope` to the first matching release image and verify its source tag. +In the same PR, remove `runners/patch_trtllm_modelscope.py`, remove its invocation and runtime package +installation from the benchmark script, and delete this waiver. diff --git a/infx/evals/thresholds.yaml b/infx/evals/thresholds.yaml index d9b41eba40..1793887a6b 100644 --- a/infx/evals/thresholds.yaml +++ b/infx/evals/thresholds.yaml @@ -50,6 +50,9 @@ "minimaxm2.5": { "gsm8k": 0.92 }, + "qwen3-0.6b": { + "gsm8k": 0.60 + }, "qwen3.5": { "gsm8k": 0.94 } diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5b4309f5f..72a1ec561c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8455,3 +8455,10 @@ description: - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274 + +- config-keys: + - qwen3-0.6b-bf16-h100-trt-modelscope + description: + - "Add H100 TensorRT-LLM 1.3.0rc27 coverage for Qwen3-0.6B in BF16, resolving the model through ModelScope with a source-matched runtime backport of SemiAnalysisAI/TensorRT-LLM#2." + - "为 Qwen3-0.6B BF16 添加 H100 TensorRT-LLM 1.3.0rc27 覆盖，通过 ModelScope 解析模型，并使用与镜像源码匹配的 SemiAnalysisAI/TensorRT-LLM#2 运行时回移补丁。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3323 diff --git a/runners/patch_trtllm_modelscope.py b/runners/patch_trtllm_modelscope.py new file mode 100755 index 0000000000..f979c840ca --- /dev/null +++ b/runners/patch_trtllm_modelscope.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Backport ModelScope loading to the pinned TensorRT-LLM runtime.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +HF_IMPORT = "from huggingface_hub import snapshot_download\n" +ALIASED_HF_IMPORT = ( + "from huggingface_hub import snapshot_download as hf_snapshot_download\n" +) + +OLD_DOWNLOAD_BLOCK = '''def download_hf_model(model: str, revision: Optional[str] = None) -> Path: + ignore_patterns = ["original/**/*"] + logger.info(f"Downloading model {model} from HuggingFace") + with get_file_lock(model): + hf_folder = snapshot_download( + model, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ignore_patterns=ignore_patterns, + revision=revision, + tqdm_class=DisabledTqdm) + logger.info(f"Finished downloading model {model} from HuggingFace") + return Path(hf_folder) + + +def download_hf_partial(model: str, + allow_patterns: List[str], + revision: Optional[str] = None) -> Path: + """Download a partial model from HuggingFace. + + Args: + model: The model name or path. + revision: The revision to use for the model. + allow_patterns: The patterns to allow for the model. + + Returns: + The path to the downloaded model. + """ + with get_file_lock(model): + hf_folder = snapshot_download( + model, + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + revision=revision, + allow_patterns=allow_patterns, + tqdm_class=DisabledTqdm) + return Path(hf_folder) + + +''' + +NEW_DOWNLOAD_BLOCK = '''def download_hf_model(model: str, revision: Optional[str] = None) -> Path: + ignore_patterns = ["original/**/*"] + hub_name = "ModelScope" if use_modelscope() else "Hugging Face" + logger.info(f"Downloading model {model} from {hub_name}") + with get_file_lock(model): + model_folder = _snapshot_download(model, + ignore_patterns=ignore_patterns, + revision=revision) + logger.info(f"Finished downloading model {model} from {hub_name}") + return Path(model_folder) + + +def download_hf_partial(model: str, + allow_patterns: List[str], + revision: Optional[str] = None) -> Path: + """Download selected model files from the configured model hub. + + Args: + model: The model name or path. + revision: The revision to use for the model. + allow_patterns: The patterns to allow for the model. + + Returns: + The path to the downloaded model. + """ + with get_file_lock(model): + model_folder = _snapshot_download(model, + revision=revision, + allow_patterns=allow_patterns) + return Path(model_folder) + + +def use_modelscope() -> bool: + """Return whether remote model IDs should resolve through ModelScope.""" + return os.environ.get("TRTLLM_USE_MODELSCOPE", "false").strip().lower( + ) in ("1", "true") + + +def _snapshot_download(model: str, + revision: Optional[str] = None, + ignore_patterns: Optional[List[str]] = None, + allow_patterns: Optional[List[str]] = None) -> str: + """Download a snapshot from ModelScope or Hugging Face. + + Keep the optional import in this boundary so standard TensorRT-LLM + installations do not need the ``modelscope`` package. + """ + local_files_only = huggingface_hub.constants.HF_HUB_OFFLINE + if use_modelscope(): + try: + from modelscope.hub.snapshot_download import snapshot_download + except ImportError as error: + raise ImportError( + "TRTLLM_USE_MODELSCOPE is enabled, but ModelScope is not " + "installed. Install it with `pip install 'modelscope>=1.20'`." + ) from error + + kwargs = { + "model_id": model, + "local_files_only": local_files_only, + "revision": revision, + } + if ignore_patterns: + kwargs["ignore_patterns"] = ignore_patterns + if allow_patterns: + kwargs["allow_patterns"] = allow_patterns + return snapshot_download(**kwargs) + + return hf_snapshot_download( + model, + local_files_only=local_files_only, + ignore_patterns=ignore_patterns, + allow_patterns=allow_patterns, + revision=revision, + tqdm_class=DisabledTqdm) + + +''' + + +def installed_package_root() -> Path: + """Return the installed TensorRT-LLM package root.""" + spec = importlib.util.find_spec("tensorrt_llm") + if spec is None or not spec.submodule_search_locations: + raise RuntimeError("tensorrt_llm package is not installed") + return Path(next(iter(spec.submodule_search_locations))) + + +def replace_once(source: str, old: str, new: str, path: Path) -> str: + """Replace one pinned source fragment or fail on an unknown runtime.""" + count = source.count(old) + if count != 1: + raise RuntimeError( + f"unsupported TensorRT-LLM source at {path}: " + f"expected one pinned fragment, found {count}" + ) + return source.replace(old, new, 1) + + +def patch_utils(path: Path) -> bool: + """Patch the hub download boundary.""" + source = path.read_text(encoding="utf-8") + if "def use_modelscope() -> bool:" in source: + if ALIASED_HF_IMPORT not in source: + raise RuntimeError(f"partial ModelScope patch found at {path}") + return False + + source = replace_once(source, HF_IMPORT, ALIASED_HF_IMPORT, path) + source = replace_once(source, OLD_DOWNLOAD_BLOCK, NEW_DOWNLOAD_BLOCK, path) + path.write_text(source, encoding="utf-8") + return True + + +def patch_llm(path: Path) -> bool: + """Load tokenizer and configuration from the downloaded snapshot.""" + source = path.read_text(encoding="utf-8") + tokenizer_marker = " model_path = self._hf_model_dir or self.args.model\n" + generation_marker = " model_dir = self._hf_model_dir or self.args.model\n" + if tokenizer_marker in source: + if source.count(generation_marker) < 3: + raise RuntimeError(f"partial ModelScope path patch found at {path}") + return False + + tokenizer_start = source.index(" def _try_load_tokenizer(") + tokenizer_end = source.index("\n @property\n def tokenizer", tokenizer_start) + tokenizer_source = source[tokenizer_start:tokenizer_end] + anchor = ( + " if self.args.tokenizer is not None:\n" + " assert isinstance(self.args.tokenizer, TokenizerBase)\n" + " return self.args.tokenizer\n\n" + ) + if tokenizer_source.count("self.args.model") != 4: + raise RuntimeError( + f"unsupported TensorRT-LLM tokenizer loader at {path}: " + "unexpected model-path reference count" + ) + tokenizer_source = tokenizer_source.replace("self.args.model", "model_path") + tokenizer_source = replace_once( + tokenizer_source, anchor, anchor + tokenizer_marker + "\n", path + ) + source = source[:tokenizer_start] + tokenizer_source + source[tokenizer_end:] + + source = replace_once( + source, + " return ModelLoader.load_hf_generation_config(self.args.model)\n", + generation_marker + + " return ModelLoader.load_hf_generation_config(model_dir)\n", + path, + ) + source = replace_once( + source, + " return ModelLoader.load_hf_model_config(\n" + " self.args.model, trust_remote_code=self.args.trust_remote_code)\n", + generation_marker + " return ModelLoader.load_hf_model_config(\n" + " model_dir, trust_remote_code=self.args.trust_remote_code)\n", + path, + ) + path.write_text(source, encoding="utf-8") + return True + + +def main(argv: list[str]) -> int: + if len(argv) > 2: + print(f"Usage: {argv[0]} [TENSORRT_LLM_PACKAGE_ROOT]", file=sys.stderr) + return 2 + + try: + package_root = ( + Path(argv[1]).resolve() if len(argv) == 2 else installed_package_root() + ) + changed = [ + patch_utils(package_root / "llmapi/utils.py"), + patch_llm(package_root / "llmapi/llm.py"), + ] + except (OSError, RuntimeError, ValueError) as error: + print( + f"ERROR: failed to patch TensorRT-LLM ModelScope support: {error}", + file=sys.stderr, + ) + return 1 + + state = "Patched" if any(changed) else "Already patched" + print(f"{state} TensorRT-LLM ModelScope support") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv))