[AgentX H200] refresh the full Kimi-K3 curve so every point carries measured power / 重测完整曲线使每个点都带实测功耗

benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-balanced.yaml

@@ -85,8 +85,7 @@ srun_options:
 
 telemetry:
   enabled: true
-  provider: dcgm-power
-  default_frequency: 1.0
+  collect_interval_ms: 1000
   storage_subdir: power
   required: true
   startup_timeout_seconds: 120

benchmarks/multi_node/srt-slurm-recipes/kimik3/vllm/h200-fp4/agentx/agg-tp8dp4ep32-vllm-simple.yaml

@@ -86,8 +86,7 @@ srun_options:
 
 telemetry:
   enabled: true
-  provider: dcgm-power
-  default_frequency: 1.0
+  collect_interval_ms: 1000
   storage_subdir: power
   required: true
   startup_timeout_seconds: 120

infx/results/power/multinode.py

@@ -89,6 +89,17 @@
 # NOT a multiple of the configured sample interval.
 MAX_SAMPLE_GAP_SECONDS = 3.0
 
+# Mirrors srt-slurm contract.MAX_SAMPLE_GAP_HARD_SECONDS and
+# MAX_OVERLONG_GAP_FRACTION. A gap past MAX_SAMPLE_GAP_SECONDS is over-long:
+# interpolated rather than measured, which bounds the per-device energy error
+# at (dynamic range) x gap / 2 -- under 0.04% for 3.3s of a 3640s window --
+# while rejecting the window discards an hour of measurement on every GPU of
+# the job. A collector that actually stopped still has to be caught, hence the
+# hard ceiling and the budget. Both values must match the producer exactly, or
+# this recompute disagrees with the stored audit and the point fails anyway.
+MAX_SAMPLE_GAP_HARD_SECONDS = 10.0
+MAX_OVERLONG_GAP_FRACTION = 0.005
+
 WORKER_ROLES = ("prefill", "decode", "agg")
 
 STATUS_COMPLETE = "complete"
@@ -742,12 +753,12 @@ def _check_coverage(
         if sequence is None:
             reasons.append("measurement_window_not_bracketed")
             continue
-        largest = max(
-            (later - earlier for earlier, later in itertools.pairwise(sequence)),
-            default=0.0,
-        )
+        observed = [later - earlier for earlier, later in itertools.pairwise(sequence)]
+        largest = max(observed, default=0.0)
         gaps[f"{device.hostname}/{device.gpu_uuids[0]}"] = largest
-        if largest > MAX_SAMPLE_GAP_SECONDS:
+        overlong = sum(gap for gap in observed if gap > MAX_SAMPLE_GAP_SECONDS)
+        budget = MAX_OVERLONG_GAP_FRACTION * (end - start)
+        if largest > MAX_SAMPLE_GAP_HARD_SECONDS or overlong > budget:
             reasons.append("sample_gap_exceeded")
 
     return gaps, reasons

perf-changelog.yaml

@@ -8455,3 +8455,16 @@
   description:
     - "Update B200 vLLM AgentX to DSpark6 and a new image with TP8 and DEP8 configurations."
   pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3274
+
+- config-keys:
+    - kimik3-fp4-h200-vllm-agentic-latency
+    - kimik3-fp4-h200-vllm-agentic-balanced
+    - kimik3-fp4-h200-vllm-agentic-simple
+  scenario-type:
+    - agentic-coding
+  description:
+    - "Refresh the complete H200 Kimi-K3 AgentX curve so every published point carries measured power. 25 of the 35 published points come from the 2026-08-07 sweep, which predates power collection on this hardware, and AgentX resolves one curve per hardware, so only a sweep selecting all three config keys can replace them together."
+    - "Migrate agg-tp8dp4ep32-balanced and agg-tp8dp4ep32-vllm-simple to the current srtctl telemetry schema: provider and default_frequency were retired for dcgm_exporter and collect_interval_ms when the submodule moved to the upstream pin, and srtctl rejected both recipes as Unknown field before submitting anything."
+    - "重测 H200 Kimi-K3 AgentX 整条曲线，使每个已发布的点都带实测功耗。已发布的 35 个点中有 25 个来自 2026-08-07 的 sweep，早于该硬件开启功耗采集；而 AgentX 每个硬件只解析出一条曲线，因此只有同时选中三个 config key 的 sweep 才能整体替换它们。"
+    - "将 agg-tp8dp4ep32-balanced 与 agg-tp8dp4ep32-vllm-simple 迁移到当前的 srtctl telemetry schema：submodule 切到上游 pin 时，provider 与 default_frequency 已被 dcgm_exporter 和 collect_interval_ms 取代，srtctl 在提交作业前就以 Unknown field 拒绝了这两个配方。"
+  pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/3309

utils/test_aggregate_power_multinode.py

@@ -653,3 +653,84 @@ def test_v2_samples_reject_mixed_versions_and_invalid_utilization(tmp_path, row)
     rows, reasons = apm.read_samples(path)
     assert not rows
     assert reasons == ("samples_csv_malformed",)
+
+
+class TestOverlongSampleGapBudget:
+    """An over-long gap is interpolated coverage, not a corrupt measurement.
+
+    H200 Kimi-K3 run 35532102407 voided its whole TP16 lane because one node's
+    exporter answered in 3.26 s and 3.10 s during a 3640 s window. The passing
+    TP8 lane in the same run had a larger 3.22 s excursion and survived only
+    because it landed in warmup, outside the window. These fix the tolerance to
+    what the coverage is actually worth instead of to that accident.
+
+    Both bounds mirror srt-slurm; the producer stores its own verdict and this
+    consumer recomputes it, so a value that drifts fails as a recompute
+    mismatch rather than silently diverging.
+    """
+
+    WINDOW = (1_000_000.0, 1_003_600.0)  # a 3600 s agentic window
+
+    def _devices(self, pauses=(), step=1.0):
+        """One host of 8 GPUs sampling at `step`, dwelling at each (offset, gap)."""
+        start, end = self.WINDOW
+        pending = sorted(pauses)
+        times, timestamp = [], start - 2.0
+        while timestamp <= end + 2.0:
+            times.append(timestamp)
+            advance = step
+            if pending and timestamp - start >= pending[0][0]:
+                advance = pending.pop(0)[1]
+            timestamp = round(timestamp + advance, 3)
+        return [
+            apm.ObservedDevice(
+                hostname="node-p", gpu_index=index, gpu_uuids=(f"GPU-{index}",),
+                first_sample_time_unix=times[0], last_sample_time_unix=times[-1],
+                sample_times=tuple(times),
+            )
+            for index in range(8)
+        ]
+
+    def _judge(self, devices):
+        start, end = self.WINDOW
+        return apm._check_coverage(start, end, {d.key for d in devices}, devices)
+
+    def test_two_isolated_excursions_are_absorbed(self):
+        # The exact H200 TP16 c6 shape: 6.36 s of 3600 s, 0.18%.
+        gaps, reasons = self._judge(self._devices([(600.0, 3.26), (1800.0, 3.10)]))
+
+        assert reasons == []
+        assert gaps["node-p/GPU-0"] == pytest.approx(3.26)
+
+    def test_gap_past_the_hard_ceiling_is_rejected(self):
+        # 11 s is 0.3% of the window, inside the budget, so only the ceiling catches it.
+        _, reasons = self._judge(self._devices([(600.0, 11.0)]))
+
+        assert "sample_gap_exceeded" in reasons
+
+    def test_excursions_are_rejected_once_they_leave_the_budget(self):
+        # Six 3.5 s gaps is 21 s of 3600 s, past 0.5%, none near the ceiling.
+        gaps, reasons = self._judge(self._devices([(300.0 * n, 3.5) for n in range(1, 7)]))
+
+        assert "sample_gap_exceeded" in reasons
+        assert gaps["node-p/GPU-0"] == pytest.approx(3.5)
+
+    def test_a_short_window_still_rejects_what_a_long_one_absorbs(self):
+        """The budget is a fraction, so tolerance tracks what is actually lost."""
+        devices = self._devices([(10.0, 3.26)])
+        start = self.WINDOW[0]
+        short_end = start + 60.0
+        _, reasons = apm._check_coverage(start, short_end, {d.key for d in devices}, devices)
+
+        assert "sample_gap_exceeded" in reasons
+
+    def test_the_rule_only_ever_relaxes(self):
+        """Nothing the old ceiling accepted can start failing.
+
+        With no gap past MAX_SAMPLE_GAP_SECONDS there is no over-long time to
+        budget and no gap near the ceiling, so the new rule is a strict
+        superset of the old one and no already-published point can regress.
+        """
+        _, reasons = self._judge(self._devices(step=apm.MAX_SAMPLE_GAP_SECONDS))
+
+        assert reasons == []