From 845310473acf94b1aeb002f4becbf9e60534f926 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Wed, 30 Sep 2026 13:56:29 +0200 Subject: [PATCH 1/7] feat: add metric observations for vLLM and SGLang Signed-off-by: Ivan Podkidyshev --- doc/reporting.rst | 55 +++++++ doc/workloads/sglang.rst | 3 + doc/workloads/vllm.rst | 3 + src/cloudai/metrics.py | 40 ++++- src/cloudai/workloads/common/llm_serving.py | 33 ++++ src/cloudai/workloads/sglang/sglang.py | 15 +- .../vllm/report_generation_strategy.py | 1 + src/cloudai/workloads/vllm/vllm.py | 10 ++ tests/workloads/common/test_llm_serving.py | 141 ++++++++++++++++++ 9 files changed, 298 insertions(+), 3 deletions(-) diff --git a/doc/reporting.rst b/doc/reporting.rst index 1bdeb8d92..5acf46e53 100644 --- a/doc/reporting.rst +++ b/doc/reporting.rst @@ -110,6 +110,61 @@ A test case can replace the targets for a metric inherited from the scenario or The precedence is test case, then scenario, then system. NCCL and NIXL comparison v2 reports include measured, SOL, and percentage-of-SOL columns and draw a shared SOL curve when every compared run resolves the same targets. +.. _llm-serving-metric-observations: + +LLM serving metric observations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +vLLM and SGLang expose structured measurements through ``TestDefinition.metric_observations(system, test_run)``. +``cloudai.metrics.assess_test_run_metrics`` compares these observations with the run's configured SOL targets. + +.. list-table:: Serving metrics + :header-rows: 1 + + * - Metric key + - Unit + - Preferred direction + * - ``request_throughput`` + - requests/s + - Higher + * - ``output_token_throughput`` + - tokens/s + - Higher + * - ``ttft`` (time to first token) + - ms + - Lower + * - ``tpot`` (time per output token) + - ms + - Lower + * - ``accuracy`` + - ratio (0–1) + - Higher + +All observations include ``backend`` (``vllm`` or ``sglang``) and ``model`` dimensions. Benchmark observations also +include ``max_concurrency`` from the result file; TTFT and TPOT have a ``statistic`` dimension with ``mean``, +``median``, or ``p99``. Accuracy comes from semantic evaluation when ``semantic_eval_cmd_args`` is configured and +does not carry benchmark concurrency or latency statistics. + +Request and output-token throughput are emitted only when their respective ``request_throughput`` and +``output_throughput`` fields are present. Missing or invalid benchmark results and results with no completed requests +produce no benchmark observations. Non-finite measurements are omitted. Semantic accuracy can be emitted +independently when an evaluation result is available. + +For example, a system or scenario can define these targets: + +.. code-block:: toml + + [[sol.output_token_throughput]] + value = 5000.0 # tokens/s + match = { model = "Qwen/Qwen3-8B", max_concurrency = 16 } + + [[sol.ttft]] + value = 100.0 # ms + match = { backend = "sglang", statistic = "p99" } + +These observations leave the existing scalar metric names and comparison reports unchanged. In particular, the +legacy ``throughput`` metric uses output tokens/s for vLLM and requests/s for SGLang. + .. _reporting-registration: Reporting Registration diff --git a/doc/workloads/sglang.rst b/doc/workloads/sglang.rst index 237537af4..d91b3cf37 100644 --- a/doc/workloads/sglang.rst +++ b/doc/workloads/sglang.rst @@ -128,6 +128,9 @@ The reported metric (``default``) is throughput. Additional supported metrics ar CloudAI also provides the scenario-level ``sglang_comparison`` report. It compares SGLang test runs in the scenario and uses ``bench_cmd_args`` values as comparison labels. +The workload also exposes :ref:`structured metric observations ` for request and +output-token throughput, TTFT, TPOT, and optional semantic accuracy, with support for SOL assessment. + Readiness health checks ----------------------- diff --git a/doc/workloads/vllm.rst b/doc/workloads/vllm.rst index 082faf70d..5fadc7a80 100644 --- a/doc/workloads/vllm.rst +++ b/doc/workloads/vllm.rst @@ -123,6 +123,9 @@ The reported metric (``default``) is throughput. Additional supported metrics ar CloudAI also provides the scenario-level ``vllm_comparison`` report. It compares vLLM test runs in the scenario and uses ``bench_cmd_args`` values as comparison labels. +The workload also exposes :ref:`structured metric observations ` for request and +output-token throughput, TTFT, TPOT, and optional semantic accuracy, with support for SOL assessment. + Controlling the Number of GPUs ------------------------------- diff --git a/src/cloudai/metrics.py b/src/cloudai/metrics.py index abf4c066a..515186bf4 100644 --- a/src/cloudai/metrics.py +++ b/src/cloudai/metrics.py @@ -105,6 +105,9 @@ def validate_dimensions(cls, values: Mapping[str, Any]) -> MetricDimensions: BACKEND = DimensionDefinition("backend", "Backend", Annotated[str, Field(strict=True, min_length=1)]) SOURCE_MEMORY = DimensionDefinition("source_memory", "Source memory", Annotated[str, Field(strict=True, min_length=1)]) TARGET_MEMORY = DimensionDefinition("target_memory", "Target memory", Annotated[str, Field(strict=True, min_length=1)]) +MODEL = DimensionDefinition("model", "Model", Annotated[str, Field(strict=True, min_length=1)]) +MAX_CONCURRENCY = DimensionDefinition("max_concurrency", "Max concurrency", Annotated[int, Field(strict=True, ge=0)]) +STATISTIC = DimensionDefinition("statistic", "Statistic", Literal["mean", "median", "p99"]) BANDWIDTH = MetricDefinition( key="bandwidth", @@ -119,8 +122,38 @@ def validate_dimensions(cls, values: Mapping[str, Any]) -> MetricDimensions: direction=OptimizationDirection.MINIMIZE, ) -MetricCatalog._metrics = {metric.key: metric for metric in (BANDWIDTH, LATENCY)} -MetricCatalog.register_metrics(BANDWIDTH, LATENCY) +REQUEST_THROUGHPUT = MetricDefinition( + key="request_throughput", + display_name="Request throughput", + unit="requests/s", + direction=OptimizationDirection.MAXIMIZE, +) +OUTPUT_TOKEN_THROUGHPUT = MetricDefinition( + key="output_token_throughput", + display_name="Output token throughput", + unit="tokens/s", + direction=OptimizationDirection.MAXIMIZE, +) +TTFT = MetricDefinition( + key="ttft", + display_name="Time to first token", + unit="ms", + direction=OptimizationDirection.MINIMIZE, +) +TPOT = MetricDefinition( + key="tpot", + display_name="Time per output token", + unit="ms", + direction=OptimizationDirection.MINIMIZE, +) +ACCURACY = MetricDefinition( + key="accuracy", + display_name="Accuracy", + unit="ratio", + direction=OptimizationDirection.MAXIMIZE, +) + +MetricCatalog.register_metrics(BANDWIDTH, LATENCY, REQUEST_THROUGHPUT, OUTPUT_TOKEN_THROUGHPUT, TTFT, TPOT, ACCURACY) MetricCatalog.register_dimensions( SIZE_BYTES, BATCH_SIZE, @@ -130,6 +163,9 @@ def validate_dimensions(cls, values: Mapping[str, Any]) -> MetricDimensions: BACKEND, SOURCE_MEMORY, TARGET_MEMORY, + MODEL, + MAX_CONCURRENCY, + STATISTIC, ) diff --git a/src/cloudai/workloads/common/llm_serving.py b/src/cloudai/workloads/common/llm_serving.py index 6b45ff854..4ad41800f 100644 --- a/src/cloudai/workloads/common/llm_serving.py +++ b/src/cloudai/workloads/common/llm_serving.py @@ -16,6 +16,7 @@ from __future__ import annotations +import math import re import shlex from abc import ABC, abstractmethod @@ -27,6 +28,7 @@ from rich.table import Table from typing_extensions import Self +import cloudai.metrics from cloudai.core import METRIC_ERROR, DockerImage, HFModel, Installable, MetricValue, ReportGenerationStrategy from cloudai.models.workload import CmdArgs, TestDefinition from cloudai.systems.slurm import SlurmCommandGenStrategy @@ -241,6 +243,37 @@ def tps_per_user(self) -> float | None: return self.throughput / self.concurrency +def llm_serving_metric_observations( + results: LLMServingBenchReport | None, + backend: str, + model: str, + accuracy: float | None = None, +) -> list[cloudai.metrics.MetricObservation]: + """Collect finite serving measurements with shared units and dimensions.""" + dimensions: dict[str, cloudai.metrics.MetricValue] = {"backend": backend, "model": model} + observations: list[cloudai.metrics.MetricObservation] = [] + if accuracy is not None and math.isfinite(accuracy): + observations.append(cloudai.metrics.MetricObservation(cloudai.metrics.ACCURACY, accuracy, dimensions)) + if results is None or results.completed <= 0: + return observations + + dimensions = {**dimensions, "max_concurrency": results.max_concurrency} + for metric, value in ( + (cloudai.metrics.REQUEST_THROUGHPUT, getattr(results, "request_throughput", None)), + (cloudai.metrics.OUTPUT_TOKEN_THROUGHPUT, getattr(results, "output_throughput", None)), + ): + if value is not None and math.isfinite(value): + observations.append(cloudai.metrics.MetricObservation(metric, value, dimensions)) + for metric in (cloudai.metrics.TTFT, cloudai.metrics.TPOT): + for statistic in ("mean", "median", "p99"): + value = getattr(results, f"{statistic}_{metric.key}_ms") + if math.isfinite(value): + observations.append( + cloudai.metrics.MetricObservation(metric, value, {**dimensions, "statistic": statistic}) + ) + return observations + + class LLMServingReportGenerationStrategy(ReportGenerationStrategy, Generic[TestDefT, ReportT], ABC): """Shared report generation strategy for LLM serving workloads.""" diff --git a/src/cloudai/workloads/sglang/sglang.py b/src/cloudai/workloads/sglang/sglang.py index 49a7af140..9b756ee61 100644 --- a/src/cloudai/workloads/sglang/sglang.py +++ b/src/cloudai/workloads/sglang/sglang.py @@ -23,7 +23,8 @@ from pydantic import ConfigDict, Field, field_validator, model_validator -from cloudai.core import JobStatusResult, TestRun +import cloudai.metrics +from cloudai.core import JobStatusResult, System, TestRun from cloudai.models.workload import CmdArgs from cloudai.workloads.common.llm_serving import ( CustomBash, @@ -31,6 +32,7 @@ LLMServingBenchReport, LLMServingCmdArgs, LLMServingTestDefinition, + llm_serving_metric_observations, validate_custom_bash_patterns, ) @@ -127,11 +129,22 @@ def was_run_successful(self, tr: TestRun) -> JobStatusResult: error_message=f"SGLang bench jsonl does not contain successful requests in {tr.output_path}.", ) + def metric_observations(self, system: System, tr: TestRun) -> list[cloudai.metrics.MetricObservation]: + del system + results = parse_sglang_bench_output(tr.output_path / SGLANG_BENCH_JSONL_FILE) + accuracy = ( + parse_sglang_semantic_accuracy(tr.output_path / SGLANG_SEMANTIC_EVAL_LOG_FILE) + if self.semantic_eval_cmd_args is not None + else None + ) + return llm_serving_metric_observations(results, "sglang", self.cmd_args.model, accuracy) + class SGLangBenchReport(LLMServingBenchReport): """Parsed benchmark data from SGLang bench_serving output.""" request_throughput: float + output_throughput: float | None = None @property def throughput(self) -> float: diff --git a/src/cloudai/workloads/vllm/report_generation_strategy.py b/src/cloudai/workloads/vllm/report_generation_strategy.py index fcecabfba..773fd524f 100644 --- a/src/cloudai/workloads/vllm/report_generation_strategy.py +++ b/src/cloudai/workloads/vllm/report_generation_strategy.py @@ -28,6 +28,7 @@ class VLLMBenchReport(LLMServingBenchReport): """Report for vLLM benchmark results.""" output_throughput: float + request_throughput: float | None = None @property def throughput(self) -> float: diff --git a/src/cloudai/workloads/vllm/vllm.py b/src/cloudai/workloads/vllm/vllm.py index af5b815af..497136f2b 100644 --- a/src/cloudai/workloads/vllm/vllm.py +++ b/src/cloudai/workloads/vllm/vllm.py @@ -25,6 +25,7 @@ from pydantic import ConfigDict, Field, field_validator +import cloudai.metrics from cloudai.core import GitRepo, Installable, JobStatusResult, System, TestRun from cloudai.models.workload import CmdArgs from cloudai.workloads.common.llm_serving import ( @@ -35,6 +36,7 @@ all_gpu_ids, calculate_decode_gpu_ids, calculate_prefill_gpu_ids, + llm_serving_metric_observations, validate_custom_bash_patterns, ) @@ -264,6 +266,14 @@ def was_run_successful(self, tr: TestRun) -> JobStatusResult: is_successful=False, error_message=f"vLLM bench log does not contain benchmark result in {tr.output_path}." ) + def metric_observations(self, system: System, tr: TestRun) -> list[cloudai.metrics.MetricObservation]: + del system + from .report_generation_strategy import parse_vllm_bench_output + + results = parse_vllm_bench_output(tr.output_path / VLLM_BENCH_JSON_FILE) + accuracy = parse_vllm_semantic_accuracy(tr.output_path) if self.semantic_eval_cmd_args is not None else None + return llm_serving_metric_observations(results, "vllm", self.cmd_args.model, accuracy) + @cache def parse_vllm_semantic_accuracy(output_path: Path) -> float | None: diff --git a/tests/workloads/common/test_llm_serving.py b/tests/workloads/common/test_llm_serving.py index f525257da..e860f8dae 100644 --- a/tests/workloads/common/test_llm_serving.py +++ b/tests/workloads/common/test_llm_serving.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -22,6 +23,7 @@ from pydantic import Field from rich.table import Table +import cloudai.metrics from cloudai.core import GitRepo, TestRun from cloudai.systems.slurm import SlurmSystem from cloudai.workloads.common.llm_serving import ( @@ -34,6 +36,8 @@ all_gpu_ids, parse_gpu_ids, ) +from cloudai.workloads.sglang import SglangCmdArgs, SglangSemanticEvalCmdArgs, SglangTestDefinition +from cloudai.workloads.vllm import VllmCmdArgs, VllmSemanticEvalCmdArgs, VllmTestDefinition class FakeLLMArgs(LLMServingArgs): @@ -461,3 +465,140 @@ def test_used_gpus_count_counts_all_cluster_gpus_for_two_node_disaggregated_run( ) assert strategy.used_gpus_count() == 8 + + +@pytest.fixture(params=["vllm", "sglang"]) +def serving_observation_run(request: pytest.FixtureRequest, tmp_path: Path) -> tuple[TestRun, Path, dict]: + backend = request.param + if backend == "vllm": + tdef = VllmTestDefinition( + name=backend, + description="Serving observations", + test_template_name=backend, + cmd_args=VllmCmdArgs(docker_image_url="image:latest", model="test/model"), + ) + else: + tdef = SglangTestDefinition( + name=backend, + description="Serving observations", + test_template_name=backend, + cmd_args=SglangCmdArgs(docker_image_url="image:latest", model="test/model"), + ) + tr = TestRun(name=backend, test=tdef, num_nodes=1, nodes=[], output_path=tmp_path) + result_path = tmp_path / ("vllm-bench.json" if backend == "vllm" else "sglang-bench.jsonl") + data = { + "num_prompts": 30, + "completed": 30, + "max_concurrency": 16, + "request_throughput": 10.0, + "output_throughput": 2400.0, + "mean_ttft_ms": 120.0, + "median_ttft_ms": 100.0, + "p99_ttft_ms": 200.0, + "mean_tpot_ms": 12.0, + "median_tpot_ms": 10.0, + "p99_tpot_ms": 20.0, + } + result_path.write_text(json.dumps(data) + "\n") + return tr, result_path, data + + +def test_serving_metric_observations(slurm_system: SlurmSystem, serving_observation_run) -> None: + tr, _, _ = serving_observation_run + observations = tr.test.metric_observations(slurm_system, tr) + + assert {(o.metric.key, o.dimensions.get("statistic")): (o.value, o.metric.unit) for o in observations} == { + ("request_throughput", None): (10.0, "requests/s"), + ("output_token_throughput", None): (2400.0, "tokens/s"), + ("ttft", "mean"): (120.0, "ms"), + ("ttft", "median"): (100.0, "ms"), + ("ttft", "p99"): (200.0, "ms"), + ("tpot", "mean"): (12.0, "ms"), + ("tpot", "median"): (10.0, "ms"), + ("tpot", "p99"): (20.0, "ms"), + } + for observation in observations: + assert {k: v for k, v in observation.dimensions.items() if k != "statistic"} == { + "backend": tr.name, + "model": "test/model", + "max_concurrency": 16, + } + + tr.metric_sol = cloudai.metrics.parse_sol_spec( + { + "output_token_throughput": [{"value": 4800, "match": {"model": "test/model", "max_concurrency": 16}}], + "ttft": [{"value": 60}, {"value": 100, "match": {"backend": tr.name, "statistic": "p99"}}], + } + ) + assessments = cloudai.metrics.assess_test_run_metrics(slurm_system, tr) + assert [a.attainment for a in assessments if a.observation.metric.key == "output_token_throughput"] == [0.5] + assert [a.attainment for a in assessments if a.observation.metric.key == "ttft"] == [0.5, 0.6, 0.5] + + +@pytest.mark.parametrize("result", ["missing", "invalid", "incomplete", "unsuccessful"]) +def test_serving_metric_observations_without_results( + slurm_system: SlurmSystem, serving_observation_run, result: str +) -> None: + tr, result_path, data = serving_observation_run + if result == "missing": + result_path.unlink() + elif result == "invalid": + result_path.write_text("{invalid\n") + elif result == "incomplete": + result_path.write_text("{}\n") + else: + result_path.write_text(json.dumps({**data, "completed": 0})) + + assert tr.test.metric_observations(slurm_system, tr) == [] + + +def test_serving_metric_observations_optional_throughput(slurm_system: SlurmSystem, serving_observation_run) -> None: + tr, result_path, data = serving_observation_run + optional_field = "request_throughput" if tr.name == "vllm" else "output_throughput" + del data[optional_field] + result_path.write_text(json.dumps(data)) + + observations = tr.test.metric_observations(slurm_system, tr) + + assert len(observations) == 7 + throughput = [o for o in observations if "throughput" in o.metric.key] + assert [o.metric.key for o in throughput] == [ + "output_token_throughput" if tr.name == "vllm" else "request_throughput" + ] + + +def test_serving_metric_observations_nonfinite_values(slurm_system: SlurmSystem, serving_observation_run) -> None: + tr, result_path, data = serving_observation_run + data.update(mean_ttft_ms=float("nan"), output_throughput=float("inf"), max_concurrency=0) + result_path.write_text(json.dumps(data)) + + observations = tr.test.metric_observations(slurm_system, tr) + + assert len(observations) == 6 + assert all(o.dimensions["max_concurrency"] == 0 for o in observations) + assert all(o.metric.key != "output_token_throughput" for o in observations) + assert all((o.metric.key, o.dimensions.get("statistic")) != ("ttft", "mean") for o in observations) + + +@pytest.mark.parametrize("evaluation", ["disabled", "missing", "valid"]) +def test_serving_metric_observations_accuracy( + slurm_system: SlurmSystem, serving_observation_run, evaluation: str +) -> None: + tr, _, _ = serving_observation_run + if evaluation != "disabled": + tr.test.semantic_eval_cmd_args = VllmSemanticEvalCmdArgs() if tr.name == "vllm" else SglangSemanticEvalCmdArgs() + if evaluation != "missing": + (tr.output_path / f"{tr.name}-semantic-eval.log").write_text("Accuracy: 0.875\n") + + observations = tr.test.metric_observations(slurm_system, tr) + accuracy = [o for o in observations if o.metric.key == "accuracy"] + + assert accuracy == ( + [ + cloudai.metrics.MetricObservation( + cloudai.metrics.ACCURACY, 0.875, {"backend": tr.name, "model": "test/model"} + ) + ] + if evaluation == "valid" + else [] + ) From 1c6ccd0e483d72fcf3039fb0a8ec588b4d70284c Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Thu, 1 Oct 2026 19:59:36 +0200 Subject: [PATCH 2/7] fix: limit LLM metric dimensions to output statistics Signed-off-by: Ivan Podkidyshev --- doc/reporting.rst | 16 +++++++------ src/cloudai/metrics.py | 4 ---- src/cloudai/workloads/common/llm_serving.py | 14 ++++------- src/cloudai/workloads/sglang/sglang.py | 2 +- src/cloudai/workloads/vllm/vllm.py | 2 +- tests/workloads/common/test_llm_serving.py | 26 +++++++++------------ 6 files changed, 26 insertions(+), 38 deletions(-) diff --git a/doc/reporting.rst b/doc/reporting.rst index 5acf46e53..2b7e21dbb 100644 --- a/doc/reporting.rst +++ b/doc/reporting.rst @@ -113,7 +113,7 @@ and percentage-of-SOL columns and draw a shared SOL curve when every compared ru .. _llm-serving-metric-observations: LLM serving metric observations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ vLLM and SGLang expose structured measurements through ``TestDefinition.metric_observations(system, test_run)``. ``cloudai.metrics.assess_test_run_metrics`` compares these observations with the run's configured SOL targets. @@ -140,10 +140,13 @@ vLLM and SGLang expose structured measurements through ``TestDefinition.metric_o - ratio (0–1) - Higher -All observations include ``backend`` (``vllm`` or ``sglang``) and ``model`` dimensions. Benchmark observations also -include ``max_concurrency`` from the result file; TTFT and TPOT have a ``statistic`` dimension with ``mean``, -``median``, or ``p99``. Accuracy comes from semantic evaluation when ``semantic_eval_cmd_args`` is configured and -does not carry benchmark concurrency or latency statistics. +Within each concrete execution (test case iteration or DSE step), the metric key and dimensions distinguish each +reported value. TTFT and TPOT use ``statistic`` (``mean``, ``median``, or ``p99``). Request throughput, output-token +throughput, and accuracy each have one value and empty dimensions. Accuracy comes from semantic evaluation when +``semantic_eval_cmd_args`` is configured. + +Workload, model, and configured concurrency belong to the test/run context. Use test-case SOL overrides when +different configurations need different targets, and ``match.statistic`` to select a latency statistic. Request and output-token throughput are emitted only when their respective ``request_throughput`` and ``output_throughput`` fields are present. Missing or invalid benchmark results and results with no completed requests @@ -156,11 +159,10 @@ For example, a system or scenario can define these targets: [[sol.output_token_throughput]] value = 5000.0 # tokens/s - match = { model = "Qwen/Qwen3-8B", max_concurrency = 16 } [[sol.ttft]] value = 100.0 # ms - match = { backend = "sglang", statistic = "p99" } + match = { statistic = "p99" } These observations leave the existing scalar metric names and comparison reports unchanged. In particular, the legacy ``throughput`` metric uses output tokens/s for vLLM and requests/s for SGLang. diff --git a/src/cloudai/metrics.py b/src/cloudai/metrics.py index 515186bf4..e13e63378 100644 --- a/src/cloudai/metrics.py +++ b/src/cloudai/metrics.py @@ -105,8 +105,6 @@ def validate_dimensions(cls, values: Mapping[str, Any]) -> MetricDimensions: BACKEND = DimensionDefinition("backend", "Backend", Annotated[str, Field(strict=True, min_length=1)]) SOURCE_MEMORY = DimensionDefinition("source_memory", "Source memory", Annotated[str, Field(strict=True, min_length=1)]) TARGET_MEMORY = DimensionDefinition("target_memory", "Target memory", Annotated[str, Field(strict=True, min_length=1)]) -MODEL = DimensionDefinition("model", "Model", Annotated[str, Field(strict=True, min_length=1)]) -MAX_CONCURRENCY = DimensionDefinition("max_concurrency", "Max concurrency", Annotated[int, Field(strict=True, ge=0)]) STATISTIC = DimensionDefinition("statistic", "Statistic", Literal["mean", "median", "p99"]) BANDWIDTH = MetricDefinition( @@ -163,8 +161,6 @@ def validate_dimensions(cls, values: Mapping[str, Any]) -> MetricDimensions: BACKEND, SOURCE_MEMORY, TARGET_MEMORY, - MODEL, - MAX_CONCURRENCY, STATISTIC, ) diff --git a/src/cloudai/workloads/common/llm_serving.py b/src/cloudai/workloads/common/llm_serving.py index 4ad41800f..793b0f4f9 100644 --- a/src/cloudai/workloads/common/llm_serving.py +++ b/src/cloudai/workloads/common/llm_serving.py @@ -245,32 +245,26 @@ def tps_per_user(self) -> float | None: def llm_serving_metric_observations( results: LLMServingBenchReport | None, - backend: str, - model: str, accuracy: float | None = None, ) -> list[cloudai.metrics.MetricObservation]: - """Collect finite serving measurements with shared units and dimensions.""" - dimensions: dict[str, cloudai.metrics.MetricValue] = {"backend": backend, "model": model} + """Use statistics to distinguish latency results; scalar results have no dimensions.""" observations: list[cloudai.metrics.MetricObservation] = [] if accuracy is not None and math.isfinite(accuracy): - observations.append(cloudai.metrics.MetricObservation(cloudai.metrics.ACCURACY, accuracy, dimensions)) + observations.append(cloudai.metrics.MetricObservation(cloudai.metrics.ACCURACY, accuracy, {})) if results is None or results.completed <= 0: return observations - dimensions = {**dimensions, "max_concurrency": results.max_concurrency} for metric, value in ( (cloudai.metrics.REQUEST_THROUGHPUT, getattr(results, "request_throughput", None)), (cloudai.metrics.OUTPUT_TOKEN_THROUGHPUT, getattr(results, "output_throughput", None)), ): if value is not None and math.isfinite(value): - observations.append(cloudai.metrics.MetricObservation(metric, value, dimensions)) + observations.append(cloudai.metrics.MetricObservation(metric, value, {})) for metric in (cloudai.metrics.TTFT, cloudai.metrics.TPOT): for statistic in ("mean", "median", "p99"): value = getattr(results, f"{statistic}_{metric.key}_ms") if math.isfinite(value): - observations.append( - cloudai.metrics.MetricObservation(metric, value, {**dimensions, "statistic": statistic}) - ) + observations.append(cloudai.metrics.MetricObservation(metric, value, {"statistic": statistic})) return observations diff --git a/src/cloudai/workloads/sglang/sglang.py b/src/cloudai/workloads/sglang/sglang.py index 9b756ee61..233c09595 100644 --- a/src/cloudai/workloads/sglang/sglang.py +++ b/src/cloudai/workloads/sglang/sglang.py @@ -137,7 +137,7 @@ def metric_observations(self, system: System, tr: TestRun) -> list[cloudai.metri if self.semantic_eval_cmd_args is not None else None ) - return llm_serving_metric_observations(results, "sglang", self.cmd_args.model, accuracy) + return llm_serving_metric_observations(results, accuracy) class SGLangBenchReport(LLMServingBenchReport): diff --git a/src/cloudai/workloads/vllm/vllm.py b/src/cloudai/workloads/vllm/vllm.py index 497136f2b..0cfecab70 100644 --- a/src/cloudai/workloads/vllm/vllm.py +++ b/src/cloudai/workloads/vllm/vllm.py @@ -272,7 +272,7 @@ def metric_observations(self, system: System, tr: TestRun) -> list[cloudai.metri results = parse_vllm_bench_output(tr.output_path / VLLM_BENCH_JSON_FILE) accuracy = parse_vllm_semantic_accuracy(tr.output_path) if self.semantic_eval_cmd_args is not None else None - return llm_serving_metric_observations(results, "vllm", self.cmd_args.model, accuracy) + return llm_serving_metric_observations(results, accuracy) @cache diff --git a/tests/workloads/common/test_llm_serving.py b/tests/workloads/common/test_llm_serving.py index e860f8dae..1f06267df 100644 --- a/tests/workloads/common/test_llm_serving.py +++ b/tests/workloads/common/test_llm_serving.py @@ -507,6 +507,7 @@ def test_serving_metric_observations(slurm_system: SlurmSystem, serving_observat tr, _, _ = serving_observation_run observations = tr.test.metric_observations(slurm_system, tr) + assert len(observations) == len({(o.metric.key, tuple(o.dimensions.items())) for o in observations}) == 8 assert {(o.metric.key, o.dimensions.get("statistic")): (o.value, o.metric.unit) for o in observations} == { ("request_throughput", None): (10.0, "requests/s"), ("output_token_throughput", None): (2400.0, "tokens/s"), @@ -518,16 +519,16 @@ def test_serving_metric_observations(slurm_system: SlurmSystem, serving_observat ("tpot", "p99"): (20.0, "ms"), } for observation in observations: - assert {k: v for k, v in observation.dimensions.items() if k != "statistic"} == { - "backend": tr.name, - "model": "test/model", - "max_concurrency": 16, - } + assert set(observation.dimensions) == ({"statistic"} if observation.metric.key in {"ttft", "tpot"} else set()) + + tr.test.cmd_args.model = "other/model" + tr.test.bench_cmd_args.max_concurrency = 32 + assert tr.test.metric_observations(slurm_system, tr) == observations tr.metric_sol = cloudai.metrics.parse_sol_spec( { - "output_token_throughput": [{"value": 4800, "match": {"model": "test/model", "max_concurrency": 16}}], - "ttft": [{"value": 60}, {"value": 100, "match": {"backend": tr.name, "statistic": "p99"}}], + "output_token_throughput": [{"value": 4800}], + "ttft": [{"value": 60}, {"value": 100, "match": {"statistic": "p99"}}], } ) assessments = cloudai.metrics.assess_test_run_metrics(slurm_system, tr) @@ -575,7 +576,7 @@ def test_serving_metric_observations_nonfinite_values(slurm_system: SlurmSystem, observations = tr.test.metric_observations(slurm_system, tr) assert len(observations) == 6 - assert all(o.dimensions["max_concurrency"] == 0 for o in observations) + assert all("max_concurrency" not in o.dimensions for o in observations) assert all(o.metric.key != "output_token_throughput" for o in observations) assert all((o.metric.key, o.dimensions.get("statistic")) != ("ttft", "mean") for o in observations) @@ -593,12 +594,7 @@ def test_serving_metric_observations_accuracy( observations = tr.test.metric_observations(slurm_system, tr) accuracy = [o for o in observations if o.metric.key == "accuracy"] + assert len(observations) == len({(o.metric.key, tuple(o.dimensions.items())) for o in observations}) assert accuracy == ( - [ - cloudai.metrics.MetricObservation( - cloudai.metrics.ACCURACY, 0.875, {"backend": tr.name, "model": "test/model"} - ) - ] - if evaluation == "valid" - else [] + [cloudai.metrics.MetricObservation(cloudai.metrics.ACCURACY, 0.875, {})] if evaluation == "valid" else [] ) From 05f44c98dbd64dcd2929cf15497d55b2f31cbe4a Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Thu, 1 Oct 2026 23:12:28 +0200 Subject: [PATCH 3/7] docs: move serving metrics reference into workload pages Signed-off-by: Ivan Podkidyshev --- doc/reporting.rst | 57 -------------------------- doc/workloads/_llm_serving_metrics.inc | 26 ++++++++++++ doc/workloads/sglang.rst | 3 +- doc/workloads/vllm.rst | 3 +- 4 files changed, 28 insertions(+), 61 deletions(-) create mode 100644 doc/workloads/_llm_serving_metrics.inc diff --git a/doc/reporting.rst b/doc/reporting.rst index 2b7e21dbb..1bdeb8d92 100644 --- a/doc/reporting.rst +++ b/doc/reporting.rst @@ -110,63 +110,6 @@ A test case can replace the targets for a metric inherited from the scenario or The precedence is test case, then scenario, then system. NCCL and NIXL comparison v2 reports include measured, SOL, and percentage-of-SOL columns and draw a shared SOL curve when every compared run resolves the same targets. -.. _llm-serving-metric-observations: - -LLM serving metric observations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -vLLM and SGLang expose structured measurements through ``TestDefinition.metric_observations(system, test_run)``. -``cloudai.metrics.assess_test_run_metrics`` compares these observations with the run's configured SOL targets. - -.. list-table:: Serving metrics - :header-rows: 1 - - * - Metric key - - Unit - - Preferred direction - * - ``request_throughput`` - - requests/s - - Higher - * - ``output_token_throughput`` - - tokens/s - - Higher - * - ``ttft`` (time to first token) - - ms - - Lower - * - ``tpot`` (time per output token) - - ms - - Lower - * - ``accuracy`` - - ratio (0–1) - - Higher - -Within each concrete execution (test case iteration or DSE step), the metric key and dimensions distinguish each -reported value. TTFT and TPOT use ``statistic`` (``mean``, ``median``, or ``p99``). Request throughput, output-token -throughput, and accuracy each have one value and empty dimensions. Accuracy comes from semantic evaluation when -``semantic_eval_cmd_args`` is configured. - -Workload, model, and configured concurrency belong to the test/run context. Use test-case SOL overrides when -different configurations need different targets, and ``match.statistic`` to select a latency statistic. - -Request and output-token throughput are emitted only when their respective ``request_throughput`` and -``output_throughput`` fields are present. Missing or invalid benchmark results and results with no completed requests -produce no benchmark observations. Non-finite measurements are omitted. Semantic accuracy can be emitted -independently when an evaluation result is available. - -For example, a system or scenario can define these targets: - -.. code-block:: toml - - [[sol.output_token_throughput]] - value = 5000.0 # tokens/s - - [[sol.ttft]] - value = 100.0 # ms - match = { statistic = "p99" } - -These observations leave the existing scalar metric names and comparison reports unchanged. In particular, the -legacy ``throughput`` metric uses output tokens/s for vLLM and requests/s for SGLang. - .. _reporting-registration: Reporting Registration diff --git a/doc/workloads/_llm_serving_metrics.inc b/doc/workloads/_llm_serving_metrics.inc new file mode 100644 index 000000000..73c7f6fd4 --- /dev/null +++ b/doc/workloads/_llm_serving_metrics.inc @@ -0,0 +1,26 @@ +Structured metrics +~~~~~~~~~~~~~~~~~~ + +.. csv-table:: + :header: "Metric", "Unit", "Preferred", "Dimensions" + + "``request_throughput``", "requests/s", "Higher", "None" + "``output_token_throughput``", "tokens/s", "Higher", "None" + "``ttft`` (time to first token)", "ms", "Lower", "``statistic``: mean, median, p99" + "``tpot`` (time per output token)", "ms", "Lower", "``statistic``: mean, median, p99" + "``accuracy``", "ratio (0–1)", "Higher", "None" + +Metric and dimensions identify each value within an iteration or DSE step. Model, backend, and configured concurrency +remain in the test/run context. Only available, finite measurements are emitted; accuracy requires +``semantic_eval_cmd_args``. + +Example scenario SOL targets (see :doc:`SOL configuration ` for precedence and test-case overrides): + +.. code-block:: toml + + [[sol.output_token_throughput]] + value = 5000.0 # tokens/s + + [[sol.ttft]] + value = 100.0 # ms + match = { statistic = "p99" } diff --git a/doc/workloads/sglang.rst b/doc/workloads/sglang.rst index d91b3cf37..921e46fa9 100644 --- a/doc/workloads/sglang.rst +++ b/doc/workloads/sglang.rst @@ -128,8 +128,7 @@ The reported metric (``default``) is throughput. Additional supported metrics ar CloudAI also provides the scenario-level ``sglang_comparison`` report. It compares SGLang test runs in the scenario and uses ``bench_cmd_args`` values as comparison labels. -The workload also exposes :ref:`structured metric observations ` for request and -output-token throughput, TTFT, TPOT, and optional semantic accuracy, with support for SOL assessment. +.. include:: _llm_serving_metrics.inc Readiness health checks diff --git a/doc/workloads/vllm.rst b/doc/workloads/vllm.rst index 5fadc7a80..9b8db716c 100644 --- a/doc/workloads/vllm.rst +++ b/doc/workloads/vllm.rst @@ -123,8 +123,7 @@ The reported metric (``default``) is throughput. Additional supported metrics ar CloudAI also provides the scenario-level ``vllm_comparison`` report. It compares vLLM test runs in the scenario and uses ``bench_cmd_args`` values as comparison labels. -The workload also exposes :ref:`structured metric observations ` for request and -output-token throughput, TTFT, TPOT, and optional semantic accuracy, with support for SOL assessment. +.. include:: _llm_serving_metrics.inc Controlling the Number of GPUs From 2869d98faea1f34f391d699a08f41192f9e4e45f Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Tue, 6 Oct 2026 13:30:51 +0200 Subject: [PATCH 4/7] docs: keep LLM metrics changes focused on observations Signed-off-by: Ivan Podkidyshev --- doc/workloads/_llm_serving_metrics.inc | 11 ----------- tests/workloads/common/test_llm_serving.py | 10 ---------- 2 files changed, 21 deletions(-) diff --git a/doc/workloads/_llm_serving_metrics.inc b/doc/workloads/_llm_serving_metrics.inc index 73c7f6fd4..3e38cc5a4 100644 --- a/doc/workloads/_llm_serving_metrics.inc +++ b/doc/workloads/_llm_serving_metrics.inc @@ -13,14 +13,3 @@ Structured metrics Metric and dimensions identify each value within an iteration or DSE step. Model, backend, and configured concurrency remain in the test/run context. Only available, finite measurements are emitted; accuracy requires ``semantic_eval_cmd_args``. - -Example scenario SOL targets (see :doc:`SOL configuration ` for precedence and test-case overrides): - -.. code-block:: toml - - [[sol.output_token_throughput]] - value = 5000.0 # tokens/s - - [[sol.ttft]] - value = 100.0 # ms - match = { statistic = "p99" } diff --git a/tests/workloads/common/test_llm_serving.py b/tests/workloads/common/test_llm_serving.py index 1f06267df..5e4c32dd5 100644 --- a/tests/workloads/common/test_llm_serving.py +++ b/tests/workloads/common/test_llm_serving.py @@ -525,16 +525,6 @@ def test_serving_metric_observations(slurm_system: SlurmSystem, serving_observat tr.test.bench_cmd_args.max_concurrency = 32 assert tr.test.metric_observations(slurm_system, tr) == observations - tr.metric_sol = cloudai.metrics.parse_sol_spec( - { - "output_token_throughput": [{"value": 4800}], - "ttft": [{"value": 60}, {"value": 100, "match": {"statistic": "p99"}}], - } - ) - assessments = cloudai.metrics.assess_test_run_metrics(slurm_system, tr) - assert [a.attainment for a in assessments if a.observation.metric.key == "output_token_throughput"] == [0.5] - assert [a.attainment for a in assessments if a.observation.metric.key == "ttft"] == [0.5, 0.6, 0.5] - @pytest.mark.parametrize("result", ["missing", "invalid", "incomplete", "unsuccessful"]) def test_serving_metric_observations_without_results( From 77d31636d9209dce1a1e44cb436cb10d1e555ef1 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Tue, 6 Oct 2026 14:13:54 +0200 Subject: [PATCH 5/7] docs: inline serving metrics tables in workload pages Signed-off-by: Ivan Podkidyshev --- doc/workloads/_llm_serving_metrics.inc | 15 --------------- doc/workloads/sglang.rst | 12 +++++++++++- doc/workloads/vllm.rst | 12 +++++++++++- 3 files changed, 22 insertions(+), 17 deletions(-) delete mode 100644 doc/workloads/_llm_serving_metrics.inc diff --git a/doc/workloads/_llm_serving_metrics.inc b/doc/workloads/_llm_serving_metrics.inc deleted file mode 100644 index 3e38cc5a4..000000000 --- a/doc/workloads/_llm_serving_metrics.inc +++ /dev/null @@ -1,15 +0,0 @@ -Structured metrics -~~~~~~~~~~~~~~~~~~ - -.. csv-table:: - :header: "Metric", "Unit", "Preferred", "Dimensions" - - "``request_throughput``", "requests/s", "Higher", "None" - "``output_token_throughput``", "tokens/s", "Higher", "None" - "``ttft`` (time to first token)", "ms", "Lower", "``statistic``: mean, median, p99" - "``tpot`` (time per output token)", "ms", "Lower", "``statistic``: mean, median, p99" - "``accuracy``", "ratio (0–1)", "Higher", "None" - -Metric and dimensions identify each value within an iteration or DSE step. Model, backend, and configured concurrency -remain in the test/run context. Only available, finite measurements are emitted; accuracy requires -``semantic_eval_cmd_args``. diff --git a/doc/workloads/sglang.rst b/doc/workloads/sglang.rst index 921e46fa9..8d0b68fc6 100644 --- a/doc/workloads/sglang.rst +++ b/doc/workloads/sglang.rst @@ -128,7 +128,17 @@ The reported metric (``default``) is throughput. Additional supported metrics ar CloudAI also provides the scenario-level ``sglang_comparison`` report. It compares SGLang test runs in the scenario and uses ``bench_cmd_args`` values as comparison labels. -.. include:: _llm_serving_metrics.inc +Structured metrics +~~~~~~~~~~~~~~~~~~ + +.. csv-table:: + :header: "Metric", "Unit", "Preferred", "Dimensions" + + "``request_throughput``", "requests/s", "Higher", "None" + "``output_token_throughput``", "tokens/s", "Higher", "None" + "``ttft`` (time to first token)", "ms", "Lower", "``statistic``: mean, median, p99" + "``tpot`` (time per output token)", "ms", "Lower", "``statistic``: mean, median, p99" + "``accuracy``", "ratio (0–1)", "Higher", "None" Readiness health checks diff --git a/doc/workloads/vllm.rst b/doc/workloads/vllm.rst index 9b8db716c..e6538f90b 100644 --- a/doc/workloads/vllm.rst +++ b/doc/workloads/vllm.rst @@ -123,7 +123,17 @@ The reported metric (``default``) is throughput. Additional supported metrics ar CloudAI also provides the scenario-level ``vllm_comparison`` report. It compares vLLM test runs in the scenario and uses ``bench_cmd_args`` values as comparison labels. -.. include:: _llm_serving_metrics.inc +Structured metrics +~~~~~~~~~~~~~~~~~~ + +.. csv-table:: + :header: "Metric", "Unit", "Preferred", "Dimensions" + + "``request_throughput``", "requests/s", "Higher", "None" + "``output_token_throughput``", "tokens/s", "Higher", "None" + "``ttft`` (time to first token)", "ms", "Lower", "``statistic``: mean, median, p99" + "``tpot`` (time per output token)", "ms", "Lower", "``statistic``: mean, median, p99" + "``accuracy``", "ratio (0–1)", "Higher", "None" Controlling the Number of GPUs From 31f39de126ab2a2ec1c9d157c919605b3b406772 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Tue, 6 Oct 2026 14:31:03 +0200 Subject: [PATCH 6/7] Require both throughput fields in serving benchmark reports Signed-off-by: Ivan Podkidyshev --- src/cloudai/workloads/common/llm_serving.py | 9 +++++---- src/cloudai/workloads/sglang/sglang.py | 2 +- .../workloads/vllm/report_generation_strategy.py | 2 +- tests/workloads/common/test_llm_serving.py | 16 ++++++---------- .../sglang/test_job_status_retrieval_strategy.py | 4 ++++ .../workloads/sglang/test_report_gen_strategy.py | 1 + tests/workloads/vllm/test_report_gen_strategy.py | 1 + 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/cloudai/workloads/common/llm_serving.py b/src/cloudai/workloads/common/llm_serving.py index 793b0f4f9..d6b574475 100644 --- a/src/cloudai/workloads/common/llm_serving.py +++ b/src/cloudai/workloads/common/llm_serving.py @@ -254,11 +254,12 @@ def llm_serving_metric_observations( if results is None or results.completed <= 0: return observations - for metric, value in ( - (cloudai.metrics.REQUEST_THROUGHPUT, getattr(results, "request_throughput", None)), - (cloudai.metrics.OUTPUT_TOKEN_THROUGHPUT, getattr(results, "output_throughput", None)), + for metric, field in ( + (cloudai.metrics.REQUEST_THROUGHPUT, "request_throughput"), + (cloudai.metrics.OUTPUT_TOKEN_THROUGHPUT, "output_throughput"), ): - if value is not None and math.isfinite(value): + value = getattr(results, field) + if math.isfinite(value): observations.append(cloudai.metrics.MetricObservation(metric, value, {})) for metric in (cloudai.metrics.TTFT, cloudai.metrics.TPOT): for statistic in ("mean", "median", "p99"): diff --git a/src/cloudai/workloads/sglang/sglang.py b/src/cloudai/workloads/sglang/sglang.py index 233c09595..65e128138 100644 --- a/src/cloudai/workloads/sglang/sglang.py +++ b/src/cloudai/workloads/sglang/sglang.py @@ -144,7 +144,7 @@ class SGLangBenchReport(LLMServingBenchReport): """Parsed benchmark data from SGLang bench_serving output.""" request_throughput: float - output_throughput: float | None = None + output_throughput: float @property def throughput(self) -> float: diff --git a/src/cloudai/workloads/vllm/report_generation_strategy.py b/src/cloudai/workloads/vllm/report_generation_strategy.py index 773fd524f..fe58e2dbc 100644 --- a/src/cloudai/workloads/vllm/report_generation_strategy.py +++ b/src/cloudai/workloads/vllm/report_generation_strategy.py @@ -28,7 +28,7 @@ class VLLMBenchReport(LLMServingBenchReport): """Report for vLLM benchmark results.""" output_throughput: float - request_throughput: float | None = None + request_throughput: float @property def throughput(self) -> float: diff --git a/tests/workloads/common/test_llm_serving.py b/tests/workloads/common/test_llm_serving.py index 5e4c32dd5..6aa002b2f 100644 --- a/tests/workloads/common/test_llm_serving.py +++ b/tests/workloads/common/test_llm_serving.py @@ -543,19 +543,15 @@ def test_serving_metric_observations_without_results( assert tr.test.metric_observations(slurm_system, tr) == [] -def test_serving_metric_observations_optional_throughput(slurm_system: SlurmSystem, serving_observation_run) -> None: +@pytest.mark.parametrize("field", ["request_throughput", "output_throughput"]) +def test_serving_metric_observations_requires_both_throughputs( + slurm_system: SlurmSystem, serving_observation_run, field: str +) -> None: tr, result_path, data = serving_observation_run - optional_field = "request_throughput" if tr.name == "vllm" else "output_throughput" - del data[optional_field] + del data[field] result_path.write_text(json.dumps(data)) - observations = tr.test.metric_observations(slurm_system, tr) - - assert len(observations) == 7 - throughput = [o for o in observations if "throughput" in o.metric.key] - assert [o.metric.key for o in throughput] == [ - "output_token_throughput" if tr.name == "vllm" else "request_throughput" - ] + assert tr.test.metric_observations(slurm_system, tr) == [] def test_serving_metric_observations_nonfinite_values(slurm_system: SlurmSystem, serving_observation_run) -> None: diff --git a/tests/workloads/sglang/test_job_status_retrieval_strategy.py b/tests/workloads/sglang/test_job_status_retrieval_strategy.py index 099e9f329..4f67b883f 100644 --- a/tests/workloads/sglang/test_job_status_retrieval_strategy.py +++ b/tests/workloads/sglang/test_job_status_retrieval_strategy.py @@ -51,6 +51,7 @@ def test_successful_job(self, base_tr: TestRun) -> None: "completed": 3, "num_prompts": 3, "request_throughput": 1.0, + "output_throughput": 2.0, "max_concurrency": 16, "mean_ttft_ms": 1.0, "median_ttft_ms": 1.0, @@ -77,6 +78,7 @@ def test_failed_job_no_successful_requests(self, base_tr: TestRun) -> None: "completed": 0, "num_prompts": 3, "request_throughput": 0.0, + "output_throughput": 0.0, "max_concurrency": 16, "mean_ttft_ms": 1.0, "median_ttft_ms": 1.0, @@ -105,6 +107,7 @@ def test_semantic_eval_successful_with_low_accuracy(self, base_tr: TestRun) -> N "completed": 3, "num_prompts": 3, "request_throughput": 1.0, + "output_throughput": 2.0, "max_concurrency": 16, "mean_ttft_ms": 1.0, "median_ttft_ms": 1.0, @@ -131,6 +134,7 @@ def test_semantic_eval_requires_parseable_accuracy(self, base_tr: TestRun) -> No "completed": 3, "num_prompts": 3, "request_throughput": 1.0, + "output_throughput": 2.0, "max_concurrency": 16, "mean_ttft_ms": 1.0, "median_ttft_ms": 1.0, diff --git a/tests/workloads/sglang/test_report_gen_strategy.py b/tests/workloads/sglang/test_report_gen_strategy.py index 86d5ed9c3..74668f9b2 100644 --- a/tests/workloads/sglang/test_report_gen_strategy.py +++ b/tests/workloads/sglang/test_report_gen_strategy.py @@ -41,6 +41,7 @@ "num_prompts": 30, "completed": 30, "request_throughput": 2400.0, + "output_throughput": 4800.0, "max_concurrency": 16, "mean_ttft_ms": 120.0, "median_ttft_ms": 100.0, diff --git a/tests/workloads/vllm/test_report_gen_strategy.py b/tests/workloads/vllm/test_report_gen_strategy.py index 7e1821067..baac6defb 100644 --- a/tests/workloads/vllm/test_report_gen_strategy.py +++ b/tests/workloads/vllm/test_report_gen_strategy.py @@ -47,6 +47,7 @@ median_tpot_ms=11.0, p99_tpot_ms=19.0, output_throughput=2400.0, + request_throughput=10.0, max_concurrency=16, ) From e653330eb8ea05bef6be755c3252a790410ad9c0 Mon Sep 17 00:00:00 2001 From: Ivan Podkidyshev Date: Tue, 6 Oct 2026 14:38:24 +0200 Subject: [PATCH 7/7] Update comparison reporter fixtures with both throughput fields Signed-off-by: Ivan Podkidyshev --- tests/workloads/common/test_llm_serving_report.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/workloads/common/test_llm_serving_report.py b/tests/workloads/common/test_llm_serving_report.py index 6d020460e..9a7fac1ff 100644 --- a/tests/workloads/common/test_llm_serving_report.py +++ b/tests/workloads/common/test_llm_serving_report.py @@ -87,6 +87,7 @@ def test_llm_comparison_report_generates_html(slurm_system: cloudai.systems.slur "median_tpot_ms": 9.0, "p99_tpot_ms": 15.0, "output_throughput": 1200.0, + "request_throughput": 12.0, "max_concurrency": 8, } ), @@ -106,6 +107,7 @@ def test_llm_comparison_report_generates_html(slurm_system: cloudai.systems.slur "median_tpot_ms": 7.0, "p99_tpot_ms": 13.0, "output_throughput": 1800.0, + "request_throughput": 18.0, "max_concurrency": 16, } ), @@ -188,6 +190,7 @@ def test_sglang_comparison_report_generates_html(slurm_system: cloudai.systems.s "median_tpot_ms": 11.0, "p99_tpot_ms": 18.0, "request_throughput": 960.0, + "output_throughput": 96000.0, "max_concurrency": 8, } ) @@ -208,6 +211,7 @@ def test_sglang_comparison_report_generates_html(slurm_system: cloudai.systems.s "median_tpot_ms": 8.0, "p99_tpot_ms": 14.0, "request_throughput": 1760.0, + "output_throughput": 176000.0, "max_concurrency": 16, } )