From 93baa2ccf1146cb19e8dc6992765d605089488f6 Mon Sep 17 00:00:00 2001 From: Matyas Sedmidubsky Date: Fri, 17 Jul 2026 19:26:20 +0200 Subject: [PATCH 1/8] fix: override pytest_start help's exit code on error --- pytest_start.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest_start.sh b/pytest_start.sh index 17a8355..6f4d716 100755 --- a/pytest_start.sh +++ b/pytest_start.sh @@ -159,8 +159,8 @@ while [ "$#" -gt 0 ]; do shift 4 ;; --) shift; read -a extra_args <<< "$@"; break ;; - *) >&2 echo unsupported option: $1 - usage + *) >&2 echo "unsupported option: $1" + : "$(usage)" # create subshell to avoid `exit 0` exit 1 ;; esac From 3a715b9240fb74f6d964229593c6cc7070b096af Mon Sep 17 00:00:00 2001 From: Matyas Sedmidubsky Date: Thu, 3 Sep 2026 15:15:13 +0200 Subject: [PATCH 2/8] feat: set -x in bash based on $LOGLEVEL --- pytest_start.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pytest_start.sh b/pytest_start.sh index 6f4d716..c0e4abb 100755 --- a/pytest_start.sh +++ b/pytest_start.sh @@ -10,7 +10,10 @@ -set -xe +set -e +if [ "$LOGLEVEL" = "DEBUG" ]; then + set -x +fi usage(){ set +x From 0f0ebd96a017a9d9d0d151e8e6b54f067a082390 Mon Sep 17 00:00:00 2001 From: Matyas Sedmidubsky Date: Thu, 3 Sep 2026 15:25:56 +0200 Subject: [PATCH 3/8] chore: make ConfigBuilder Suricata agnostic --- .../traffic_profiles/trex_client_manager.py | 4 +- conftest.py | 4 +- util/config_builder.py | 45 ++++++++++--------- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/assets/trex/traffic_profiles/trex_client_manager.py b/assets/trex/traffic_profiles/trex_client_manager.py index faaf5b9..50750e1 100644 --- a/assets/trex/traffic_profiles/trex_client_manager.py +++ b/assets/trex/traffic_profiles/trex_client_manager.py @@ -35,7 +35,7 @@ from pytest import FixtureRequest from util.add_vlan import edit_vlan -from util.config_builder import ConfigBuilder +from util.config_builder import DEFAULT_TREX_CONF, ConfigBuilder from util.suri_util import RunInfo from util.trex_util import ( PcapList, @@ -204,7 +204,7 @@ def __init__( os.makedirs("tmp", exist_ok=True) config = ConfigBuilder( "tmp/trex_cfg.yaml", - str(Path(__file__).parent / "default_trex.yaml"), + str(DEFAULT_TREX_CONF), ) config.set_option("[0].interfaces", [trex_pcie, "dummy"]) config.set_option("[0].port_info[.=dest_mac].dest_mac", target_mac) diff --git a/conftest.py b/conftest.py index a32d206..12816a4 100644 --- a/conftest.py +++ b/conftest.py @@ -28,7 +28,7 @@ from pathlib import Path from itertools import product from param import filter -from util.config_builder import ConfigBuilder +from util.config_builder import DEFAULT_SURICATA_CONF, ConfigBuilder from util.log_util import get_logger, setup_logging TIME_STR = time.strftime("-".join(["%Y", "%m", "%d", "%H:%M"])) @@ -527,7 +527,7 @@ def suricata_conf_file(request) -> ConfigBuilder: editable_yaml, request.config.getoption("--suricata-cfg") ) else: - builder = ConfigBuilder(editable_yaml) + builder = ConfigBuilder(editable_yaml, str(DEFAULT_SURICATA_CONF)) return builder diff --git a/util/config_builder.py b/util/config_builder.py index 13ccb57..410c381 100644 --- a/util/config_builder.py +++ b/util/config_builder.py @@ -17,6 +17,15 @@ logger = logging.getLogger(__name__) +DEFAULT_SURICATA_CONF = Path(__file__).resolve().parent.parent / "default_suricata.yaml" +DEFAULT_TREX_CONF = ( + Path(__file__).resolve().parent.parent + / "assets" + / "trex" + / "traffic_profiles" + / "default_trex.yaml" +) + def update_recursively(destination: Dict, source: Dict, extend_lists=True) -> Dict: for k, v in source.items(): @@ -41,6 +50,21 @@ class ConfigBuilder: __proc: Processor output: str + def __init__(self, output: str, input: str) -> None: + self.output = output + logger.debug("Loading configuration builder: output=%s input=%s", output, input) + + self.__yaml = YAML() + self.__yaml.indent(sequence=4, offset=2) + self.__yaml.preserve_quotes = True + + with open(input, mode="r") as f: + data = self.__yaml.load(f) + + log_args = SimpleNamespace(quiet=True, verbose=False, debug=False) + log = ConsolePrinter(log_args) + self.__proc = Processor(log, data) + def add_option(self, key: str, value: Any) -> Self: """ Allows for nested keys to be added using dot notation, e.g. "app-layer.protocols.dns.tcp.enabled" @@ -126,24 +150,3 @@ def build(self) -> str: self.__yaml.dump(self.__proc.data, out) return self.output - - def __init__(self, output: str, input: str | None = None) -> None: - self.output = output - logger.debug("Loading configuration builder: output=%s input=%s", output, input) - - self.__yaml = YAML() - self.__yaml.indent(sequence=4, offset=2) - self.__yaml.preserve_quotes = True - - if input is not None: - with open(input, mode="r") as f: - data = self.__yaml.load(f) - else: - root_dir = Path(__file__).resolve().parent.parent - default_config_path = root_dir / "default_suricata.yaml" - with default_config_path.open(mode="r") as f: - data = self.__yaml.load(f) - - log_args = SimpleNamespace(quiet=True, verbose=False, debug=False) - log = ConsolePrinter(log_args) - self.__proc = Processor(log, data) From e2a115317babc959e1d93bc975ed44a0a63db605 Mon Sep 17 00:00:00 2001 From: Matyas Sedmidubsky Date: Thu, 3 Sep 2026 15:29:54 +0200 Subject: [PATCH 4/8] chore: static analysis improvements --- .../traffic_profiles/trex_client_manager.py | 50 +++++++++++-------- conftest.py | 34 ++++++------- util/config_builder.py | 8 +-- util/make-graphs.py | 2 +- util/suri_util.py | 13 ++--- util/trex_util.py | 4 -- 6 files changed, 57 insertions(+), 54 deletions(-) diff --git a/assets/trex/traffic_profiles/trex_client_manager.py b/assets/trex/traffic_profiles/trex_client_manager.py index 50750e1..e37e776 100644 --- a/assets/trex/traffic_profiles/trex_client_manager.py +++ b/assets/trex/traffic_profiles/trex_client_manager.py @@ -12,7 +12,7 @@ import warnings from pathlib import Path from time import sleep, time -from typing import Callable, Dict, Literal, NamedTuple, Self +from typing import Any, Callable, Literal, NamedTuple, Self, cast from lbr_testsuite.trex import ( TRexAdvancedStateful, @@ -38,7 +38,6 @@ from util.config_builder import DEFAULT_TREX_CONF, ConfigBuilder from util.suri_util import RunInfo from util.trex_util import ( - PcapList, TrexMode, get_trex_mac, merge_pcaps, @@ -63,13 +62,13 @@ class BaseTrexClientManager: Subclasses are created as `MyProfile(BaseTrexClientManager, pcaps)`. - `pcaps: PcapList` is a list of (str, int) tuples, where int is: + `pcaps: list[Pcap]` is a list of (str, int) tuples, where int is: - cps in STF - cps in ASTF - the divisor for `self.BASE_IPG_USEC` in STL """ - pcaps: PcapList + pcaps: list[Pcap] multiplier: float | None = None duration: int | None = None _stf_config_path: Path | None = None @@ -84,7 +83,7 @@ def __new__(cls, *args, **kwargs) -> Self: ) return super().__new__(cls) - def __init_subclass__(cls, pcaps: PcapList) -> None: + def __init_subclass__(cls, pcaps: list[Pcap]) -> None: cls.profile_pcaps = pcaps def __init__( @@ -97,13 +96,13 @@ def __init__( ) -> None: # self.pcaps holds (absolute local Path, weight) Pcap objects; the # class-level `pcaps`/`profile_pcaps` are (relative str, weight). - self.pcaps: list[Pcap] = [ + self.pcaps = [ Pcap(self.PCAP_PATH_PREFIX / p[0], p[1]) for p in self.profile_pcaps ] self.mode = mode self.vlan_id = target_vlan self.request = request - self.multiplier: float | None = None + self.multiplier = None # warn once per profile instead of on every run()/multiplier iteration if ( @@ -127,15 +126,18 @@ def __init__( ) trex_gen = request.config.getoption("--trex-generator") + assert trex_gen is not None trex_host = trex_gen[0].split(",") trex_hostname = trex_host[0] trex_pcie = trex_host[1] match self.mode: case TrexMode.STL: - self.stl_generator: TRexStateless = manager.request_stateless(request) + self.stl_generator = cast( + TRexStateless, manager.request_stateless(request) + ) self.trex_version = ( - self.stl_generator.get_handler().get_server_version()["version"] + self.stl_generator.get_handler().get_server_version()["version"] # pyright: ignore[reportOptionalMemberAccess] ) self.stl_generator.set_dst_mac(target_mac) @@ -172,17 +174,21 @@ def __init__( pcap.path, trex_hostname, pcap_remote_path, - force=self.request.config.getoption("--force-pcap-upload"), + force=cast( + bool, self.request.config.getoption("--force-pcap-upload") + ), ) case TrexMode.ASTF: - self.client: TRexAdvancedStateful = manager.request_stateful( - request, role="client" + self.client = cast( + TRexAdvancedStateful, + manager.request_stateful(request, role="client"), ) - self.server: TRexAdvancedStateful = manager.request_stateful( - request, role="server" + self.server = cast( + TRexAdvancedStateful, + manager.request_stateful(request, role="server"), ) - self.trex_version = self.server.get_handler().get_server_version()[ + self.trex_version = self.server.get_handler().get_server_version()[ # pyright: ignore[reportOptionalMemberAccess] "version" ] @@ -222,7 +228,9 @@ def __init__( config_path = Path(config.build()) config_remote_path = self.get_remote_data_path(config_path) self.remote_stf_config = config_remote_path - force_upload = self.request.config.getoption("--force-pcap-upload") + force_upload = cast( + bool, self.request.config.getoption("--force-pcap-upload") + ) send_to_remote( config_path, trex_hostname, config_remote_path, force=force_upload ) @@ -386,8 +394,8 @@ def prepare(self) -> None: ) profile = self.get_astf_profile(self.multiplier) - client_handler: ASTFClient = self.client.get_handler() - server_handler: ASTFClient = self.server.get_handler() + client_handler = cast(ASTFClient, self.client.get_handler()) + server_handler = cast(ASTFClient, self.server.get_handler()) client_handler.load_profile(profile) server_handler.load_profile(profile) @@ -439,7 +447,7 @@ def _mark_measurement_start() -> None: match self.mode: case TrexMode.STL: - client: STLClient = self.stl_generator.get_handler() + client = cast(STLClient, self.stl_generator.get_handler()) burst = self.request.config.getoption("--trex-stl-burst") if burst is not None: @@ -640,7 +648,7 @@ def get_tx_pps(self) -> float: ]["data"] return float(data.get("m_tx_pps", 0.0)) - def get_stats(self, role: Literal["server"] | Literal["client"] = "server") -> Dict: + def get_stats(self, role: Literal["server", "client"] = "server") -> dict[str, Any]: assert role in ("server", "client") match self.mode: @@ -667,7 +675,7 @@ class BaseAdHocTrex(BaseTrexClientManager, pcaps=[]): def __init__( self, - pcaps: PcapList, + pcaps: list[Pcap], manager: TRexManager, request: FixtureRequest, target_mac: str, diff --git a/conftest.py b/conftest.py index 12816a4..3e28b2b 100644 --- a/conftest.py +++ b/conftest.py @@ -24,7 +24,6 @@ from dataclasses import dataclass from lbr_testsuite.executable import executable, remote_executor from lbr_trex_client.interactive import trex -from typing import Tuple from pathlib import Path from itertools import product from param import filter @@ -36,7 +35,7 @@ logger = get_logger(__name__) # Defaults for --trex-stl-burst when it is given without arguments: (PPS, PACKET_COUNT). -STL_BURST_DEFAULTS: Tuple[float, int] = (200_000, 10_000_000) +STL_BURST_DEFAULTS: tuple[float, int] = (200_000, 10_000_000) # alias lbr_trex_client.interactive.trex to trex for importing native TRex profiles sys.modules["trex"] = trex @@ -350,7 +349,7 @@ def get_trex_executor(request): return remote_executor.RemoteExecutor(host=trex_name, user=user) -def get_host_internal(request) -> Tuple[str, str]: +def get_host_internal(request) -> str: return request.config.getoption("--remote-host") @@ -490,7 +489,7 @@ def suri_interface_bind(request): elif af_packet_match is not None: return (request.node.callspec.params["params"][parameter_path], "af-packet") - assert dpdk_match is not None or af_packet_match is not None + raise ValueError("No interfaces to bind") @pytest.fixture(autouse=True) @@ -708,6 +707,7 @@ def import_module(param_file): spec = importlib.util.spec_from_file_location( module_name_of_param_file, module_path ) + assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module @@ -834,6 +834,7 @@ def get_capture_modes(param_file): module = import_module(param_file) if hasattr(module, "capture_modes"): return module.capture_modes + return [] def make_combinations_for_af_packet(queues, rx_descriptors): @@ -886,17 +887,18 @@ def setup_af_packet(request): def af_packet_get_queues_rx_descriptors(param_file, params): + parameters = None + key = None for parameter_path in params[-1].keys(): af_packet_match = re.match(r"af-packet\[[0-9]+\].interface", parameter_path) if af_packet_match is not None: - key = af_packet_match.group(0) parameters = params[-1] - else: - return + key = af_packet_match.group(0) + break - queues_not_empty = False - rx_descriptors_not_empty = False + if parameters is None or key is None: + return file_is_accessible(param_file) module = import_module(param_file) @@ -912,9 +914,8 @@ def af_packet_get_queues_rx_descriptors(param_file, params): .replace("[", "") .replace("]", "") ) - if query_result: # empty str - queues = [int(i) for i in query_result.split(",")] - queues_not_empty = True + assert query_result, "queues cannot be empty because of settings" + queues = [int(i) for i in query_result.split(",")] query_result = ( str( @@ -927,13 +928,8 @@ def af_packet_get_queues_rx_descriptors(param_file, params): .replace("[", "") .replace("]", "") ) - if query_result: # empty str - rx_descriptors = [int(i) for i in query_result.split(",")] - rx_descriptors_not_empty = True - - assert ( - queues_not_empty and rx_descriptors_not_empty - ) # cannot be empty because of settings + assert query_result, "rx_descriptors cannot be empty because of settings" + rx_descriptors = [int(i) for i in query_result.split(",")] combinations = make_combinations_for_af_packet(queues, rx_descriptors) params.pop() diff --git a/util/config_builder.py b/util/config_builder.py index 410c381..9a35924 100644 --- a/util/config_builder.py +++ b/util/config_builder.py @@ -8,7 +8,7 @@ import logging from pathlib import Path from types import SimpleNamespace -from typing import Any, Dict, Self +from typing import Any, Self from ruamel.yaml import YAML from yamlpath import Processor @@ -27,7 +27,9 @@ ) -def update_recursively(destination: Dict, source: Dict, extend_lists=True) -> Dict: +def update_recursively( + destination: dict[str, Any], source: dict[str, Any], extend_lists=True +) -> dict[str, Any]: for k, v in source.items(): if isinstance(v, dict): existing = destination.get(k) @@ -136,7 +138,7 @@ def delete_option(self, key: str) -> Self: return self - def with_params(self, params: Dict) -> Self: + def with_params(self, params: dict[str, Any]) -> Self: for k, v in params.items(): if k == "queues" or k == "rx_descriptors": continue diff --git a/util/make-graphs.py b/util/make-graphs.py index 89c6dd6..33dad7a 100755 --- a/util/make-graphs.py +++ b/util/make-graphs.py @@ -135,7 +135,7 @@ def main(*args): if agg_dict.get("event", "") == "test_results": parameters = { key.split(".")[-1]: value - for (key, value) in agg_dict.get("parameters").items() + for (key, value) in agg_dict.get("parameters", {}).items() } process_results_line(x_axis, y_axis, agg_dict) if agg_dict.get("event", "") == "test_info": diff --git a/util/suri_util.py b/util/suri_util.py index 0c2a7ca..8176006 100644 --- a/util/suri_util.py +++ b/util/suri_util.py @@ -387,7 +387,9 @@ def make_graph( plt.savefig(path_to_graph) -def get_trex_suri_stats(path: str = None, stats_to_get: List[str] = None): +def get_trex_suri_stats( + result_path: str | None = None, stats_to_get: List[str] | None = None +): """ Gets stats from the latest result (or specified path) in the results/artefacts directory. @@ -400,7 +402,7 @@ def get_trex_suri_stats(path: str = None, stats_to_get: List[str] = None): can trace where the data originated. Inputs: - path -> Optional path to a specific result folder (e.g. + result_path -> Optional path to a specific result folder (e.g. "results/artefacts/2026-07-03-12:00/test_https_simple"). If None, the `results/artefacts/latest` symlink is resolved. stats_to_get -> List of stat names to extract (e.g., ["suricata_rx_packets", @@ -408,16 +410,15 @@ def get_trex_suri_stats(path: str = None, stats_to_get: List[str] = None): Output: Dictionary with requested stats and their values, plus a "_source_path" key. """ - if path is None: + if result_path is None: latest_symlink = ( Path(__file__).resolve().parent.parent / "results" / "artefacts" / "latest" ) if not latest_symlink.exists(): raise GetStatsError(f"Latest symlink does not exist: {latest_symlink}") - path = str(latest_symlink.resolve()) + result_path = str(latest_symlink.resolve()) - path = Path(path) - path = path / "aggregated.json" + path = Path(result_path) / "aggregated.json" if not path.exists(): raise GetStatsError(f"No aggregated.json found in: {path}") diff --git a/util/trex_util.py b/util/trex_util.py index e8aee65..c54d907 100644 --- a/util/trex_util.py +++ b/util/trex_util.py @@ -11,7 +11,6 @@ import subprocess from enum import Enum from pathlib import Path -from typing import Sequence, Tuple from scapy.all import PcapWriter, PcapReader import pytest @@ -26,9 +25,6 @@ class TrexMode(Enum): STF = 2 -PcapList = Sequence[Tuple[str, int | float]] - - def _packet_generator( pcap_paths: list[Path], per_round: list[int], From 03684a7b763057c43c2b1bb1ed20eb6b8e9fca64 Mon Sep 17 00:00:00 2001 From: Matyas Sedmidubsky Date: Fri, 4 Sep 2026 13:27:37 +0200 Subject: [PATCH 5/8] lint: ruff formats .md files, too --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 28ce581..6933ce6 100644 --- a/README.md +++ b/README.md @@ -360,7 +360,7 @@ suri_cmd_params = {"capture-mode": ["dpdk"]} filter = { "dpdk": [lambda x: x["dpdk.interfaces[0].mtu"] <= 3000], - "af-packet": [lambda x: True] + "af-packet": [lambda x: True], } ``` @@ -396,7 +396,7 @@ filter = { lambda x: x["dpdk.interfaces[0].mtu"] <= 3000, lambda x: x["dpdk.interfaces[0].rx-descriptors"] >= 4096, ], - "af-packet": [lambda x: True] + "af-packet": [lambda x: True], } ``` From 5c93b2d3c95151bf982a76653dd1a9313575724d Mon Sep 17 00:00:00 2001 From: Matyas Sedmidubsky Date: Fri, 4 Sep 2026 13:29:04 +0200 Subject: [PATCH 6/8] feat: more info in logs --- .../traffic_profiles/trex_client_manager.py | 13 +++++-- conftest.py | 37 +++++++++++++++++-- .../test_http_https_smb_simple.py | 1 - .../http_simple/test_http_simple.py | 1 - .../https_simple/test_https_simple.py | 1 - .../nfs_smb_simple/test_nfs_smb_simple.py | 1 - .../pcap_replay/test_pcap_replay.py | 1 - .../web_50_sites/test_web_50_sites.py | 1 - util/suri_util.py | 24 ++++++++---- util/test_runner.py | 16 +++++++- util/trex_util.py | 2 + 11 files changed, 78 insertions(+), 20 deletions(-) diff --git a/assets/trex/traffic_profiles/trex_client_manager.py b/assets/trex/traffic_profiles/trex_client_manager.py index e37e776..cd3804f 100644 --- a/assets/trex/traffic_profiles/trex_client_manager.py +++ b/assets/trex/traffic_profiles/trex_client_manager.py @@ -27,6 +27,7 @@ # against in `isinstance()` (e.g. STLClient.add_streams). Importing from # `lbr_trex_client.interactive.trex.*` instead would create distinct class # objects and break those checks. +from conftest import fmt_bytes, fmt_thousands from trex.astf import trex_astf_profile from trex.astf.trex_astf_client import ASTFClient from trex.common.trex_exceptions import TRexError @@ -122,7 +123,7 @@ def __init__( "Initializing TRex client manager: mode=%s vlan_id=%d pcaps=%s", self.mode.name, self.vlan_id, - [p.path for p in self.pcaps], + [str(p.path.relative_to(self.PCAP_PATH_PREFIX)) for p in self.pcaps], ) trex_gen = request.config.getoption("--trex-generator") @@ -551,10 +552,11 @@ def wait_on_traffic(self) -> None: match self.mode: case TrexMode.STL: self.stl_generator.wait_on_traffic() + self.stop() case TrexMode.ASTF: self.client.wait_on_traffic() - self.server.stop() + self.stop() case TrexMode.STF: assert self.duration is not None @@ -566,7 +568,12 @@ def wait_on_traffic(self) -> None: self.stop() def stop(self) -> None: - logger.info("Stopping TRex traffic: mode=%s", self.mode.name) + logger.info( + "Stopping TRex traffic (%s, %s pkts, %s)", + self.mode.name, + fmt_thousands(self.get_tx_packets()), + fmt_bytes(self.get_tx_bytes()), + ) match self.mode: case TrexMode.STL: self.stl_generator.stop() diff --git a/conftest.py b/conftest.py index 3e28b2b..44c3f75 100644 --- a/conftest.py +++ b/conftest.py @@ -10,6 +10,7 @@ import argparse import logging +from math import log2 import sys import pytest import os.path @@ -88,11 +89,41 @@ def _log_level_type(value: str) -> str | int: ) -def _fmt_thousands(value: int) -> str: +def fmt_thousands(value: int) -> str: """Format an integer with space thousands separators (e.g. 200000 -> '200 000').""" return f"{value:,}".replace(",", " ") +def fmt_bytes(value: int) -> str: + """Format an integer as an SI prefixed amount of bytes. + + If the value is an integer multiple of the -ibby (base 2) + prefixes, then those are used. For example 6GiB. + + Otherwise normal (base 10) prefixes are used. + For example 42.67KB. + """ + if value == 0: + return "0B" + + sign = "-" if value < 0 else "" + value = abs(value) + + binary_prefixes = ["KiB", "MiB", "GiB", "TiB", "PiB", "EiB"] + for i in range(len(binary_prefixes), 0, -1): + divisor = 1024**i + if value % divisor == 0: + return f"{sign}{value // divisor}{binary_prefixes[i - 1]}" + + decimal_prefixes = ["B", "KB", "MB", "GB", "TB", "PB", "EB"] + index = min(int(log2(value) // log2(1000)), len(decimal_prefixes) - 1) + if index == 0: + return f"{sign}{value}B" + + scaled = value / 1000**index + return f"{sign}{scaled:.2f}{decimal_prefixes[index]}" + + def _validate_stl_burst_option(config) -> None: """Validate ``--trex-stl-burst`` and store the typed ``(float, int)`` tuple. @@ -291,8 +322,8 @@ def pytest_addoption(parser): help=( "In STL mode, send a fixed burst of PACKET_COUNT packets at PPS " "instead of replaying for the configured duration. With no " - f"arguments, defaults to {_fmt_thousands(int(STL_BURST_DEFAULTS[0]))} " - f"PPS and {_fmt_thousands(STL_BURST_DEFAULTS[1])} packets. Only " + f"arguments, defaults to {fmt_thousands(int(STL_BURST_DEFAULTS[0]))} " + f"PPS and {fmt_thousands(STL_BURST_DEFAULTS[1])} packets. Only " "applies to STL mode; ignored (with a warning) for other modes." ), ) diff --git a/performance_tests/http_https_smb_simple/test_http_https_smb_simple.py b/performance_tests/http_https_smb_simple/test_http_https_smb_simple.py index 8a3673e..ba6ec43 100644 --- a/performance_tests/http_https_smb_simple/test_http_https_smb_simple.py +++ b/performance_tests/http_https_smb_simple/test_http_https_smb_simple.py @@ -96,7 +96,6 @@ def test_http_https_smb( ) tester.execute(multiplier) mult_iter.set_result(get_drop_rate()) - logger.info("Run ended.") if mult_iter.result is not None: logger.progress( diff --git a/performance_tests/http_simple/test_http_simple.py b/performance_tests/http_simple/test_http_simple.py index 5ca66aa..c5564fc 100644 --- a/performance_tests/http_simple/test_http_simple.py +++ b/performance_tests/http_simple/test_http_simple.py @@ -96,7 +96,6 @@ def test_http_simple( ) tester.execute(multiplier) mult_iter.set_result(get_drop_rate()) - logger.info("Run ended.") if mult_iter.result is not None: logger.progress( diff --git a/performance_tests/https_simple/test_https_simple.py b/performance_tests/https_simple/test_https_simple.py index a7d2c64..a11473e 100644 --- a/performance_tests/https_simple/test_https_simple.py +++ b/performance_tests/https_simple/test_https_simple.py @@ -96,7 +96,6 @@ def test_https_simple( ) tester.execute(multiplier) mult_iter.set_result(get_drop_rate()) - logger.info("Run ended.") if mult_iter.result is not None: logger.progress( diff --git a/performance_tests/nfs_smb_simple/test_nfs_smb_simple.py b/performance_tests/nfs_smb_simple/test_nfs_smb_simple.py index 457ce80..50f99d1 100644 --- a/performance_tests/nfs_smb_simple/test_nfs_smb_simple.py +++ b/performance_tests/nfs_smb_simple/test_nfs_smb_simple.py @@ -97,7 +97,6 @@ def test_nfs_smb( ) tester.execute(multiplier) mult_iter.set_result(get_drop_rate()) - logger.info("Run ended.") if mult_iter.result is not None: logger.progress( diff --git a/performance_tests/pcap_replay/test_pcap_replay.py b/performance_tests/pcap_replay/test_pcap_replay.py index 50cfaf4..c6134b6 100644 --- a/performance_tests/pcap_replay/test_pcap_replay.py +++ b/performance_tests/pcap_replay/test_pcap_replay.py @@ -100,7 +100,6 @@ def test_pcap_replay( ) tester.execute(multiplier) mult_iter.set_result(get_drop_rate()) - logger.info("Run ended.") if mult_iter.result is not None: logger.progress( diff --git a/performance_tests/web_50_sites/test_web_50_sites.py b/performance_tests/web_50_sites/test_web_50_sites.py index 5b1ddf2..f0f8f68 100644 --- a/performance_tests/web_50_sites/test_web_50_sites.py +++ b/performance_tests/web_50_sites/test_web_50_sites.py @@ -95,7 +95,6 @@ def test_web_50_sites( ) tester.execute(multiplier) mult_iter.set_result(get_drop_rate()) - logger.info("Run ended.") if mult_iter.result is not None: logger.progress( diff --git a/util/suri_util.py b/util/suri_util.py index 8176006..d997f77 100644 --- a/util/suri_util.py +++ b/util/suri_util.py @@ -17,7 +17,7 @@ import matplotlib.pyplot as plt from file_read_backwards import FileReadBackwards -from typing import List +from typing import Any, List from pathlib import Path from shutil import copy as copy_content @@ -92,7 +92,9 @@ def get_rx_packets_from_file(file: str, skip=0) -> int: pkts = jq.compile(".stats.decoder.pkts").input(json_loaded).first() try: - return int(pkts) - get_rx_packets_until(file, skip) + skipped = get_rx_packets_until(file, skip) + logger.debug("Ignored %d packets", skipped) + return int(pkts) - skipped except ValueError: return 0 @@ -102,7 +104,9 @@ def get_rx_bytes_from_file(file: str, skip=0) -> int: bytes = jq.compile(".stats.decoder.bytes").input(json_loaded).first() try: - return int(bytes) - get_rx_bytes_until(file, skip) + skipped = get_rx_bytes_until(file, skip) + logger.debug("Ignored %d bytes", skipped) + return int(bytes) - skipped except ValueError: return 0 @@ -160,7 +164,9 @@ def get_flow_filtered_packets_from_file(file: str, skip=0) -> int: ) try: - return int(flow_filtered) - get_flow_filtered_packets_until(file, skip) + skipped = get_flow_filtered_packets_until(file, skip) + logger.debug("Ignored %d flow filtered packets", skipped) + return int(flow_filtered) - skipped except (ValueError, TypeError): return 0 @@ -215,7 +221,9 @@ def convert_multiplier_to_str(multiplier: float) -> str: ) -def save_stats(params, request, test_info: TestInfo, run_info: RunInfo): +def save_stats( + params, request, test_info: TestInfo, run_info: RunInfo +) -> dict[str, Any]: multiplier_str: str = convert_multiplier_to_str(run_info.multiplier) output_dir: str = os.path.join(test_info.result_path, multiplier_str) aggregated_output_path = os.path.join(test_info.result_path, "aggregated.json") @@ -233,7 +241,7 @@ def save_stats(params, request, test_info: TestInfo, run_info: RunInfo): save_suricata_stats(request, output_dir) save_trex_stats(run_info, output_dir) - save_aggregated_stats( + return save_aggregated_stats( test_info, run_info, output_dir, aggregated_output_path, params ) @@ -291,7 +299,7 @@ def save_aggregated_stats( suri_stats_path: str, aggregated_output_path: str, params, -): +) -> dict[str, Any]: logger.debug("Saving aggregated stats to %s", aggregated_output_path) out_params = params.copy() @@ -329,6 +337,8 @@ def save_aggregated_stats( json.dump(output, output_file) output_file.write("\n") + return output + def save_test_info(request, test_info: TestInfo, aggregated_output_path: str) -> None: logger.debug("Saving test info to %s", aggregated_output_path) diff --git a/util/test_runner.py b/util/test_runner.py index 076d540..e0a2f6b 100644 --- a/util/test_runner.py +++ b/util/test_runner.py @@ -9,11 +9,17 @@ Provide a common interface for running Suricata tests, including setup, traffic generation, and stats collection. """ +from time import time + import pytest +import logging +from conftest import fmt_bytes, fmt_thousands from util.suricata_manager import Suricata_manager, SuriDown from util.suri_util import RunInfo, save_stats, TestInfo +logger = logging.getLogger(__name__) + class TestRun: def __init__( @@ -49,6 +55,7 @@ def execute(self, multiplier: float, duration: int | None = None): except SuriDown: pytest.fail("Suricata is down.") + start_time = time() run_info = RunInfo(multiplier=multiplier) try: self._run_traffic(multiplier, duration, run_info) @@ -60,7 +67,14 @@ def execute(self, multiplier: float, duration: int | None = None): self._collect_stats(run_info) run_info.suricata_start_delay = self.suri_daemon.last_start_delay - save_stats(self.params, self.request, self.test_info, run_info) + stats = save_stats(self.params, self.request, self.test_info, run_info) + + logger.info( + "Run ended (%ds, %s pkts, %s)", + int(time() - start_time), + fmt_thousands(stats.get("suricata_rx_packets", 0)), + fmt_bytes(stats.get("suricata_rx_bytes", 0)), + ) class TrexTestRun(TestRun): diff --git a/util/trex_util.py b/util/trex_util.py index c54d907..3a50159 100644 --- a/util/trex_util.py +++ b/util/trex_util.py @@ -145,6 +145,8 @@ def merge_pcaps( if total_w <= 0: raise ValueError("sum of weights must be positive") + logger.info("Merging %d pcaps. This might take a while.", len(pcap_paths)) + # weighted round-robin: per-round packet count proportional to weight share quotas = [w / total_w for w in weights] min_q = min(q for q in quotas if q > 0) From 6048ae76714e673eccf5c2a25da6af1fe9b3b58e Mon Sep 17 00:00:00 2001 From: Matyas Sedmidubsky Date: Fri, 4 Sep 2026 13:39:42 +0200 Subject: [PATCH 7/8] fix: safe stat getters in trex_client_manager --- .../traffic_profiles/trex_client_manager.py | 63 ++++++++++++------- 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/assets/trex/traffic_profiles/trex_client_manager.py b/assets/trex/traffic_profiles/trex_client_manager.py index cd3804f..0fae651 100644 --- a/assets/trex/traffic_profiles/trex_client_manager.py +++ b/assets/trex/traffic_profiles/trex_client_manager.py @@ -608,31 +608,41 @@ def get_tx_packets(self) -> int: """Current cumulative TRex transmit packet count.""" match self.mode: case TrexMode.STL: - return int(self.stl_generator.get_stats()["total"]["opackets"]) - case TrexMode.ASTF: - return int(self.server.get_stats()["total"]["opackets"]) + int( - self.client.get_stats()["total"]["opackets"] + return int( + self.stl_generator.get_stats().get("total", {}).get("opackets", 0) ) + case TrexMode.ASTF: + return int( + self.server.get_stats().get("total", {}).get("opackets", 0) + ) + int(self.client.get_stats().get("total", {}).get("opackets", 0)) case TrexMode.STF: - data = self.stf_generator.get_result_obj().get_latest_dump()[ - "trex-global" - ]["data"] - return int(data["m_total_tx_pkts"]) + return int( + self.stf_generator.get_result_obj() + .get_latest_dump() + .get("trex-global", {}) + .get("data", {}) + .get("m_total_tx_pkts", 0) + ) def get_tx_bytes(self) -> int: """Current cumulative TRex transmit byte count.""" match self.mode: case TrexMode.STL: - return int(self.stl_generator.get_stats()["total"]["obytes"]) - case TrexMode.ASTF: - return int(self.server.get_stats()["total"]["obytes"]) + int( - self.client.get_stats()["total"]["obytes"] + return int( + self.stl_generator.get_stats().get("total", {}).get("obytes", 0) ) + case TrexMode.ASTF: + return int( + self.server.get_stats().get("total", {}).get("obytes", 0) + ) + int(self.client.get_stats().get("total", {}).get("obytes", 0)) case TrexMode.STF: - data = self.stf_generator.get_result_obj().get_latest_dump()[ - "trex-global" - ]["data"] - return int(data["m_total_tx_bytes"]) + return int( + self.stf_generator.get_result_obj() + .get_latest_dump() + .get("trex-global", {}) + .get("data", {}) + .get("m_total_tx_bytes", 0) + ) def get_tx_pps(self) -> float: """Current instantaneous TRex transmit rate (packets per second). @@ -644,16 +654,21 @@ def get_tx_pps(self) -> float: """ match self.mode: case TrexMode.STL: - return float(self.stl_generator.get_stats()["total"]["tx_pps"]) - case TrexMode.ASTF: - return float(self.server.get_stats()["total"]["tx_pps"]) + float( - self.client.get_stats()["total"]["tx_pps"] + return float( + self.stl_generator.get_stats().get("total", {}).get("tx_pps", 0.0) ) + case TrexMode.ASTF: + return float( + self.server.get_stats().get("total", {}).get("tx_pps", 0.0) + ) + float(self.client.get_stats().get("total", {}).get("tx_pps", 0.0)) case TrexMode.STF: - data = self.stf_generator.get_result_obj().get_latest_dump()[ - "trex-global" - ]["data"] - return float(data.get("m_tx_pps", 0.0)) + return float( + self.stf_generator.get_result_obj() + .get_latest_dump() + .get("trex-global", {}) + .get("data", {}) + .get("m_tx_pps", 0.0) + ) def get_stats(self, role: Literal["server", "client"] = "server") -> dict[str, Any]: assert role in ("server", "client") From 904d71ff639a504ac86b2952070b5310fcc312a1 Mon Sep 17 00:00:00 2001 From: Matyas Sedmidubsky Date: Fri, 4 Sep 2026 14:07:13 +0200 Subject: [PATCH 8/8] fix: measure suricata delay accurately --- util/suricata_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/suricata_manager.py b/util/suricata_manager.py index 58a255d..8f29912 100644 --- a/util/suricata_manager.py +++ b/util/suricata_manager.py @@ -215,10 +215,8 @@ def is_alive(self): def wait_on_start(self) -> None: """Wait until Suricata is started, then continue""" can_continue = False - self.last_start_delay = 0 + start_time = time.time() while not can_continue: - time.sleep(1) - self.last_start_delay += 1 process_wait_on_start = executable.Tool( "suricatasc -c uptime", sudo=True, @@ -234,7 +232,9 @@ def wait_on_start(self) -> None: can_continue = False logger.debug("Suricata is not started yet") self.is_alive() + time.sleep(1) + self.last_start_delay = int(time.time() - start_time) logger.info("Suricata started after %d seconds", self.last_start_delay) def _wait_for_clean_start(self) -> None: