From 1f3ac917bb58b34cc1abcc5329fc3982ae3b8093 Mon Sep 17 00:00:00 2001 From: Samuel Williams Date: Mon, 9 Nov 2026 20:19:05 +1300 Subject: [PATCH] Run the hello benchmark's closed-loop client with two workers on two cores, as a client layout screen on hana found, and document each measuring host's hardware and CPU placement in benchmarks/hosts. --- benchmarks/hello/Cargo.toml | 10 +- benchmarks/hello/readme.md | 26 +- .../results/client-layout-20261109/layout.py | 282 ++++++++++++++++++ .../results/client-layout-20261109/readme.md | 98 ++++++ .../client-layout-20261109/samples.csv | 169 +++++++++++ benchmarks/hosts/hana.md | 75 +++++ benchmarks/hosts/readme.md | 13 + 7 files changed, 655 insertions(+), 18 deletions(-) create mode 100644 benchmarks/hello/results/client-layout-20261109/layout.py create mode 100644 benchmarks/hello/results/client-layout-20261109/readme.md create mode 100644 benchmarks/hello/results/client-layout-20261109/samples.csv create mode 100644 benchmarks/hosts/hana.md create mode 100644 benchmarks/hosts/readme.md diff --git a/benchmarks/hello/Cargo.toml b/benchmarks/hello/Cargo.toml index 931965d..15a8d55 100644 --- a/benchmarks/hello/Cargo.toml +++ b/benchmarks/hello/Cargo.toml @@ -41,7 +41,7 @@ base-scheduler-iouring = "=0.5.0" [profile.release] debug = 1 -# Scenarios for `base-benchmark` 0.4 and `cargo bake benchmark:* --manifest-path benchmarks/hello/Cargo.toml`. Each is a multi-process scenario: the server is the subject, a service whose ready line is the address it bound, and its backends are the variants; the client is base-benchmark-load, an external program installed at a pinned version and given with `--program load=PATH`, the same for every subject. CPU placement is given per run with `--role-cpus`, so no CPU numbers appear here; see readme.md. +# Scenarios for `base-benchmark` 0.4 and `cargo bake benchmark:* --manifest-path benchmarks/hello/Cargo.toml`. Each is a multi-process scenario: the server is the subject, a service whose ready line is the address it bound, and its backends are the variants; the client is base-benchmark-load, an external program installed at a pinned version and given with `--program load=PATH`, the same for every subject. CPU placement is given per run with `--role-cpus`, so no CPU numbers appear here; each measuring host's document in ../hosts gives its placement. # Every client validates every response. A smoke run warns about invalid responses rather than failing on them, so the smoke arguments add `--fail-on invalid`, failing the sample instead: a correct server, raw controls included at one request outstanding per connection, never sends one. Measured runs mark the window's start 500 ms before it (`--window-lead`), rather than the client's default 100 ms, since an acknowledgement after the window's start fails the run: it tolerates five times the default's delay, though not the roughly 650 ms of one load spike, whose acknowledgement came 551 ms after the window began and which only an idle host avoids. The services' window then also covers up to 500 ms of the warmup's load, about 6% of the 8-second window, at the window's load for every subject. See readme.md. [package.metadata.base-benchmark] binary = "base-http-hello-benchmark" @@ -55,7 +55,7 @@ variants = [ { name = "hyper-task-tokio", arguments = ["hyper-task-tokio"] }, ] -# Closed loop: 64 persistent connections, one request outstanding on each, as the default wrk comparison in run.py. +# Closed loop: 64 persistent connections, one request outstanding on each, as the default wrk comparison in run.py. The client runs two workers, which need two physical cores: on hana, one worker was busy for 96–100% of the window in both closed-loop scenarios, and limited raw-closed to about 94% of what two workers on two cores drove, with neither of them saturated (results/client-layout-20261109). [[package.metadata.base-benchmark.scenario]] name = "http-closed" @@ -70,7 +70,7 @@ name = "client" kind = "driver" external = "load" invalid-responses = "responses_invalid" -arguments = ["closed", "--address", "{server.address}", "--connections", "64", "--workers", "1", "--body", "Hello, world!", "--expect-header", "Content-Type: text/plain", "--expect-header", "Content-Length: 13"] +arguments = ["closed", "--address", "{server.address}", "--connections", "64", "--workers", "2", "--body", "Hello, world!", "--expect-header", "Content-Type: text/plain", "--expect-header", "Content-Length: 13"] smoke-arguments = ["--warmup", "100ms", "--duration", "200ms", "--fail-on", "invalid"] measure-arguments = ["--warmup", "2s", "--duration", "8s", "--window-lead", "500ms"] @@ -93,7 +93,7 @@ arguments = ["rate", "--address", "{server.address}", "--rate", "100k", "--conne smoke-arguments = ["--warmup", "100ms", "--duration", "200ms", "--fail-on", "invalid"] measure-arguments = ["--warmup", "2s", "--duration", "8s", "--window-lead", "500ms"] -# The raw controls read once and write one fixed response, ignoring request framing, so they serve an HTTP client correctly only with one request outstanding per connection, the client's default depth. Their response is byte-for-byte the HTTP modes', so it is checked the same way. +# The raw controls read once and write one fixed response, ignoring request framing, so they serve an HTTP client correctly only with one request outstanding per connection, the client's default depth. Their response is byte-for-byte the HTTP modes', so it is checked the same way. The client runs two workers, as in http-closed. [[package.metadata.base-benchmark.scenario]] name = "raw-closed" variants = [ @@ -116,7 +116,7 @@ name = "client" kind = "driver" external = "load" invalid-responses = "responses_invalid" -arguments = ["closed", "--address", "{server.address}", "--connections", "64", "--workers", "1", "--depth", "1", "--body", "Hello, world!", "--expect-header", "Content-Type: text/plain", "--expect-header", "Content-Length: 13"] +arguments = ["closed", "--address", "{server.address}", "--connections", "64", "--workers", "2", "--depth", "1", "--body", "Hello, world!", "--expect-header", "Content-Type: text/plain", "--expect-header", "Content-Length: 13"] smoke-arguments = ["--warmup", "100ms", "--duration", "200ms", "--fail-on", "invalid"] measure-arguments = ["--warmup", "2s", "--duration", "8s", "--window-lead", "500ms"] diff --git a/benchmarks/hello/readme.md b/benchmarks/hello/readme.md index c39d3e6..e510024 100644 --- a/benchmarks/hello/readme.md +++ b/benchmarks/hello/readme.md @@ -31,7 +31,7 @@ The Hyper service follows the [official HTTP/1 hello-world guide](https://hyper. | `raw-closed` | As `http-closed`. | The raw controls: `base-native-raw` (the baseline), `base-iouring-raw`, `base-iouring-owned-raw`, `base-tokio-raw`, `tokio-raw`, `tokio-task-raw`. | | `raw-rate` | As `http-rate`. | As `raw-closed`. | -The client runs one worker, validates every response's status, body and fields, and measures a 2-second warmup and an 8-second window, or 100 ms and 200 ms in smoke runs. A smoke run only warns about invalid responses, so the smoke arguments add `--fail-on invalid`, which fails the sample on any: a correct server never sends one. Measured runs mark the window's start 500 ms before it (`--window-lead 500ms`), as [a late window start](#a-late-window-start) explains. The raw controls ignore request framing, so they are driven with one request outstanding per connection, the only load they serve correctly. In [base-benchmark-load's evaluation on hana](https://github.com/base-rs/base-benchmark-load/blob/main/results/2026-11-09-hana/readme.md), its one worker was busy for about 65% of the window at the fixed rate, and base-native and hyper-tokio served 185,000–205,000 requests per second in the closed loop, so the rate leaves headroom on both sides. +The client runs two workers in the closed loop and one at the fixed rate, validates every response's status, body and fields, and measures a 2-second warmup and an 8-second window, or 100 ms and 200 ms in smoke runs. A smoke run only warns about invalid responses, so the smoke arguments add `--fail-on invalid`, which fails the sample on any: a correct server never sends one. Measured runs mark the window's start 500 ms before it (`--window-lead 500ms`), as [a late window start](#a-late-window-start) explains. The raw controls ignore request framing, so they are driven with one request outstanding per connection, the only load they serve correctly. In [base-benchmark-load's evaluation on hana](https://github.com/base-rs/base-benchmark-load/blob/main/results/2026-11-09-hana/readme.md), its one worker was busy for about 65% of the window at the fixed rate, and base-native and hyper-tokio served 185,000–205,000 requests per second in the closed loop, so the rate leaves headroom on both sides. The package stays outside the workspace, so every command names its manifest with `--manifest-path`. Install the client at its pinned version anywhere; its path is given to every run, rather than in package metadata: @@ -56,26 +56,26 @@ cargo bake benchmark:prepare --manifest-path benchmarks/hello/Cargo.toml \ cargo bake benchmark:run http-closed --manifest-path benchmarks/hello/Cargo.toml \ --artifacts benchmarks/hello/target/base-benchmark/prepared-TIME/artifacts.json \ --program load=$HOME/.cache/base-benchmark-load/bin/base-benchmark-load \ - --role-cpus client=1,17 --role-cpus server=2,3 + --role-cpus client=1,2 --role-cpus server=3 ``` -`--role-cpus ROLE=CPUS` confines each role, and the client's worker and controller run within its set. Give the client one physical core and the server two or three others, with no role on another's SMT sibling: base-benchmark warns about shared cores in screening and refuses them in recorded campaigns. The CPUs above suit a host whose SMT siblings are CPUs `N` and `N + 16`; check `/sys/devices/system/cpu/cpuN/topology/thread_siblings_list`. The `base-benchmark` command (`cargo install --locked base-benchmark --version 0.4.0`) takes the same options, as `base-benchmark smoke`, `prepare` and `run`. +`--role-cpus ROLE=CPUS` confines each role, and the client's workers and controller run within its set. Give each role physical cores of its own, with no role on another's SMT sibling: base-benchmark warns about shared cores in screening and refuses them in recorded campaigns. The closed-loop client's two workers need two cores, not the two threads of one: [on hana](results/client-layout-20261109/readme.md), two workers on one core's SMT siblings were both saturated just to match what two workers on two cores drove at two-thirds busy. The hello backends are single-threaded, so one server core suffices. The CPUs above suit a host where CPUs 1, 2 and 3 are on different cores; check `/sys/devices/system/cpu/cpuN/topology/thread_siblings_list`. The `base-benchmark` command (`cargo install --locked base-benchmark --version 0.4.0`) takes the same options, as `base-benchmark smoke`, `prepare` and `run`. -### On hana +### On a measuring host -Measure on hana, a dedicated measuring host with 4 cores and 8 threads whose SMT siblings are CPUs `N` and `N + 4`, with `--host`. Install the client there once; a relative `--program` path is relative to the remote home directory: +Each measuring host has a document in [benchmarks/hosts](../hosts/readme.md) that gives its hardware and the `--role-cpus` to use on it for each scenario, such as [hana's](../hosts/hana.md). Measure there with `--host`. Install the client there once; a relative `--program` path is relative to the remote home directory: ```sh -ssh hana cargo install --locked --root .cache/base-benchmark-load base-benchmark-load --version 0.1.0 -cargo bake benchmark:prepare --host hana --manifest-path benchmarks/hello/Cargo.toml \ - --tree main=$PWD/../base-http-main/benchmarks/hello --tree candidate=$PWD/benchmarks/hello --scenario http-rate -cargo bake benchmark:run http-rate --host hana --manifest-path benchmarks/hello/Cargo.toml \ +ssh HOST cargo install --locked --root .cache/base-benchmark-load base-benchmark-load --version 0.1.0 +cargo bake benchmark:prepare --host HOST --manifest-path benchmarks/hello/Cargo.toml \ + --tree main=$PWD/../base-http-main/benchmarks/hello --tree candidate=$PWD/benchmarks/hello --scenario http-closed +cargo bake benchmark:run http-closed --host HOST --manifest-path benchmarks/hello/Cargo.toml \ --artifacts benchmarks/hello/target/base-benchmark/prepared-TIME/artifacts.json \ --program load=.cache/base-benchmark-load/bin/base-benchmark-load \ - --role-cpus client=1,5 --role-cpus server=2,3 + --role-cpus client=CPUS --role-cpus server=CPUS ``` -The client gets physical core 1, both its threads, and the server cores 2 and 3, whose siblings 6 and 7 stay idle. The trees, and the libraries they depend on outside `benchmarks/hello`, are mirrored on hana with rsync, the host runs the crates.io release of the same `base-benchmark`, and the campaign comes back to `benchmarks/hello/target/base-benchmark` locally. Run prepared artifacts on the host that built them. [Measuring on another host](https://github.com/base-rs/base-benchmark/blob/main/context/usage.md#measuring-on-another-host) lists what both machines need. A recorded campaign also needs at least 10 rounds (`--rounds 10`), `--mode recorded`, a `--decision-rule`, `--idle-host true`, and committed trees. +The trees, and the libraries they depend on outside `benchmarks/hello`, are mirrored on the host with rsync, the host runs the crates.io release of the same `base-benchmark`, and the campaign comes back to `benchmarks/hello/target/base-benchmark` locally. Run prepared artifacts on the host that built them. [Measuring on another host](https://github.com/base-rs/base-benchmark/blob/main/context/usage.md#measuring-on-another-host) lists what both machines need. A recorded campaign also needs at least 10 rounds (`--rounds 10`), `--mode recorded`, a `--decision-rule`, `--idle-host true`, and committed trees. ### A late window start @@ -83,13 +83,13 @@ The client's measured window and the server's CPU time are tied by a handshake: The measured arguments therefore use a 500 ms lead, which tolerates five times the default's delay. The cost is that the server's CPU time also covers up to 500 ms of the warmup's load, besides the 8-second window, which inflates CPU per request by up to about 6%. The warmup's load is the window's, so the inflation is alike for every subject, and comparisons between them stand. A lead at least as long as the 2-second warmup would mark the start before the load and could never be late, but would count all of the warmup. Smoke runs keep the default lead, as long as their warmup, so theirs is never late. -No lead covers every stall: the spike above delayed the acknowledgement by about 650 ms, the 100 ms lead and 551 ms more. Avoid it by measuring on an idle host. On hana, run nothing else there while measuring, prepare before a campaign rather than during one, and declare it with `--idle-host true`, as recorded campaigns must. On a shared development machine, screen when it is quiet, and run again after such a failure rather than lengthening the lead further. +No lead covers every stall: the spike above delayed the acknowledgement by about 650 ms, the 100 ms lead and 551 ms more. Avoid it by measuring on an idle host. On a measuring host, run nothing else there while measuring, prepare before a campaign rather than during one, and declare it with `--idle-host true`, as recorded campaigns must. On a shared development machine, screen when it is quiet, and run again after such a failure rather than lengthening the lead further. ### Reading the results Read the warnings first. Operations are the client's valid responses, so `wall_time` per operation is the inverse of the throughput and every per-operation value is per valid response; a screening comparison whose subject received any invalid response is reported as invalid, and a recorded campaign fails on one. `server/cpu_time_per_operation`, with its user and system parts, is the server's CPU per request, marked approximate: its CPU time over the services' window, which includes up to the [lead](#a-late-window-start) of the warmup's load and, in the fixed-rate scenarios, the drain after the window, divided by the valid responses in the client's window. It also leaves out softirq time spent on the server's network traffic, 16–24% of its CPU in base-benchmark-load's closed-loop evaluation on hana, so compare it between subjects measured alike rather than read it as the whole cost. -The closed loop can flag the client as saturated (`client_saturated`), since a faster server makes it busier. In `raw-closed`, the client is the limit: its one worker is busy for the whole window, and in an independent review two workers served about 4% more requests per second, so throughput there compares the raw controls only as far as one client worker can drive them. In `http-closed`, the servers are the limit: two workers served the same throughput. Prefer the fixed-rate scenarios for latency. +The closed loop can flag the client as saturated (`client_saturated`), since a faster server makes it busier. With one worker, the client was the limit in `raw-closed`, busy for the whole window and about 6% below two workers on hana, and close to it in `http-closed`, busy for 96–98% of it. With two workers on two cores, each was busy for about two-thirds of the window in `raw-closed` and about half in `http-closed`, so the servers are the limit; the [client layout screen](results/client-layout-20261109/readme.md) records them. If busy time grows in proportion to throughput, a raw control about 20% faster than today's would reach the 80% busy limit there, so check `client_saturated` all the same. Prefer the fixed-rate scenarios for latency. `hyper-tokio` and `hyper-task-tokio` are variants built from each tree, with its lockfile, so they are controls within every build rather than competitors run against every build. diff --git a/benchmarks/hello/results/client-layout-20261109/layout.py b/benchmarks/hello/results/client-layout-20261109/layout.py new file mode 100644 index 0000000..65d4507 --- /dev/null +++ b/benchmarks/hello/results/client-layout-20261109/layout.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Screen client CPU layouts for the hello benchmark's closed-loop scenarios. + +`layout.py campaign` runs every configuration once per round, in a seeded shuffled order, each sample as +`flock /tmp/base-benchmark-host.lock layout.py sample ...`, after checking with `pgrep -af` that no other benchmark is running. +`layout.py sample` starts the server under `taskset`, runs base-benchmark-load under `taskset`, acts as base-benchmark's runner for +the window markers (`BASE_BENCHMARK_CONTROL=stdio`), snapshotting the server's CPU time from /proc at the start and stop markers, +and prints one JSON line. `layout.py summarize` prints the Markdown table, and `layout.py table` the samples as CSV. +""" +import argparse, json, os, random, signal, statistics, subprocess, sys, threading, time + +LOCK = "/tmp/base-benchmark-host.lock" +VALIDATION = ["--body", "Hello, world!", "--expect-header", "Content-Type: text/plain", "--expect-header", "Content-Length: 13"] +CONTROLLER = "0" + +# Layouts: (client process CPUs, worker arguments). Pinned layouts give each worker a CPU with --cpus and the controller CPU 0; +# unpinned ones (-u) run `--workers 2` within the process's CPUs, as base-benchmark's --role-cpus would place them. +LAYOUTS = { + "A": ("1," + CONTROLLER, ["--cpus", "1", "--controller-cpu", CONTROLLER]), + "B": ("1,5," + CONTROLLER, ["--cpus", "1,5", "--controller-cpu", CONTROLLER]), + "C": ("1,2," + CONTROLLER, ["--cpus", "1,2", "--controller-cpu", CONTROLLER]), + "B-u": ("1,5", ["--workers", "2"]), + "C-u": ("1,2", ["--workers", "2"]), +} +SCENARIOS = { + "raw-closed": (["base-native-raw", "base-iouring-raw"], ["--depth", "1"]), + "http-closed": (["base-native", "hyper-tokio"], []), +} + + +def configurations(name): + if name == "main": + for scenario, (backends, extra) in SCENARIOS.items(): + for backend in backends: + for layout in LAYOUTS: + yield {"scenario": scenario, "backend": backend, "layout": layout, "server_cpus": "3"} + for server_cpus in ["2,3", "2,3,6,7"]: + yield {"scenario": "http-closed", "backend": "base-native", "layout": "A", "server_cpus": server_cpus} + elif name == "supplementary": + # Layout C leaves the server core 3 and its sibling 7: does the sibling help the server? And B against C, with frequencies. + for scenario, backend in [("http-closed", "base-native"), ("raw-closed", "base-iouring-raw")]: + yield {"scenario": scenario, "backend": backend, "layout": "B", "server_cpus": "3"} + yield {"scenario": scenario, "backend": backend, "layout": "C", "server_cpus": "3"} + yield {"scenario": scenario, "backend": backend, "layout": "C", "server_cpus": "3,7"} + + +def frequencies(): + """Each CPU's current frequency in MHz, as /proc/cpuinfo reports it (from APERF/MPERF on recent kernels).""" + result, cpu = {}, None + for line in open("/proc/cpuinfo"): + if line.startswith("processor"): + cpu = int(line.split(":")[1]) + elif line.startswith("cpu MHz"): + result[cpu] = float(line.split(":")[1]) + return result + + +def process_cpu(pid): + """User and system ticks of the whole process, and the sum of its threads' scheduler run time in ns.""" + with open(f"/proc/{pid}/stat") as file: + fields = file.read().rsplit(")", 1)[1].split() + utime, stime = int(fields[11]), int(fields[12]) + runtime = 0 + threads = 0 + for task in os.listdir(f"/proc/{pid}/task"): + try: + with open(f"/proc/{pid}/task/{task}/schedstat") as file: + runtime += int(file.read().split()[0]) + threads += 1 + except FileNotFoundError: + pass + return utime, stime, runtime, threads + + +def stop(process): + if process.poll() is None: + process.send_signal(signal.SIGTERM) + try: + process.wait(5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def sample(arguments): + configuration = json.loads(arguments.configuration) + scenario_extra = SCENARIOS[configuration["scenario"]][1] + client_cpus, worker_arguments = LAYOUTS[configuration["layout"]] + tick = os.sysconf("SC_CLK_TCK") + server = subprocess.Popen(["taskset", "-c", configuration["server_cpus"], arguments.server, configuration["backend"]], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, start_new_session=True) + client = None + try: + address = server.stdout.readline().strip() + command = ["taskset", "-c", client_cpus, arguments.load, "closed", "--address", address, "--connections", "64", + *worker_arguments, *scenario_extra, *VALIDATION, + "--warmup", arguments.warmup, "--duration", arguments.duration, "--window-lead", arguments.window_lead, "--quiet"] + environment = dict(os.environ, BASE_BENCHMARK_CONTROL="stdio") + loadavg = open("/proc/loadavg").read().split()[:3] + client = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + env=environment, start_new_session=True) + start = stop_snapshot = None + record = None + samples = [] + sampling = threading.Event() + + def sample_frequencies(): + # Sample every 250 ms from the start marker to the stop marker. + while not sampling.wait(0.25): + samples.append(frequencies()) + + sampler = threading.Thread(target=sample_frequencies) + for line in client.stdout: + line = line.rstrip("\n") + if line == "base-benchmark window start": + start = (time.monotonic_ns(), process_cpu(server.pid)) + client.stdin.write("base-benchmark window started\n") + client.stdin.close() + if arguments.frequencies: + sampler.start() + elif line == "base-benchmark window stop": + stop_snapshot = (time.monotonic_ns(), process_cpu(server.pid)) + sampling.set() + if sampler.is_alive(): + sampler.join() + elif line.startswith("{"): + record = json.loads(line) + stderr = client.stderr.read() + status = client.wait() + finally: + if client is not None and client.poll() is None: + client.kill() + client.wait() + stop(server) + result = dict(configuration, client_cpus=client_cpus, command=command[3:], status=status, loadavg=loadavg, + stderr=stderr[-2000:] if status else "") + if record is None or start is None or stop_snapshot is None: + result["failed"] = True + print(json.dumps(result)) + return + metrics = {metric["name"]: metric["measurement"]["value"] for metric in record["samples"][0]["metrics"]} + load = record["extensions"]["base-benchmark-load"] + valid = metrics["responses_valid"] + (t0, (u0, s0, r0, _)), (t1, (u1, s1, r1, threads)) = start, stop_snapshot + server_cpu_ns = (u1 - u0 + s1 - s0) * 1e9 / tick + result.update( + valid=valid, + throughput=valid / (metrics["wall_time"] / 1e9), + invalid=metrics["responses_invalid"], + errors=metrics["errors"] + metrics["connect_errors"], + saturated=load["saturation"]["saturated"], + saturation_reasons=load["saturation"]["reasons"], + busy=[worker["busy_fraction"] for worker in load["workers"]], + worker_cpus=[worker["cpu"] for worker in load["workers"]], + worker_cpu_ns=[worker["cpu_ns"] for worker in load["workers"]], + worker_softirq_ns=[worker.get("softirq_ns") for worker in load["workers"]], + dispatch_lag_p99_ns=metrics["dispatch_lag_p99"], + service_time_p50_ns=metrics["service_time_p50"], + service_time_p99_ns=metrics["service_time_p99"], + client_cpu_ns=metrics["cpu_time"], + server_window_ns=t1 - t0, + server_user_ns=(u1 - u0) * 1e9 / tick, + server_system_ns=(s1 - s0) * 1e9 / tick, + server_cpu_per_request_ns=server_cpu_ns / valid, + server_runtime_per_request_ns=(r1 - r0) / valid, + server_threads=threads, + ) + if samples: + result["frequency_mhz"] = {cpu: statistics.median(sample[cpu] for sample in samples) for cpu in samples[0]} + print(json.dumps(result)) + + +def others(own): + """Other benchmark processes on the host, as `pgrep -af` reports them.""" + found = subprocess.run(["pgrep", "-af", "base-benchmark|wrk|hello-benchmark"], capture_output=True, text=True).stdout + return [line for line in found.splitlines() if int(line.split()[0]) not in own] + + +def campaign(arguments): + order = list(configurations(arguments.set)) + rng = random.Random(arguments.seed) + with open(arguments.output, "a") as output: + for round_index in range(arguments.rounds): + rng.shuffle(order) + for position, configuration in enumerate(order): + while busy := others({os.getpid(), os.getppid()}): + print("waiting for:", *busy, sep="\n ", file=sys.stderr) + time.sleep(30) + command = ["flock", LOCK, sys.executable, "-I", os.path.abspath(__file__), "sample", + "--server", arguments.server, "--load", arguments.load, "--warmup", arguments.warmup, + "--duration", arguments.duration, "--window-lead", arguments.window_lead, + "--configuration", json.dumps(configuration)] + (["--frequencies"] if arguments.frequencies else []) + line = subprocess.run(command, capture_output=True, text=True, check=True).stdout.strip() + result = dict(json.loads(line), round=round_index, position=position) + output.write(json.dumps(result) + "\n") + output.flush() + print(round_index, position, configuration["scenario"], configuration["backend"], configuration["layout"], + configuration["server_cpus"], f"{result.get('throughput', 0):,.0f}", result.get("busy"), + result.get("saturated"), file=sys.stderr) + time.sleep(arguments.pause) + + +ORDER = ["A", "B", "C", "B-u", "C-u"] +FIELDS = ["round", "position", "scenario", "backend", "layout", "server_cpus", "client_cpus", "valid", "throughput", "busy", + "saturated", "dispatch_lag_p99_ns", "server_user_ns", "server_system_ns", "server_cpu_per_request_ns", + "worker_softirq_ns", "errors", "invalid"] + + +def load(paths): + results = [] + for path in paths: + for line in open(path): + results.append(dict(json.loads(line), set=os.path.basename(path).split(".")[0])) + return results + + +def summarize(arguments): + """One row per configuration and set: medians over rounds, and the median of each round's ratio to layout C on server CPU 3.""" + rows = {} + for result in load(arguments.output): + key = (result["set"], result["scenario"], result["backend"], result["server_cpus"], result["layout"]) + rows.setdefault(key, {})[result["round"]] = result + print("| Set | Scenario | Backend | Layout | Server CPUs | Requests/s, median | Min–max | CV | vs C | Busy | Saturated | Dispatch lag p99 µs | Server CPU µs/request | Client softirq µs/request | Errors/invalid |") + print("| --- | --- | --- | --- | --- | ---: | --- | ---: | ---: | --- | ---: | ---: | ---: | ---: | --- |") + for key in sorted(rows, key=lambda key: (key[0], key[1] != "raw-closed", key[2], key[3] != "3", ORDER.index(key[4]))): + by_round = rows[key] + good = [r for r in by_round.values() if not r.get("failed")] + rates = [r["throughput"] for r in good] + reference = rows.get(key[:3] + ("3", "C")) + ratios = [r["throughput"] / reference[n]["throughput"] - 1 for n, r in by_round.items() if reference and n in reference] + versus = "—" if key[3:] == ("3", "C") else f"{statistics.median(ratios) * 100:+.1f}%" + busy = " ".join(f"{statistics.median(r['busy'][i] for r in good):.2f}" for i in range(len(good[0]["busy"]))) + softirq = [sum(r["worker_softirq_ns"]) / r["valid"] for r in good if None not in r["worker_softirq_ns"]] + print(f"| {key[0]} | {key[1]} | {key[2]} | {key[4]} | {key[3]} | {statistics.median(rates):,.0f} | {min(rates):,.0f}–{max(rates):,.0f} " + f"| {statistics.stdev(rates) / statistics.mean(rates) * 100:.1f}% | {versus} | {busy} | {sum(r['saturated'] for r in good)}/{len(good)} " + f"| {statistics.median(r['dispatch_lag_p99_ns'] for r in good) / 1000:.1f} " + f"| {statistics.median(r['server_cpu_per_request_ns'] for r in good) / 1000:.2f} " + f"| {f'{statistics.median(softirq) / 1000:.2f}' if softirq else '—'} " + f"| {sum(r['errors'] for r in good)}/{sum(r['invalid'] for r in good)} |") + + +def table(arguments): + """The samples as CSV, without commands or frequencies.""" + print(",".join(["set"] + FIELDS + ["frequency_mhz_cpu3"])) + for result in load(arguments.output): + values = [result["set"]] + for field in FIELDS: + value = result.get(field) + if isinstance(value, list): + value = " ".join(f"{item:.4f}" if isinstance(item, float) else str(item) for item in value) + elif isinstance(value, float): + value = f"{value:.6g}" if field == "busy" else f"{value:.1f}" + values.append(str(value)) + values.append(str(result.get("frequency_mhz", {}).get("3", ""))) + print(",".join(f'"{value}"' if "," in value else value for value in values)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + for name in ["campaign", "sample"]: + command = commands.add_parser(name) + command.add_argument("--server", required=True) + command.add_argument("--load", required=True) + command.add_argument("--warmup", default="2s") + command.add_argument("--duration", default="8s") + command.add_argument("--window-lead", default="500ms") + command.add_argument("--frequencies", action="store_true", help="sample each CPU's frequency over the window") + commands.choices["campaign"].add_argument("--rounds", type=int, default=6) + commands.choices["campaign"].add_argument("--seed", type=int, default=20261109) + commands.choices["campaign"].add_argument("--pause", type=float, default=1.0) + commands.choices["campaign"].add_argument("--output", required=True) + commands.choices["campaign"].add_argument("--set", choices=["main", "supplementary"], default="main") + commands.choices["sample"].add_argument("--configuration", required=True) + commands.add_parser("summarize").add_argument("--output", required=True, nargs="+") + commands.add_parser("table").add_argument("--output", required=True, nargs="+") + arguments = parser.parse_args() + {"campaign": campaign, "sample": sample, "summarize": summarize, "table": table}[arguments.command](arguments) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/hello/results/client-layout-20261109/readme.md b/benchmarks/hello/results/client-layout-20261109/readme.md new file mode 100644 index 0000000..1b09a30 --- /dev/null +++ b/benchmarks/hello/results/client-layout-20261109/readme.md @@ -0,0 +1,98 @@ +# Client layout for the closed-loop scenarios on hana + +Screening evidence, gathered 9 November 2026 on [hana](../../../hosts/hana.md): which CPUs base-benchmark-load's workers should run on in `raw-closed` and `http-closed`, with the server on one CPU. It is one host, one day and six rounds, on a host shared with light services, run by the script here rather than by base-benchmark, so it chooses a placement rather than recording a performance result. + +## Decision + +Run two client workers on two physical cores: `--workers 2` with `--role-cpus client=1,2 --role-cpus server=3` on hana, the layout called C-u below. Of the layouts measured, only two workers on two cores drove each server without the client saturated: each worker was busy for about two-thirds of the window in `raw-closed` and about half in `http-closed`. In `raw-closed` that makes it the only layout that measures the server rather than the client. + +- **One worker (A)** is the limit in `raw-closed`: busy for the whole window, 5–6% below two workers in every round. In `http-closed` it was busy for 96–98% of the window, flagged as saturated, and within noise of two cores (−0.2% for hyper-tokio, −1.4% for base-native, with a CV of 3.2%). +- **Two workers on one core's SMT siblings (B)** help over one worker in `raw-closed`, +6–7%, but only by saturating both threads: the second thread gives the core at least that much more capacity, just enough to match what two cores drove at two-thirds busy, with no headroom left for a faster server. In `http-closed` B was 1–2% faster than two cores, but with both workers flagged as saturated (85–90% busy) and the server spending 0.7–2.8% less CPU per request. The server's CPU ran at the same frequency, about 4.0 GHz, in both layouts, so turbo does not explain it. The difference is in the server's workload, consistent with a slower client sending requests in larger batches, though this screen did not test that. A layout that changes the server's work is not the one to compare servers with. The siblings' contention also shows in the client: against C, its 99th percentile dispatch lag rose from 0.7–0.8 µs to 1.1–1.3 µs, and its CPUs' softirq time per request from 1.2 µs to 1.8–2.1 µs. +- **Two cores (C, C-u)** had a CV of 0.3–1.4%, except base-native in `http-closed` (C, at 2.4% and 4.0%, from one or two low rounds per cohort); C-u, the form base-benchmark runs, had 1.0% there. B and B-u's CVs were 0.4–2.1%. +- **Pinned or not**: C-u, two workers left to the kernel within CPUs 1 and 2 with the controller unpinned, as `--role-cpus client=1,2` places them, was within 0.6% of C, its workers pinned and its controller on CPU 0, for every backend. B-u was within 1.1% of B. +- **Server CPUs**: the backends are single-threaded, with helper threads that do little. Giving the server its sibling (3,7 against 3, under C) changed nothing measurable (−0.6% for base-native and base-iouring-raw). Under A, two cores (2,3) or two cores and their siblings (2,3,6,7) were +3.2% and +1.8% against CPU 3 by the median of rounds, but rounds ranged from −8.8% to +9.1% with the client saturated, which is inconclusive. One server CPU stays. + +The fixed-rate scenarios keep one worker: this screen did not measure them. + +## Results + +Requests per second are medians of six rounds, with their range and coefficient of variation. "vs C" is the median over rounds of each layout's ratio to layout C in the same round, on server CPU 3. "Busy" is each worker's median busy fraction; the client flags itself as saturated above 0.8. Server CPU is per valid response; client softirq is the time the workers' CPUs spent in software interrupts per valid response, reported only for pinned workers. No sample had an error or an invalid response. + +| Layout | Client CPUs | Workers | Controller | +| --- | --- | --- | --- | +| A | 1 | 1, pinned to 1 | Pinned to 0 | +| B | 1 and its sibling 5 | 2, pinned to 1 and 5 | Pinned to 0 | +| C | 1 and 2, two cores | 2, pinned to 1 and 2 | Pinned to 0 | +| B-u | 1 and 5 | 2, unpinned within them | Unpinned within them | +| C-u | 1 and 2 | 2, unpinned within them | Unpinned within them | + +### Main campaign + +| Scenario | Backend | Layout | Server CPUs | Requests/s | Min–max | CV | vs C | Busy | Saturated | Dispatch lag p99 µs | Server CPU µs/request | Client softirq µs/request | +| --- | --- | --- | --- | ---: | --- | ---: | ---: | --- | ---: | ---: | ---: | ---: | +| raw-closed | base-iouring-raw | A | 3 | 240,355 | 235,525–243,824 | 1.1% | −5.3% | 1.00 | 6/6 | 0.5 | 3.25 | 1.07 | +| raw-closed | base-iouring-raw | B | 3 | 257,585 | 251,364–259,282 | 1.1% | +0.9% | 1.00 1.00 | 6/6 | 1.1 | 3.02 | 1.84 | +| raw-closed | base-iouring-raw | C | 3 | 254,059 | 248,153–255,792 | 1.2% | — | 0.66 0.66 | 0/6 | 0.7 | 3.10 | 1.20 | +| raw-closed | base-iouring-raw | B-u | 3 | 256,619 | 254,327–258,483 | 0.6% | +1.0% | 1.00 1.00 | 6/6 | 1.1 | 3.02 | — | +| raw-closed | base-iouring-raw | C-u | 3 | 255,592 | 252,509–256,978 | 0.6% | +0.5% | 0.67 0.67 | 0/6 | 0.8 | 3.08 | — | +| raw-closed | base-native-raw | A | 3 | 240,353 | 237,362–243,353 | 1.0% | −5.8% | 1.00 | 6/6 | 0.6 | 3.26 | 1.07 | +| raw-closed | base-native-raw | B | 3 | 255,474 | 250,402–257,333 | 1.0% | +0.4% | 0.99 0.99 | 6/6 | 1.1 | 3.06 | 1.87 | +| raw-closed | base-native-raw | C | 3 | 254,649 | 253,775–255,994 | 0.3% | — | 0.67 0.66 | 0/6 | 0.7 | 3.10 | 1.20 | +| raw-closed | base-native-raw | B-u | 3 | 255,694 | 254,568–257,395 | 0.4% | +0.4% | 0.99 0.99 | 6/6 | 1.1 | 3.05 | — | +| raw-closed | base-native-raw | C-u | 3 | 254,649 | 251,312–256,174 | 0.7% | −0.1% | 0.67 0.67 | 0/6 | 0.8 | 3.09 | — | +| http-closed | base-native | A | 3 | 204,909 | 193,714–211,028 | 3.2% | −1.4% | 0.97 | 6/6 | 0.7 | 3.93 | 1.10 | +| http-closed | base-native | B | 3 | 212,486 | 207,326–213,566 | 1.1% | +1.8% | 0.90 0.90 | 6/6 | 1.2 | 3.84 | 1.98 | +| http-closed | base-native | C | 3 | 209,814 | 193,329–213,195 | 4.0% | — | 0.55 0.55 | 0/6 | 0.8 | 3.92 | 1.20 | +| http-closed | base-native | B-u | 3 | 212,123 | 207,972–214,491 | 1.1% | +1.4% | 0.90 0.90 | 6/6 | 1.3 | 3.84 | — | +| http-closed | base-native | C-u | 3 | 209,272 | 206,843–212,167 | 1.0% | +0.6% | 0.55 0.54 | 0/6 | 0.8 | 3.94 | — | +| http-closed | base-native | A | 2,3 | 210,785 | 192,047–215,353 | 4.0% | +0.9% | 0.97 | 6/6 | 0.6 | 3.82 | 1.09 | +| http-closed | base-native | A | 2,3,6,7 | 210,063 | 199,608–212,992 | 2.3% | +0.2% | 0.98 | 6/6 | 0.7 | 3.84 | 1.11 | +| http-closed | hyper-tokio | A | 3 | 194,532 | 186,210–197,992 | 2.1% | −0.2% | 0.96 | 6/6 | 0.6 | 4.19 | 1.11 | +| http-closed | hyper-tokio | B | 3 | 195,779 | 186,891–198,363 | 2.1% | +1.0% | 0.85 0.86 | 6/6 | 1.3 | 4.22 | 2.07 | +| http-closed | hyper-tokio | C | 3 | 194,276 | 188,308–196,461 | 1.4% | — | 0.51 0.51 | 0/6 | 0.8 | 4.25 | 1.20 | +| http-closed | hyper-tokio | B-u | 3 | 198,860 | 196,354–201,082 | 1.0% | +2.1% | 0.85 0.86 | 6/6 | 1.3 | 4.14 | — | +| http-closed | hyper-tokio | C-u | 3 | 194,961 | 191,202–197,046 | 1.3% | −0.4% | 0.50 0.50 | 0/6 | 0.8 | 4.26 | — | + +Against base-native's layout A on server CPU 3 in the same round, the server-side check's 2,3 and 2,3,6,7 were +3.2% (rounds −8.8% to +7.6%) and +1.8% (−3.5% to +9.1%). + +### Supplementary campaign + +Run afterwards, to check whether turbo frequency explains B's lead in `http-closed`, and whether the server gains from its own sibling under C. Each CPU's frequency was sampled from `/proc/cpuinfo` every 250 ms over the window. The median for the server's CPU 3 was 4,000 MHz under B and 3,976–3,997 MHz under C, and the client's workers' CPUs at 3,997–4,000 MHz in both. + +| Scenario | Backend | Layout | Server CPUs | Requests/s | Min–max | CV | vs C | Busy | Saturated | Dispatch lag p99 µs | Server CPU µs/request | Client softirq µs/request | +| --- | --- | --- | --- | ---: | --- | ---: | ---: | --- | ---: | ---: | ---: | ---: | +| raw-closed | base-iouring-raw | B | 3 | 256,312 | 252,724–258,220 | 0.8% | +0.5% | 1.00 1.00 | 6/6 | 1.1 | 3.03 | 1.84 | +| raw-closed | base-iouring-raw | C | 3 | 255,256 | 251,561–255,854 | 0.6% | — | 0.66 0.66 | 0/6 | 0.7 | 3.08 | 1.19 | +| raw-closed | base-iouring-raw | C | 3,7 | 253,625 | 248,557–254,037 | 0.9% | −0.6% | 0.66 0.66 | 0/6 | 0.8 | 3.10 | 1.20 | +| http-closed | base-native | B | 3 | 212,147 | 207,876–213,336 | 0.9% | +2.1% | 0.89 0.89 | 6/6 | 1.3 | 3.85 | 1.97 | +| http-closed | base-native | C | 3 | 208,026 | 196,886–210,749 | 2.4% | — | 0.55 0.55 | 0/6 | 0.8 | 3.95 | 1.20 | +| http-closed | base-native | C | 3,7 | 207,755 | 204,992–209,691 | 0.8% | −0.6% | 0.55 0.55 | 0/6 | 0.8 | 3.95 | 1.21 | + +## Conditions + +- hana as its [host document](../../../hosts/hana.md) describes it on this date: Xeon E3-1240 v6, Linux 7.2.14-arch1-1, the `powersave` governor with `balance_performance` energy preference, turbo enabled. No host setting was changed. It was idle apart from its usual light services; no other benchmark ran, and every sample held the host lock, `/tmp/base-benchmark-host.lock`. +- base-http `eaa13fd`, the hello server built on hana with `cargo build --release --locked` and Rust 1.99.1 (SHA-256 `27c64823d6ff9d64f5343e4ff708e7825393440ee4432ca706fa85e098467615`). base-benchmark-load 0.1.0, `cargo install --locked` on hana (SHA-256 `3e0129da3fa1776d7d8ec714a8aed399fc71706438767e32fb128c7afb9f6adb`). +- The scenarios' measured arguments: closed loop, 64 connections, `--depth 1` for the raw controls, a 2-second warmup, an 8-second window and a 500 ms `--window-lead`, validating the body, `Content-Type` and `Content-Length` of every response. Only the worker and controller placement differed. +- The server was started under `taskset -c SERVER_CPUS`, fresh for every sample, and stopped with `SIGTERM` after it. The client ran under `taskset -c` its CPUs, with `--cpus` and `--controller-cpu 0` in the pinned layouts. The script itself, acting as the runner, ran on CPU 0 with the OS. The controller went on CPU 0 rather than 5, so that it was in the same place in every pinned layout and on neither a worker's CPU nor its sibling; it wakes only to mark the window. +- The script plays base-benchmark's runner for the window markers (`BASE_BENCHMARK_CONTROL=stdio`): at the client's start marker it reads the server's user and system time from `/proc/PID/stat`, every thread included, then acknowledges; it reads them again at the stop marker. So server CPU covers the window and up to 500 ms of the warmup before it, as base-benchmark's does, at the clock tick's 10 ms resolution, and excludes softirq time spent on the server's traffic. +- Six rounds per campaign. Each round ran every configuration once, in an order shuffled by a seeded generator (seeds 20261109 and 20261110), with a 1-second pause between samples. +- Schedule lag applies only at a fixed rate; the closed loop reports dispatch lag, the delay from a worker learning a connection may send to sending. + +## Files + +- `layout.py`: the campaign driver, which runs each sample as `flock /tmp/base-benchmark-host.lock layout.py sample ...` after checking `pgrep -af` for other benchmarks, and the summaries above. +- `samples.csv`: every sample's figures, one row each, from `layout.py table`. The full JSON records stayed on hana and were not kept. + +On hana, from a scratch directory holding a clone of base-http at `eaa13fd` and `layout.py`: + +```sh +cargo install --locked --root $PWD/load base-benchmark-load --version 0.1.0 +(cd base-http/benchmarks/hello && cargo build --release --locked) +SERVER=base-http/benchmarks/hello/target/release/base-http-hello-benchmark +taskset -c 0 python3 -I layout.py campaign --rounds 6 --seed 20261109 \ + --server $SERVER --load load/bin/base-benchmark-load --output samples.jsonl +taskset -c 0 python3 -I layout.py campaign --set supplementary --frequencies --rounds 6 --seed 20261110 \ + --server $SERVER --load load/bin/base-benchmark-load --output supplementary.jsonl +python3 -I layout.py summarize --output samples.jsonl supplementary.jsonl +python3 -I layout.py table --output samples.jsonl supplementary.jsonl > samples.csv +``` diff --git a/benchmarks/hello/results/client-layout-20261109/samples.csv b/benchmarks/hello/results/client-layout-20261109/samples.csv new file mode 100644 index 0000000..7c7d9d5 --- /dev/null +++ b/benchmarks/hello/results/client-layout-20261109/samples.csv @@ -0,0 +1,169 @@ +set,round,position,scenario,backend,layout,server_cpus,client_cpus,valid,throughput,busy,saturated,dispatch_lag_p99_ns,server_user_ns,server_system_ns,server_cpu_per_request_ns,worker_softirq_ns,errors,invalid,frequency_mhz_cpu3 +samples,0,0,raw-closed,base-iouring-raw,B-u,3,"1,5",2052565,256570.5,0.9949 0.9950,True,1055.0,1520000000.0,4690000000.0,3025.5,None None,0,0, +samples,0,1,http-closed,hyper-tokio,B-u,3,"1,5",1570837,196354.4,0.8508 0.8585,True,1288.0,2210000000.0,4350000000.0,4176.1,None None,0,0, +samples,0,2,http-closed,hyper-tokio,B,3,"1,5,0",1564733,195590.7,0.8510 0.8610,True,1302.0,2210000000.0,4410000000.0,4230.8,1630000000 1620000000,0,0, +samples,0,3,http-closed,hyper-tokio,C,3,"1,2,0",1571693,196461.3,0.5097 0.5107,False,822.0,2210000000.0,4410000000.0,4212.0,930000000 940000000,0,0, +samples,0,4,http-closed,base-native,C-u,3,"1,2",1678956,209869.4,0.5501 0.5447,False,792.0,2260000000.0,4320000000.0,3919.1,None None,0,0, +samples,0,5,http-closed,hyper-tokio,C-u,3,"1,2",1562193,195274.0,0.5034 0.5022,False,711.0,2230000000.0,4380000000.0,4231.2,None None,0,0, +samples,0,6,raw-closed,base-iouring-raw,B,3,"1,5,0",2010915,251364.1,0.9944 0.9946,True,1118.0,1580000000.0,4670000000.0,3108.0,1880000000 1890000000,0,0, +samples,0,7,http-closed,base-native,A,"2,3,6,7","1,0",1703937,212991.7,0.9911,True,666.0,2090000000.0,4370000000.0,3791.2,1920000000,0,0, +samples,0,8,http-closed,base-native,A,3,"1,0",1684733,210591.5,0.9815,True,679.0,2170000000.0,4320000000.0,3852.2,1860000000,0,0, +samples,0,9,http-closed,base-native,C,3,"1,2,0",1682346,210293.7,0.5492 0.5474,False,675.0,2250000000.0,4320000000.0,3905.3,1000000000 1010000000,0,0, +samples,0,10,raw-closed,base-native-raw,B-u,3,"1,5",2036543,254567.6,0.9938 0.9940,True,1067.0,1530000000.0,4740000000.0,3078.7,None None,0,0, +samples,0,11,http-closed,base-native,A,"2,3","1,0",1536381,192047.3,0.9587,True,726.0,2320000000.0,4250000000.0,4276.3,1710000000,0,0, +samples,0,12,raw-closed,base-iouring-raw,C,3,"1,2,0",2008364,251045.3,0.6563 0.6531,False,795.0,1580000000.0,4700000000.0,3126.9,1200000000 1190000000,0,0, +samples,0,13,raw-closed,base-native-raw,A,3,"1,0",1910446,238805.6,0.9999,True,641.0,1560000000.0,4730000000.0,3292.4,2080000000,0,0, +samples,0,14,raw-closed,base-native-raw,C,3,"1,2,0",2040955,255119.1,0.6641 0.6611,False,663.0,1510000000.0,4810000000.0,3096.6,1220000000 1220000000,0,0, +samples,0,15,http-closed,base-native,B,3,"1,5,0",1707330,213416.1,0.9056 0.9029,True,1240.0,2150000000.0,4400000000.0,3836.4,1690000000 1690000000,0,0, +samples,0,16,http-closed,base-native,B-u,3,"1,5",1663777,207971.9,0.8868 0.8839,True,1252.0,2240000000.0,4330000000.0,3948.8,None None,0,0, +samples,0,17,raw-closed,base-native-raw,C-u,3,"1,2",2034838,254354.6,0.6663 0.6658,False,693.0,1500000000.0,4800000000.0,3096.1,None None,0,0, +samples,0,18,raw-closed,base-iouring-raw,C-u,3,"1,2",2049174,256146.6,0.6688 0.6678,False,759.0,1630000000.0,4670000000.0,3074.4,None None,0,0, +samples,0,19,http-closed,hyper-tokio,A,3,"1,0",1565001,195625.0,0.9562,True,621.0,2130000000.0,4410000000.0,4178.9,1730000000,0,0, +samples,0,20,raw-closed,base-native-raw,B,3,"1,5,0",2037992,254748.7,0.9855 0.9872,True,1105.0,1530000000.0,4730000000.0,3071.7,1870000000 1870000000,0,0, +samples,0,21,raw-closed,base-iouring-raw,A,3,"1,0",1924967,240620.5,0.9984,True,571.0,1580000000.0,4670000000.0,3246.8,2080000000,0,0, +samples,1,0,http-closed,hyper-tokio,C-u,3,"1,2",1529617,191202.0,0.5017 0.5002,False,790.0,2210000000.0,4420000000.0,4334.4,None None,0,0, +samples,1,1,raw-closed,base-iouring-raw,B-u,3,"1,5",2043106,255388.0,0.9966 0.9965,True,1136.0,1570000000.0,4660000000.0,3049.3,None None,0,0, +samples,1,2,raw-closed,base-native-raw,B-u,3,"1,5",2049884,256235.3,0.9963 0.9964,True,1138.0,1480000000.0,4760000000.0,3044.1,None None,0,0, +samples,1,3,http-closed,hyper-tokio,B-u,3,"1,5",1596367,199545.7,0.8531 0.8660,True,1282.0,2210000000.0,4360000000.0,4115.6,None None,0,0, +samples,1,4,http-closed,hyper-tokio,C,3,"1,2,0",1556955,194619.3,0.5094 0.5072,False,753.0,2210000000.0,4410000000.0,4251.9,940000000 930000000,0,0, +samples,1,5,http-closed,base-native,B,3,"1,5,0",1708533,213565.6,0.9544 0.9537,True,1442.0,2230000000.0,4300000000.0,3822.0,1720000000 1720000000,0,0, +samples,1,6,http-closed,base-native,A,3,"1,0",1624285,203035.5,0.9621,True,541.0,2200000000.0,4230000000.0,3958.7,1760000000,0,0, +samples,1,7,raw-closed,base-native-raw,B,3,"1,5,0",2041106,255138.1,0.9951 0.9952,True,1112.0,1480000000.0,4760000000.0,3057.2,1910000000 1910000000,0,0, +samples,1,8,raw-closed,base-native-raw,C-u,3,"1,2",2010496,251311.9,0.7057 0.6991,False,905.0,1530000000.0,4760000000.0,3128.6,None None,0,0, +samples,1,9,http-closed,base-native,A,"2,3,6,7","1,0",1663185,207897.9,0.9706,True,646.0,2200000000.0,4240000000.0,3872.1,1820000000,0,0, +samples,1,10,http-closed,base-native,C,3,"1,2,0",1674669,209333.4,0.5480 0.5459,False,745.0,2230000000.0,4360000000.0,3935.1,1010000000 1000000000,0,0, +samples,1,11,raw-closed,base-native-raw,C,3,"1,2,0",2034576,254321.8,0.6662 0.6650,False,797.0,1420000000.0,4880000000.0,3096.5,1230000000 1230000000,0,0, +samples,1,12,http-closed,base-native,C-u,3,"1,2",1697338,212167.1,0.5533 0.5572,False,738.0,2180000000.0,4390000000.0,3870.8,None None,0,0, +samples,1,13,http-closed,base-native,A,"2,3","1,0",1707740,213467.3,0.9805,True,586.0,2120000000.0,4310000000.0,3765.2,1860000000,0,0, +samples,1,14,http-closed,base-native,B-u,3,"1,5",1715926,214490.6,0.9171 0.9155,True,1310.0,2190000000.0,4350000000.0,3811.4,None None,0,0, +samples,1,15,http-closed,hyper-tokio,A,3,"1,0",1570745,196343.4,0.9672,True,577.0,2180000000.0,4340000000.0,4150.9,1700000000,0,0, +samples,1,16,http-closed,hyper-tokio,B,3,"1,5,0",1571564,196444.8,0.8527 0.8601,True,1292.0,2230000000.0,4360000000.0,4193.3,1610000000 1610000000,0,0, +samples,1,17,raw-closed,base-iouring-raw,A,3,"1,0",1950600,243824.1,0.9996,True,526.0,1560000000.0,4670000000.0,3193.9,2070000000,0,0, +samples,1,18,raw-closed,base-iouring-raw,C-u,3,"1,2",2035759,254469.8,0.6732 0.6684,False,815.0,1590000000.0,4710000000.0,3094.7,None None,0,0, +samples,1,19,raw-closed,base-iouring-raw,B,3,"1,5,0",2074265,259282.4,0.9988 0.9988,True,1111.0,1510000000.0,4710000000.0,2998.7,1900000000 1900000000,0,0, +samples,1,20,raw-closed,base-iouring-raw,C,3,"1,2,0",2022041,252754.9,0.6553 0.6597,False,706.0,1630000000.0,4690000000.0,3125.6,1210000000 1210000000,0,0, +samples,1,21,raw-closed,base-native-raw,A,3,"1,0",1898897,237362.1,0.9999,True,670.0,1590000000.0,4700000000.0,3312.4,2030000000,0,0, +samples,2,0,http-closed,base-native,A,"2,3,6,7","1,0",1672803,209100.0,0.9760,True,705.0,2150000000.0,4270000000.0,3837.9,1860000000,0,0, +samples,2,1,raw-closed,base-iouring-raw,B,3,"1,5,0",2066775,258346.5,0.9976 0.9977,True,1124.0,1550000000.0,4680000000.0,3014.4,1910000000 1900000000,0,0, +samples,2,2,raw-closed,base-native-raw,A,3,"1,0",1937594,242199.0,0.9999,True,629.0,1530000000.0,4720000000.0,3225.6,2060000000,0,0, +samples,2,3,http-closed,base-native,B-u,3,"1,5",1688496,211061.8,0.8964 0.8952,True,1250.0,2290000000.0,4220000000.0,3855.5,None None,0,0, +samples,2,4,http-closed,base-native,A,"2,3","1,0",1687587,210948.2,0.9746,True,548.0,2140000000.0,4320000000.0,3828.0,1840000000,0,0, +samples,2,5,raw-closed,base-native-raw,C,3,"1,2,0",2039812,254976.3,0.6617 0.6582,False,785.0,1520000000.0,4790000000.0,3093.4,1220000000 1220000000,0,0, +samples,2,6,http-closed,base-native,B,3,"1,5,0",1693319,211664.6,0.8891 0.8794,True,1242.0,2240000000.0,4280000000.0,3850.4,1650000000 1640000000,0,0, +samples,2,7,http-closed,hyper-tokio,A,3,"1,0",1545821,193227.5,0.9587,True,540.0,2180000000.0,4390000000.0,4250.2,1740000000,0,0, +samples,2,8,raw-closed,base-iouring-raw,C-u,3,"1,2",2055829,256978.4,0.6704 0.6706,False,774.0,1570000000.0,4720000000.0,3059.6,None None,0,0, +samples,2,9,raw-closed,base-native-raw,C-u,3,"1,2",2034379,254297.2,0.6690 0.6690,False,822.0,1530000000.0,4800000000.0,3111.5,None None,0,0, +samples,2,10,http-closed,base-native,C-u,3,"1,2",1669398,208674.6,0.5396 0.5389,False,762.0,2230000000.0,4380000000.0,3959.5,None None,0,0, +samples,2,11,http-closed,hyper-tokio,B-u,3,"1,5",1608660,201082.3,0.8575 0.8672,True,1252.0,2280000000.0,4360000000.0,4127.7,None None,0,0, +samples,2,12,http-closed,hyper-tokio,B,3,"1,5,0",1567733,195966.3,0.8498 0.8629,True,1270.0,2180000000.0,4420000000.0,4209.9,1650000000 1630000000,0,0, +samples,2,13,raw-closed,base-iouring-raw,C,3,"1,2,0",1985223,248152.8,0.6520 0.6475,False,759.0,1540000000.0,4690000000.0,3138.2,1210000000 1190000000,0,0, +samples,2,14,http-closed,hyper-tokio,C-u,3,"1,2",1531500,191437.4,0.5012 0.5051,False,799.0,2230000000.0,4400000000.0,4329.1,None None,0,0, +samples,2,15,raw-closed,base-native-raw,B,3,"1,5,0",2003219,250402.1,0.9846 0.9864,True,1104.0,1520000000.0,4690000000.0,3100.0,1890000000 1900000000,0,0, +samples,2,16,http-closed,hyper-tokio,C,3,"1,2,0",1551467,193933.2,0.4968 0.4966,False,696.0,2200000000.0,4410000000.0,4260.5,910000000 910000000,0,0, +samples,2,17,raw-closed,base-iouring-raw,B-u,3,"1,5",2053349,256668.4,0.9935 0.9935,True,1137.0,1490000000.0,4720000000.0,3024.3,None None,0,0, +samples,2,18,raw-closed,base-native-raw,B-u,3,"1,5",2042288,255286.3,0.9920 0.9916,True,1072.0,1500000000.0,4740000000.0,3055.4,None None,0,0, +samples,2,19,http-closed,base-native,C,3,"1,2,0",1691560,211444.9,0.5475 0.5487,False,759.0,2140000000.0,4450000000.0,3895.8,1010000000 1010000000,0,0, +samples,2,20,http-closed,base-native,A,3,"1,0",1618251,202281.3,0.9656,True,718.0,2230000000.0,4200000000.0,3973.4,1780000000,0,0, +samples,2,21,raw-closed,base-iouring-raw,A,3,"1,0",1884204,235525.3,0.9985,True,707.0,1610000000.0,4650000000.0,3322.4,2070000000,0,0, +samples,3,0,http-closed,base-native,A,3,"1,0",1654307,206783.5,0.9721,True,718.0,2160000000.0,4280000000.0,3892.9,1850000000,0,0, +samples,3,1,http-closed,base-native,C-u,3,"1,2",1654745,206843.0,0.5411 0.5425,False,734.0,2280000000.0,4330000000.0,3994.6,None None,0,0, +samples,3,2,raw-closed,base-native-raw,C-u,3,"1,2",2045318,255664.1,0.6689 0.6678,False,747.0,1510000000.0,4790000000.0,3080.2,None None,0,0, +samples,3,3,http-closed,base-native,A,"2,3","1,0",1684984,210622.8,0.9753,True,681.0,2140000000.0,4300000000.0,3822.0,1850000000,0,0, +samples,3,4,raw-closed,base-iouring-raw,C-u,3,"1,2",2040293,255036.5,0.6602 0.6613,False,786.0,1540000000.0,4750000000.0,3082.9,None None,0,0, +samples,3,5,raw-closed,base-native-raw,C,3,"1,2,0",2032182,254022.3,0.6681 0.6666,False,721.0,1500000000.0,4800000000.0,3100.1,1250000000 1240000000,0,0, +samples,3,6,http-closed,hyper-tokio,B,3,"1,5,0",1495133,186891.5,0.8393 0.8299,True,1296.0,2240000000.0,4340000000.0,4400.9,1700000000 1690000000,0,0, +samples,3,7,http-closed,hyper-tokio,C-u,3,"1,2",1557186,194648.1,0.5039 0.5024,False,713.0,2260000000.0,4400000000.0,4276.9,None None,0,0, +samples,3,8,raw-closed,base-native-raw,B,3,"1,5,0",2056498,257062.0,0.9910 0.9906,True,1095.0,1490000000.0,4740000000.0,3029.4,1890000000 1880000000,0,0, +samples,3,9,http-closed,hyper-tokio,B-u,3,"1,5",1585402,198175.0,0.8531 0.8540,True,1260.0,2190000000.0,4390000000.0,4150.4,None None,0,0, +samples,3,10,raw-closed,base-iouring-raw,C,3,"1,2,0",2042920,255364.0,0.6708 0.6731,False,690.0,1520000000.0,4760000000.0,3074.0,1230000000 1220000000,0,0, +samples,3,11,raw-closed,base-native-raw,B-u,3,"1,5",2059165,257395.4,0.9929 0.9930,True,1071.0,1480000000.0,4750000000.0,3025.5,None None,0,0, +samples,3,12,http-closed,hyper-tokio,C,3,"1,2,0",1560254,195031.5,0.5100 0.5089,False,781.0,2210000000.0,4400000000.0,4236.5,940000000 930000000,0,0, +samples,3,13,raw-closed,base-iouring-raw,B-u,3,"1,5",2034616,254326.7,0.9950 0.9948,True,1083.0,1530000000.0,4670000000.0,3047.3,None None,0,0, +samples,3,14,http-closed,base-native,C,3,"1,2,0",1579358,197419.7,0.5311 0.5302,False,855.0,2140000000.0,4410000000.0,4147.3,980000000 970000000,0,0, +samples,3,15,raw-closed,base-iouring-raw,A,3,"1,0",1915439,239429.6,0.9987,True,538.0,1610000000.0,4660000000.0,3273.4,2050000000,0,0, +samples,3,16,raw-closed,base-native-raw,A,3,"1,0",1946830,243353.3,0.9981,True,613.0,1520000000.0,4740000000.0,3215.5,2050000000,0,0, +samples,3,17,http-closed,base-native,B,3,"1,5,0",1699717,212464.3,0.9125 0.9106,True,1280.0,2210000000.0,4300000000.0,3830.0,1700000000 1690000000,0,0, +samples,3,18,raw-closed,base-iouring-raw,B,3,"1,5,0",2067182,258397.7,0.9975 0.9975,True,1073.0,1500000000.0,4720000000.0,3008.9,1880000000 1880000000,0,0, +samples,3,19,http-closed,base-native,A,"2,3,6,7","1,0",1596900,199608.0,0.9651,True,658.0,2130000000.0,4200000000.0,3963.9,1750000000,0,0, +samples,3,20,http-closed,hyper-tokio,A,3,"1,0",1489683,186210.2,0.9333,True,640.0,2100000000.0,4280000000.0,4282.8,1650000000,0,0, +samples,3,21,http-closed,base-native,B-u,3,"1,5",1699509,212438.4,0.8880 0.8865,True,1191.0,2260000000.0,4250000000.0,3830.5,None None,0,0, +samples,4,0,http-closed,base-native,B,3,"1,5,0",1700062,212507.5,0.9021 0.8954,True,1231.0,2270000000.0,4270000000.0,3846.9,1680000000 1670000000,0,0, +samples,4,1,http-closed,base-native,A,"2,3","1,0",1722822,215352.8,0.9827,True,552.0,2110000000.0,4340000000.0,3743.9,1880000000,0,0, +samples,4,2,raw-closed,base-iouring-raw,B-u,3,"1,5",2058144,257267.8,0.9957 0.9957,True,1118.0,1520000000.0,4700000000.0,3022.1,None None,0,0, +samples,4,3,http-closed,base-native,C-u,3,"1,2",1667263,208407.7,0.5518 0.5469,False,780.0,2250000000.0,4350000000.0,3958.6,None None,0,0, +samples,4,4,http-closed,hyper-tokio,C,3,"1,2,0",1549329,193665.8,0.5115 0.5122,False,772.0,2260000000.0,4330000000.0,4253.5,950000000 960000000,0,0, +samples,4,5,raw-closed,base-iouring-raw,A,3,"1,0",1929838,241229.4,0.9980,True,551.0,1590000000.0,4650000000.0,3233.4,2070000000,0,0, +samples,4,6,raw-closed,base-native-raw,B-u,3,"1,5",2046457,255806.9,0.9940 0.9939,True,1074.0,1540000000.0,4690000000.0,3044.3,None None,0,0, +samples,4,7,raw-closed,base-iouring-raw,C,3,"1,2,0",2044095,255511.6,0.6604 0.6562,False,696.0,1510000000.0,4770000000.0,3072.3,1220000000 1210000000,0,0, +samples,4,8,http-closed,base-native,C,3,"1,2,0",1546635,193329.1,0.5189 0.5142,False,774.0,2290000000.0,4380000000.0,4312.6,960000000 940000000,0,0, +samples,4,9,http-closed,base-native,A,3,"1,0",1688226,211027.9,0.9774,True,661.0,2140000000.0,4290000000.0,3808.7,1860000000,0,0, +samples,4,10,raw-closed,base-native-raw,C,3,"1,2,0",2030197,253774.6,0.6559 0.6540,False,683.0,1540000000.0,4760000000.0,3103.1,1210000000 1200000000,0,0, +samples,4,11,raw-closed,base-native-raw,C-u,3,"1,2",2039553,254944.0,0.6730 0.6746,False,815.0,1500000000.0,4810000000.0,3093.8,None None,0,0, +samples,4,12,http-closed,hyper-tokio,C-u,3,"1,2",1568500,196062.3,0.5168 0.5119,False,736.0,2240000000.0,4410000000.0,4239.7,None None,0,0, +samples,4,13,raw-closed,base-iouring-raw,C-u,3,"1,2",2051356,256419.3,0.6808 0.6767,False,745.0,1540000000.0,4740000000.0,3061.4,None None,0,0, +samples,4,14,http-closed,hyper-tokio,B-u,3,"1,5",1575032,196878.8,0.8530 0.8606,True,1231.0,2190000000.0,4420000000.0,4196.7,None None,0,0, +samples,4,15,http-closed,base-native,B-u,3,"1,5",1694467,211808.2,0.9078 0.8990,True,1302.0,2200000000.0,4350000000.0,3865.5,None None,0,0, +samples,4,16,raw-closed,base-native-raw,A,3,"1,0",1904542,238067.0,0.9996,True,608.0,1590000000.0,4700000000.0,3302.6,2060000000,0,0, +samples,4,17,raw-closed,base-iouring-raw,B,3,"1,5,0",2054585,256822.8,0.9964 0.9965,True,1123.0,1570000000.0,4660000000.0,3032.2,1890000000 1910000000,0,0, +samples,4,18,raw-closed,base-native-raw,B,3,"1,5,0",2046481,255809.9,0.9945 0.9946,True,1132.0,1430000000.0,4820000000.0,3054.0,1900000000 1910000000,0,0, +samples,4,19,http-closed,base-native,A,"2,3,6,7","1,0",1688210,211026.1,0.9809,True,550.0,2190000000.0,4280000000.0,3832.5,1890000000,0,0, +samples,4,20,http-closed,hyper-tokio,B,3,"1,5,0",1586904,198362.9,0.8487 0.8553,True,1263.0,2210000000.0,4400000000.0,4165.3,1650000000 1620000000,0,0, +samples,4,21,http-closed,hyper-tokio,A,3,"1,0",1547518,193439.5,0.9567,True,572.0,2150000000.0,4360000000.0,4206.7,1720000000,0,0, +samples,5,0,raw-closed,base-iouring-raw,B,3,"1,5,0",2052904,256612.7,0.9952 0.9950,True,1086.0,1570000000.0,4660000000.0,3034.7,1890000000 1890000000,0,0, +samples,5,1,http-closed,base-native,A,3,"1,0",1549718,193714.5,0.9541,True,563.0,2230000000.0,4340000000.0,4239.5,1710000000,0,0, +samples,5,2,raw-closed,base-iouring-raw,A,3,"1,0",1920720,240089.5,0.9998,True,517.0,1500000000.0,4750000000.0,3254.0,2060000000,0,0, +samples,5,3,http-closed,base-native,A,"2,3,6,7","1,0",1690402,211299.9,0.9810,True,684.0,2170000000.0,4290000000.0,3821.6,1860000000,0,0, +samples,5,4,raw-closed,base-native-raw,C,3,"1,2,0",2047956,255994.3,0.6808 0.6801,False,719.0,1520000000.0,4780000000.0,3076.2,1290000000 1290000000,0,0, +samples,5,5,http-closed,base-native,A,"2,3","1,0",1667953,208493.8,0.9730,True,562.0,2190000000.0,4280000000.0,3879.0,1830000000,0,0, +samples,5,6,http-closed,base-native,C,3,"1,2,0",1705423,213195.2,0.5501 0.5454,False,730.0,2210000000.0,4360000000.0,3852.4,1020000000 1000000000,0,0, +samples,5,7,http-closed,hyper-tokio,C-u,3,"1,2",1576367,197045.7,0.5393 0.5289,False,788.0,2240000000.0,4420000000.0,4224.9,None None,0,0, +samples,5,8,http-closed,hyper-tokio,A,3,"1,0",1583938,197992.0,0.9629,True,504.0,2180000000.0,4390000000.0,4147.9,1720000000,0,0, +samples,5,9,http-closed,hyper-tokio,C,3,"1,2,0",1506464,188307.8,0.4934 0.4915,False,748.0,2210000000.0,4390000000.0,4381.1,910000000 900000000,0,0, +samples,5,10,http-closed,base-native,B-u,3,"1,5",1710225,213777.9,0.9191 0.9220,True,1273.0,2200000000.0,4350000000.0,3829.9,None None,0,0, +samples,5,11,raw-closed,base-native-raw,B-u,3,"1,5",2044651,255581.1,0.9963 0.9963,True,1132.0,1540000000.0,4710000000.0,3056.8,None None,0,0, +samples,5,12,http-closed,base-native,B,3,"1,5,0",1658614,207325.8,0.8773 0.8785,True,1248.0,2180000000.0,4360000000.0,3943.1,1610000000 1610000000,0,0, +samples,5,13,raw-closed,base-native-raw,B,3,"1,5,0",2058664,257332.9,0.9964 0.9964,True,1241.0,1430000000.0,4800000000.0,3026.2,1950000000 1940000000,0,0, +samples,5,14,http-closed,base-native,C-u,3,"1,2",1696036,212004.4,0.5460 0.5414,False,670.0,2190000000.0,4390000000.0,3879.6,None None,0,0, +samples,5,15,raw-closed,base-native-raw,A,3,"1,0",1935211,241901.2,0.9997,True,621.0,1530000000.0,4720000000.0,3229.6,2070000000,0,0, +samples,5,16,http-closed,hyper-tokio,B,3,"1,5,0",1551411,193926.8,0.8455 0.8534,True,1320.0,2240000000.0,4350000000.0,4247.7,1590000000 1570000000,0,0, +samples,5,17,raw-closed,base-native-raw,C-u,3,"1,2",2049394,256174.2,0.6615 0.6572,False,744.0,1520000000.0,4780000000.0,3074.1,None None,0,0, +samples,5,18,raw-closed,base-iouring-raw,C,3,"1,2,0",2046336,255791.8,0.6701 0.6668,False,735.0,1640000000.0,4640000000.0,3068.9,1230000000 1220000000,0,0, +samples,5,19,raw-closed,base-iouring-raw,B-u,3,"1,5",2067862,258482.5,0.9961 0.9962,True,1076.0,1560000000.0,4660000000.0,3007.9,None None,0,0, +samples,5,20,raw-closed,base-iouring-raw,C-u,3,"1,2",2020076,252509.5,0.6553 0.6533,False,677.0,1610000000.0,4700000000.0,3123.6,None None,0,0, +samples,5,21,http-closed,hyper-tokio,B-u,3,"1,5",1608141,201017.4,0.8572 0.8711,True,1276.0,2180000000.0,4410000000.0,4097.9,None None,0,0, +supplementary,0,0,http-closed,base-native,B,3,"1,5,0",1684532,210560.6,0.8935 0.8921,True,1283.0,2210000000.0,4340000000.0,3888.3,1670000000 1680000000,0,0,3999.998 +supplementary,0,1,raw-closed,base-iouring-raw,C,3,"1,2,0",2044616,255569.1,0.6587 0.6584,False,694.0,1580000000.0,4710000000.0,3076.4,1220000000 1210000000,0,0,3996.868 +supplementary,0,2,raw-closed,base-iouring-raw,C,"3,7","1,2,0",2007945,250985.6,0.6508 0.6512,False,749.0,1640000000.0,4660000000.0,3137.5,1190000000 1200000000,0,0,3984.2 +supplementary,0,3,raw-closed,base-iouring-raw,B,3,"1,5,0",2043002,255370.9,0.9970 0.9969,True,1081.0,1490000000.0,4740000000.0,3049.4,1860000000 1850000000,0,0,3999.997 +supplementary,0,4,http-closed,base-native,C,3,"1,2,0",1649139,206136.7,0.5381 0.5406,False,791.0,2210000000.0,4370000000.0,3990.0,990000000 970000000,0,0,3996.822 +supplementary,0,5,http-closed,base-native,C,"3,7","1,2,0",1674958,209363.7,0.5526 0.5482,False,793.0,2250000000.0,4320000000.0,3922.5,1030000000 1010000000,0,0,3974.474 +supplementary,1,0,raw-closed,base-iouring-raw,B,3,"1,5,0",2021845,252723.5,0.9945 0.9945,True,1079.0,1610000000.0,4650000000.0,3096.2,1840000000 1840000000,0,0,4000.0 +supplementary,1,1,http-closed,base-native,C,3,"1,2,0",1575133,196885.8,0.5126 0.5069,False,780.0,2320000000.0,4310000000.0,4209.2,950000000 930000000,0,0,3991.918 +supplementary,1,2,http-closed,base-native,B,3,"1,5,0",1706734,213336.0,0.9098 0.9064,True,1254.0,2250000000.0,4300000000.0,3837.7,1700000000 1690000000,0,0,3999.997 +supplementary,1,3,http-closed,base-native,C,"3,7","1,2,0",1677576,209691.3,0.5543 0.5522,False,755.0,2270000000.0,4310000000.0,3922.3,1010000000 1000000000,0,0,3962.042 +supplementary,1,4,raw-closed,base-iouring-raw,C,3,"1,2,0",2040980,255114.3,0.6705 0.6662,False,746.0,1500000000.0,4770000000.0,3072.1,1230000000 1240000000,0,0,3996.321 +supplementary,1,5,raw-closed,base-iouring-raw,C,"3,7","1,2,0",2032354,254036.8,0.6799 0.6780,False,800.0,1560000000.0,4710000000.0,3085.1,1290000000 1290000000,0,0,3977.359 +supplementary,2,0,http-closed,base-native,C,3,"1,2,0",1667362,208414.7,0.5547 0.5538,False,848.0,2190000000.0,4370000000.0,3934.4,1010000000 1020000000,0,0,3993.074 +supplementary,2,1,raw-closed,base-iouring-raw,B,3,"1,5,0",2065825,258219.8,0.9968 0.9969,True,1166.0,1530000000.0,4680000000.0,3006.1,1910000000 1910000000,0,0,3999.996 +supplementary,2,2,http-closed,base-native,B,3,"1,5,0",1700303,212531.2,0.8939 0.8938,True,1256.0,2210000000.0,4330000000.0,3846.4,1670000000 1670000000,0,0,3999.995 +supplementary,2,3,http-closed,base-native,C,"3,7","1,2,0",1655707,206958.0,0.5501 0.5474,False,774.0,2170000000.0,4360000000.0,3943.9,1010000000 1000000000,0,0,3978.282 +supplementary,2,4,raw-closed,base-iouring-raw,C,"3,7","1,2,0",1988517,248557.0,0.6530 0.6520,False,733.0,1570000000.0,4730000000.0,3168.2,1200000000 1190000000,0,0,3987.271 +supplementary,2,5,raw-closed,base-iouring-raw,C,3,"1,2,0",2046896,255853.7,0.6675 0.6661,False,762.0,1500000000.0,4780000000.0,3068.1,1230000000 1220000000,0,0,3997.04 +supplementary,3,0,http-closed,base-native,C,3,"1,2,0",1673291,209155.8,0.5515 0.5508,False,732.0,2260000000.0,4320000000.0,3932.4,1030000000 1000000000,0,0,3996.917 +supplementary,3,1,raw-closed,base-iouring-raw,B,3,"1,5,0",2058284,257253.1,0.9953 0.9953,True,1158.0,1440000000.0,4770000000.0,3017.1,1890000000 1900000000,0,0,3999.997 +supplementary,3,2,raw-closed,base-iouring-raw,C,3,"1,2,0",2012560,251561.4,0.6617 0.6639,False,709.0,1540000000.0,4760000000.0,3130.3,1220000000 1230000000,0,0,3996.979 +supplementary,3,3,http-closed,base-native,C,"3,7","1,2,0",1665111,208133.4,0.5480 0.5534,False,772.0,2250000000.0,4340000000.0,3957.7,1010000000 1010000000,0,0,3967.825 +supplementary,3,4,raw-closed,base-iouring-raw,C,"3,7","1,2,0",2030503,253805.8,0.6611 0.6599,False,762.0,1570000000.0,4710000000.0,3092.8,1220000000 1210000000,0,0,3992.671 +supplementary,3,5,http-closed,base-native,B,3,"1,5,0",1695351,211913.2,0.8894 0.8907,True,1243.0,2230000000.0,4320000000.0,3863.5,1660000000 1660000000,0,0,3999.996 +supplementary,4,0,http-closed,base-native,C,3,"1,2,0",1686038,210749.3,0.5464 0.5439,False,673.0,2180000000.0,4400000000.0,3902.6,1000000000 1000000000,0,0,3997.099 +supplementary,4,1,http-closed,base-native,C,"3,7","1,2,0",1659058,207376.2,0.5391 0.5419,False,720.0,2260000000.0,4340000000.0,3978.2,1000000000 990000000,0,0,3988.669 +supplementary,4,2,raw-closed,base-iouring-raw,C,3,"1,2,0",2043247,255398.6,0.6588 0.6586,False,678.0,1540000000.0,4760000000.0,3083.3,1210000000 1200000000,0,0,3996.878 +supplementary,4,3,raw-closed,base-iouring-raw,C,"3,7","1,2,0",2028901,253604.6,0.6658 0.6675,False,770.0,1590000000.0,4710000000.0,3105.1,1220000000 1220000000,0,0,3996.822 +supplementary,4,4,raw-closed,base-iouring-raw,B,3,"1,5,0",2061058,257620.2,0.9972 0.9971,True,1061.0,1520000000.0,4700000000.0,3017.9,1900000000 1900000000,0,0,3999.994 +supplementary,4,5,http-closed,base-native,B,3,"1,5,0",1663051,207875.9,0.8792 0.8803,True,1259.0,2220000000.0,4310000000.0,3926.5,1630000000 1640000000,0,0,4000.0 +supplementary,5,0,raw-closed,base-iouring-raw,C,"3,7","1,2,0",2029222,253645.3,0.6737 0.6711,False,741.0,1550000000.0,4730000000.0,3094.8,1240000000 1240000000,0,0,3996.625 +supplementary,5,1,raw-closed,base-iouring-raw,C,3,"1,2,0",2031025,253870.3,0.6566 0.6520,False,693.0,1550000000.0,4750000000.0,3101.9,1210000000 1190000000,0,0,3996.999 +supplementary,5,2,http-closed,base-native,C,3,"1,2,0",1661140,207636.6,0.5447 0.5471,False,775.0,2330000000.0,4260000000.0,3967.2,1000000000 1000000000,0,0,3996.812 +supplementary,5,3,http-closed,base-native,B,3,"1,5,0",1699086,212380.0,0.8898 0.8835,True,1228.0,2230000000.0,4290000000.0,3837.4,1640000000 1640000000,0,0,3999.997 +supplementary,5,4,http-closed,base-native,C,"3,7","1,2,0",1639984,204992.3,0.5420 0.5445,False,850.0,2200000000.0,4380000000.0,4012.2,1000000000 1000000000,0,0,3992.848 +supplementary,5,5,raw-closed,base-iouring-raw,B,3,"1,5,0",2033634,254195.0,0.9934 0.9933,True,1089.0,1540000000.0,4660000000.0,3048.7,1880000000 1880000000,0,0,3997.55 diff --git a/benchmarks/hosts/hana.md b/benchmarks/hosts/hana.md new file mode 100644 index 0000000..c889729 --- /dev/null +++ b/benchmarks/hosts/hana.md @@ -0,0 +1,75 @@ +# hana + +Gathered 9 November 2026. hana, reached as `hana.local`, is a physical Linux server used as a measuring host. It is shared with light services, so it is quiet rather than isolated: a media server with occasional transcoding, SMB file sharing, a key-value store, a mail transfer agent, monitoring and endpoint security agents, ZFS's event daemon, SSH, and other users' sessions. Don't change its configuration, and don't use root. + +## Hardware + +| | | +| --- | --- | +| CPU | Intel Xeon E3-1240 v6 (Kaby Lake, family 6, model 158, stepping 9), microcode `0xf8`, one socket | +| Cores and threads | 4 cores, 8 threads; SMT siblings are CPUs `N` and `N + 4`: 0 and 4, 1 and 5, 2 and 6, 3 and 7 | +| Caches | 32 KiB L1 data and 32 KiB L1 instruction per core, 256 KiB L2 per core, 8 MiB L3 shared | +| Frequency | 3.7 GHz base, 4.1 GHz maximum turbo, 0.8 GHz minimum; busy CPUs ran at about 4.0 GHz with two to three cores loaded | +| Memory | 62 GiB (`MemTotal` 65,653,900 kB), 93 GiB swap; type and speed need root (`dmidecode`) and are not recorded | +| NUMA | One node, CPUs 0–7 | + +## Software and settings + +| | | +| --- | --- | +| Distribution and kernel | Arch Linux (rolling), Linux 7.2.14-arch1-1, `PREEMPT_DYNAMIC`, root on ZFS | +| Kernel command line | No CPU isolation, `nohz_full` or mitigation options: `isolated` and `nohz_full` are empty | +| Frequency scaling | `intel_pstate`, active, with hardware P-states; governor `powersave` and energy preference `balance_performance` on every CPU; turbo enabled (`no_turbo` 0), `hwp_dynamic_boost` 1 | +| SMT | On (`smt/control`) | +| Mitigations | Kernel defaults: PTI for Meltdown, IBRS for Spectre v2 and Retbleed, buffer clearing for MDS and MMIO stale data, which report "SMT vulnerable", TSX disabled; see `/sys/devices/system/cpu/vulnerabilities` | +| Network | Benchmarks use loopback: MTU 65,536, qdisc `noqueue`. Two four-queue Ethernet interfaces are bonded; their eight queue interrupts are spread one per CPU over CPUs 1–7 (CPU 4 has two), so every measuring CPU also takes the host's network interrupts. The SATA controller's interrupt is on CPU 0. Default IRQ affinity is all CPUs. | +| Network settings | `net.core.default_qdisc` `fq_codel`, `tcp_congestion_control` `cubic`, `somaxconn` 4096, `busy_poll` and `busy_read` 0 | +| Rust | 1.99.1, the distribution's package (`/usr/bin/cargo`) | + +## CPU placement + +For base-http's hello benchmark, from the [client layout screen](../hello/results/client-layout-20261109/readme.md): + +| CPUs | Use | +| --- | --- | +| 0 and 4 | The OS, the runner (`base-benchmark`) and the client's controller when it is pinned | +| 1 and 2 | The closed-loop client's two workers, one core each | +| 3 | The server; the hello backends are single-threaded | +| 5, 6 and 7 | Idle: the client's and the server's SMT siblings | + +- `raw-closed` and `http-closed`: `--role-cpus client=1,2 --role-cpus server=3`. base-benchmark-load runs `--workers 2` within CPUs 1 and 2, its controller with them; it wakes only to mark the window. Run directly, without base-benchmark, pin them instead with `--cpus 1,2 --controller-cpu 0`, which measured the same. +- Not two workers on one core's siblings (`client=1,5`): both saturated to drive what two cores drove at two-thirds busy, and that layout changed the server's CPU per request. Not one worker: it limited `raw-closed` by 5–6%. +- The server's sibling (3,7) did not help a single-threaded backend. +- `http-rate` and `raw-rate` run one worker, and the screen did not measure them. They have used `--role-cpus client=1,5 --role-cpus server=2,3`, the client's one core with its controller on the sibling and the server on two cores, until a screen of them says otherwise. + +Earlier investigations used other placements, recorded with their results, such as the server on CPU 0 and the client on CPUs 2, 3, 6 and 7 in the [Linux epoll and io_uring investigation](../hello/hana-linux.md) (Linux 7.2.11 then), the [equal offered-rate follow-up](../hello/fixed-rate.md) and the [echo benchmark](../echo/readme.md). + +## Measuring + +- Take the host lock for every measurement, `flock /tmp/base-benchmark-host.lock`, as base-benchmark does, and check `pgrep -af 'base-benchmark|wrk|hello-benchmark'` for another session's run, waiting for it rather than interfering. +- Build and prepare before a campaign, not during one, and run nothing else while measuring. +- Single rounds in the screen still ran up to 8% below their cohort's median, so interleave rounds and read ranges as well as medians. + +## Gathered with + +All without root, on 9 November 2026: + +```sh +lscpu; lscpu -e; lscpu -C +grep -m1 'model name' /proc/cpuinfo; grep -m1 microcode /proc/cpuinfo +cat /sys/devices/system/cpu/cpu[0-3]/topology/thread_siblings_list +cat /sys/devices/system/cpu/cpu0/cpufreq/{scaling_driver,scaling_governor,energy_performance_preference,cpuinfo_min_freq,cpuinfo_max_freq,base_frequency} +cat /sys/devices/system/cpu/cpu*/cpufreq/{scaling_governor,energy_performance_preference} | sort | uniq -c +cat /sys/devices/system/cpu/intel_pstate/{status,no_turbo,hwp_dynamic_boost} +cat /sys/devices/system/cpu/{smt/control,isolated,nohz_full} +grep . /sys/devices/system/cpu/vulnerabilities/* +cat /proc/cmdline; uname -a; head -4 /etc/os-release +free -h; grep MemTotal /proc/meminfo; ls /sys/devices/system/node/ +dmidecode -t memory # needs root: not readable +ip -br link; ip link show lo +cat /proc/sys/net/core/{default_qdisc,somaxconn,busy_poll,busy_read} /proc/sys/net/ipv4/tcp_congestion_control +awk 'NR == 1 || /eno|enp|ahci/' /proc/interrupts; cat /proc/irq/default_smp_affinity +systemctl list-units --type=service --state=running +rustc --version; cargo --version +grep 'cpu MHz' /proc/cpuinfo # sampled during measurement, by layout.py --frequencies +``` diff --git a/benchmarks/hosts/readme.md b/benchmarks/hosts/readme.md new file mode 100644 index 0000000..45cea79 --- /dev/null +++ b/benchmarks/hosts/readme.md @@ -0,0 +1,13 @@ +# Benchmark hosts + +Each host that base-http's benchmarks are measured on has one document here, named after the host, such as [hana.md](hana.md). Benchmark readmes link to it rather than repeat its hardware or CPU placement. + +A host document records: + +- what the host is and how to reach it, and what else runs on it, by kind of service, without naming people; +- its hardware: CPU model, cores, threads and the SMT sibling layout, caches, base and turbo frequency, memory size and, where readable, type and speed, and NUMA nodes; +- its software and settings: frequency scaling driver, governor and energy preference, kernel, distribution and command line, CPU vulnerability mitigations, network interfaces and their interrupt placement, and loopback settings, since the benchmarks run over loopback, and the Rust toolchain used; +- the CPU placement for client and server benchmarks: which CPUs the client's workers, its controller, the server, and the runner and OS use, which SMT siblings stay idle, and the measurements that support it; +- the date it was gathered and the commands that gathered it, all readable without root, so that it can be gathered again. + +Gather it again when the host's kernel, firmware, settings or toolchain change, and keep the previous values where results depend on them. A placement changes only with measurements behind it.