docs(loader): remove stale versions and counts

.cargo/config.toml

@@ -1,17 +1,8 @@
-# Cargo build configuration for dfe-loader
-#
-# Target platforms: Linux x86_64 and aarch64 only
-# SIMD optimizations enabled for sonic-rs
-
-# =============================================================================
-# Per-target CPU optimizations
-# =============================================================================
-# x86_64: AVX2 + SSE4.2 (available on all modern Intel/AMD since ~2013)
-# aarch64: NEON (baseline for all ARM64)
+# SIMD for sonic-rs: x86-64-v3 (AVX2) on x86_64, NEON is baseline on aarch64.
 
 [target.x86_64-unknown-linux-gnu]
-rustflags = ["-C", "target-cpu=x86-64-v3", "-C", "link-arg=-fuse-ld=mold"]  # AVX2, BMI1/2, FMA (Haswell+)
+rustflags = ["-C", "target-cpu=x86-64-v3", "-C", "link-arg=-fuse-ld=mold"]
 
 [target.aarch64-unknown-linux-gnu]
 linker = "aarch64-linux-gnu-gcc"
-rustflags = ["-C", "target-cpu=generic"]    # NEON is baseline for ARM64
+rustflags = ["-C", "target-cpu=generic"]

.config/nextest.toml

@@ -17,19 +17,14 @@ not (
     or test(/^integration::container_hygiene::(a_started_container_carries_the_name_and_label_then_goes_away|the_shared_container_is_named_and_labelled_for_the_run|the_watcher_removes_the_container_once_its_owner_exits|a_sweep_leaves_another_owners_live_container_and_takes_a_lapsed_one)$/)
 )
 '''
-# No retries, deliberately. Retries here previously masked three real defects:
-# an amd64-only Kafka image running under emulation, a broker probed before it
-# served consumers, and tests asserting a cold consumer's first poll is clean.
-# A retry would have kept all three green and unfixed.
+# No retries: a retry keeps a real defect green.
 retries = 0
 # These tests are legitimately slower than unit tests; warn rather than kill,
 # but do not let a hung container run forever.
 slow-timeout = { period = "60s", terminate-after = 5 }
 
-# Tests that start a container of their own get their own concurrency budget:
-# each waits for a cold server or broker, and letting all of them start at once
-# starves each other of CPU and pushes them past the readiness timeout. The
-# shared instances start once per run, so tests on them stay fully parallel.
+# Tests that start their own container get a concurrency budget, or cold starts
+# starve each other past the readiness timeout.
 [test-groups]
 own-containers = { max-threads = 2 }
 

.gitleaks.toml

@@ -1,29 +1,4 @@
-# Project-specific gitleaks allowlist
-# Extends the CI base configuration (ci/.gitleaks.toml)
-#
-# The CI runs gitleaks to detect secrets in your code. This file lets you
-# exclude false positives specific to your project.
-#
-# Documentation: https://github.com/gitleaks/gitleaks#configuration
+# Project gitleaks allowlist: https://github.com/gitleaks/gitleaks#configuration
 
 [extend]
 useDefault = true
-
-# Example allowlist - uncomment [[allowlist]] blocks as needed
-# See: https://github.com/gitleaks/gitleaks#configuration
-
-# [[allowlist]]
-# description = "Test files with fake tokens"
-# paths = ['''tests/.*\.rs''', '''tests/fixtures/.*''']
-
-# [[allowlist]]
-# description = "Example configuration files"
-# paths = ['''.env.example''', '''.env.sample''']
-
-# [[allowlist]]
-# description = "Known false positive patterns"
-# regexes = ['''YOUR_.*_TOKEN''', '''test-token''']
-
-# [[allowlist]]
-# description = "Common false positive stopwords"
-# stopwords = ["example", "sample", "test", "mock", "fake", "dummy", "placeholder"]

.hyperi-ci.yaml

@@ -6,44 +6,28 @@
 # Copyright: (c) 2026 HYPERI PTY LIMITED
 classification: product
 
-# Project language (auto-detected if not specified)
-# Values: python, typescript, golang, rust, bash, none
 language: rust
 
 # Quality checks
 quality:
   enabled: true
-  # Advisory ignores fed to cargo-audit and osv-scanner. cargo-deny reads
-  # deny.toml separately; osv-scanner does NOT, so the paste unmaintained
-  # advisory must be ignored here too or the release Quality stage fails.
+  # osv-scanner does not read deny.toml, so its ignores repeat here.
   ignore:
     - tool: osv-scanner
       id: RUSTSEC-2024-0436
       reason: "paste (dtolnay) unmaintained, no replacement; transitive dep. Mirrors the deny.toml ignore. Drop when a maintained fork lands."
-    # GHSA-w9wp-h8wv-79jx / CVE-2026-48504: opentelemetry_sdk 0.31.0 is in the
-    # lock through scalo, in the affected range (<= 0.32.0, fixed in 0.32.1).
-    # The flaw is in BaggagePropagator::extract_with_context, which parses an
-    # inbound W3C Baggage header before enforcing its size limits. It is
-    # unreachable: no Baggage propagator is registered in dfe-loader or in scalo,
-    # so the function is never called. Accepted as not reachable in dfe-infra
-    # docs/THREAT-MODEL.md, accepted risks. scalo cannot move to 0.32 while
-    # metrics-exporter-opentelemetry 0.2.1 requires opentelemetry ^0.31;
-    # hyperi-io/scalo-rs#281 tracks the move. A GitHub advisory with no RustSec
-    # entry, so only osv-scanner sees it and deny.toml needs no matching ignore.
+    # No RustSec entry, so deny.toml needs no matching ignore.
     - tool: osv-scanner
       id: GHSA-w9wp-h8wv-79jx
       reason: "CVE-2026-48504, opentelemetry_sdk 0.31.0 via scalo: unreachable, no Baggage propagator is registered in dfe-loader or scalo. Accepted risk in dfe-infra docs/THREAT-MODEL.md. Blocked on metrics-exporter-opentelemetry 0.2.1 requiring opentelemetry ^0.31; tracked in hyperi-io/scalo-rs#281."
   # Rust: test specific feature sets (avoids --all-features conflicts).
-  # DFE policy 2026-04-17 — jemalloc only; mimalloc removed.
   rust:
     features:
-      - "default"                    # Default features
-      - "default,jemalloc"           # With jemalloc allocator
-      - "default,testcontainers"     # With testcontainers integration tests
-      # Required: transport-memory gates MemoryTransportAdapter, so without
-      # this set the transport-adapter suite is compiled out rather than run.
-      # helm_contract.rs asserts it stays listed.
-      - "default,transport-memory"   # With the in-memory transport adapter
+      - "default"
+      - "default,jemalloc"
+      - "default,testcontainers"
+      # Without it the transport-adapter suite is compiled out. helm_contract.rs asserts it stays.
+      - "default,transport-memory"
 
 # Testing
 test:
@@ -56,7 +40,6 @@ test:
   # matrix and the peer tools. tests/TESTING.md says which tests are which.
   full:
     required_for_release: true
-  # min_coverage: 80       # Fail if coverage below this percentage
 
 # Building
 build:

BENCHMARKS.md

@@ -178,7 +178,7 @@ open target/criterion/report/index.html
 
 Mison-style structural indexing for schema-guided extraction. Based on the VLDB 2017 paper.
 
-### Current Benchmarks (v0.11.0)
+### Benchmarks at v0.11.0
 
 | Benchmark | Mison | Traditional | Ratio |
 |-----------|-------|-------------|-------|

Cargo.toml

@@ -17,10 +17,7 @@ path = "src/lib.rs"
 name = "dfe-loader"
 path = "src/main.rs"
 
-# PGO workload driver — Kafka producer that drives realistic load against a
-# running dfe-loader during hyperi-ci Tier 2 PGO/BOLT builds.
-# `required-features` keeps it out of default builds; hyperi-ci ≥v1.9.2
-# respects this and skips the binary in detect-build-config.
+# PGO/BOLT workload driver, kept out of default builds by `required-features`.
 [[bin]]
 name = "pgo-driver"
 path = "src/bin/pgo_driver.rs"
@@ -31,24 +28,18 @@ required-features = ["pgo-driver"]
 tokio = { version = ">=1.54.1, <2", features = ["full"] }
 tokio-util = { version = ">=0.7.20, <0.8", features = ["rt"] }
 
-# Shared library with transport abstraction. The floor moves with
-# `release.helm.library` in .hyperi-ci.yaml: the library renders only the
-# contract version its scalo release writes.
+# The floor moves with `release.helm.library` in .hyperi-ci.yaml.
 scalo = { version = ">=2.14.29, <3", features = ["config-schema", "transport-kafka", "transport-grpc", "dlq-kafka", "config", "config-reload", "deployment", "version-check", "scaling", "cli-service", "top", "logger", "metrics", "service-metrics", "expression", "memory", "worker-batch", "governor", "secrets", "sink-stack", "geoip-download"] }
 
 # Steps scalo's jittered retry schedule (sink_stack backoff). Range MUST track
 # scalo's: BackoffBuilder only applies to the ExponentialBuilder scalo returns.
 backon = { version = ">=1.6.0, <2", default-features = false, features = ["std"] }
 
-# JSON Schema derivation for the reflectable config-schema capability
-# (scalo-rs#6). Same version range as scalo so cargo unifies to one schemars.
+# Config-schema derivation. Same range as scalo so cargo unifies to one schemars.
 schemars = { version = ">=1.2.3, <2", default-features = false, features = ["std", "derive", "preserve_order", "indexmap2"] }
 
-# CEL expression evaluation (same crate scalo uses, needed for Program type).
-# Range MUST track scalo's, which is `>=0.14.5, <0.15`: we hand cel::Program across
-# the scalo boundary, so a wider range here lets cargo resolve two semver-
-# incompatible cels and Program stops being the same type. In ignoreDeps so it
-# moves only with scalo.
+# Range MUST track scalo's: cel::Program crosses the scalo boundary, and two
+# semver-incompatible cels make it two types. In ignoreDeps, so it moves with scalo.
 cel = { version = ">=0.14.5, <0.15", features = ["json"] }
 
 # JSON parsing (SIMD-accelerated)
@@ -68,9 +59,7 @@ thiserror = ">=2.0.21, <3"
 tracing = ">=0.1.44, <0.2"
 metrics = ">=0.24.6, <0.25"
 
-# Memory management - jemalloc allocator (DFE policy 2026-04-17: jemalloc only)
-# `stats` builds jemalloc with --enable-stats; without it there is no
-# stats.allocated for the memory guard's heap source to read (#162).
+# `stats` gives the memory guard's heap source its stats.allocated.
 tikv-jemalloc-ctl = { version = ">=0.7.0, <0.8", optional = true, features = ["stats"] }
 tikv-jemallocator = { version = ">=0.7.0, <0.8", optional = true }
 
@@ -106,11 +95,7 @@ parking_lot = ">=0.13.0, <0.14"
 crossbeam-channel = ">=0.5.17, <0.6"
 indexmap = { version = ">=2.14.2, <3", features = ["serde"] }
 
-# File watching (config hot-reload) - using polling for S3/NFS/FUSE compatibility
-# notify = ">=6.1"  # Not needed - polling works on all filesystems
-
-# PGO workload driver — Kafka producer for hyperi-ci Tier 2 PGO/BOLT builds.
-# Optional, gated by the `pgo-driver` feature. Main loader binary doesn't link it.
+# The PGO workload driver's producer, behind the `pgo-driver` feature.
 rdkafka = { version = ">=0.39.0, <0.40", features = ["dynamic-linking"], optional = true }
 
 [dev-dependencies]
@@ -123,9 +108,7 @@ test-case = ">=3.4.1, <4"
 # Held on 0.12: 0.13 drops the rustls-tls-native-roots feature.
 reqwest = { version = ">=0.12, <0.13", default-features = false, features = ["blocking", "rustls-tls-native-roots"] }
 
-# The Kafka broker image pin and the TLS / mTLS / SASL_SSL broker fixture the
-# Kafka tests share, so neither is copied here. The production dependency above
-# does not carry this feature.
+# The Kafka tests' broker image pin and TLS/SASL broker fixture.
 scalo = { version = ">=2.14.29, <3", features = ["transport-kafka-test-support"] }
 
 # The status code a raw Push is refused with, through scalo's generated gRPC
@@ -186,37 +169,11 @@ name = "json_depth"
 harness = false
 
 [profile.release]
-lto = "thin"           # Link-time optimization (use "fat" for max perf, slower compile)
-codegen-units = 1      # Single codegen unit for better optimization
-panic = "abort"        # Smaller binary, no unwinding overhead
-strip = true           # Remove debug symbols
-opt-level = 3          # Maximum optimization (default for release, explicit for clarity)
-
-# =============================================================================
-# Advanced Optimization Options (Manual)
-# =============================================================================
-#
-# 1. Profile-Guided Optimization (PGO) - 10-20% improvement:
-#    cargo install cargo-pgo
-#    cargo pgo build
-#    ./target/x86_64-unknown-linux-gnu/release/dfe-loader  # Run with real workload
-#    cargo pgo optimize
-#
-# 2. BOLT Post-Link Optimization (additional 5-15% on top of PGO):
-#    cargo pgo bolt build
-#    ./target/x86_64-unknown-linux-gnu/release/dfe-loader-bolt-instrumented  # Run workload
-#    cargo pgo bolt optimize
-#
-# 3. Combined PGO + BOLT:
-#    cargo pgo optimize
-#    cargo pgo bolt optimize --with-pgo
-#
-# 4. Fat LTO (slower compile, sometimes faster runtime):
-#    Change lto = "thin" to lto = "fat"
-#
-# References:
-# - https://github.com/Kobzol/cargo-pgo
-# - https://nnethercote.github.io/perf-book/build-configuration.html
+lto = "thin"
+codegen-units = 1
+panic = "abort"
+strip = true
+opt-level = 3
 
 [profile.bench]
 inherits = "release"
@@ -229,13 +186,10 @@ strip = false
 
 [lints.rust]
 # Tests require unsafe for env::set_var/remove_var (edition 2024).
-# Production code has zero unsafe blocks — the lib target is clean.
 unsafe_code = "allow"
-# `#[cfg(tarpaulin)]` is set by cargo-tarpaulin at coverage time. Declaring
-# it here silences `unexpected_cfgs` without a build.rs.
+# cargo-tarpaulin sets `cfg(tarpaulin)`.
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tarpaulin)'] }
-# Some clippy allows below are 1.95+-only (e.g. `duration_suboptimal_units`).
-# 1.94 toolchains would otherwise fail with `unknown lint`.
+# Some clippy allows below name lints an older toolchain does not know.
 unknown_lints = "allow"
 
 [lints.clippy]
@@ -247,74 +201,55 @@ dbg_macro = "deny"
 module_name_repetitions = "allow"
 # #[must_use] goes where dropping the value is a bug, not on every pure function.
 must_use_candidate = "allow"
-# 117 doc comments name ClickHouse types and columns without backticks.
+# Doc comments name ClickHouse types and columns without backticks.
 doc_markdown = "allow"
-# 41 public Result fns have no # Errors section, and 4 no # Panics section.
 missing_errors_doc = "allow"
 missing_panics_doc = "allow"
-# 16 sites.
 items_after_statements = "allow"
-# 13 methods would become associated functions, which rewrites every caller.
+# Turning a method into an associated function rewrites every caller.
 unused_self = "allow"
-# 10 sites, and changing a signature rewrites its callers.
 needless_pass_by_value = "allow"
-# 9 sites.
 unreadable_literal = "allow"
-# 8 sites.
 doc_link_with_quotes = "allow"
-# 7 sites.
 match_same_arms = "allow"
-# 5 sites.
 struct_excessive_bools = "allow"
-# 4 functions run long, the largest at 665 lines, and splitting them needs its own change.
 too_many_lines = "allow"
-# 4 sites, and boxing a future changes the hot path, which needs a measurement first.
+# Boxing a future changes the hot path, which needs a measurement first.
 large_futures = "allow"
-# 3 sites.
 similar_names = "allow"
-# 3 sites.
 trivially_copy_pass_by_ref = "allow"
-# 2 sites.
 unused_async = "allow"
-# 20 sites, all in tests/.
+# The next three are tests only.
 manual_let_else = "allow"
-# 8 sites, all in tests/.
 default_trait_access = "allow"
-# 7 sites, all in tests/.
 format_push_string = "allow"
 # Dynamic Map<String, Value> patterns need flexible casting
 cast_possible_truncation = "allow"
 cast_sign_loss = "allow"
 cast_precision_loss = "allow"
 cast_lossless = "allow"
 cast_possible_wrap = "allow"
-# Tests use `let mut cfg = X::default(); cfg.field = ...` for readability over
-# struct-init. Production code is unaffected.
+# Tests set fields after `X::default()` for readability.
 field_reassign_with_default = "allow"
 # Test data sometimes uses literal 3.14 etc. for clarity over `std::f64::consts::PI`.
 approx_constant = "allow"
 # A handful of test/utility helpers exceed the default 7-arg threshold.
 too_many_arguments = "allow"
 # Mostly affects internal types in benches/tests where TupleX is fine.
 type_complexity = "allow"
-# `&Option<T>` appears in a few config setters where call-site ergonomics
-# (`set_x(&self.x)`) outweigh the API guideline.
+# Config setters take `&Option<T>` so call sites read `set_x(&self.x)`.
 ref_option = "allow"
-# Tests use `Duration::from_millis(30000)` etc. for symmetry with config
-# values stored in millis. Style-only.
+# Tests write durations in millis, as config stores them.
 duration_suboptimal_units = "allow"
-# `.map(|x| ...).unwrap_or(false)` is clearer than `.is_ok_and()` for some
-# readers. Stylistic.
 map_unwrap_or = "allow"
 
 [features]
 default = []
-# Allocator (DFE policy 2026-04-17: jemalloc at every channel, no mimalloc)
+# Allocator
 jemalloc = ["dep:tikv-jemallocator", "dep:tikv-jemalloc-ctl"]
 # Transport backends
 transport-memory = ["scalo/transport-memory"]  # Memory transport for unit tests
 # Testing
 testcontainers = []  # The tests that run real ClickHouse and Kafka containers (Docker required)
-# PGO workload driver (hyperi-ci Tier 2 — see scripts/pgo-workload.sh)
-# Pulls in rdkafka for the producer binary; main loader binary is unaffected.
+# PGO workload driver (scripts/pgo-workload.sh). Only that binary links rdkafka.
 pgo-driver = ["dep:rdkafka"]

docker-compose.dev.yaml

@@ -12,8 +12,7 @@ services:
   # ClickHouse for testing inserts
   clickhouse:
     # Matches CLICKHOUSE_TAG in tests/common/containers.rs and dfe-infra's
-    # services.clickhouse-version. The 26.3 minor was a moving tag: every
-    # rebuild of it changed the server without changing the pin.
+    # services.clickhouse-version.
     image: clickhouse/clickhouse-server:26.3.42.3@sha256:21d572843e59539c7d100286b6f5a6053c341fe4b33c2d7b74b1ff7cb24c5399
     container_name: clickhouse-dev
     ports:
@@ -39,8 +38,7 @@ services:
   # Kafka (Redpanda - Kafka-compatible, lighter weight)
   redpanda:
     # Matches KAFKA_TAG in scripts/pgo-workload.sh and dfe-infra's
-    # services.redpanda-version. Two pins of the same broker on different
-    # patches makes Renovate track two lines and open a PR per line.
+    # services.redpanda-version, so Renovate tracks one line.
     image: redpandadata/redpanda:v26.2.6@sha256:8b1076e0d61bfceb66594dc01536427ce0b529f8ead35fdab9763ab6085891ce
     container_name: redpanda-dev
     command:

docs/config-schema.json

@@ -500,7 +500,7 @@
           "default": null
         },
         "allow_insecure_transport": {
-          "description": "Opt in to an unencrypted Kafka transport (`plaintext` / `sasl_plaintext`)\nin production. scalo (>=2.8) rejects unencrypted transports under a\nproduction profile at transport construction unless this is set —\ndata and SASL/PLAIN credentials would otherwise ship in the clear. Leave\n`false` (the default) and configure TLS, or set `true` only when the\nin-cluster traffic is already mesh-encrypted.",
+          "description": "Opt in to an unencrypted Kafka transport (`plaintext` / `sasl_plaintext`)\nin production. scalo rejects unencrypted transports under a\nproduction profile at transport construction unless this is set —\ndata and SASL/PLAIN credentials would otherwise ship in the clear. Leave\n`false` (the default) and configure TLS, or set `true` only when the\nin-cluster traffic is already mesh-encrypted.",
           "type": "boolean",
           "default": false
         },