Add --count N to radius test: repeat checks and report aggregate flak…

README.md

@@ -189,6 +189,8 @@ $ authhound-probe radsec test --server radius.corp.com \
 | `--password-file FILE` | Password for a `user`-only `--pap/--peap/--ttls`, from a file (non-interactive). |
 | `--client-cert FILE` `--client-key FILE` | Run an EAP-TLS test with this client certificate + key (PEM). |
 | `--mtu` | Run the path-MTU / fragmentation probe (sends a few padded packets). |
+| `--count N` | Run the checks `N` times (2–50) and report aggregate statistics — see [Chasing intermittent failures](#chasing-intermittent-failures). |
+| `--interval DURATION` | Pause between `--count` iterations (default `2s`; a hard-coded safety floor applies). |
 | `--nas-port-type wireless\|ethernet\|virtual` | How the probe presents itself, so server policies match (default `wireless`). |
 | `--server-name NAME` | Expected server-certificate name (TLS SNI). |
 | `--nas-id NAME` | NAS-Identifier to send (default `authhound-probe`). |
@@ -197,6 +199,53 @@ $ authhound-probe radsec test --server radius.corp.com \
 | `--strict` | Exit non-zero on **warnings** too (e.g. a soon-to-expire cert), for scheduled monitoring. |
 | `--no-color` | Force plain output. Colour is auto-detected otherwise — see [Colour](#colour-and-windows-terminals). |
 
+### Chasing intermittent failures
+
+A single-shot PASS proves nothing about the failure that hits one user in ten.
+When the complaint is "Wi-Fi drops people randomly" or "auth works, except when
+it doesn't", run the same checks repeatedly and look at the distribution:
+
+```console
+$ export AUTHHOUND_SECRET='shared-secret'
+$ authhound-probe radius test --server radius.corp.com --peap alice --count 10
+
+Running 10 iterations, 2s apart
+
+run  1/10  reachability PASS 3ms · shared-secret PASS · peap-mschapv2 PASS
+run  2/10  reachability PASS 41ms · shared-secret PASS · peap-mschapv2 LOST (no reply)
+run  3/10  reachability PASS 2ms · shared-secret PASS · peap-mschapv2 PASS
+...
+
+Aggregate over 10 runs:
+
+PASS  reachability: 10/10 succeeded — stable
+        Latency over 10 answered runs: min 2ms, median 3ms, p95 41ms, max 41ms.
+PASS  shared-secret: 10/10 succeeded — stable
+FAIL  peap-mschapv2: 8/10 succeeded, 2/10 requests lost — consistent with an
+        unstable path or an overloaded/failing server, not a config error
+        Latency over 8 answered runs: min 9ms, median 12ms, p95 96ms, max 96ms.
+```
+
+How to read it:
+
+- **Requests lost** (timeouts) with the rest succeeding → the configuration is
+  fine; suspect the network path or an overloaded/failing server. A p95 far
+  above the median is the same story told by latency.
+- **Failed every run** → not flaky at all; it's a configuration problem
+  (secret, credentials, policy) that a single run would also have caught.
+- Exit code stays `1` if *any* iteration failed, so a flaky server fails a
+  scripted run loudly instead of depending on which iteration you got.
+
+Iterations run sequentially, `--interval` apart (default `2s`). The probe's
+hard-coded rate ceiling still bounds everything: intervals below the safety
+floor are stretched (and the stretch announced), and `--count` is capped at 50.
+This is a diagnosis loop you babysit, not monitoring — it never schedules,
+repeats forever, or stores anything between runs.
+
+With `--json`, each iteration's results plus per-check aggregate statistics
+(success counts, timeouts, latency min/median/p95/max) appear in an additive
+`repeat` block — see the [schema](docs/json-schema.md).
+
 ### Where credentials come from
 
 This tool is meant to run on shared jump boxes, so it never *requires* a secret or

cmd/authhound-probe/main.go

@@ -16,6 +16,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"os/signal"
 	"runtime/debug"
 	"strings"
 	"time"
@@ -186,6 +187,8 @@ func cmdRadiusTest(args []string) int {
 	nasPortType := fs.String("nas-port-type", "wireless", "NAS-Port-Type: wireless, ethernet, or virtual")
 	serverName := fs.String("server-name", "", "expected server certificate name (TLS SNI); optional")
 	mtu := fs.Bool("mtu", false, "run the path-MTU / fragmentation probe (sends a few padded packets)")
+	count := fs.Int("count", 1, "run the checks N times (2..50) and report aggregate statistics — for chasing intermittent failures")
+	interval := fs.Duration("interval", 2*time.Second, "pause between --count iterations (a hard-coded safety floor applies)")
 	timeout := fs.Duration("timeout", 5*time.Second, "per-request timeout")
 	jsonOut := fs.Bool("json", false, "emit results as JSON instead of text")
 	noColor := fs.Bool("no-color", false, "disable ANSI colour")
@@ -216,6 +219,18 @@ func cmdRadiusTest(args []string) int {
 		return 2
 	}
 
+	// --count 1 (the default) is exactly the classic single run; repeat mode is
+	// hard-capped at RepeatCountMax — this is a diagnosis loop a human watches,
+	// not a monitor (that's `connect`).
+	if *count != 1 && (*count < check.RepeatCountMin || *count > check.RepeatCountMax) {
+		fmt.Fprintf(os.Stderr, "error: --count must be between %d and %d\n", check.RepeatCountMin, check.RepeatCountMax)
+		return 2
+	}
+	if provided["interval"] && *count == 1 {
+		fmt.Fprintln(os.Stderr, "error: --interval only makes sense with --count")
+		return 2
+	}
+
 	prompter := credential.Default()
 	secretValue, err := prompter.Resolve(credential.Spec{
 		Name:      "shared secret",
@@ -269,7 +284,6 @@ func cmdRadiusTest(args []string) int {
 		sink = report.NewTextSink(os.Stdout, report.UseColor(os.Stdout, *noColor))
 	}
 
-	runner := check.Runner{Sink: sink}
 	plan := check.Plan{
 		Target: target,
 		Checks: []check.Check{
@@ -283,12 +297,67 @@ func cmdRadiusTest(args []string) int {
 			check.MTUProbe{Enabled: *mtu},
 		},
 	}
+
+	if *count != 1 {
+		return runRepeat(plan, sink, *count, *interval, *strict, *jsonOut)
+	}
+
+	runner := check.Runner{Sink: sink}
 	runner.Run(context.Background(), plan)
 	_ = sink.Close()
 
 	return exitCode(sink, *strict)
 }
 
+// runRepeat drives --count: N sequential iterations, then the aggregate
+// verdicts through the normal sink, so text/JSON rendering and the exit-code
+// contract are identical to a single run. Ctrl-C mid-loop aggregates the
+// iterations that completed instead of throwing them away.
+func runRepeat(plan check.Plan, sink resultSink, count int, interval time.Duration, strict, jsonOut bool) int {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer stop()
+
+	effective, stretched := check.EffectiveInterval(interval)
+	if stretched {
+		// Stderr in both modes: with --json, stdout stays pure JSON (the
+		// document carries interval_stretched for scripts).
+		fmt.Fprintf(os.Stderr, "note: --interval %s is below the probe's hard-coded safety floor; running %s apart instead\n", interval, effective)
+	}
+	opts := check.RepeatOptions{Count: count, Interval: interval}
+	if !jsonOut {
+		fmt.Printf("Running %d iterations, %s apart\n\n", count, effective)
+		opts.OnIteration = func(i int, results []check.Result) {
+			fmt.Println(report.IterationLine(i, count, results))
+		}
+	}
+
+	runner := check.Runner{}
+	run, err := check.RunRepeated(ctx, &runner, plan, opts)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "error:", err)
+		return 2
+	}
+	if completed := len(run.Iterations); completed < count {
+		fmt.Fprintf(os.Stderr, "interrupted — aggregating the %d completed iteration(s)\n", completed)
+		if completed == 0 {
+			return 1
+		}
+	}
+
+	stats := check.AggregateRepeat(run)
+	if !jsonOut {
+		fmt.Printf("\nAggregate over %d runs:\n\n", len(run.Iterations))
+	}
+	for _, s := range stats {
+		sink.Emit(s.Verdict())
+	}
+	if js, ok := sink.(*report.JSONSink); ok {
+		js.SetRepeat(count, run, stats)
+	}
+	_ = sink.Close()
+	return exitCode(sink, strict)
+}
+
 // resultSink is the report-sink surface both subcommands use: the check.ResultSink
 // contract plus the tallies that drive the process exit code.
 type resultSink interface {

cmd/authhound-probe/main_test.go

@@ -38,6 +38,26 @@ func TestExitCode(t *testing.T) {
 	}
 }
 
+// TestCountFlagValidation pins the --count contract: out-of-range counts and a
+// stray --interval are usage errors (exit 2) caught before any credential
+// prompting or network I/O.
+func TestCountFlagValidation(t *testing.T) {
+	cases := []struct {
+		name string
+		args []string
+	}{
+		{"count too high", []string{"--server", "192.0.2.1", "--count", "51"}},
+		{"count zero", []string{"--server", "192.0.2.1", "--count", "0"}},
+		{"count negative", []string{"--server", "192.0.2.1", "--count", "-3"}},
+		{"interval without count", []string{"--server", "192.0.2.1", "--interval", "5s"}},
+	}
+	for _, c := range cases {
+		if got := cmdRadiusTest(c.args); got != 2 {
+			t.Errorf("%s: exit = %d, want 2", c.name, got)
+		}
+	}
+}
+
 func TestResolveVersion(t *testing.T) {
 	orig := version
 	t.Cleanup(func() { version = orig })

docs/json-schema.md

@@ -42,8 +42,9 @@ is a deliberate, reviewed act rather than an accident.
 | Field | Type | Presence | Notes |
 |---|---|---|---|
 | `schema_version` | string | always | Major version of this document shape. Currently `"1"`. |
-| `results` | array of result objects | always | One entry per check, in the order they ran. |
+| `results` | array of result objects | always | One entry per check, in the order they ran. With `--count`, one **aggregate verdict** per check (see the `repeat` block below). |
 | `summary` | object | always | Per-status tally. **All five keys are always present**, including zeros — address `.summary.warn` without checking it exists first. |
+| `repeat` | object | only with `--count` | Additive: per-iteration results and aggregate statistics for a repeated run. Absent on single runs, whose documents are unchanged. |
 
 ### `summary` object
 
@@ -66,9 +67,44 @@ Each entry in `results`:
 | `summary` | string | always | One plain-English line describing the outcome. |
 | `detail` | string | when present | Extra context. Omitted when empty. |
 | `hint` | string | when present | Multi-line, paste-ready remediation. Newline formatting is significant. Omitted when empty. Never contains secrets. |
-| `fields` | object (string→string) | when present | Structured extras such as `rtt_ms`, `tls_version`, `not_after`, `subject`, `san`, `chain_len`, `source_ip`. Keys vary by check; values are always strings. Omitted when there are none. |
+| `fields` | object (string→string) | when present | Structured extras such as `rtt_ms`, `tls_version`, `not_after`, `subject`, `san`, `chain_len`, `source_ip`. `timeout: "true"` marks a request that got no reply at all (a *lost* request, as opposed to a processed rejection). Aggregate verdicts under `--count` add `success_rate`, `attempts`, `successes`, `timeouts`, and `latency_{min,median,p95,max}_ms`. Keys vary by check; values are always strings. Omitted when there are none. |
 | `duration_ns` | integer | when present | How long the check took, in nanoseconds. Omitted when zero. |
 
+## `repeat` block (`--count`)
+
+`radius test --count N` runs the checks N times; the document then carries one
+**aggregate verdict per check** in `results` (so `summary` and the exit code
+reflect the whole run — any failed iteration fails the run), and this block with
+the raw material:
+
+```json
+"repeat": {
+  "count": 10,
+  "completed": 10,
+  "interval_ms": 2000,
+  "requested_interval_ms": 2000,
+  "interval_stretched": false,
+  "iterations": [ { "results": [ /* result objects, one per check */ ] } ],
+  "aggregate": [
+    {
+      "check": "peap-mschapv2",
+      "attempts": 10, "successes": 8, "failures": 2, "timeouts": 2, "skipped": 0,
+      "latency_ms": { "min": 9, "median": 12, "p95": 96, "max": 96 }
+    }
+  ]
+}
+```
+
+| Field | Type | Notes |
+|---|---|---|
+| `count` | integer | Requested iterations (2–50). |
+| `completed` | integer | Iterations that actually finished (lower than `count` after Ctrl-C). |
+| `interval_ms` | integer | Pause between iterations actually used. |
+| `requested_interval_ms` | integer | The `--interval` that was asked for. |
+| `interval_stretched` | boolean | `true` when the requested interval was below the hard-coded safety floor and got stretched. |
+| `iterations` | array | One entry per completed iteration, each with its `results` (same result-object shape as the top level). |
+| `aggregate` | array | Per-check tallies. `successes` = the server answered and processed the request (pass/warn/info); `timeouts` = the subset of `failures` where no reply arrived at all. `latency_ms` (nearest-rank percentiles over answered runs) is omitted when nothing was answered. |
+
 ### `status` values
 
 | Value | Meaning | Effect on exit code |
@@ -96,6 +132,9 @@ authhound-probe radius test --server r --json | jq '.summary.fail'
 
 # Pin to the schema you coded against:
 ... --json | jq -e '.schema_version=="1"' >/dev/null || echo "schema changed"
+
+# --count: which checks lost requests, from the aggregate block:
+... --count 10 --json | jq -r '.repeat.aggregate[] | select(.timeouts > 0) | "\(.check): \(.timeouts) lost"'
 ```
 
 ## Exit codes

internal/check/common.go

@@ -33,3 +33,17 @@ func addCommon(p *radius.Packet, t Target) {
 		p.Add(a.Type, a.Value)
 	}
 }
+
+// TimeoutField marks a Result whose underlying request got no reply, so
+// aggregate reporting (--count) can count lost requests separately from
+// processed rejections. Additive within schema major "1".
+const TimeoutField = "timeout"
+
+// markTimeout tags r as a timeout result (see TimeoutField).
+func markTimeout(r Result) Result {
+	if r.Fields == nil {
+		r.Fields = map[string]string{}
+	}
+	r.Fields[TimeoutField] = "true"
+	return r
+}

internal/check/eaptls.go

@@ -2,6 +2,7 @@ package check
 
 import (
 	"context"
+	"errors"
 
 	"github.com/authhound/probe/internal/radius"
 )
@@ -31,13 +32,17 @@ func (c ServerCert) Run(ctx context.Context, t Target) Result {
 
 	captured, err := sess.InspectServerCert(ctx, c.ServerName)
 	if err != nil {
-		return Result{
+		r := Result{
 			Check: "server-cert", Status: StatusSkip,
 			Summary: "Could not inspect the server certificate",
 			Detail: "The PEAP/TLS handshake didn't get far enough to read the certificate: " +
 				err.Error() + ". This is expected if the server doesn't offer PEAP or EAP-TLS, " +
 				"or if reachability/secret checks above failed.",
 		}
+		if errors.Is(err, radius.ErrTimeout) {
+			r = markTimeout(r)
+		}
+		return r
 	}
 	return analyzeCert("server-cert", captured.Chain, captured.TLSVersion)
 }

internal/check/eaptls_auth.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 
 	"github.com/authhound/probe/internal/radius"
@@ -64,7 +65,11 @@ func (c EAPTLS) Run(ctx context.Context, t Target) Result {
 
 	res, err := sess.AuthEAPTLS(ctx, cert, c.ServerName)
 	if err != nil {
-		return Result{Check: "eap-tls", Status: StatusFail, Summary: "EAP-TLS exchange failed: " + err.Error()}
+		r := Result{Check: "eap-tls", Status: StatusFail, Summary: "EAP-TLS exchange failed: " + err.Error()}
+		if errors.Is(err, radius.ErrTimeout) {
+			r = markTimeout(r)
+		}
+		return r
 	}
 
 	fields := map[string]string{"identity": identity}

internal/check/pap.go

@@ -41,7 +41,7 @@ func (c PAP) Run(ctx context.Context, t Target) Result {
 	reply, _, _, err := radius.Exchange(t.Address, t.Secret, p, t.Timeout)
 	if err != nil {
 		if errors.Is(err, radius.ErrTimeout) {
-			return Result{Check: "pap-auth", Status: StatusSkip, Summary: "No reply — resolve reachability first"}
+			return markTimeout(Result{Check: "pap-auth", Status: StatusSkip, Summary: "No reply — resolve reachability first"})
 		}
 		return Result{Check: "pap-auth", Status: StatusFail, Summary: "PAP exchange failed: " + err.Error()}
 	}

internal/check/peap.go

@@ -2,6 +2,7 @@ package check
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"strconv"
 
@@ -41,13 +42,17 @@ func (c PEAPMSCHAPv2) Run(ctx context.Context, t Target) Result {
 
 	res, err := sess.AuthPEAPMSCHAPv2(ctx, c.User, c.Pass, c.ServerName)
 	if err != nil {
-		return Result{
+		r := Result{
 			Check: "peap-mschapv2", Status: StatusFail,
 			Summary: "PEAP-MSCHAPv2 exchange did not complete",
 			Detail: "The tunnel or inner exchange broke before a verdict: " + err.Error() +
 				". If reachability/secret above failed, fix those first; otherwise the server " +
 				"may not offer PEAP-MSCHAPv2.",
 		}
+		if errors.Is(err, radius.ErrTimeout) {
+			r = markTimeout(r)
+		}
+		return r
 	}
 
 	fields := map[string]string{}

internal/check/reachability.go

@@ -37,6 +37,7 @@ func (Reachability) Run(ctx context.Context, t Target) Result {
 		}
 	}
 	if errors.Is(err, radius.ErrTimeout) {
+		fields[TimeoutField] = "true"
 		srcIP := "<this host's IP>"
 		var te *radius.TimeoutError
 		if errors.As(err, &te) && te.LocalIP != "" {