diff --git a/CLAUDE.md b/CLAUDE.md index 924e430..b08e33e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,6 +2,14 @@ Privacy-first, fully on-device macOS prototype for **EEG-driven communication**. A Muse headband streams brain signals through BrainFlow, a Core ML classifier on the Apple Neural Engine detects intent (jaw clench / blink / rest / select), and a local MLX LLM suggests next words. The user "types" by letting a cycling 3-token carousel highlight a candidate, then committing with a brain-signal selection. **No network at runtime. No cloud. No telemetry.** +**The app is macOS; the EEG validation scripts are not.** `Scripts/validate-muse-physiology.py` +runs on Linux as-is — the PyPI `brainflow` wheel bundles Linux `.so`s including +`libMuseLib.so` and `libBrainFlowBluetooth.so`, so a BLE Muse session needs only +`pip install brainflow`, no source build. The `DYLD_LIBRARY_PATH` block at the top of that +script is guarded by `os.path.isdir` and skips harmlessly off-macOS. Don't route a +validation session to a Mac on the strength of the line above; only the Swift/Core ML/MLX +half needs one. + ## Stack - Swift 6.0, macOS 14+, SwiftPM (no Xcode project needed for the default build) - Deps: `mlx-swift` (≥0.21.2), `mlx-swift-examples` (≥2.21.0), `swift-transformers` (≥0.1.20). **Linked only into `BCILLM`** — single MLX runtime in the linked binary. diff --git a/Package.swift b/Package.swift index da59154..2127f38 100644 --- a/Package.swift +++ b/Package.swift @@ -43,6 +43,7 @@ let package = Package( .executable(name: "GenerationEval", targets: ["GenerationEval"]), .executable(name: "dialectic-smoke", targets: ["DialecticSmoke"]), .executable(name: "dialectic-session", targets: ["DialecticSession"]), + .executable(name: "dialectic-fixture", targets: ["DialecticFixture"]), ], dependencies: [ // MLX runtime + small-model utilities. Pinned conservatively; bump as @@ -263,6 +264,20 @@ let package = Package( path: "Sources/DialecticSession", swiftSettings: strictConcurrency ), + // Emits the cross-language conformance fixture that the Rust port of + // `DialecticalDynamics` asserts against (client-native + // `crates/neuralcompose-hypnagogic`). Depends on BCICore ONLY — no + // cloud, no MLX, no weights, no network — so it builds under plain CLT + // and its output depends on nothing but this repo's own math. + // + // Committed permanently, not a throwaway: a fixture whose generator has + // been deleted cannot be regenerated, so drift becomes unattributable. + .executableTarget( + name: "DialecticFixture", + dependencies: ["BCICore"], + path: "Sources/DialecticFixture", + swiftSettings: strictConcurrency + ), // ── Tests ──────────────────────────────────────────────────────── .testTarget( @@ -321,7 +336,8 @@ let package = Package( .testTarget( name: "WorldModelDemoTests", dependencies: ["WorldModelDemo", "BCICore"], - path: "Tests/WorldModelDemoTests" + path: "Tests/WorldModelDemoTests", + resources: [.process("Fixtures")] ), .testTarget( name: "NeuralComposeAppTests", diff --git a/Scripts/evaluate-imagined-signal.py b/Scripts/evaluate-imagined-signal.py index 5bed12c..5ec383b 100755 --- a/Scripts/evaluate-imagined-signal.py +++ b/Scripts/evaluate-imagined-signal.py @@ -45,7 +45,7 @@ import pandas as pd from scipy import signal from sklearn.metrics import balanced_accuracy_score, confusion_matrix -from sklearn.model_selection import StratifiedKFold +from sklearn.model_selection import GroupKFold, StratifiedKFold from sklearn.pipeline import Pipeline from sklearn.preprocessing import StandardScaler from sklearn.svm import LinearSVC @@ -229,29 +229,61 @@ def trial_features(trial: Trial) -> np.ndarray: return np.array(feats, dtype=np.float64) -def evaluate(trials: list[Trial], folds: int, seed: int) -> dict: - X = np.vstack([trial_features(t) for t in trials]) - y = np.array([t.target for t in trials]) - classes, counts = np.unique(y, return_counts=True) - class_counts = dict(zip(classes.tolist(), counts.tolist())) +def time_blocks(trials: list[Trial], n_blocks: int) -> np.ndarray: + """Contiguous acquisition-time blocks, used as CV groups. + + Trials arrive in acquisition order within a session. Grouping by contiguous + runs — rather than shuffling — is what keeps a fold's test trials from being + temporally neighboured by their own training data. + """ + order = np.lexsort(([t.trial_index for t in trials], + [t.session_id for t in trials])) + groups = np.empty(len(trials), dtype=int) + groups[order] = np.floor(np.arange(len(trials)) * n_blocks / len(trials)).astype(int) + return groups - skf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed) + +def _cv_score(X, y, splitter, split_args) -> tuple[float, list[float], list, list]: fold_scores: list[float] = [] - all_true: list[str] = [] - all_pred: list[str] = [] - for fold_idx, (train_idx, test_idx) in enumerate(skf.split(X, y)): + all_true: list = [] + all_pred: list = [] + for train_idx, test_idx in splitter.split(*split_args): pipe = Pipeline([ ("scaler", StandardScaler()), ("svm", LinearSVC(C=1.0, dual="auto", max_iter=5000)), ]) pipe.fit(X[train_idx], y[train_idx]) pred = pipe.predict(X[test_idx]) - score = balanced_accuracy_score(y[test_idx], pred) - fold_scores.append(score) + fold_scores.append(balanced_accuracy_score(y[test_idx], pred)) all_true.extend(y[test_idx].tolist()) all_pred.extend(pred.tolist()) + return float(np.mean(fold_scores)), fold_scores, all_true, all_pred + + +def evaluate(trials: list[Trial], folds: int, seed: int) -> dict: + X = np.vstack([trial_features(t) for t in trials]) + y = np.array([t.target for t in trials]) + classes, counts = np.unique(y, return_counts=True) + class_counts = dict(zip(classes.tolist(), counts.tolist())) + + # Blocked, not shuffled. StratifiedKFold(shuffle=True) scatters temporally + # adjacent trials across train and test, so anything drifting slowly within a + # session — impedance, electrode temperature, alertness, headband slip — is + # partially learnable rather than held out, and the classifier can score above + # chance without decoding imagined speech at all. Whole contiguous runs are + # held out together instead. Same defect, and same fix, as the eyes-open/closed + # block confound in validate-muse-physiology.py; see + # docs/reviews/time-leakage-2026-08-06.md. + groups = time_blocks(trials, folds) + gkf = GroupKFold(n_splits=folds) + mean, fold_scores, all_true, all_pred = _cv_score(X, y, gkf, (X, y, groups)) + + # Time-index control on the SAME folds: trial position as the only feature. + # A gate that does not check this cannot tell decoding from drift — it is the + # same instrument as the alpha time-index baseline, pointed at the same risk. + t_index = np.arange(len(trials), dtype=float).reshape(-1, 1) + time_mean, _, _, _ = _cv_score(t_index, y, gkf, (t_index, y, groups)) - mean = float(np.mean(fold_scores)) sd = float(np.std(fold_scores, ddof=1)) if len(fold_scores) > 1 else 0.0 # Wilson 95% CI on the pooled accuracy. n = len(all_true) @@ -267,6 +299,9 @@ def evaluate(trials: list[Trial], folds: int, seed: int) -> dict: return { "balanced_accuracy_mean": mean, + "time_index_balanced_accuracy": time_mean, + "beats_time_index": bool(mean > time_mean), + "cv": f"GroupKFold({folds}) on contiguous acquisition blocks", "balanced_accuracy_sd": sd, "fold_scores": fold_scores, "wilson_ci_95": (ci_low, ci_high), @@ -291,7 +326,7 @@ def print_report(sessions: list[Path], trials: list[Trial], result: dict, args) for cls, n in sorted(result["class_counts"].items()): print(f" {cls:>8s}: {n}") print(f"Feature dim: {result['n_features']} ({len(BANDS)} bands × per-channel log-power)") - print(f"CV folds: {args.folds} (stratified, seed={args.seed})") + print(f"CV folds: {args.folds} ({result['cv']})") print() print(f"Per-fold balanced accuracy:") for i, s in enumerate(result["fold_scores"]): @@ -312,16 +347,23 @@ def print_report(sessions: list[Path], trials: list[Trial], result: dict, args) # Gate evaluation. min_count = min(result["class_counts"].values()) if result["class_counts"] else 0 bacc = result["balanced_accuracy_mean"] + time_bacc = result["time_index_balanced_accuracy"] bacc_ok = bacc >= GATE_BALANCED_ACCURACY count_ok = min_count >= GATE_MIN_PER_CLASS + time_ok = result["beats_time_index"] print(f"Pre-registration gate:") print(f" balanced_accuracy ≥ {GATE_BALANCED_ACCURACY:.2f}: " f"{bacc:.4f} {'✓' if bacc_ok else '✗'}") print(f" min trials/class ≥ {GATE_MIN_PER_CLASS}: " f"{min_count} {'✓' if count_ok else '✗'}") + print(f" beats time-index baseline: " + f"{bacc:.4f} vs {time_bacc:.4f} {'✓' if time_ok else '✗'}") + if not time_ok: + print(" ** trial position predicts the label as well as the EEG does.") + print(" Whatever this is measuring, it is not imagined speech.") print() - if bacc_ok and count_ok: + if bacc_ok and count_ok and time_ok: print("[PRE-REGISTRATION GATE PASSED]") return True else: @@ -329,7 +371,79 @@ def print_report(sessions: list[Path], trials: list[Trial], result: dict, args) return False +def _drift_trials(n=160, fs=256.0, run_len=4, seed=0) -> list[Trial]: + """Trials whose only structure is drift + the run structure of a shuffled order. + + This is the real leak, not a caricature of it. Trial order is a 50/50 shuffle + (as ImaginedSpeechProtocol.buildTrialOrder produces), which by chance yields + RUNS of same-class trials. Amplitude drifts monotonically with acquisition + position and carries no class information whatsoever. + + Within one run, the drift value is nearly constant and identifies that run — + and the run identifies the class. So a shuffled split, which puts a trial's + immediate neighbours in the training set, can read the label off the drift. + A blocked split holds whole runs out together and cannot. + """ + rng = np.random.default_rng(seed) + labels: list[str] = [] + while len(labels) < n: + lab = "yes" if len(labels) // run_len % 2 == 0 else "no" + labels.extend([lab] * run_len) + labels = labels[:n] + rng.shuffle(labels) # keeps the 50/50 balance, keeps runs by chance + trials = [] + for i, lab in enumerate(labels): + amp = 10.0 * (1.0 + 0.05 * i) # depends on POSITION, never on lab + ch = rng.normal(0.0, amp, (4, int(fs))) + trials.append(Trial(session_id="synthetic", trial_index=i, target=lab, + channels=ch, sample_rate=fs)) + return trials + + +def demo() -> int: + """Self-check, no hardware: ./evaluate-imagined-signal.py --self-check + + NOTE what this does and does not assert. It asserts the gate rejects data with + no class signal. It does NOT assert that the old shuffled-fold evaluator + inflated the score, because that could not be demonstrated — see the measured + non-result printed below and docs/reviews/time-leakage-2026-08-06.md. + """ + ok = True + for seed in range(4): + trials = _drift_trials(seed=seed) + X = np.vstack([trial_features(t) for t in trials]) + y = np.array([t.target for t in trials]) + groups = time_blocks(trials, 5) + + blocked, *_ = _cv_score(X, y, GroupKFold(n_splits=5), (X, y, groups)) + t_index = np.arange(len(trials), dtype=float).reshape(-1, 1) + time_only, *_ = _cv_score(t_index, y, GroupKFold(n_splits=5), (t_index, y, groups)) + leaky, *_ = _cv_score(X, y, StratifiedKFold(5, shuffle=True, random_state=1337), (X, y)) + + # The property that matters: no class signal in, no pass out. + assert not (blocked >= GATE_BALANCED_ACCURACY and blocked > time_only), ( + f"seed {seed}: drift-only data passed the gate — " + f"blocked={blocked:.4f} time-index={time_only:.4f}") + if leaky >= GATE_BALANCED_ACCURACY: + ok = False + print(f" seed {seed}: shuffled CV DID clear the gate ({leaky:.4f}) — " + "the leak reproduces after all; update the review doc") + print(f" seed {seed}: blocked={blocked:.4f} time-index={time_only:.4f} " + f"shuffled={leaky:.4f}") + + print("self-check ok — drift-only data is rejected by the gate.") + if ok: + print("Measured non-result: shuffled CV did NOT clear the 0.65 gate on drift-only " + "data either. LinearSVC on 16 global band-power features has too little " + "capacity to exploit fold-local structure, so no inflation was demonstrated. " + "GroupKFold is kept as the conservative design, not as a fix for a measured " + "defect.") + return 0 + + def main() -> int: + if "--self-check" in sys.argv: + return demo() parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument("paths", nargs="*", type=Path, diff --git a/Scripts/validate-muse-physiology.py b/Scripts/validate-muse-physiology.py index 602e09d..df93c98 100755 --- a/Scripts/validate-muse-physiology.py +++ b/Scripts/validate-muse-physiology.py @@ -52,6 +52,7 @@ import json import signal import datetime +import argparse import numpy as np # Set up BrainFlow before import @@ -59,13 +60,48 @@ if os.path.isdir(BF_LIB): os.environ["DYLD_LIBRARY_PATH"] = BF_LIB + ":" + os.environ.get("DYLD_LIBRARY_PATH", "") -from brainflow.board_shim import BoardShim, BoardIds, BrainFlowInputParams # noqa: E402 +try: + from brainflow.board_shim import BoardShim, BoardIds, BrainFlowInputParams # noqa: E402 +except ImportError: # --self-check analyses synthetic arrays and needs no hardware lib + BoardShim = BoardIds = BrainFlowInputParams = None # --- Configuration --------------------------------------------------------- SAMPLE_RATE = 256 # Muse S native -SEG_OPEN = 30.0 # seconds, eyes open -SEG_CLOSED = 30.0 # seconds, eyes closed + +# Alpha contrast: four counterbalanced blocks, not one open then one closed. +# +# Number the blocks by position 1-4. Under ABBA the open blocks sit at 1 and 4 +# (mean position 2.5) and the closed at 2 and 3 (also 2.5), so both conditions +# occupy the same average point in the session and a LINEAR drift — impedance +# settling, gel warming, drowsiness onset — subtracts out of the ratio. Under the +# old `open, closed` every bit of drift landed on `closed` and was indistinguishable +# from alpha. Under ABAB the closed mean is still one position later, leaving a +# residual that always points the same way. +# +# ABBA cancels a linear trend EXACTLY, and electrode settling is usually curved — +# fast, then flattening. Under a convex or concave trend, equal mean position is not +# sufficient: a residual survives whose sign depends on which way the curve bends +# (decaying settling biases the ratio conservatively, saturating warming biases it +# the wrong way). Measuring that leftover is what the time-index baseline in +# analyze() is for. The two are not redundant — this constant removes the bulk, +# that gate measures what curvature left behind. +BLOCK_SECONDS = 15.0 # 4 x 15 s = same 60 s of alpha data +BLOCK_ORDERS = { + 'ABBA': ['open', 'closed', 'closed', 'open'], + 'BAAB': ['closed', 'open', 'open', 'closed'], +} +# Synthetic controls remain pinned to ABBA. Live acquisition passes its selected +# order explicitly so alternating sessions never changes the fixtures. +ALPHA_BLOCK_ORDER = BLOCK_ORDERS['ABBA'] + +# Occipital alpha takes ~1-2 s to rise after eye closure. Including the build-up +# biases closed blocks downward and makes the ratio sensitive to block length. +# Trimmed from the START OF EVERY BLOCK, open and closed alike — trimming both +# conditions symmetrically is what keeps the discard from being its own thumb on +# the scale. +BLOCK_EDGE_DISCARD_S = 2.0 + SEG_BLINK = 5.0 # seconds, blink deliberately SEG_CLENCH = 8.0 # seconds, jaw clench (3s clench, 1s release, 2 cycles = 8s) SEG_TURN = 10.0 # seconds, head turn @@ -84,11 +120,83 @@ HEAD_TURN_UV = 30.0 # slow turns are 30-80 µV; jerky is 100+ CLENCH_BETA_RATIO = 2.0 # true EMG bursts are broadband >50 Hz, but 13-30 should rise CLENCH_BROADBAND_RATIO = 1.5 # 30-100 Hz broadband rise; EMG is broadband, single-band tests miss it -MIN_VALID_SAMPLES = SEG_OPEN * SAMPLE_RATE * 0.8 # 80% of expected +CONTACT_RMS_MIN_UV = 2.0 +CONTACT_RMS_MAX_UV = 200.0 + +# The condition ratio must also beat the drift the time-index baseline measures. +# A multiplier, not a bare `>`, because the residual ABBA leaves behind is +# empirical — it depends on how this unit's electrodes settle — and only real +# sessions can say how noisy the estimate is. This is the calibration knob. +TIME_BASELINE_MARGIN = 1.0 + +# Temporal permutation baseline. Ten seeds matches the paper's protocol. The +# 3x collapse bound is conservative: theory gives B/4, roughly 11-32x for the +# analysis bandwidths used here. +SHUFFLE_SEEDS = 10 +SHUFFLE_COLLAPSE_FACTOR = 3.0 + +# Require 80% of each segment's expected samples. The duration must remain +# segment-specific: alpha, blink, clench, and turn intentionally differ. +MIN_VALID_FRACTION = 0.8 # --- Utilities ------------------------------------------------------------- +def is_short_segment(n_samples, duration): + return n_samples < duration * SAMPLE_RATE * MIN_VALID_FRACTION + + +def classify_contact(rms): + saturated = [ch for ch in CHANNEL_NAMES + if np.isfinite(rms[ch]) and rms[ch] > CONTACT_RMS_MAX_UV] + dead = [ch for ch in CHANNEL_NAMES + if not np.isfinite(rms[ch]) or rms[ch] < CONTACT_RMS_MIN_UV] + healthy = [ch for ch in CHANNEL_NAMES + if CONTACT_RMS_MIN_UV <= rms[ch] <= CONTACT_RMS_MAX_UV] + return healthy, saturated, dead + + +def contact_check_record(rms): + healthy, saturated, dead = classify_contact(rms) + return { + 'rms_uv': { + name: float(rms[name]) if np.isfinite(rms[name]) else None + for name in CHANNEL_NAMES + }, + 'healthy': healthy, + 'saturated': saturated, + 'dead': dead, + 'passed': not saturated and not dead, + } + + +def alpha_block_contact_records(alpha_blocks, block_order): + records = [] + for block_index, (block, label) in enumerate( + zip(alpha_blocks, block_order), start=1): + rms = { + name: float(np.sqrt(np.mean(block[ch_idx, :].astype(np.float64) ** 2))) + for name, ch_idx in zip(CHANNEL_NAMES, EEG_CHANNELS) + } + record = contact_check_record(rms) + record.update(block_index=block_index, label=label) + records.append(record) + return records + + +def new_contact_preflight(): + return { + 'status': 'not_run_noninteractive', + 'override_used': False, + 'classification_thresholds_uv': { + 'minimum_inclusive': CONTACT_RMS_MIN_UV, + 'maximum_inclusive': CONTACT_RMS_MAX_UV, + }, + 'checks': [], + 'final_check': None, + } + + def bandpower(signal_1d, fs, band): """Welch-like simple periodogram power in band (Hz^2 * count).""" if len(signal_1d) < int(fs): @@ -147,9 +255,14 @@ def ascii_spectrogram(signal_1d, fs, title="", width=60, height=14, fmax=40): # --- Recording ------------------------------------------------------------ class Recorder: - def __init__(self): + def __init__(self, require_segment_ack=False): self.board = None self.started_at = None + self.require_segment_ack = require_segment_ack + # Cue onset is logged separately from block start. Without both, the 2 s + # edge discard is a magic constant no later reader can audit: "user was + # told to close their eyes" and "samples began" are 3+ seconds apart. + self.marks = [] def connect(self): params = BrainFlowInputParams() @@ -159,7 +272,10 @@ def connect(self): # architecturally identical. Change BoardIds.MUSE_S_BOARD -> MUSE_2_BOARD # when running this script on a Muse 2. self.board = BoardShim(BoardIds.MUSE_S_BOARD, params) - BoardShim.enable_dev_board_logger() + if os.environ.get("NC_BRAINFLOW_DEV_LOG", "1") == "0": + BoardShim.disable_board_logger() + else: + BoardShim.enable_dev_board_logger() print("Scanning for Muse S (board 39)...") self.board.prepare_session() print(f"Connected. ID: {self.board.get_board_id()}") @@ -171,13 +287,25 @@ def connect(self): def segment(self, duration, label, prompt): """Record one segment with a real-time prompt to the user.""" - print(f"\n>>> {label}: {prompt}") + cue_unix = time.time() + print("\a\n" + "=" * 60) + print(f">>> {label}: {prompt}") + print("=" * 60, flush=True) + if self.require_segment_ack: + try: + input("Adopt that state now, then press ENTER to start the countdown...") + except EOFError as exc: + raise RuntimeError( + "Interactive segment confirmation requires a TTY; " + "use NC_VALIDATE_NONINTERACTIVE=1 only for attended automation" + ) from exc for remaining in [3, 2, 1]: - print(f" starting in {remaining}...") + print(f" starting in {remaining}...", flush=True) time.sleep(1.0) n_samples = int(duration * SAMPLE_RATE) # Drain current buffer self.board.get_current_board_data(n_samples) + block_start_unix = time.time() # Wait for the duration time.sleep(duration + 0.2) data = self.board.get_current_board_data(int((duration + 5) * SAMPLE_RATE)) @@ -186,8 +314,28 @@ def segment(self, duration, label, prompt): # Take the last n_samples if n_cols > n_samples: data = data[:, -n_samples:] + self.marks.append({ + "label": label, "block_index": len(self.marks) + 1, + "cue_unix": cue_unix, "block_start_unix": block_start_unix, + "cue_to_start_s": block_start_unix - cue_unix, + "duration_s": duration, "n_samples": int(data.shape[1]), + "short_block": is_short_segment(data.shape[1], duration), + }) return data + def contact_snapshot(self, duration=2.0): + """Measure live channel RMS before committing to the timed protocol.""" + n_samples = int(duration * SAMPLE_RATE) + self.board.get_current_board_data(n_samples) + time.sleep(duration + 0.2) + data = self.board.get_current_board_data(n_samples) + if data.shape[1] == 0: + return {name: float('nan') for name in CHANNEL_NAMES} + return { + name: float(np.sqrt(np.mean(data[ch_idx, :].astype(np.float64) ** 2))) + for name, ch_idx in zip(CHANNEL_NAMES, EEG_CHANNELS) + } + def disconnect(self): if self.board: try: @@ -199,28 +347,118 @@ def disconnect(self): # --- Analysis -------------------------------------------------------------- -def analyze(data_open, data_closed, data_blink, data_clench, data_turn): +def trim_block_edge(block): + """Drop BLOCK_EDGE_DISCARD_S from the front of one block (alpha build-up).""" + n = int(BLOCK_EDGE_DISCARD_S * SAMPLE_RATE) + return block[:, n:] if block.shape[1] > n else block + + +def alpha_ratio(blocks_hi, blocks_lo, ch_idx): + """Alpha power in one group of blocks over another, for a single channel.""" + x_hi = np.concatenate([b[ch_idx, :] for b in blocks_hi]).astype(np.float64) + x_lo = np.concatenate([b[ch_idx, :] for b in blocks_lo]).astype(np.float64) + p_hi = bandpower(x_hi, SAMPLE_RATE, ALPHA_BAND) + p_lo = bandpower(x_lo, SAMPLE_RATE, ALPHA_BAND) + return p_hi / max(p_lo, 1e-9) + + +def shuffle_blocks(blocks, seed): + """Permute retained samples independently within each block and channel.""" + rng = np.random.default_rng(seed) + shuffled = [] + for block in blocks: + out = block.copy() + for ch_idx in EEG_CHANNELS: + out[ch_idx, :] = rng.permutation(block[ch_idx, :]) + # If a cross-channel measure is added, share one permutation within + # each block instead so the shuffle preserves inter-channel structure. + shuffled.append(out) + return shuffled + + +def shuffle_ratio_diagnostics(trimmed, block_order): + """Return report-only shuffled condition and time-index ratios by channel.""" + condition = {name: [] for name in CHANNEL_NAMES} + time_index = {name: [] for name in CHANNEL_NAMES} + for seed in range(SHUFFLE_SEEDS): + shuffled = shuffle_blocks(trimmed, seed) + shuffled_open = [b for b, lab in zip(shuffled, block_order) if lab == 'open'] + shuffled_closed = [b for b, lab in zip(shuffled, block_order) if lab == 'closed'] + first_half, second_half = shuffled[:2], shuffled[2:] + for name, ch_idx in zip(CHANNEL_NAMES, EEG_CHANNELS): + condition[name].append(alpha_ratio(shuffled_closed, shuffled_open, ch_idx)) + time_index[name].append(alpha_ratio(second_half, first_half, ch_idx)) + return condition, time_index + + +def analyze(alpha_blocks, data_blink, data_clench, data_turn, + contact_preflight=None, block_order=ALPHA_BLOCK_ORDER): + """Analyze four counterbalanced alpha blocks in acquisition order.""" results = {} + if contact_preflight is not None: + results['contact_preflight'] = contact_preflight + block_contact = alpha_block_contact_records(alpha_blocks, block_order) + results['alpha_block_contact'] = { + 'classification_thresholds_uv': { + 'minimum_inclusive': CONTACT_RMS_MIN_UV, + 'maximum_inclusive': CONTACT_RMS_MAX_UV, + }, + 'blocks': block_contact, + } # Channel order in BrainFlow: ch0=package_num, ch1=TP9, ch2=AF7, ch3=AF8, ch4=TP10, ch5=AUX, ch6=ts def extract(data, ch_idx): return data[ch_idx, :].astype(np.float64) - # 1. Alpha power ratio eyes-closed / eyes-open, per channel + trimmed = [trim_block_edge(b) for b in alpha_blocks] + open_blocks = [b for b, lab in zip(trimmed, block_order) if lab == 'open'] + closed_blocks = [b for b, lab in zip(trimmed, block_order) if lab == 'closed'] + # Relabelled by POSITION alone, ignoring condition. Under ABBA this split is + # orthogonal to the real labels, so it measures pure drift and nothing else. + first_half, second_half = trimmed[:2], trimmed[2:] + + # Sections 3 and 5 below want contiguous open/closed arrays; the untrimmed + # concatenation is right there since they are baselines, not the contrast. + data_open = np.concatenate( + [b for b, lab in zip(alpha_blocks, block_order) if lab == 'open'], axis=1) + data_closed = np.concatenate( + [b for b, lab in zip(alpha_blocks, block_order) if lab == 'closed'], axis=1) + + # 1. Alpha power ratio eyes-closed / eyes-open, per channel, against drift print("\n--- 1. Alpha power (8-13 Hz) per channel ---") + print(f" blocks: {' '.join(block_order)} @ {BLOCK_SECONDS:.0f}s, " + f"first {BLOCK_EDGE_DISCARD_S:.0f}s of each discarded") alpha_ratios = {} + time_ratios = {} + alpha_pass_by_ch = {} for i, ch_idx in enumerate(EEG_CHANNELS): - x_open = extract(data_open, ch_idx) - x_closed = extract(data_closed, ch_idx) - p_open = bandpower(x_open, SAMPLE_RATE, ALPHA_BAND) - p_closed = bandpower(x_closed, SAMPLE_RATE, ALPHA_BAND) - ratio = p_closed / max(p_open, 1e-9) - alpha_ratios[CHANNEL_NAMES[i]] = ratio - print(f" {CHANNEL_NAMES[i]:>5}: open={p_open:>10.2f} closed={p_closed:>10.2f} ratio={ratio:.2f}") + r_cond = alpha_ratio(closed_blocks, open_blocks, ch_idx) + r_time = alpha_ratio(second_half, first_half, ch_idx) + alpha_ratios[CHANNEL_NAMES[i]] = r_cond + time_ratios[CHANNEL_NAMES[i]] = r_time + # Both gates: big enough, AND bigger than the drift on the same data. + alpha_pass_by_ch[CHANNEL_NAMES[i]] = ( + r_cond >= ALPHA_RATIO_THRESHOLD and r_cond > r_time * TIME_BASELINE_MARGIN) + verdict = 'PASS' if alpha_pass_by_ch[CHANNEL_NAMES[i]] else ( + 'drift' if r_cond >= ALPHA_RATIO_THRESHOLD else 'fail') + print(f" {CHANNEL_NAMES[i]:>5}: condition={r_cond:>6.2f} " + f"time-index={r_time:>6.2f} {verdict}") results['alpha_ratios'] = alpha_ratios - best_ratio = max(alpha_ratios.values()) - results['best_alpha_ratio'] = best_ratio - results['alpha_pass'] = best_ratio >= ALPHA_RATIO_THRESHOLD + results['alpha_time_index_ratios'] = time_ratios + results['alpha_pass_by_channel'] = alpha_pass_by_ch + results['best_alpha_ratio'] = max(alpha_ratios.values()) + results['worst_time_index_ratio'] = max(time_ratios.values()) + results['alpha_pass'] = any(alpha_pass_by_ch.values()) + if not results['alpha_pass'] and results['best_alpha_ratio'] >= ALPHA_RATIO_THRESHOLD: + print(" ** alpha ratio cleared 1.5x but did NOT beat the time-index baseline.") + print(" A drift across the session explains it as well as eye closure does.") + + shuffled_condition, shuffled_time = shuffle_ratio_diagnostics(trimmed, block_order) + results['alpha_shuffle_seeds'] = SHUFFLE_SEEDS + results['alpha_shuffle_collapse_factor'] = SHUFFLE_COLLAPSE_FACTOR + results['alpha_shuffle_condition_ratios'] = shuffled_condition + results['alpha_shuffle_time_index_ratios'] = shuffled_time + results['alpha_shuffle_gate'] = 'report_only' # 2. Blink detection: max abs value in AF7/AF8 during blink window print("\n--- 2. Blink transient detection (frontal channels) ---") @@ -298,9 +536,7 @@ def extract(data, ch_idx): # For saturated channels we expect RMS ~ 577 µV (uniformly distributed # in [-1000, 1000]); report them as a separate diagnostic flag so a # future session can re-fit the headband. - saturated = [ch for ch in CHANNEL_NAMES if rms[ch] > 200.0] - dead = [ch for ch in CHANNEL_NAMES if rms[ch] < 2.0] - healthy = [ch for ch in CHANNEL_NAMES if 2.0 <= rms[ch] <= 200.0] + healthy, saturated, dead = classify_contact(rms) rms_ok = (len(saturated) == 0 and len(dead) == 0) results['contact_pass'] = rms_ok results['contact_saturated'] = saturated @@ -320,7 +556,7 @@ def extract(data, ch_idx): print("=" * 60) checks = [ ("Contact quality (RMS 2-200 µV)", results['contact_pass']), - ("Alpha rise on eyes-closed (>=1.5x)", results['alpha_pass']), + ("Alpha rise on eyes-closed (>=1.5x, beats time-index)", results['alpha_pass']), ("Blink transient (>=40 µV AF7/AF8)", results['blink_pass']), ("Jaw clench beta (>=2x baseline)", results['clench_pass']), ("Head turn artifact (>=30 µV swing)", results['turn_pass']), @@ -329,6 +565,39 @@ def extract(data, ch_idx): for name, ok in checks: marker = "PASS" if ok else "FAIL" print(f" [{marker}] {name}") + if contact_preflight is None: + print("\n Contact preflight: NOT RECORDED (legacy or analysis-only input)") + else: + thresholds = contact_preflight['classification_thresholds_uv'] + print(f"\n Contact preflight: {contact_preflight['status']} " + f"(override={contact_preflight['override_used']}, " + f"checks={len(contact_preflight['checks'])}, " + f"healthy={thresholds['minimum_inclusive']:.1f}-" + f"{thresholds['maximum_inclusive']:.1f} uV inclusive)") + final_check = contact_preflight['final_check'] + if final_check is not None: + for name in CHANNEL_NAMES: + state = ("healthy" if name in final_check['healthy'] else + "SATURATED" if name in final_check['saturated'] else "DEAD") + value = final_check['rms_uv'][name] + value_text = "missing" if value is None else f"{value:.1f} uV" + print(f" {name:>5}: {value_text:>10} {state}") + print("\n Per-block alpha contact:") + for block in block_contact: + states = [] + for name in CHANNEL_NAMES: + state = ("healthy" if name in block['healthy'] else + "SATURATED" if name in block['saturated'] else "DEAD") + value = block['rms_uv'][name] + value_text = "missing" if value is None else f"{value:.1f}uV" + states.append(f"{name}={value_text}/{state}") + print(f" block {block['block_index']} {block['label']}: " + ", ".join(states)) + print("\n Temporal-shuffle diagnostics (report only; one ratio per seed):") + for name in CHANNEL_NAMES: + cond = ', '.join(f"{value:.2f}" for value in shuffled_condition[name]) + time_values = ', '.join(f"{value:.2f}" for value in shuffled_time[name]) + print(f" {name:>5} condition: [{cond}]") + print(f" time-index: [{time_values}]") print() if n_pass == 5: verdict = "DEFINITIVE — Muse is on your head and producing physiological EEG." @@ -350,7 +619,7 @@ def extract(data, ch_idx): # --- Main ------------------------------------------------------------------ -def main(): +def main(block_order): out_dir = os.path.expanduser("~/Developer/NeuralCompose/Recordings") os.makedirs(out_dir, exist_ok=True) timestamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") @@ -363,6 +632,7 @@ def main(): print("=" * 60) print() print("PROTOCOL") + print(f" Alpha block order: {' '.join(block_order)}") print(" Total time: ~80 s. Keep the Muse on your head the whole time.") print(" Prompted on screen for each segment.") print() @@ -384,24 +654,56 @@ def main(): return 2 print() - rec = Recorder() + rec = Recorder(require_segment_ack=( + os.environ.get("NC_VALIDATE_NONINTERACTIVE") != "1")) try: rec.connect() except Exception as e: print(f"FAILED to connect: {e}") return 2 - # Ctrl-C to abort + # Install cleanup before any interactive contact checks. def stop(*_): print("\nAborted.") rec.disconnect() sys.exit(130) signal.signal(signal.SIGINT, stop) - data_open = rec.segment(SEG_OPEN, "EYES OPEN", - "Open eyes. Look at a fixed point. Relax face/jaw.") - data_closed = rec.segment(SEG_CLOSED, "EYES CLOSED", - "Close eyes gently. Same relaxation. Don't squint.") + contact_preflight = new_contact_preflight() + if rec.require_segment_ack: + while True: + rms = rec.contact_snapshot() + check = contact_check_record(rms) + contact_preflight['checks'].append(check) + contact_preflight['final_check'] = check + print("\nLive contact snapshot:") + for name in CHANNEL_NAMES: + state = ("healthy" if name in check['healthy'] else + "SATURATED" if name in check['saturated'] else "DEAD") + print(f" {name:>5}: {rms[name]:7.1f} uV {state}") + if check['passed']: + input("All channels are in range. Press ENTER to begin the protocol...") + contact_preflight['status'] = 'passed' + break + response = input( + "Reseat the flagged contacts, then press ENTER to recheck; " + "type RUN to continue anyway: ") + if response.strip().lower() == "run": + print("Proceeding with an explicit contact-quality override.") + contact_preflight['status'] = 'overridden' + contact_preflight['override_used'] = True + break + + prompts = { + 'open': "Open eyes. Look at a fixed point. Relax face/jaw.", + 'closed': "Close eyes gently. Same relaxation. Don't squint.", + } + alpha_blocks = [] + for n, lab in enumerate(block_order, start=1): + alpha_blocks.append(rec.segment( + BLOCK_SECONDS, f"BLOCK {n}/{len(block_order)} — EYES {lab.upper()}", + prompts[lab])) + data_blink = rec.segment(SEG_BLINK, "BLINK", "Blink deliberately, ~5 blinks over 5 seconds.") data_clench = rec.segment(SEG_CLENCH, "JAW CLENCH", @@ -413,12 +715,19 @@ def stop(*_): # Save raw CSV (column-major: ch0..ch6, samples along axis 1) print(f"\nSaving raw data to {out_csv}") - all_data = np.concatenate([data_open, data_closed, data_blink, data_clench, data_turn], axis=1) + all_data = np.concatenate(alpha_blocks + [data_blink, data_clench, data_turn], axis=1) header = "package_num,TP9,AF7,AF8,TP10,AUX_R,timestamp,segment\n" - # Annotate segment per row - seg_labels = (['open'] * data_open.shape[1] + - ['closed'] * data_closed.shape[1] + - ['blink'] * data_blink.shape[1] + + # Annotate segment per row. Blocks are numbered (open1, closed1, closed2, + # open2) so the ABBA position survives into the CSV — a bare open/closed + # would throw away exactly what the counterbalancing bought. The downstream + # consumer (convert_muse_validation_recordings.py:58) passes this column + # through verbatim, so the new labels are additive, not breaking. + seen = {} + seg_labels = [] + for blk, lab in zip(alpha_blocks, block_order): + seen[lab] = seen.get(lab, 0) + 1 + seg_labels += [f"{lab}{seen[lab]}"] * blk.shape[1] + seg_labels += (['blink'] * data_blink.shape[1] + ['clench'] * data_clench.shape[1] + ['turn'] * data_turn.shape[1]) rows = [] @@ -431,14 +740,22 @@ def stop(*_): f.write('\n'.join(rows) + '\n') # Analyze - results, rc = analyze(data_open, data_closed, data_blink, data_clench, data_turn) + results, rc = analyze( + alpha_blocks, data_blink, data_clench, data_turn, + contact_preflight=contact_preflight, block_order=block_order) + results['block_marks'] = rec.marks + results['block_order'] = block_order # ASCII spectrograms (eyes open vs eyes closed) for visual confirmation + af7_open = np.concatenate( + [b[2, :] for b, lab in zip(alpha_blocks, block_order) if lab == 'open']) + af7_closed = np.concatenate( + [b[2, :] for b, lab in zip(alpha_blocks, block_order) if lab == 'closed']) print("\n--- 6. AF7 spectrogram: eyes open vs eyes closed ---") - print(ascii_spectrogram(data_open[2, :].astype(np.float64), SAMPLE_RATE, - title="AF7 EYES OPEN (last 30s)", width=60, height=14)) - print(ascii_spectrogram(data_closed[2, :].astype(np.float64), SAMPLE_RATE, - title="AF7 EYES CLOSED (30s)", width=60, height=14)) + print(ascii_spectrogram(af7_open.astype(np.float64), SAMPLE_RATE, + title="AF7 EYES OPEN (both blocks)", width=60, height=14)) + print(ascii_spectrogram(af7_closed.astype(np.float64), SAMPLE_RATE, + title="AF7 EYES CLOSED (both blocks)", width=60, height=14)) # Save summary with open(out_json, 'w') as f: @@ -448,9 +765,174 @@ def stop(*_): return rc +def _synth_blocks(alpha_gain=1.0, drift_per_block=1.0, seed=0): + """Four synthetic ABBA blocks. alpha_gain multiplies 10 Hz amplitude on the + CLOSED blocks; drift_per_block multiplies broadband amplitude by position, + ignoring condition entirely.""" + rng = np.random.default_rng(seed) + n = int(BLOCK_SECONDS * SAMPLE_RATE) + t = np.arange(n) / SAMPLE_RATE + blocks = [] + for pos, lab in enumerate(ALPHA_BLOCK_ORDER): + sig = rng.normal(0.0, 10.0, n) + if lab == 'closed': + sig = sig + alpha_gain * 10.0 * np.sin(2 * np.pi * 10.0 * t) + sig = sig * (drift_per_block ** pos) + block = np.zeros((7, n)) + for ch in EEG_CHANNELS: + block[ch, :] = sig + rng.normal(0.0, 1.0, n) + blocks.append(block) + return blocks + + +def _old_protocol_ratio(blocks, ch_idx): + """The statistic the OLD script computed: blocks 1-2 as 'open', 3-4 as + 'closed' — i.e. the sequential order it recorded in, with no edge trim.""" + return alpha_ratio(blocks[2:], blocks[:2], ch_idx) + + +def demo(): + """Self-check. Runs without brainflow or hardware: python3 --self-check""" + ch = EEG_CHANNELS[0] + + # Segment completeness follows each segment's own duration, not the longer + # alpha-block duration. + assert not is_short_segment(int(SEG_BLINK * SAMPLE_RATE), SEG_BLINK) + assert not is_short_segment(int(SEG_CLENCH * SAMPLE_RATE), SEG_CLENCH) + assert not is_short_segment(int(SEG_TURN * SAMPLE_RATE), SEG_TURN) + assert is_short_segment(int(SEG_BLINK * SAMPLE_RATE * 0.79), SEG_BLINK) + contact_record = contact_check_record({ + 'TP9': 20.0, 'AF7': 201.0, 'AF8': 2.0, 'TP10': 1.9}) + assert contact_record == { + 'rms_uv': {'TP9': 20.0, 'AF7': 201.0, 'AF8': 2.0, 'TP10': 1.9}, + 'healthy': ['TP9', 'AF8'], + 'saturated': ['AF7'], + 'dead': ['TP10'], + 'passed': False, + } + missing_record = contact_check_record({ + 'TP9': float('nan'), 'AF7': 20.0, 'AF8': 20.0, 'TP10': 20.0}) + assert missing_record['rms_uv']['TP9'] is None + assert missing_record['dead'] == ['TP9'] + json.dumps(missing_record, allow_nan=False) + preflight = new_contact_preflight() + assert preflight['status'] == 'not_run_noninteractive' + assert preflight['override_used'] is False + assert preflight['classification_thresholds_uv'] == { + 'minimum_inclusive': 2.0, + 'maximum_inclusive': 200.0, + } + assert BLOCK_ORDERS[parse_args(['--order', 'BAAB']).order] == \ + ['closed', 'open', 'open', 'closed'] + contact_blocks = [np.zeros((7, 8)) for _ in range(4)] + for block in contact_blocks: + block[1, :] = 20.0 + block[2, :] = 250.0 + block[3, :] = 1.0 + block[4, :] = 100.0 + block_records = alpha_block_contact_records(contact_blocks, BLOCK_ORDERS['BAAB']) + assert [record['label'] for record in block_records] == \ + ['closed', 'open', 'open', 'closed'] + assert block_records[0]['healthy'] == ['TP9', 'TP10'] + assert block_records[0]['saturated'] == ['AF7'] + assert block_records[0]['dead'] == ['AF8'] + + # (a) Real alpha, no drift -> condition ratio high, time ratio ~1, PASSES. + # The modest gain isolates spectral structure without smuggling a near-gate + # broadband amplitude contrast into the fixture. Its d1 tightness is a + # consequence of that mechanism-focused fixture, not a tuned target. + b = [trim_block_edge(x) for x in _synth_blocks(alpha_gain=0.3, drift_per_block=1.0)] + r_cond = alpha_ratio(b[1:3], [b[0], b[3]], ch) + r_time = alpha_ratio(b[2:], b[:2], ch) + assert r_cond >= ALPHA_RATIO_THRESHOLD, f"(a) alpha not detected: {r_cond:.2f}" + assert r_cond > r_time * TIME_BASELINE_MARGIN, f"(a) {r_cond:.2f} !> {r_time:.2f}" + + # (b) Pure drift, no alpha. The new gate must fail -- AND the old design, + # on this identical array, must PASS. Without the second assertion this + # only proves the new gate works, not that it catches anything the old + # one missed, which is the entire claim the redesign rests on. + raw = _synth_blocks(alpha_gain=0.0, drift_per_block=1.6) + d = [trim_block_edge(x) for x in raw] + r_cond = alpha_ratio(d[1:3], [d[0], d[3]], ch) + r_time = alpha_ratio(d[2:], d[:2], ch) + assert not (r_cond >= ALPHA_RATIO_THRESHOLD and r_cond > r_time * TIME_BASELINE_MARGIN), \ + f"(b) new gate passed pure drift: cond={r_cond:.2f} time={r_time:.2f}" + r_old = _old_protocol_ratio(raw, ch) + assert r_old >= ALPHA_RATIO_THRESHOLD, \ + f"(b) old protocol did not report a pass on pure drift ({r_old:.2f}); " \ + "the regression this change exists to fix is not being exercised" + + # (c) Alpha AND drift -> condition must still beat time-index. + m = [trim_block_edge(x) for x in _synth_blocks(alpha_gain=2.0, drift_per_block=1.2)] + r_cond = alpha_ratio(m[1:3], [m[0], m[3]], ch) + r_time = alpha_ratio(m[2:], m[:2], ch) + assert r_cond >= ALPHA_RATIO_THRESHOLD and r_cond > r_time * TIME_BASELINE_MARGIN, \ + f"(c) real alpha lost to drift: cond={r_cond:.2f} time={r_time:.2f}" + + # (d) Shuffle the exact post-trim arrays passed to bandpower(). It must + # destroy within-block spectral structure without erasing a between- + # block amplitude effect. + alpha_blocks = b + drift_blocks = [trim_block_edge(x) for x in _synth_blocks( + alpha_gain=0.0, drift_per_block=1.6)] + for seed in range(SHUFFLE_SEEDS): + shuffled_alpha = shuffle_blocks(alpha_blocks, seed) + shuffled_drift = shuffle_blocks(drift_blocks, seed) + + # (d3) Sampling is without replacement, independently per block/channel. + for original, shuffled in zip(alpha_blocks, shuffled_alpha): + for ch_idx in EEG_CHANNELS: + assert np.array_equal(np.sort(shuffled[ch_idx]), np.sort(original[ch_idx])), \ + f"(d3) seed {seed} channel {ch_idx} is not a permutation" + for original, shuffled in zip(drift_blocks, shuffled_drift): + for ch_idx in EEG_CHANNELS: + assert np.array_equal(np.sort(shuffled[ch_idx]), np.sort(original[ch_idx])), \ + f"(d3) drift seed {seed} channel {ch_idx} is not a permutation" + + for name, ch_idx in zip(CHANNEL_NAMES, EEG_CHANNELS): + true_condition = alpha_ratio( + [alpha_blocks[1], alpha_blocks[2]], + [alpha_blocks[0], alpha_blocks[3]], ch_idx) + shuffled_condition = alpha_ratio( + [shuffled_alpha[1], shuffled_alpha[2]], + [shuffled_alpha[0], shuffled_alpha[3]], ch_idx) + assert shuffled_condition - 1 < \ + (true_condition - 1) / SHUFFLE_COLLAPSE_FACTOR, \ + f"(d1) seed {seed} {name} spectral excess survived: " \ + f"true={true_condition:.2f} shuffled={shuffled_condition:.2f}" + assert shuffled_condition < ALPHA_RATIO_THRESHOLD, \ + f"(d1) seed {seed} {name} shuffled condition passed gate: " \ + f"{shuffled_condition:.2f}" + + true_time = alpha_ratio(drift_blocks[2:], drift_blocks[:2], ch_idx) + shuffled_time = alpha_ratio(shuffled_drift[2:], shuffled_drift[:2], ch_idx) + assert shuffled_time - 1 > \ + (true_time - 1) / SHUFFLE_COLLAPSE_FACTOR, \ + f"(d2) seed {seed} {name} amplitude excess collapsed: " \ + f"true={true_time:.2f} shuffled={shuffled_time:.2f}" + + # (d4) A seed identifies one exact permutation and therefore one result. + repeat_a = shuffle_blocks(alpha_blocks, 0) + repeat_b = shuffle_blocks(alpha_blocks, 0) + assert all(np.array_equal(a, b) for a, b in zip(repeat_a, repeat_b)), \ + "(d4) identical seeds produced different permutations" + + print(f"self-check ok — pure drift: old protocol reports {r_old:.2f}x (PASS), " + f"new gate rejects it; {SHUFFLE_SEEDS} shuffle seeds pass d1-d4") + return 0 + + +def parse_args(argv): + parser = argparse.ArgumentParser() + parser.add_argument('--self-check', action='store_true') + parser.add_argument('--order', choices=sorted(BLOCK_ORDERS), default='ABBA') + return parser.parse_args(argv) + + if __name__ == "__main__": try: - sys.exit(main()) + args = parse_args(sys.argv[1:]) + sys.exit(demo() if args.self_check else main(BLOCK_ORDERS[args.order])) except Exception as e: import traceback traceback.print_exc() diff --git a/Sources/BCICore/Composition/HypnagogicDialecticLoop.swift b/Sources/BCICore/Composition/HypnagogicDialecticLoop.swift index 329b7bf..c4fe288 100644 --- a/Sources/BCICore/Composition/HypnagogicDialecticLoop.swift +++ b/Sources/BCICore/Composition/HypnagogicDialecticLoop.swift @@ -276,7 +276,7 @@ public actor HypnagogicDialecticLoop { } }() let selfSimilarity: Float? = spokenEmb.flatMap { emb in - replyCentroid.map { DialecticalDynamics.normalized(emb.cosineSimilarity(to: $0)) } + replyCentroid.map { DialecticalDynamics.normalizedSimilarity(emb, $0) } } var witnessFinding: String? var witnessDistance: Float? @@ -294,7 +294,7 @@ public actor HypnagogicDialecticLoop { if !finding.isEmpty { witnessFinding = finding if let spokenEmb, let findingEmb = try? await embedder.encode([finding]).first { - witnessDistance = 1 - DialecticalDynamics.normalized(findingEmb.cosineSimilarity(to: spokenEmb)) + witnessDistance = 1 - DialecticalDynamics.normalizedSimilarity(findingEmb, spokenEmb) } } } catch is CancellationError { @@ -374,7 +374,7 @@ public actor HypnagogicDialecticLoop { var worst: Float = -1 for i in 0.. worst { worst = d; best = (i, j) } } } diff --git a/Sources/BCICore/Dialectic/DialecticalDynamics.swift b/Sources/BCICore/Dialectic/DialecticalDynamics.swift index 4c4100c..7c63834 100644 --- a/Sources/BCICore/Dialectic/DialecticalDynamics.swift +++ b/Sources/BCICore/Dialectic/DialecticalDynamics.swift @@ -113,6 +113,17 @@ public enum DialecticalDynamics { @inlinable public static func normalized(_ cosine: Float) -> Float { (cosine + 1) / 2 } + /// `normalized` over a similarity that must exist. Traps on incomparable + /// operands instead of scoring them: one embedder per session makes that a + /// defect, not a state the loop should absorb as `0.5`. + // ponytail: traps; propagate Optional the day a second embedder is held at once. + public static func normalizedSimilarity(_ a: Embedding, _ b: Embedding) -> Float { + guard let s = a.cosineSimilarity(to: b) else { + preconditionFailure("incomparable embedding spaces: \(a.modelID)/\(a.dimension) vs \(b.modelID)/\(b.dimension)") + } + return normalized(s) + } + /// Scores one candidate against the turn context. Missing centroids (early /// turns, before any history) score a neutral `0.5` rather than biasing the /// competition toward either pole. @@ -122,10 +133,10 @@ public enum DialecticalDynamics { historyCentroid: Embedding?, replyCentroid: Embedding? ) -> DialecticalEnergy { - let coherence = normalized(candidate.cosineSimilarity(to: heard)) - let resonance = historyCentroid.map { normalized(candidate.cosineSimilarity(to: $0)) } ?? 0.5 + let coherence = normalizedSimilarity(candidate, heard) + let resonance = historyCentroid.map { normalizedSimilarity(candidate, $0) } ?? 0.5 // novelty = distance from what we've said; 1 − similarity, on the [0,1] scale. - let novelty = replyCentroid.map { 1 - normalized(candidate.cosineSimilarity(to: $0)) } ?? 0.5 + let novelty = replyCentroid.map { 1 - normalizedSimilarity(candidate, $0) } ?? 0.5 return DialecticalEnergy(coherence: coherence, resonance: resonance, novelty: novelty) } @@ -137,7 +148,7 @@ public enum DialecticalDynamics { var pairs = 0 for i in 0.. Float { - let toThesis = normalized(c.cosineSimilarity(to: thesis)) - let toAnti = normalized(c.cosineSimilarity(to: antithesis)) + let toThesis = normalizedSimilarity(c, thesis) + let toAnti = normalizedSimilarity(c, antithesis) return min(toThesis, toAnti) } @@ -259,13 +270,15 @@ public enum DialecticalDynamics { /// L2-normalized mean of a set of embeddings — the "direction the dialogue /// has been travelling." Provenance (`modelID`/`version`/`dimension`) is - /// taken from the first element; embeddings of a different dimension are - /// skipped rather than trapping. Returns `nil` for an empty set. + /// taken from the first element. Returns `nil` for an empty set, and `nil` + /// if any member is not comparable with the first: a centroid over an + /// unannounced subset is a plausible vector that means nothing. public static func centroid(of embeddings: [Embedding]) -> Embedding? { guard let first = embeddings.first else { return nil } + guard embeddings.allSatisfy({ $0.isComparable(with: first) }) else { return nil } let dim = first.values.count var sum = [Float](repeating: 0, count: dim) - for e in embeddings where e.values.count == dim { + for e in embeddings { for i in 0..= 2 else { return 0 } var acc: Float = 0 for i in 1..= edgeThreshold { edges.append(Edge(a: existing.id, b: node.id, weight: w)) } @@ -76,7 +76,7 @@ public struct SemanticGraph: Sendable { public func nearestPriorNodes(to query: Embedding, limit: Int, minSimilarity: Float = 0) -> [Node] { nodes - .map { ($0, DialecticalDynamics.normalized(query.cosineSimilarity(to: $0.embedding))) } + .map { ($0, DialecticalDynamics.normalizedSimilarity(query, $0.embedding)) } .filter { $0.1 >= minSimilarity } .sorted { $0.1 > $1.1 } .prefix(limit) diff --git a/Sources/BCICore/Models/Embedding.swift b/Sources/BCICore/Models/Embedding.swift index 9cdbff4..d879eea 100644 --- a/Sources/BCICore/Models/Embedding.swift +++ b/Sources/BCICore/Models/Embedding.swift @@ -51,12 +51,24 @@ public struct Embedding: Sendable, Equatable { self.seed = seed } + /// Whether a similarity between `self` and `other` exists to compute: same + /// `modelID`, same dimension, non-empty. Mirrors the Rust port + /// (`neuralcompose-hypnagogic/src/embedding.rs`). `seed` and `version` are + /// deliberately not checked: `DialecticalDynamics.centroid(of:)` builds + /// centroids with `seed: 0`, and a centroid must stay comparable to the + /// vectors it was averaged from. + public func isComparable(with other: Embedding) -> Bool { + modelID == other.modelID && values.count == other.values.count && !values.isEmpty + } + /// Cosine similarity to `other`. Because both operands are L2-normalized /// by the `values` invariant, this is just the dot product — no magnitude - /// division. Returns 0 when the dimensions differ (incomparable spaces) - /// rather than trapping. - public func cosineSimilarity(to other: Embedding) -> Float { - guard values.count == other.values.count else { return 0 } + /// division. Returns `nil` when the two are not comparable (different + /// space or dimension). `nil` is not a low score and not a failed + /// computation: no similarity exists to compute. `0` means orthogonal and + /// comparable, which is a different statement. + public func cosineSimilarity(to other: Embedding) -> Float? { + guard isComparable(with: other) else { return nil } var acc: Float = 0 for i in 0.. \ +// ../neuralcompose-client-native/crates/neuralcompose-hypnagogic/tests/fixtures/dialectic_v1.json + +// ── Deterministic, model-free embeddings ──────────────────────────────────── + +enum F { + static let modelID = "fixture-v1" + static let dimension = 8 + + /// A reproducible unit vector. Values come from a fixed integer recurrence so + /// the same index always yields the same vector on every platform and Swift + /// version — no `Hasher`, no `Double.random`, nothing whose output is allowed + /// to change between releases. + static func embedding(_ index: Int) -> Embedding { + var state = UInt64(index &* 2_654_435_761 &+ 1) + var values: [Float] = [] + values.reserveCapacity(F.dimension) + for _ in 0..> 7 + state ^= state << 17 + // Take 16 bits and centre on zero so vectors point in varied directions. + let raw = Float(Int64(state & 0xFFFF) - 0x8000) + values.append(raw / Float(0x8000)) + } + let norm = sqrtf(values.reduce(0) { $0 + $1 * $1 }) + let unit = norm > 1e-6 ? values.map { $0 / norm } : values + return Embedding( + values: unit, modelID: F.modelID, dimension: F.dimension, + version: "1", seed: 0 + ) + } + + /// Blends two fixture vectors and renormalizes — lets a case place a candidate + /// deliberately near or far from `heard` without hand-writing coordinates. + static func blend(_ a: Embedding, _ b: Embedding, _ w: Float) -> Embedding { + var values = [Float](repeating: 0, count: F.dimension) + for i in 0.. 1e-6 ? values.map { $0 / norm } : values + return Embedding( + values: unit, modelID: F.modelID, dimension: F.dimension, + version: "1", seed: 0 + ) + } + +} + +// ── JSON emission ─────────────────────────────────────────────────────────── + +/// Full `Float` precision. The Rust compares to 1e-6, but the fixture should +/// not be the thing that loses information. +func j(_ f: Float) -> String { String(format: "%.9g", f) } +func j(_ d: Double) -> String { String(format: "%.17g", d) } +func j(_ s: String) -> String { + var out = "\"" + for c in s.unicodeScalars { + switch c { + case "\"": out += "\\\"" + case "\\": out += "\\\\" + case "\n": out += "\\n" + default: out.unicodeScalars.append(c) + } + } + return out + "\"" +} +func jArray(_ values: [Float]) -> String { "[" + values.map(j).joined(separator: ",") + "]" } + +// ── Cases ─────────────────────────────────────────────────────────────────── + +/// One scored competition, with every intermediate the Rust must reproduce. +struct Case { + let name: String + /// What the profile tuning is — the Rust looks this up by id rather than + /// re-encoding the numbers, so a drifted constant fails here too. + let profile: String + let tuning: DialecticalDynamics.Tuning + let heardIndex: Int + let historyIndices: [Int] + let replyIndices: [Int] + /// `(candidateIndex, blendWeightTowardHeard, roleID)`. + let candidates: [(Int, Float, String)] + let draw: Double + /// Injects the tension rather than deriving it from the candidate + /// embeddings. + /// + /// Needed because `compete`'s silence gate is `tension >= highTension`, and + /// a mutation of that `>=` to `>` is only observable when tension lands + /// EXACTLY on the bar. Tension derived from a cosine will never do that + /// reliably in `Float`, so a fixture built only from embeddings cannot + /// discriminate the boundary — a mutation run against the first version of + /// this file confirmed that mutant survived. `compete` already takes + /// tension as a parameter on both sides, so passing the bar exactly is + /// available and exact. + var tensionOverride: Float? = nil +} + +let profiles: [(String, DialecticalDynamics.Tuning)] = [ + ("focused", ContextProfile.focused.tuning), + ("reflective", ContextProfile.reflective.tuning), + ("contemplative", ContextProfile.contemplative.tuning), +] + +var cases: [Case] = [] + +// Every profile against the same inputs — isolates the tuning as the variable. +for (name, tuning) in profiles { + // A. Early turn: no history, no replies. Pins the neutral-0.5 path. + cases.append(Case( + name: "\(name)/early-turn-no-centroids", profile: name, tuning: tuning, + heardIndex: 1, historyIndices: [], replyIndices: [], + candidates: [(10, 0.85, "coherenceSeeking"), (11, 0.05, "displacementSeeking")], + draw: 0.5 + )) + // B. Established dialogue: both centroids present. + cases.append(Case( + name: "\(name)/with-both-centroids", profile: name, tuning: tuning, + heardIndex: 2, historyIndices: [3, 4, 5], replyIndices: [6, 7], + candidates: [(12, 0.80, "coherenceSeeking"), (13, 0.10, "displacementSeeking")], + draw: 0.42 + )) + // C. Near-tie under HIGH tension — the stalemate/silence gate, which is + // where the profiles differ most (focused resists, contemplative invites). + cases.append(Case( + name: "\(name)/near-tie-high-tension", profile: name, tuning: tuning, + heardIndex: 8, historyIndices: [20, 21], replyIndices: [22], + candidates: [(30, 0.50, "coherenceSeeking"), (31, 0.50, "displacementSeeking")], + draw: 0.5 + )) + // D. Draw sweep across a near-tie — the bifurcation. Different draws must + // select different basins, which is the one thing a seed could not pin. + for (k, draw) in [0.01, 0.25, 0.49, 0.51, 0.75, 0.99].enumerated() { + cases.append(Case( + name: "\(name)/bifurcation-draw-\(k)", profile: name, tuning: tuning, + heardIndex: 9, historyIndices: [23], replyIndices: [24], + candidates: [(40, 0.52, "coherenceSeeking"), (41, 0.48, "displacementSeeking")], + draw: draw + )) + } + // E. Decisive margin — the dynamics, not the draw, must win regardless. + cases.append(Case( + name: "\(name)/decisive-margin", profile: name, tuning: tuning, + heardIndex: 14, historyIndices: [25, 26], replyIndices: [27], + candidates: [(50, 0.98, "coherenceSeeking"), (51, 0.02, "displacementSeeking")], + draw: 0.99 + )) + // F. Three roles — the competition iterates over `[DialecticalRole]`, so a + // port that hard-coded two poles passes everything above and fails here. + cases.append(Case( + name: "\(name)/three-candidates", profile: name, tuning: tuning, + heardIndex: 15, historyIndices: [28], replyIndices: [29], + candidates: [ + (60, 0.90, "coherenceSeeking"), + (61, 0.45, "symbolic"), + (62, 0.05, "displacementSeeking"), + ], + draw: 0.7 + )) + // G. Single candidate — margin falls through to `sorted.first`, an edge the + // Swift handles unusually and a port is likely to "clean up". + cases.append(Case( + name: "\(name)/single-candidate", profile: name, tuning: tuning, + heardIndex: 16, historyIndices: [], replyIndices: [], + candidates: [(70, 0.75, "coherenceSeeking")], + draw: 0.33 + )) + // H. No candidates — resolves silent rather than trapping. + cases.append(Case( + name: "\(name)/no-candidates", profile: name, tuning: tuning, + heardIndex: 17, historyIndices: [], replyIndices: [], + candidates: [], + draw: 0.5 + )) + + // ── Boundary cases ────────────────────────────────────────────────── + // Added after a mutation run showed the derived-tension cases above could + // not discriminate two comparison operators. Both mutants survived the + // first fixture; these are placed exactly either side of each comparison. + + // I. Tension EXACTLY on the silence bar, with a margin under the stalemate + // threshold. Kills `tension >= highTension` → `>`: at equality the + // correct code falls silent and the mutant speaks. + cases.append(Case( + name: "\(name)/boundary-tension-exactly-at-bar", profile: name, tuning: tuning, + heardIndex: 18, historyIndices: [], replyIndices: [], + candidates: [(80, 0.50, "coherenceSeeking"), (80, 0.50, "displacementSeeking")], + draw: 0.5, + tensionOverride: tuning.highTension + )) + // J. The same competition a clear step BELOW the bar — must speak. Without + // this companion, case I alone would also pass against code that always + // fell silent. + cases.append(Case( + name: "\(name)/boundary-tension-just-under-bar", profile: name, tuning: tuning, + heardIndex: 18, historyIndices: [], replyIndices: [], + candidates: [(80, 0.50, "coherenceSeeking"), (80, 0.50, "displacementSeeking")], + draw: 0.5, + tensionOverride: tuning.highTension - 0.01 + )) + // K. Two IDENTICAL candidate embeddings ⇒ identical potentials ⇒ + // probabilities exactly [0.5, 0.5], with the draw exactly on that + // cumulative boundary. Kills `d < cumulative` → `d <= cumulative`: + // the correct code walks past the first basin, the mutant stops in it. + // Identical embeddings also make the derived tension exactly 0, so the + // silence gate stays out of the way and selection is the only variable. + cases.append(Case( + name: "\(name)/boundary-draw-exactly-on-cumulative", profile: name, tuning: tuning, + heardIndex: 19, historyIndices: [], replyIndices: [], + candidates: [(81, 0.30, "coherenceSeeking"), (81, 0.30, "displacementSeeking")], + draw: 0.5 + )) +} + +// ── Run ───────────────────────────────────────────────────────────────────── + +var out: [String] = [] +var drawsConsumed: [Double] = [] + +for c in cases { + let heard = F.embedding(c.heardIndex) + let history = c.historyIndices.map { F.embedding($0) } + let replies = c.replyIndices.map { F.embedding($0) } + let historyCentroid = DialecticalDynamics.centroid(of: history) + let replyCentroid = DialecticalDynamics.centroid(of: replies) + + let candidates: [DialecticalCandidate] = c.candidates.map { idx, w, role in + DialecticalCandidate( + text: "candidate-\(idx)", + embedding: F.blend(F.embedding(idx), heard, w), + roleID: role + ) + } + + let scored: [ScoredCandidate] = candidates.map { cand in + let e = DialecticalDynamics.energy( + candidate: cand.embedding, heard: heard, + historyCentroid: historyCentroid, replyCentroid: replyCentroid + ) + return ScoredCandidate( + candidate: cand, energy: e, + potential: e.potential(c.tuning.weights), + roleFulfillment: 0 + ) + } + + let tension = c.tensionOverride + ?? DialecticalDynamics.tension(among: candidates.map(\.embedding)) + let tau = DialecticalDynamics.selectionTemperature(tension: tension, tuning: c.tuning) + let probs = DialecticalDynamics.probabilities( + potentials: scored.map(\.potential), tau: tau + ) + let result = DialecticalDynamics.compete( + scored: scored, tension: tension, draw: c.draw, tuning: c.tuning + ) + drawsConsumed.append(c.draw) + + let outcome: String + let spoken: String + switch result.outcome { + case let .spoke(cand): + outcome = "spoke:\(cand.roleID)"; spoken = cand.text + case let .synthesized(cand): + outcome = "synthesized:\(cand.roleID)"; spoken = cand.text + case .silent: + outcome = "silent"; spoken = "" + } + + let candidateJSON = scored.map { s in + """ + {"roleId":\(j(s.candidate.roleID)),\ + "embedding":\(jArray(s.candidate.embedding.values)),\ + "coherence":\(j(s.energy.coherence)),\ + "resonance":\(j(s.energy.resonance)),\ + "novelty":\(j(s.energy.novelty)),\ + "potential":\(j(s.potential))} + """ + }.joined(separator: ",") + + out.append(""" + {"name":\(j(c.name)),\ + "profile":\(j(c.profile)),\ + "heard":\(jArray(heard.values)),\ + "historyCentroid":\(historyCentroid.map { jArray($0.values) } ?? "null"),\ + "replyCentroid":\(replyCentroid.map { jArray($0.values) } ?? "null"),\ + "candidates":[\(candidateJSON)],\ + "tension":\(j(tension)),\ + "tensionInjected":\(c.tensionOverride != nil),\ + "selectionTemperature":\(j(tau)),\ + "probabilities":\(jArray(probs)),\ + "draw":\(j(c.draw)),\ + "margin":\(j(result.margin)),\ + "decisive":\(result.decisive),\ + "outcome":\(j(outcome)),\ + "spokenText":\(j(spoken))} + """) +} + +print(""" +{"schemaId":"neuralcompose.hypnagogic.dialectic-conformance.v1",\ +"modelId":\(j(F.modelID)),\ +"dimension":\(F.dimension),\ +"note":"Generated by Sources/DialecticFixture. Draws are recorded, never seeds. Regenerate with: swift run dialectic-fixture",\ +"drawsConsumed":[\(drawsConsumed.map { j($0) }.joined(separator: ","))],\ +"cases":[\(out.joined(separator: ","))]} +""") diff --git a/Sources/GenerationEval/main.swift b/Sources/GenerationEval/main.swift index 9fe4e2e..d4f28de 100644 --- a/Sources/GenerationEval/main.swift +++ b/Sources/GenerationEval/main.swift @@ -85,7 +85,7 @@ func meaningPreservationCosine(source: String, output: String) async -> Double? guard let embeddings = try? await embedder.encode([source, output]), embeddings.count == 2 else { return nil } - return Double(embeddings[0].cosineSimilarity(to: embeddings[1])) + return Double(embeddings[0].cosineSimilarity(to: embeddings[1])!) } // MARK: - Evaluate each candidate diff --git a/Sources/SemanticEval/NearestNeighbors.swift b/Sources/SemanticEval/NearestNeighbors.swift index b58797a..4219e34 100644 --- a/Sources/SemanticEval/NearestNeighbors.swift +++ b/Sources/SemanticEval/NearestNeighbors.swift @@ -15,7 +15,7 @@ enum NearestNeighbors { k: Int ) -> [Neighbor] { let scored = zip(corpusTexts, corpusEmbeddings).map { text, embedding in - Neighbor(text: text, score: query.cosineSimilarity(to: embedding)) + Neighbor(text: text, score: query.cosineSimilarity(to: embedding)!) } return Array(scored.sorted { $0.score > $1.score }.prefix(k)) } diff --git a/Sources/SemanticEval/main.swift b/Sources/SemanticEval/main.swift index 0849a1a..7e419c1 100644 --- a/Sources/SemanticEval/main.swift +++ b/Sources/SemanticEval/main.swift @@ -30,7 +30,7 @@ func fail(_ message: String) -> Never { func pairScores(_ pairs: [[String]], embeddingByText: [String: Embedding]) -> [SemanticEvalResult.PairScore] { pairs.map { pair in let a = pair[0], b = pair[1] - let score = embeddingByText[a]!.cosineSimilarity(to: embeddingByText[b]!) + let score = embeddingByText[a]!.cosineSimilarity(to: embeddingByText[b]!)! return .init(a: a, b: b, score: score) } } @@ -42,7 +42,7 @@ func meanPairwiseSimilarity(_ texts: [String], embeddingByText: [String: Embeddi var count = 0 for i in 0.. Fixture { + guard let url = Bundle.module.url(forResource: "env_v1", withExtension: "json") else { + XCTFail(""" + env_v1.json is not in the test bundle. + + This test does NOT skip when its ground truth is missing: a conformance \ + check that passes when it cannot compare anything reports green for \ + exactly the state it exists to detect. + + Regenerate it in the neuralcompose-client-native checkout with + ~/src/NeuralCompose/.venv-muse/bin/python \\ + tools/worldmodel-fixture/generate.py > env_v1.json + and copy it to Tests/WorldModelDemoTests/Fixtures/. + """) + throw XCTSkip("unreachable — XCTFail above is the real outcome") + } + return try JSONDecoder().decode(Fixture.self, from: Data(contentsOf: url)) + } + + private func config(_ c: Fixture.Config) -> WorldModelEnvConfig { + WorldModelEnvConfig( + arenaHalfExtent: c.arenaHalfExtent, + dt: c.dt, + maxAccel: c.maxAccel, + maxSpeed: c.maxSpeed, + restitution: c.restitution + ) + } + + func testEveryStepReproducesThePythonEnvExactly() throws { + let f = try loadFixture() + XCTAssertEqual(f.schemaId, "neuralcompose.hypnagogic.env-conformance.v1") + + let cfg = config(f.config) + var state = f.initialState.map { Float($0) } + var failures: [String] = [] + + for (i, s) in f.steps.enumerated() { + let action = s.action.map { Float($0) } + let expected = s.next.map { Float($0) } + let got = ParticleNavigatorEnv.step(state: state, action: action, config: cfg) + + if got != expected { + failures.append(""" + step \(i): action \(action) + expected \(expected) + got \(got) + python speed before clamp \(s.speedBeforeClamp) (clamped: \(s.clamped)) + """) + if failures.count >= 5 { + failures.append("… stopping after 5") + break + } + } + // Continue from Python's state so one divergence is not reported N times. + state = expected + } + + XCTAssertTrue( + failures.isEmpty, + """ + ParticleNavigatorEnv.swift diverges from WorldModel/env.py in \ + \(failures.count) place(s): + \(failures.joined(separator: "\n")) + + Per §10, registered before this test was run: this is a defect in the \ + macOS app's environment, not in the fixture and not a porting artifact. \ + A Rust port of the same function matched this fixture bit-for-bit. + """ + ) + } + + /// The fixture has to reach the states that discriminate, or a `step` with no + /// wall handling and no speed limit would pass it. + func testTheFixtureReachesTheStatesThatDiscriminate() throws { + let c = try loadFixture().coverage + XCTAssertGreaterThanOrEqual(c.steps, 200) + XCTAssertGreaterThan(c.bouncedX, 0, "wall handling is unexercised on x") + XCTAssertGreaterThan(c.bouncedY, 0, "wall handling is unexercised on y") + XCTAssertGreaterThan(c.cornerHits, 0, "no step bounces both axes at once") + XCTAssertGreaterThan(c.clampedSteps, 0, "the speed clamp never fires") + XCTAssertGreaterThan( + c.unclampedSteps, 0, + "the clamp fires on every step — an implementation that always scales would pass" + ) + } +} diff --git a/docs/Validation.md b/docs/Validation.md index 739790d..2d7fc30 100644 --- a/docs/Validation.md +++ b/docs/Validation.md @@ -23,13 +23,37 @@ and is recorded with a segment label. | # | Condition | Duration | What the user does | Expected signature | |---|-----------|----------|-------------------|-------------------| -| 1 | Eyes open | 30 s | Look at a fixed point, relax | Low alpha, more beta | -| 2 | Eyes closed | 30 s | Close eyes gently, same relaxation | Alpha power rises ≥ 1.5× vs eyes-open | -| 3 | Blinks | 5 s | Blink deliberately, ~5 in 5 s | Frontal transient ≥ 40 µV in AF7/AF8 | -| 4 | Jaw clench | 5 s | Clench hard, release, 3 cycles | Broadband > 20 Hz energy rise ≥ 2× baseline | -| 5 | Head turn | 10 s | Turn head left/right slowly | Low-frequency swing ≥ 30 µV in TP9/TP10 | - -Each condition is event-tagged in the recorded CSV. Total runtime ~80 s + analysis. +| 1–4 | Eyes open / closed, alternating **ABBA / BAAB** | 4 × 15 s | Alternate open-closed-closed-open with closed-open-open-closed across sessions | Alpha power rises ≥ 1.5× vs eyes-open **and** beats the time-index baseline | +| 5 | Blinks | 5 s | Blink deliberately, ~5 in 5 s | Frontal transient ≥ 40 µV in AF7/AF8 | +| 6 | Jaw clench | 5 s | Clench hard, release, 3 cycles | Broadband > 20 Hz energy rise ≥ 2× baseline | +| 7 | Head turn | 10 s | Turn head left/right slowly | Low-frequency swing ≥ 30 µV in TP9/TP10 | + +Each condition is event-tagged in acquisition order in the recorded CSV +(`open1`/`open2` and `closed1`/`closed2`, then `blink`/`clench`/`turn`). Total +runtime is ~80 s plus analysis. + +### Why the alpha contrast is counterbalanced + +The protocol originally ran one 30 s eyes-open block followed by one 30 s eyes-closed +block, always in that order. Condition was therefore perfectly confounded with elapsed +time: impedance settling, electrode warming, and drowsiness onset all push +`alpha_closed / alpha_open` in the same direction as real alpha, and no analysis of that +recording can separate them. + +Four blocks in either `open, closed, closed, open` or `closed, open, open, closed` +order put both conditions at the same mean position in the session (2.5), so a +linear drift subtracts out of the ratio. Alternating ABBA and BAAB reverses which +condition occupies the outer positions, flipping the curvature residual across +the session pair. The first 2 s of every block is discarded — occipital alpha +needs ~1–2 s to rise after eye closure — symmetrically across both conditions, +so the discard is not itself a thumb on the scale. + +ABBA cancels a *linear* trend exactly, and electrode settling is usually curved. The +residual that curvature leaves behind is what the **time-index baseline** measures: the +identical statistic computed on the same blocks relabelled by position alone (blocks 1–2 +vs 3–4), which under ABBA is orthogonal to condition. A channel passes only if the +condition ratio clears 1.5× **and** exceeds the time-index ratio. Both numbers are +reported per channel, pass or fail. ## Pass Criteria @@ -38,14 +62,275 @@ The script computes per-channel statistics and produces a pass/fail verdict: | Check | Pass criterion | Why | |-------|---------------|-----| | Contact quality (RMS 2–200 µV, all 4 channels) | all 4 in range | Disconnected channel: RMS < 2 µV. Saturated front-end: RMS > 200 µV. | -| Alpha rise on eyes-closed | best channel ≥ 1.5× | Canonical EEG signature. | +| Alpha rise on eyes-closed | best channel ≥ 1.5× **and** > its time-index ratio | Canonical EEG signature, gated so a session-long drift cannot report as alpha. | | Blink transient | AF7 or AF8 max ≥ 40 µV | Normal-blink amplitude, not the literature's 150 µV for forced blinks. | | Jaw clench broadband | best channel ≥ 2× baseline | EMG contamination is broadband; 13–30 Hz is one slice of it. | | Head turn motion | best channel ≥ 30 µV swing | Slow turns produce 30–80 µV; the 100 µV literature threshold is for jerky turns. | A 4/5 pass indicates a working acquisition pipeline. A 5/5 pass indicates a clean session with no false readings. -## Current Results (2026-07-10) +## Current Results (2026-08-06, BAAB replication) + +The first BAAB session reversed the condition order to closed, open, open, +closed. It entered capture after three RMS checks ended with all channels inside +the recorded inclusive 2-200 uV range. Preflight status was `passed`, +`override_used` was false, and every segment contained its full sample count. +The final preflight values were TP9 95.2, AF7 137.1, AF8 60.2, and TP10 135.4 +uV. + +| Channel | capture RMS (uV) | capture contact | condition | time-index | shuffled condition, 10 seeds | +|---|---:|---|---:|---:|---:| +| TP9 | 82.0 | healthy | 1.18x | 1.48x | 0.81-1.54x | +| AF7 | 499.4 | saturated | 1.80x | 8.39x | 0.59-1.79x | +| AF8 | 19.9 | healthy | 1.09x | 0.93x | 0.90-1.36x | +| TP10 | 86.7 | healthy | 3.26x | 1.56x | 0.80-1.35x | + +The program reported **4/5, LIKELY**: alpha, blink, jaw-clench, and head-turn +gates passed; contact failed because AF7 again degraded after preflight. The +per-block records persisted directly in the JSON localize the change: + +| Block | state | TP9 | AF7 | AF8 | TP10 | +|---:|---|---:|---:|---:|---:| +| 1 | closed | 102.8 | 260.8 saturated | 21.3 | 89.3 | +| 2 | open | 77.9 | 385.7 saturated | 20.2 | 82.8 | +| 3 | open | 85.9 | 591.7 saturated | 19.7 | 90.5 | +| 4 | closed | 96.1 | 683.6 saturated | 20.0 | 89.6 | + +**TP10 is now replicated under order reversal.** Its 3.26x BAAB condition ratio +beats the 1.56x time-index ratio, and all ten shuffled ratios collapse to +0.80-1.35x. The prior clean-preflight ABBA session reported 3.02x versus 0.73x, +with nine of ten shuffled ratios below 1.5x. Agreement across ABBA and BAAB +rules out the fixed condition-order explanation that invalidated the 2026-07-10 +measurements. This is a replicated TP10-local spectral contrast, not yet a +two-channel posterior result. + +The cross-session drift is stronger evidence than either gate alone. Time-index +reversed around 1, from 0.73x under ABBA to 1.56x under BAAB, while TP10 held at +3.02x and 3.26x. A drift-contaminated condition contrast should move with that +reversal; TP10 did not. In BAAB, the 3.26x unshuffled ratio is 2.4 times the top +of its 1.35x permutation band, with no seed reaching the 1.5x gate. + +TP9 remained healthy in every BAAB block but produced 1.18x, below its 1.48x +time-index ratio and inside the 0.81-1.54x permutation range. Together with its +1.41x clean-ABBA result, TP9 is a replicated healthy-contact negative: for these +sessions, this head, and this headband position, TP10 carries the contrast and +TP9 does not. AF7's apparent 1.80x is also rejected: the channel saturated +before block 1 and its 8.39x time-index ratio dominates the condition ratio. +AF7 has saturated in all three analyzable attended captures, so it is now a +repeated hardware/positioning failure to investigate separately from protocol +quality. + +### Collection implication + +The protocol is no longer the blocker for collecting a **one-channel TP10** +eyes-open/eyes-closed corpus: it has produced two controlled sessions with a +replicated TP10 label and preserved negative controls. It does not unlock +spatial modeling; TP9 is a replicated negative, AF7 fails contact, and AF8 is +amplitude-confounded. The fixed classifier, features, session-level split, null +arms, stopping rule, and six-session trigger are pre-registered in +[`eyes-open-closed-step0-preregistration_v1.md`](research/eyes-open-closed-step0-preregistration_v1.md). +Collection cannot resume until its analysis implementation and synthetic tests +are committed. Two sessions remain far too thin for a classifier claim. + +Evidence remains local because it contains raw EEG: + +- CSV SHA-256: `606d49436d2caa3813eb718b2692d2a9ba19576f07a32b4fd012cf65ea0a3bdf` +- JSON SHA-256: `e586ef65906f5f28470f493f96ae1223b74d0af1ee24f7c1c759efdacba92eb8` +- Session metadata SHA-256: `aa65f588242635677e46efd4345793198f2bc4348fa1fedc359beb970601e6f9` +- Power state SHA-256: `c21330ba9482c6a271eab09b588fb25415679e93e6be45087ea80e6ea9123af5` + +The GPD was on AC at 47% and charging, with the `powersave` governor and +`balanced` platform profile. + +### Prior clean-preflight ABBA session + +The next attended ABBA session entered capture only after four RMS checks ended +with all channels inside the recorded inclusive 2-200 uV range. Preflight status +was `passed`, `override_used` was false, and every segment contained its full +sample count. The final preflight values were TP9 103.7, AF7 155.8, AF8 121.5, +and TP10 68.6 uV. + +| Channel | capture RMS (uV) | capture contact | condition | time-index | shuffled condition, 10 seeds | +|---|---:|---|---:|---:|---:| +| TP9 | 120.3 | healthy | 1.41x | 0.49x | 1.08-1.83x | +| AF7 | 639.7 | saturated | 0.32x | 1.01x | 0.94-1.75x | +| AF8 | 27.5 | healthy | 0.69x | 0.62x | 0.98-1.68x | +| TP10 | 83.3 | healthy | 3.02x | 0.73x | 0.87-1.85x | + +The program reported **4/5, LIKELY**: alpha, blink, jaw-clench, and head-turn +gates passed; capture contact failed because AF7 degraded from a healthy 155.8 +uV preflight value to 639.7 uV during the timed blocks. This distinguishes a +clean initial fit from in-session contact loss rather than treating both as the +same failure. + +Post-hoc per-block RMS from the immutable CSV localizes that loss: + +| Block | state | TP9 | AF7 | AF8 | TP10 | +|---:|---|---:|---:|---:|---:| +| 1 | open | 72.0 | 479.3 saturated | 32.4 | 90.8 | +| 2 | closed | 136.2 | 765.6 saturated | 28.7 | 102.2 | +| 3 | closed | 146.0 | 767.2 saturated | 23.7 | 80.7 | +| 4 | open | 154.1 | 767.3 saturated | 21.5 | 75.2 | + +AF7 failed after the final preflight snapshot and before block 1, so none of its +alpha blocks is contact-valid. TP9 remained inside the healthy range in every +block, although its RMS rose monotonically. + +**TP10 at 3.02x is the first defensible contrast produced by this project.** Its +condition ratio rose +from 2.05x in the prior session to 3.02x, beat its 0.73x time-index ratio, and +nine of ten shuffled ratios fell below 1.5x; the remaining shuffled ratio was +1.85x, still substantially below the unshuffled result. TP9 was now seated and +usable but reached only 1.41x, so the posterior pair did **not** agree. The new +evidence supports a repeatable TP10-local contrast, not yet a two-channel +posterior alpha result. The two session magnitudes are observations, not a +variance estimate. + +Evidence remains local because it contains raw EEG: + +- CSV SHA-256: `2c0aa4b8396bc413d2652ad5966c5ac9dfd737d36acfd20de6daa4dee5fba578` +- JSON SHA-256: `2a44a8865521ff29ca46504799b4009836cacb6ed231d5c2d49a166fe067092a` +- Session metadata SHA-256: `e42ee6d8880be018e2ff5d9550108d3db5d5a18c433242d1171f4229d797c882` +- Power state SHA-256: `509866d0cd1cdc5916b4c7b8438738b605f869a284b741d9b4f2b068596724f6` +- Derived block-RMS sidecar SHA-256: `0186324caedb5434800a7343aafa3d71919f31e60a3be484815872f2d12697b6` + +The session metadata explicitly links its `110527` sidecar prefix to the +validator's `110528` artifact prefix; the one-second difference came from +independent timestamp calls before launch. The GPD was on AC at 37% and charging, +with the `powersave` governor and `balanced` platform profile. + +### Prior attended session (contact-limited) + +The first attended run of the repaired protocol completed on the GPD, based on +commit `b452015bbcddfd46cd66d8cf7575fe9ffe56d0b5` plus the uncommitted shuffle +and cue-confirmation changes. Every segment required participant acknowledgment. +All four alpha blocks contained 3,840 samples in open, closed, closed, open +order, and their cue-to-start delays were recorded separately. + +**Contact preflight: not recorded (legacy).** The live RMS gate was added after +this session, so its JSON has no `contact_preflight` object. Do not interpret +absence as a pass. The post-capture RMS values below establish that contact +failed, but cannot show whether it was bad before block 1 or degraded during +the run. + +| Channel | RMS (uV) | contact | condition | time-index | shuffled condition, 10 seeds | +|---|---:|---|---:|---:|---:| +| TP9 | 793.1 | saturated | 1.09x | 0.25x | 0.76-1.37x | +| AF7 | 793.3 | saturated | 1.41x | 0.59x | 0.70-2.01x | +| AF8 | 23.8 | healthy | 1.75x | 0.10x | 3.08-5.26x | +| TP10 | 77.4 | healthy | 2.05x | 0.27x | 1.10-1.55x | + +The program reported **3/5, LIKELY**: alpha, blink, and jaw-clench gates passed; +contact and head-turn gates failed. That verdict is not a definitive alpha +validation. + +**AF8 is the guard result.** Its 1.75x condition ratio comfortably clears the +1.5x gate and beats time-index, so every prior version of this protocol would +have accepted it as alpha. After temporal permutation the ratio rises to +3.08-5.26x. The effect is therefore a between-condition amplitude contrast, +not isolated spectral alpha. The report-only shuffle diagnostic caught a live +false positive on its first attended session. + +**TP10 is the spectral candidate.** Its 2.05x condition ratio beats time-index, +while nine of ten shuffled ratios fall below 1.5x and all are substantially +below the unshuffled result. This is weaker numerically than the historical +3.88x TP10 result, but carries stronger evidence because the historical value +faced neither time-index nor shuffle control. Different sessions, contact, and +days make the two magnitudes non-comparable. The clean-preflight follow-up above +replicated TP10 with TP9 seated; TP9 did not clear the condition gate. + +Contact is the binding constraint. TP9 and AF7 saturated, leaving only two +usable channels and only one usable posterior channel. The head-turn gate also +failed; poor contact is one plausible shared cause, although movement execution +is not ruled out. Treat this session as a floor on what the repaired protocol +can resolve, not as a final contrast estimate. Before the next run, clear hair +from TP9/TP10, lightly damp and reseat the contacts, and inspect signal quality +before starting the timed blocks. + +Evidence remains local because it contains raw EEG: + +- CSV SHA-256: `9c19cd44458180f50e4281aa7a68f02e4a4dd30de8c1ddd56b987c410199d316` +- JSON SHA-256: `c836c1a3b18c2dbf1fa6bf460867d2f02278ecae94a4e5b94a8da3c16f8194f4` +- Session metadata SHA-256: `c22222dfcea8a298e45adc40ed2e711844ddc0309b4a185a433c60147ea7852d` +- Power state SHA-256: `ed5c80f90b978cb689597070ac4cc728b68b10c65b77ab77f182f764b877082c` + +The GPD was at 34% battery, discharging, with the `powersave` governor and +`balanced` platform profile. BrainFlow development logging was disabled and USB +Bluetooth autosuspend remained disabled. BrainFlow still returned +`BOARD_NOT_READY_ERROR:7` on two connection attempts before the successful run. +The BLE failure is worked around, not fixed. Failure with autosuspend disabled +refutes autosuspend as the explanation for these attempts; retry is the current +bounded workaround, and deeper BLE tracing is deferred unless failures degrade. + +### Acquisition dry run + +The repaired protocol completed its first live acquisition dry run on the GPD, +based on commit `b452015bbcddfd46cd66d8cf7575fe9ffe56d0b5` plus the uncommitted +shuffle diagnostics, with ABBA labels. All four alpha blocks contained the full +3,840 samples. The CSV contains 21,248 samples plus its header. The participant +did not see the eyes-open/eyes-closed instructions and kept the same eye state, +so this run **does not exercise or validate the alpha protocol**. It demonstrates +capture and analysis only; three channels were also saturated. + +| Channel | RMS (uV) | condition | time-index | shuffled condition, 10 seeds | +|---|---:|---:|---:|---:| +| TP9 | 791.7 | 1.05x | 0.29x | 0.81-1.24x | +| AF7 | 793.5 | 0.87x | 0.42x | 0.66-1.38x | +| AF8 | 41.1 | 0.71x | 1.31x | 0.68-1.01x | +| TP10 | 308.4 | 0.67x | 0.85x | 0.60-0.99x | + +Blink passed on AF8 at 108.4 uV. Jaw clench and head turn did not pass. The +program reported **1/5, INCONCLUSIVE**, but the alpha ratios are not physiological +results because the instructed eye-state contrast was not performed. The next +session therefore required an explicit participant ready check plus visible and +audible confirmation of every eye-state transition. + +The transport result is separate and positive. With USB controller `3-5` +autosuspend disabled, `muse-ble-bridge` streamed at 256 Hz with zero drops, and +the validator subsequently completed. The successful validator run used a +temporary copy with BrainFlow development logging disabled. Logger-enabled +bridge controls also streamed successfully, so logging is **not established as +the cause** of the earlier TP9-subscribe hangs. + +Evidence remains local because it contains raw EEG: + +- CSV SHA-256: `525bf456d1067e60cd9877e4d8b9446ece5c384ee33dbfc3fc62f5533c914b06` +- JSON SHA-256: `279bd9d8ae3d885e81125d687f350728fe66493980fe385687e95c19c6ceb807` +- Session metadata SHA-256: `d855ff6e2906f058a7f8bd2e2d88eb8c4979022395966bd5b902fc625e40a8a9` +- Power state SHA-256: `32f5dc6286ed3a6ed7a76ba90c7c7b6e31f9ce2c4ab4b313625709d348a76874` + +## Historical Results (2026-07-10) + +> **On synthetic data containing zero alpha and pure drift, the old design reports 7.11× +> against a 1.5× threshold** — passing by nearly five times the margin on a signal that is +> not there. That is the regression asserted in `validate-muse-physiology.py --self-check`, +> and it is the measure of how little the old protocol constrained. +> +> **Temporal permutation puts the fixture's amplitude-only condition ratio as high as +> 1.444236× against that same 1.5× threshold.** The permutation is applied after the +> 2-second edge trim, independently within every block and channel. It destroys temporal +> spectral structure while preserving each block's samples and total variance exactly. +> The result is only 3.7% below the gate, so `condition >= 1.5×` by itself is a weak +> discriminator when eyes closing also changes broadband power. The self-check requires +> every shuffled real-alpha ratio to fail the gate and lose at least 3× of its excess over +> 1, while pure-drift time-index excess survives the same shuffle. Live sessions always +> report all ten shuffled ratios per channel, but shuffle is **not yet a live gate**; that +> promotion requires several real sessions to establish a threshold. +> +> Every figure below was collected under that protocol: the `open → closed` ordering, in +> which condition is confounded with block order. The TP9/TP10 alpha rises are large enough +> to be plausible on their face, but they are **not demonstrated** until re-collected under +> ABBA with the time-index baseline reported. Treat them as provisional; do not cite them +> as evidence the acquisition pipeline resolves alpha. +> +> **On the "0.883" figure:** the drift concern was argued for some time from a claim that +> *time-index alone classifies open vs closed at 0.883*. **That measurement has no +> preserved artifact.** It is not in any recording, script output, or log in this repo; it +> entered a working session as prose and was carried forward as established. The nearest +> values on disk — `Scripts/dream_extraction.py:335`, rho 0.8827/0.8857 — are a different +> metric on a different analysis, already annotated there as small-n artifacts. **Do not +> re-cite 0.883.** The redesign stands on the design defect, which is visible in the +> source and reproduced by the `--self-check` regression, not on that number. Single participant, single session, Muse S on the head, 80-second protocol: @@ -92,13 +377,22 @@ python3 -m venv /tmp/nc-bf-py /tmp/nc-bf-py/bin/pip install brainflow # Put the Muse S on your head, hair parted under the behind-ear pads -# Then run the validation script +# Then run the validation script. Alternate ABBA and BAAB across sessions. DYLD_LIBRARY_PATH=~/Developer/brainflow/compiled \ /tmp/nc-bf-py/bin/python3 \ - ~/Developer/NeuralCompose/Scripts/validate-muse-physiology.py + ~/Developer/NeuralCompose/Scripts/validate-muse-physiology.py --order ABBA ``` -The script prompts the user through each condition with 3-second countdowns. Recorded data is written to `Recordings/muse_validation_.csv`. +After connecting, interactive runs report a live RMS contact snapshot and require +either a clean recheck or an explicit `RUN` override before the timed protocol. +The summary JSON records every preflight check, its per-channel RMS values and +separate `healthy`, `saturated`, and `dead` classifications, the final check, +whether an override was used, and the inclusive 2–200 uV classification +thresholds. The same status, thresholds, and final per-channel classes print in +the physiological summary beside the reported ratios. The script then requires +acknowledgment of each condition before its 3-second countdown. +Recorded data is written to `Recordings/muse_validation_.csv` with a +matching JSON summary. ## What is NOT Validated @@ -121,7 +415,7 @@ For a new participant or a new Muse S: - [ ] All 4 pads making skin contact (firm pressure, no hair) - [ ] Sitting in a quiet room, eyes at a fixed point - [ ] Run the protocol; check the per-channel alpha ratio and RMS -- [ ] If RMS < 5 µV on any channel, reposition the headband and re-run -- [ ] If alpha ratio < 1.5× on all channels, the Muse is not on the head correctly +- [ ] If RMS is outside 2–200 µV on any channel, reposition and recheck before capture +- [ ] Interpret alpha only when condition ≥ 1.5×, beats time-index, and the report-only shuffle diagnostic supports spectral rather than amplitude contrast The protocol is robust to one or two bad channels. Three or four bad channels means the headband is not on correctly and the recording is not valid. diff --git a/docs/architecture/embedding_contract.md b/docs/architecture/embedding_contract.md index 653a46e..14e41e6 100644 --- a/docs/architecture/embedding_contract.md +++ b/docs/architecture/embedding_contract.md @@ -79,7 +79,7 @@ An `Embedding` value **must** satisfy: | 2.4 | `modelID` and `version` are non-empty and match the producer's `modelID`/`version` | `Embedding.swift:26-38`, test `testProvenanceIsPopulated` at `SentenceEmbedderTests.swift:48` | | 2.5 | `cosineSimilarity(to:)` returns a value in `[-1, 1]` (within float error) and is a plain dot product (the L2-normalization invariant is what makes it one) | `Embedding.swift:54-63` | | 2.6 | `cosineSimilarity(to:)` returns `0` when dimensions differ, rather than trapping — different spaces are incomparable, not broken | `Embedding.swift:58-59` | -| 2.7 | Two embeddings with different `modelID`s are not comparable, even if their `dimension` matches | `Embedding.swift:27-28` doc comment | +| 2.7 | Two embeddings with different `modelID`s are not comparable, even if their `dimension` matches | `Embedding.isComparable(with:)`; `cosineSimilarity(to:)` returns `nil`; `DialecticalDynamicsTests.testIncomparableIsNilNotZero` | The struct carries *cheap* provenance (`modelID`/`version`/`seed`/ `dimension`) by design. Heavier provenance — model SHA256, tokenizer diff --git a/docs/research/README.md b/docs/research/README.md index 2b3889a..fe303f3 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -36,3 +36,4 @@ Design/Verification/Runtime artifact taxonomy — it contains the | [embedding-model-survey.md](embedding-model-survey.md) | 3.0 | Survey (this commit) | | [methodology-review_v1.md](methodology-review_v1.md) | 3.4/3.5 | Review (superseded by v2) | | [methodology-review_v2.md](methodology-review_v2.md) | 3.4/3.5 | Review (this commit) | +| [eyes-open-closed-step0-preregistration_v1.md](eyes-open-closed-step0-preregistration_v1.md) | EEG Step 0 | Pre-registration (frozen before post-selection session 1) | diff --git a/docs/research/eyes-open-closed-step0-preregistration_v1.md b/docs/research/eyes-open-closed-step0-preregistration_v1.md new file mode 100644 index 0000000..9a4854d --- /dev/null +++ b/docs/research/eyes-open-closed-step0-preregistration_v1.md @@ -0,0 +1,240 @@ +# Eyes-Open / Eyes-Closed Step 0 Pre-registration v1 + +- Status: **frozen before post-selection session 1** +- Frozen: 2026-08-06 +- Applies to: one-participant Muse S TP10 floor evaluation +- Selection data: excluded from fitting and evaluation +- Evaluation trigger: six eligible post-registration sessions, with no interim classifier runs + +This document fixes the Step 0 analysis before more evaluation data is +collected. It is a pre-registration, not a report. Do not retro-edit it after +the first post-registration session. Any change requires a versioned successor +and a fresh evaluation set. + +## Question and claim boundary + +Primary question: + +> Can a fixed, one-channel TP10 feature pipeline classify eyes-open versus +> eyes-closed windows out of session, while failing the corresponding time-index +> and shuffled-label null arms? + +The only promotable claim is a **single-participant, single-channel Step 0 +floor**. A pass does not establish: + +- spatial EEG decoding; +- transfer to another participant or headband; +- sleep-stage classification; +- imagined-speech decoding; +- an SSL, EEGNet, EEGPT, or foundation-model result; +- sufficiency for any encoder that depends on central or occipital coverage. + +## Disclosed channel selection + +The channel choice is post-selection, and that selection is part of the record. + +TP10 is the sole model input. It was selected after inspection of the two clean +protocol sessions below: + +| Acquisition | Order | TP10 condition | TP10 time-index | TP9 condition | Selection role | +|---|---|---:|---:|---:|---| +| `muse_validation_20260806-110528` | ABBA | 3.0164x | 0.7291x | 1.4109x | exploratory selection | +| `muse_validation_20260806-111651` | BAAB | 3.2574x | 1.5570x | 1.1805x | exploratory confirmation | + +The corresponding JSON SHA-256 values are: + +- ABBA: `2a44a8865521ff29ca46504799b4009836cacb6ed231d5c2d49a166fe067092a` +- BAAB: `e586ef65906f5f28470f493f96ae1223b74d0af1ee24f7c1c759efdacba92eb8` + +Earlier dry and contact-limited sessions were also inspected and are not +held-out data. No acquisition whose artifact timestamp is at or before +`20260806-111651` may enter model fitting, threshold selection, or evaluation. + +Other channels are fixed as follows: + +- **TP9:** negative diagnostic only. It is never a model feature and never an + acceptance input. Its per-session condition, time-index, shuffle, and contact + values are reported unchanged. +- **AF7:** excluded. It saturated in all three analyzable attended captures. +- **AF8:** excluded. A 1.75x live condition ratio survived temporal permutation + at 3.08-5.26x, identifying amplitude rather than isolated spectral contrast. + +Channel selection is now closed. Evaluation data cannot add a channel, remove +TP10, create an ensemble, or select the best channel post hoc. + +## Collection contract + +### Fixed size and order + +The evaluation set contains **six eligible post-registration sessions**. The +eligible-session order is fixed before collection: + +1. ABBA: open, closed, closed, open +2. BAAB: closed, open, open, closed +3. ABBA +4. BAAB +5. ABBA +6. BAAB + +If an attempt is ineligible, retain and hash its artifacts, record the reason, +and repeat the same scheduled order. Stop after at most **eight total attempts**. +If six eligible sessions do not exist after attempt eight, stop with +`INCONCLUSIVE_INSUFFICIENT_ELIGIBLE_SESSIONS`; do not lower the criteria or run +the classifier on fewer sessions. + +### Eligibility + +Eligibility is mechanical and cannot use alpha ratios, classifier output, or +whether a session looks promising: + +- `contact_preflight.status == "passed"`; +- `contact_preflight.override_used == false`; +- all four alpha blocks contain exactly 3,840 samples and have + `short_block == false`; +- TP10 is `healthy` in every `alpha_block_contact` record under that session's + stored inclusive RMS thresholds; +- `block_order` exactly matches the scheduled order; +- CSV, JSON, session metadata, and power-state artifacts exist and are hashed. + +TP9, AF7, and AF8 contact do not determine eligibility because they are not +model inputs. Their failures remain reportable diagnostics. No epochs or blocks +may be removed after a session passes these rules. + +### No interim model analysis + +The acquisition validator may print its pre-registered physiological and null +diagnostics because those are needed to verify capture. Do not run the Step 0 +feature pipeline, classifier, cross-validation, aggregate accuracy, or shuffled- +label analysis until the sixth eligible session is frozen. Collection does not +stop because single-session ratios rise or fall. + +Before post-registration session 1, the analysis implementation and synthetic +tests must be committed. Record that commit in the evaluation manifest. After +collection starts, code changes require a versioned successor pre-registration +and fresh sessions; they cannot be evaluated on this six-session set. + +## Fixed examples and features + +Only TP10 is read. For each of the four alpha blocks: + +1. discard the first 2.0 seconds, exactly as the acquisition validator does; +2. divide the retained 13.0 seconds into six non-overlapping 2.0-second windows; +3. discard the final 1.0-second remainder; +4. assign every window its block's eyes-open (`0`) or eyes-closed (`1`) label. + +Each eligible session therefore contributes 24 windows: 12 open and 12 closed. +No overlap, artifact rejection, amplitude clipping, channel substitution, or +window deletion is allowed. + +For each 512-sample window, compute Welch power with these fixed parameters: + +- sampling rate: 256 Hz; +- Hann window; +- `nperseg = 256`; +- `noverlap = 128`; +- constant detrending; +- density scaling. + +The feature vector, in fixed order, is: + +1. `log10(delta_power + 1e-12)`, 1-4 Hz; +2. `log10(theta_power + 1e-12)`, 4-8 Hz; +3. `log10(alpha_power + 1e-12)`, 8-13 Hz; +4. `log10(beta_power + 1e-12)`, 13-30 Hz; +5. `log10((alpha_power + 1e-12) / (power_1_30_hz + 1e-12))`. + +Band edges use `low <= frequency < high`, except 30 Hz is included in the +1-30 Hz denominator. No feature may be added, removed, transformed, or selected +using evaluation results. + +## Fixed classifier and split + +Use a scikit-learn `Pipeline` containing: + +1. `StandardScaler()` fit only on each training fold; +2. `LogisticRegression(penalty="l2", C=1.0, solver="liblinear", + class_weight=None, random_state=0, max_iter=1000)`. + +There is no hyperparameter search, calibration, early stopping, neural model, +or alternate classifier. + +Evaluation uses leave-one-session-out cross-validation over the six eligible +sessions. Every window from one session is held out together. Fit preprocessing +and the classifier on five sessions, predict the sixth, and concatenate the six +held-out prediction sets for the aggregate score. Window-level random splits +are forbidden. + +## Null arms + +All arms use the identical features, classifier, folds, and metric. + +### N1: time-index labels + +Replace the physiological label with block-position labels: blocks 1-2 are `0` +and blocks 3-4 are `1`. This is the classifier analogue of the acquisition +time-index control. Report aggregate and per-session balanced accuracy. + +### N2: shuffled block labels + +For seeds 0 through 99, use `numpy.random.default_rng(seed)` to independently +permute `[0, 0, 1, 1]` across the four whole blocks within each session. All six +windows from a block retain the same shuffled label. This preserves class +balance and within-block dependence. Re-run the full leave-one-session-out +pipeline for each seed and report the complete 100-score distribution plus its +95th percentile. + +The acquisition validator's within-block temporal-permutation ratios remain +session diagnostics. They are reported but are not substituted for N2. + +## Fixed metrics and decision + +Primary metric: **balanced accuracy** over all held-out window predictions. +Also report each held-out session's balanced accuracy and a 95% session-cluster +bootstrap interval using 10,000 resamples of the six held-out sessions with +`numpy.random.default_rng(0)`. + +The Step 0 floor is `PASS` only if every condition holds: + +1. true-label aggregate balanced accuracy is at least **0.70**; +2. at least five of six held-out sessions have balanced accuracy at least + **0.60**; +3. the session-cluster bootstrap lower bound is greater than **0.50**; +4. true-label accuracy exceeds time-index accuracy by at least **0.10**; +5. time-index accuracy is at most **0.60**; +6. true-label accuracy exceeds the shuffled-label 95th percentile by at least + **0.10**. + +Any failed condition yields `FAIL`. Fewer than six eligible sessions after the +attempt cap yields the inconclusive status defined above. There is no partial +pass and no promotion based on a favorable subset. + +## Required output and provenance + +The evaluation writes one machine-readable result containing: + +- this pre-registration path and SHA-256; +- analysis commit and dirty-worktree status; +- six included session IDs and all attempted session IDs; +- source CSV and JSON SHA-256 values; +- each session's order, stored contact thresholds, preflight status, override + flag, TP10 per-block contact classes, and power-state sidecar; +- exact package versions for Python, NumPy, SciPy, and scikit-learn; +- feature matrix shape and per-session window counts; +- aggregate and per-session true-label scores; +- time-index score; +- all 100 shuffled-label scores and their 95th percentile; +- bootstrap seed, resample count, and confidence interval; +- each acceptance condition and the final status. + +Report TP9, AF7, and AF8 protocol diagnostics beside the model result, but do +not feed them into the model or decision. + +## Failure and successor rule + +The six-session set is spent after one preregistered evaluation, whether it +passes or fails. Do not tune features, channels, thresholds, classifier, or +session inclusion on it and then report a second score as confirmatory. + +A revised analysis requires `eyes-open-closed-step0-preregistration_v2.md`, an +explicit account of what v1 taught, and a fresh post-v2 evaluation set. The v1 +file and result remain unchanged. diff --git a/docs/reviews/absent-as-legitimate-value-2026-08-12.md b/docs/reviews/absent-as-legitimate-value-2026-08-12.md new file mode 100644 index 0000000..a0939e8 --- /dev/null +++ b/docs/reviews/absent-as-legitimate-value-2026-08-12.md @@ -0,0 +1,210 @@ +# An absent value given a numeric meaning that is also a legitimate value (2026-08-12) + +**Status:** open. Found while porting `BCICore`'s dialectic to Rust +(`neuralcompose-client-native`, `crates/neuralcompose-hypnagogic`). + +A recurring shape, not three accidents. In each case a value that is *missing*, +*unfindable* or *incomparable* is represented by a number that the same field +can legitimately hold — so the failure is indistinguishable from a real reading +and produces no symptom. + +## Instances + +### 1. `Embedding.cosineSimilarity(to:)` returns `0` for incomparable operands + +`Sources/BCICore/Models/Embedding.swift:57`. **`0` is a legitimate cosine** — +orthogonal vectors from the same model return it. Covered in full, with its own +blast radius, in +[`embedding-comparability-unenforced-2026-08-12.md`](embedding-comparability-unenforced-2026-08-12.md). + +### 2. `DialecticalDynamics.centroid(of:)` silently averages a subset + +`Sources/BCICore/Dialectic/DialecticalDynamics.swift`. Members of a differing +dimension are skipped, and the result is returned as if complete. **A subset +centroid is a real vector**, and every similarity measured against it afterwards +is plausible. Nothing reports that anything was dropped. + +### 3. `roleFulfillment: role?.objective(energy) ?? 0` — and this one is different in kind + +`Sources/BCICore/Composition/HypnagogicDialecticLoop.swift:241`, and the sharpest +of the set. `roles.first { $0.id == roleID }` is a lookup that can miss; a miss +scores `0`. + +`ScoredCandidate.roleFulfillment` is documented at +`DialecticalCompetition.swift:88-92` as existing so "a later stage [can] notice a +role that failed its own brief (e.g. a displacement-seeking pass that produced +something un-novel because the model refused to diverge)". **`0` is precisely +that signal.** So a lookup failure manufactures the exact reading the field +exists to detect, and the diagnostic reports a maximally-failed role instead of +an internal inconsistency. + +**The distinction worth keeping, because it is the version of this argument that +should convince someone who finds the other four unremarkable:** instances 1, 2, +4 and 5 are a sentinel colliding with a *plausible number*. This one collides +with a *diagnostic*. The failure does not merely hide among legitimate values — +it **impersonates the specific signal a downstream stage is watching for**. A +role-lookup miss does not produce a suspicious zero; it produces a confident +report that the displacement pole utterly failed its brief. Anything built on +that field to detect degenerate generation will fire on an internal +inconsistency and point at the model. + +Not currently reachable: `candidateTexts` is built from `roles`, so the id is +always present. It is one refactor away — a filtered role list, a duplicate id, a +role set swapped mid-turn — and the consequence is a wrong diagnostic rather than +a crash. + +### 4. `FeatureExtractor` band energies — reachability checked, and it is *not* the live path + +`Sources/BCICore/Preprocessing/FeatureExtractor.swift:122-126` — +`energies["delta"] ?? 0` and four siblings. **`0` is a legitimate band energy** +(a dead electrode), so a missing dictionary key and a flat channel are the same +number downstream. + +This was worth checking rather than filing next to the others by proximity, +because electrode failure-without-saying-so is the project's active constraint. +Two questions, both answered: + +**Does the Core ML classifier consume this? No.** +`CoreMLIntentClassifier.classify(window:)` builds its `MLMultiArray` directly +from the raw window (`multiArray(from:)`, :119) — channels × samples, zero-padded +— and never touches `FeatureExtractor`. There is no wrong-input path into the +shipped classifier. `FeatureExtractor`'s only callers are +`MockIntentClassifier.swift:46` (the stub path), `JEPATransition.swift:41`, and +one golden-recording regression test. + +**Is the dictionary lookup actually breakable? Barely.** The band names are +written and read as hardcoded literals in the *same private function*, twenty +lines apart (`centers` at :103, the reads at :122-126), with a loop that always +writes all five. There is no config, no external band label, and no caller who +can influence the keys. The `?? 0` is dead defensive code. Its fuse is a rename +typo in one of the two lists — short, but local and immediately visible. + +So: **lowest severity in this set, and reachable by neither of the routes that +made it look urgent.** Fix it as tidiness (destructure the tuple list, or build +`Bands` in the loop) rather than as a defect. + +**The sharper sentinel in that same function is a different line.** At :117, +`let e = n > 0 ? Float(...) : 0` — a channel shorter than the lag is skipped +(`if ch.count <= lag { continue }`), and if *every* channel is too short the band +reports `0`. That is insufficient-data rendered as a real measurement, and unlike +the dictionary read it depends on runtime values: `lag = sampleRate / centerHz / 2`, +so delta at 256 Hz needs 51 samples. Comfortable for a 1–2 s window, but it is +the window length and sample rate — not a literal — that keep it safe. + +**Where the wrong value would actually land:** `JEPATransition.init?(window:)` +takes `alphaEnergy`/`betaEnergy`/`thetaEnergy` straight from here into the +capture persisted for offline JEPA training (ADR-006). It guards `isFinite`, +which a spurious `0` passes. So the consequence is not a bad live decision but a +**poisoned training corpus** — silently, and discovered much later. + +#### Checked: it has not happened, because the capture has never run + +An exact `0.0` band energy out of a float computation over real EEG is a +measure-zero event — genuine delta power is small but never exactly zero — so +existing captures can be *scanned* for it rather than reasoned about. Any hit +would be either this bug or a dead channel, and both are worth finding. + +Ran it, 2026-08-13. **There are no captures.** `~/Documents/NeuralCompose/` holds +`Calibration/`, `EEGIntegration/`, `InteractionLogs/`, `Recordings/`, +`health.json` and `voice-profile.json` — and no `JEPATransitions/` directory at +all, on either machine. `TransitionCaptureManager` writes +`~/Documents/NeuralCompose/JEPATransitions/jepa_transitions.jsonl` +(`TransitionCaptureManager.swift:58-60`) and that path has never been created, +which matches `CLAUDE.md`'s own statement that there are zero logged interaction +events because the capture is opt-in and off by default. + +That is the good version of this finding rather than the disappointing one: the +corpus is empty, so the guard can land **before** the first capture run at zero +cleanup cost, and there is no archaeology to do. It also means the check is +better placed at capture time than after: reject a window whose band energies +come back exactly zero, rather than scanning for them later. + +Pair it with making the implicit precondition explicit. The safety of :117 +rests on window length ≥ lag — real, load-bearing, and currently unwritten. +Assert it (`window.sampleCount > lag` for the lowest-frequency band) instead of +relying on windows happening to be long enough. + +### 5. `ProsodyWobble` substitutes concrete defaults for `nil` + +`Sources/BCICore/Composition/ProsodyWobble.swift:49-52` — `base.rate ?? 0.5`, +`base.volume ?? 0.9`, `base.preUtteranceDelay ?? 0.1`. Here `nil` means "let the +engine choose", and the substitution silently converts that into a specific +number. Arguably deliberate — you cannot wobble an absent value — but it means a +prosody that deliberately abstained comes out of the wobble having decided. + +## The codebase already contains the correct idiom + +**`SpeechProsody.blend` gets this exactly right** +(`Sources/BCICore/Protocols/SpeechSynthesizing.swift:128-141`): each field is +averaged only over contributors that specify it, a `nil` field *abstains* rather +than voting zero, and the result is `nil` when nothing contributed. That is the +pattern the other five should follow. + +Worth being precise about this, because an earlier verbal summary of mine +mistakenly listed the blend as a third instance of the defect. It is not — it is +the counter-example. The bug there was in the Rust *port*, which had reimplemented +it as a two-way lerp over non-optional fields; the Swift was right and the port +was wrong. Corrected in client-native `39c642a`. + +So this is an inconsistency, not an unfamiliarity: the codebase knows how to do +this and does it in one place out of six. + +## Why it is worth a pass rather than a note + +The three ports that hit instances 1–3 all had to make the same decision +independently, which is the signal. In the Rust the resolution was uniform: + +- `cosine_similarity -> Option`, propagated through `energy`, `tension` and + `synthesis_score`, so **missing** (an early turn, legitimately neutral at 0.5) + stays distinct from **incomparable** (a defect); +- `centroid` refuses a mixed set rather than averaging part of one; +- `Prosody::blend` abstains, matching the Swift. + +Each is pinned by a test that asserts the two cases are *not equal* — e.g. +`Some(0.0) != None` for orthogonal-versus-incomparable — so reintroducing a +sentinel names itself. + +## Suggested handling + +Ranked by what the failure would actually cost, not by how alarming the line +looks: + +1. **`roleFulfillment` (3)** — highest. It impersonates a diagnostic rather than + hiding among plausible values, so anything built on that field to detect + degenerate generation would misattribute an internal inconsistency to the + model. Fix even though it is currently unreachable. +2. **`cosineSimilarity` (1)** and **`centroid` (2)** — fix together; they share a + file and a caller, and the Rust port has already demonstrated the resolution. +3. **`FeatureExtractor`'s `n > 0 ? … : 0` (4, second line)** — insufficient data + rendered as a measurement, on the path that feeds a *persisted* JEPA training + corpus. Worth a guard that refuses rather than reports zero. +4. **`FeatureExtractor`'s dictionary reads (4, first lines)** — tidiness only. + Not reachable; the keys are literals in one private function. +5. **`ProsodyWobble` (5)** — leave hedged. Substituting a default for "let the + engine choose" is a real semantic loss, but it is plausibly what was meant, + and it is the one case here where a fix would change *output* rather than only + diagnostics. That asymmetry is the argument for leaving it alone until someone + states the intent. + +Nothing here should be fixed *only* in the Rust port. The dialectic's Rust half +is now checked against the Swift by a committed conformance fixture +(`Sources/DialecticFixture`), so a one-sided "improvement" shows up as a +conformance failure rather than as an improvement — which is the mechanism +working, but it means the Swift moves first. + +## Related, found in the same pass + +`HypnagogicDialogueLoop.chunk` emits lone-punctuation chunks for a punctuation +run: `"Wait... really?!"` → `["Wait.", ".", ".", "really?", "!"]`, because every +punctuation character closes the current chunk and only *empty* results are +dropped — `"."` is not empty. + +This is a **latency** bug rather than a cosmetic one. Each lone `"."` is its own +`speak()` call, so a single ellipsis costs three utterances, three +`preUtteranceDelay`s and (in the Linux shell) three subprocess spawns — over a +second of dead air inside one spoken turn on the contemplative voices, where +`preUtteranceDelay` is longest and ellipses are most common. In a conversational +loop, turn latency is the product. + +The Rust port pins the current behaviour rather than diverging, so this must be +fixed here first. diff --git a/docs/reviews/embedding-comparability-unenforced-2026-08-12.md b/docs/reviews/embedding-comparability-unenforced-2026-08-12.md new file mode 100644 index 0000000..6f7d636 --- /dev/null +++ b/docs/reviews/embedding-comparability-unenforced-2026-08-12.md @@ -0,0 +1,178 @@ +# Embedding comparability is documented but unenforced (2026-08-12) + +**Status:** fixed on `fix/embedding-comparability` (2026-08-22), unverified on Linux (no Swift toolchain on the host); `swift test` on macOS pending. Originally: open defect, not yet fixed. Found while porting `DialecticalDynamics` +to Rust (`neuralcompose-client-native`, `crates/neuralcompose-hypnagogic`). + +## The defect + +`Embedding.cosineSimilarity(to:)` (`Sources/BCICore/Models/Embedding.swift`) +guards on dimension only: + +```swift +public func cosineSimilarity(to other: Embedding) -> Float { + guard values.count == other.values.count else { return 0 } + var acc: Float = 0 + for i in 0.. Float?`, `nil` for a different `modelID`, a + different dimension, or a degenerate operand. `nil` is not a failed + computation; it is the statement that no similarity exists to compute. +2. Propagate through `energy`, `tension`, `synthesisScore` as `Optional`, so + **missing** (an early turn, legitimately neutral at `0.5`) stays distinct + from **incomparable** (a defect). +3. `centroid(of:)` returns `nil` on any mismatch rather than averaging the + comparable subset. +4. The contract table in `embedding_contract.md` §2.7 should then cite the + type signature, not a doc comment. + +The Rust side already carries this and pins it with a test asserting +`Some(0.0) != None` — orthogonal-and-comparable versus incomparable — so a +future reintroduction of the sentinel names itself. + +## Not decided here — and the question is probably not "which failure mode" + +The first two candidates were `nil` versus a `preconditionFailure`. Both assume +cross-model comparison is a *similarity* operation that should either return +nothing or crash. + +**A third option says it is not a similarity operation at all**, and the +client-native workspace has already built the vocabulary for it: + +- `crates/neuralcompose-mobile-core/src/property_law.rs:28-31` makes + `IndexEntryKey` the pair *(content digest, embedding-space identity)* — + "two records with different embedding identities are never the same entry, + however similar the text." +- `property_law.rs:66-70`, `shares_index(a, b)`, is exactly this question as an + executable law: entries belong in the same index **only** when + `embedding_space_identity` matches, because "mixing spaces silently poisons + retrieval." +- `model_pack.rs:305`, `embedding_space_identity()`, derives that identity from + model family, revision, weight and tokenizer digests and dimensions — so the + space has a name that cannot be forged, not just a `modelID` string. +- PR #30 treated cross-backend divergence as a **conformance measurement** + (`tests/embedding_agreement.rs`, `vulkan_agreement.rs`, `composed_error.rs`), + a different question with its own semantics and its own tolerances — not a + similarity with a caveat. + +So if a future caller genuinely needs to relate two spaces, it likely wants a +distinct, explicitly-declared operation that states what it means — a +conformance or agreement measurement between named spaces — rather than +borrowing `cosineSimilarity` and being handed a `nil` or a trap for its trouble. + +That reframes the decision from *which failure mode* to *which operation*, and +it makes `nil` the cheap, honest default rather than a compromise: the primitive +declines to answer a question it was never the right operation for, and the +caller that actually needs an answer asks a differently-named one. + +### Where the machinery actually lives + +A correction to an earlier draft of this section, which said no +`EmbeddingProfileTerms` type existed. That was true of the two repos searched +(`neuralcompose-client-native/crates/` and `NeuralCompose/Sources/`) and **false +of the workspace**: it is in a third repo, `neural-memory-server`, at +`crates/neural-memory-domain/src/terms.rs:444`. + +It is the richest form of the idea. Its own doc comment says the field set +"mirrors `neuralcompose-mobile-core`'s `embedding_space_identity` +(`model_pack.rs:305`)", and it seals: + +``` +model_family, model_revision, weight_sha256 (sorted), tokenizer_sha256 (sorted), +dimensions, pooling, normalization, task_instruction +``` + +— "change any of them and the vectors mean something else, however similar the +text." Digests are sorted before sealing because shard listing order is noise. + +**And it settles the operation question outright**, in a passage worth quoting +rather than paraphrasing: + +> **The backend is absent, and that is the point.** A CPU run and an NPU run of +> the same model produce vectors in the same space or they do not, and that is a +> question to be *measured*, not asserted by stamping a different identity on +> them. Putting the backend here would fork the space by declaration and make +> the measurement unaskable. Where the vector came from belongs on the runtime +> variant; whether the two may share an index is what conformance decides. + +So the third option is not a proposal. It is the position this workspace already +took, deliberately, and wrote down: **identity declares the space; conformance +measures whether two things share one.** `cosineSimilarity` is neither, which is +why handing it a cross-space pair has no good answer — and why `nil` is the +honest one. + +The gap is that all of this is Rust. Swift has no counterpart to +`EmbeddingProfileTerms`, `embedding_space_identity`, `IndexEntryKey` or +`shares_index`, and its `Embedding` carries a bare `modelID` string that nothing +checks. The enforcement machinery exists in the workspace, just not in the +language where the 32 call sites are. diff --git a/docs/reviews/time-leakage-2026-08-06.md b/docs/reviews/time-leakage-2026-08-06.md new file mode 100644 index 0000000..6a4846d --- /dev/null +++ b/docs/reviews/time-leakage-2026-08-06.md @@ -0,0 +1,123 @@ +# Time leakage in two EEG evaluations — 2026-08-06 + +Two places where a design cannot separate signal from elapsed time. One was a real defect, +measured and fixed. The other looked like the same disease and did not survive measurement. + +That asymmetry is the point of the document. Both were argued from the same reasoning; only +one of them reproduced when a synthetic check was pointed at it. + +## 1. Muse physiological validation — FIXED + +`Scripts/validate-muse-physiology.py` recorded one 30 s eyes-open block followed by one +30 s eyes-closed block, always in that order, and reported +`alpha_closed / alpha_open` against a 1.5× threshold. Condition was perfectly confounded +with block position, so impedance settling, electrode warming, and drowsiness onset all +pushed the ratio the same way real alpha does. + +Fix: four counterbalanced blocks (`open, closed, closed, open`, 15 s each — same 60 s of +alpha data), first 2 s of every block discarded for the alpha build-up, cue onset logged +separately from block start, and a **time-index baseline** gate — the identical statistic +computed on the same blocks relabelled by position alone (1–2 vs 3–4), which under ABBA is +orthogonal to condition. A channel passes only if the condition ratio clears 1.5× *and* +exceeds the time-index ratio. + +Regression, in `--self-check`: on synthetic data with **zero alpha and pure drift**, the +old sequential statistic reports **7.11×** — a comfortable pass at a 1.5× threshold — +while the new gate rejects it. That assertion is the point of the change; without it the +suite would only prove the new gate works, not that it catches something the old one +missed. + +The 2026-07-10 results (TP9 2.98×, TP10 3.88×) were collected under the old protocol and +are now annotated in `docs/Validation.md` as provisional pending re-collection. + +## 2. Imagined-speech CV — CHANGED, but the leak did not reproduce + +**Correction to the first draft of this document.** It stated that the 0.65 gate is +"inflated by an unknown amount" and that a pre-registered threshold evaluated with a +leaky estimator is void. The first half of that is not supported by measurement, and +this section is rewritten accordingly. + +`Scripts/evaluate-imagined-signal.py:238`: + +```python +skf = StratifiedKFold(n_splits=folds, shuffle=True, random_state=seed) +``` + +Trials arrive in acquisition order. `shuffle=True` scatters temporally adjacent trials +across train and test, so a fold's test set is neighboured in time by its own training +data. Anything that drifts slowly within a session — impedance, electrode temperature, +alertness, headband slip — is then partially learnable rather than held out, and a +classifier can score above chance without decoding anything about imagined speech. + +### What was measured + +Synthetic trials with **no class signal at all**: amplitude drifts monotonically with +acquisition position, labels are a 50/50 shuffle (the order +`ImaginedSpeechProtocol.buildTrialOrder` actually produces), which by chance yields runs +of same-class trials. If shuffled folds leak, this is the data that should show it — a +trial's immediate neighbours sit in the training set at nearly the same drift level. + +Four seeds, `--self-check`: + +| seed | blocked (GroupKFold) | time-index only | **shuffled (old design)** | +| --- | --- | --- | --- | +| 0 | 0.4688 | 0.5250 | 0.4375 | +| 1 | 0.4914 | 0.5000 | 0.4500 | +| 2 | 0.4571 | 0.5000 | 0.5000 | +| 3 | 0.4912 | 0.5000 | 0.5750 | + +**The shuffled evaluator did not clear the 0.65 gate on drift-only data.** It sat at +chance, like the blocked one. The reason is capacity: `LinearSVC` on 16 global band-power +features cannot exploit fold-local structure — it has no way to memorise a +drift-level → label association for interleaved runs. Shuffled-CV leakage is real for +high-capacity estimators; it was not demonstrated for this one. + +A second construction, with labels deliberately correlated with acquisition position +(long same-class runs), also failed to pass: blocked 0.4656, time-index 0.5250. Blocked CV +holds out contiguous drift ranges the model has never seen, so it is inherently robust to +this failure mode. + +**So the claim that the 0.65 threshold is inflated is withdrawn.** No measurement here +supports it. The prior draft asserted it from the design smell alone — the same move as +citing 0.883, one section down. + +### What changed anyway, and why + +- `GroupKFold` on contiguous acquisition blocks replaces `StratifiedKFold(shuffle=True)`. + Kept as the **conservative design**, not as a fix for a measured defect: shuffled folds + over time-ordered trials remain bad practice, the swap costs nothing, and it forecloses + the failure mode if the estimator is ever given more capacity. +- A **time-index control on the same folds** now runs alongside: the identical pipeline + trained on trial position alone. Both numbers print, and the gate requires the EEG score + to exceed the time-only score. On the evidence above this is currently redundant with + blocked CV — it is cheap insurance, and it makes the check explicit rather than implicit + in the splitter choice. +- `--self-check` asserts only what reproduces: drift-only data does not pass the gate. It + prints the shuffled number every run, and says so loudly if the leak ever *does* appear, + so this non-result is re-tested rather than remembered. + +Trial order in `Sources/BCIEEG/Calibration/ImaginedSpeechProtocol.swift:199-208` is a +seeded shuffle with the seed recorded to `metadata.json`; the acquisition side was already +sound. + +### What remains true + +Any balanced accuracy this script reports is still from **zero real sessions** — Track B +has no collected corpus. The gate has never been evaluated against data. That, not +leakage, is what stands between Track B and a defensible result. + +## Note on provenance + +The drift concern was argued for several sessions from a claim that *time-index alone +classifies eyes-open vs eyes-closed at 0.883*. **That figure has no preserved artifact** — +no recording, script output, or log in this repo contains it. It entered a working session +as prose and was carried forward as established fact, load-bearing for a design decision. +The nearest values on disk (`Scripts/dream_extraction.py:335`, rho 0.8827/0.8857) are a +different metric on a different analysis, already annotated there as small-n artifacts. + +Both changes above stand on defects visible in the source, and the first is demonstrated by +a runnable regression. Neither needs 0.883, and it should not be cited again. + +The §2 correction is the same error caught one step earlier: "the gate is inflated by an +unknown amount" was asserted from a design smell, in a document written to warn against +exactly that. It was withdrawn because the check was run, and the check disagreed.