Make the seed settable, pre-register its reading, and launch both seed-variance campaigns (#51, #135)

rampnet/seeding.py

@@ -0,0 +1,51 @@
+"""Seed bookkeeping for Stage 2 training.
+
+Stage 2 had two independent sources of run-to-run randomness and only one of them was
+ever set:
+
+* ``torch`` / ``numpy`` / ``random`` were seeded to a hardcoded ``42`` -- that governs
+  weight initialization for the head, dropout, and the augmentation draws.
+* ``DistributedSampler`` carries its **own** ``seed`` (default ``0``) and derives each
+  epoch's permutation from ``seed + epoch`` inside ``set_epoch()``. Nothing in
+  ``train.py`` touched it, so it stayed at ``0``.
+
+So every published run is the pair ``(42, 0)``, and the recipe's spread across seeds has
+never been measured -- it is n=1. That was a footnote while the RampNet-vs-YOLO gap was
+0.252 F1; at the matched-operating-point gap of 0.039 it is the binding number. See
+``docs/seed_variance_51_135.md``.
+
+The trap this module exists to close: a sweep that varies only the torch seed reuses one
+data order across every arm, which understates the true spread, and **does it silently**
+-- no log line distinguishes the two. So the two seeds move together, with one exception
+that has to be exact: at the historical torch seed the sampler must stay at its
+historical ``0``, or the default stops reproducing the published runs.
+"""
+
+HISTORICAL_SEED = 42
+"""The torch/numpy/random seed every published Stage 2 run used."""
+
+HISTORICAL_SAMPLER_SEED = 0
+"""``DistributedSampler``'s default, which every published Stage 2 run inherited."""
+
+
+def sampler_seed_for(seed: int) -> int:
+    """Return the ``DistributedSampler`` seed that pairs with ``seed``.
+
+    At :data:`HISTORICAL_SEED` this is :data:`HISTORICAL_SAMPLER_SEED`, so the default
+    reproduces published runs exactly. Every other seed maps to itself, so a sweep gets
+    a genuinely different data order as well as different initialization.
+
+    The asymmetry is deliberate and is the whole point of the function: it is the only
+    way to add a seed flag without silently changing what the default does.
+
+    .. warning::
+       The mapping is therefore NOT injective: ``sampler_seed_for(0)`` and
+       ``sampler_seed_for(42)`` are both ``0``. A replicate at seed ``0`` gets a fresh
+       initialization but the **published run's data order**, which makes it less
+       independent of the published run than its seed column suggests. Seed 0 is the
+       natural first pick -- it is the ultralytics default every #51 YOLO arm used -- so
+       extend a Stage 2 sweep with 4, 5, ..., never with 0.
+    """
+    if seed == HISTORICAL_SEED:
+        return HISTORICAL_SAMPLER_SEED
+    return seed

scripts/model_comparison/run_yolo_train_tillicum.slurm

@@ -86,6 +86,21 @@ EPOCHS="${EPOCHS:-60}"
 BATCH="${BATCH:--1}"                          # pin per config to match the klone runs
 PATIENCE="${PATIENCE:-20}"
 
+# SEED. Every #51 arm ran seed=0 (the ultralytics default), so the whole baseline is
+# n=1 and its run-to-run spread has never been measured. That was a footnote while the
+# RampNet-vs-YOLO gap was 0.252 F1; at the matched-operating-point gap of 0.039
+# (docs/operating_point_parity_51.md -- on branch fix/yolo-label-cache-rescue-51 / PR
+# #154, not on main yet) it is the binding question, and #51's own rule is that
+# differences under ~0.02 should not be read.
+#
+# Ultralytics seeds torch/numpy/random from this AND sets deterministic=True by default,
+# so it also governs augmentation and the initial head weights -- i.e. it is the whole
+# run-to-run knob, not just the shuffle.
+#
+# Leave it 0 for anything meant to reproduce an existing arm. Set it for a seed sweep:
+#   SEED=1 NAME=y11x_tiles_s1 ... sbatch <this script>
+SEED="${SEED:-0}"
+
 # Keep a checkpoint every N epochs. Ultralytics defaults save_period=-1, keeping ONLY
 # last.pt and best.pt -- which forecloses, permanently and retroactively, any analysis
 # that needs a checkpoint from a specific epoch.
@@ -144,7 +159,7 @@ APPTAINER_IMG="${APPTAINER_IMG:-}"
 echo "--- YOLO train on TILLICUM (issue #51 / #70) ---"
 echo "base:   ${YOLO_CKPT}"
 echo "data:   ${YOLO_DATA}"
-echo "imgsz:  ${YOLO_IMGSZ}   epochs: ${EPOCHS}   batch: ${BATCH}   patience: ${PATIENCE}"
+echo "imgsz:  ${YOLO_IMGSZ}   epochs: ${EPOCHS}   batch: ${BATCH}   patience: ${PATIENCE}   seed: ${SEED}"
 echo "alloc:  ${SLURM_GPUS_ON_NODE:-?} GPU(s), ${SLURM_CPUS_PER_TASK:-?} CPUs on ${SLURMD_NODENAME:-?}"
 echo "device: ${DEVICE}   workers: ${WORKERS}   (allocation and device differ on purpose -- see header)"
 echo "chain:  ${CHAIN} follow-on job(s) queued after this one"
@@ -173,11 +188,12 @@ fi
 
 run_train() {
     "$@" - "$YOLO_CKPT" "$YOLO_DATA" "$YOLO_IMGSZ" "$EPOCHS" "$BATCH" "$PATIENCE" \
-            "$PROJECT" "$NAME" "$TRAIN_HOURS" "$DEVICE" "$SAVE_PERIOD" "$WORKERS" <<'PY'
+            "$PROJECT" "$NAME" "$TRAIN_HOURS" "$DEVICE" "$SAVE_PERIOD" "$WORKERS" \
+            "$SEED" <<'PY'
 import os, sys
 from ultralytics import YOLO
 (ckpt, data, imgsz, epochs, batch, patience, project, name, hours, device,
- save_period, workers) = sys.argv[1:13]
+ save_period, workers, seed) = sys.argv[1:14]
 last = os.path.join(project, name, "weights", "last.pt")
 # Still needed on Tillicum, but for a different reason than on klone: not preemption,
 # but the 24 h normal-QoS ceiling. A 60-epoch tiles schedule does not fit in one job,
@@ -206,7 +222,7 @@ else:
     kw = dict(
         data=data, imgsz=int(imgsz), epochs=int(epochs), batch=batch_arg,
         patience=int(patience), project=project, name=name, device=dev, exist_ok=True,
-        save_period=int(save_period), workers=int(workers),
+        save_period=int(save_period), workers=int(workers), seed=int(seed),
     )
     if float(hours) > 0:
         kw["time"] = float(hours)

stage_two/run_train_seed.slurm

@@ -0,0 +1,141 @@
+#!/bin/bash
+# Train ONE Stage 2 seed replicate on klone, for the seed-variance campaign
+# (docs/seed_variance_51_135.md; issues #51 and #135).
+#
+# WHY THIS IS A SEPARATE FILE AND NOT A FLAG ON run_train.slurm
+# run_train.slurm is the preserved record of the published run and of #135's rungs. It
+# should not churn, and more importantly a seed replicate needs two things that file
+# deliberately does not do: a per-seed working directory, and a bounded epoch count. The
+# same reasoning kept run_yolo_train_tillicum.slurm separate from its klone original.
+#
+# WHY A PER-SEED WORKING DIRECTORY -- THIS IS THE LOAD-BEARING PART
+# train.py writes `best_model.pth` and `latest_checkpoint.pth` to the CURRENT DIRECTORY,
+# not to --checkpoint-dir (see stage_two/train.py: torch.save(..., "best_model.pth")).
+# Three seeds launched from one directory would therefore overwrite each other's best
+# model AND each other's resume state, and the second failure is worse than the first:
+# a resume file from another seed is silently loaded as if it were this run's own, so
+# the arms converge on one lineage and nothing in the log says so. Each seed gets its
+# own RUNDIR and cd's into it before torchrun.
+#
+# USAGE. Submit from the repo root; logs/ must ALREADY exist, because Slurm opens
+# --output relative to the SUBMIT directory before this script runs (the mkdir below
+# cannot help with that -- it is there for a RUNDIR that does not exist yet).
+#   mkdir -p logs
+#   SEED=1 sbatch stage_two/run_train_seed.slurm
+#   SEED=2 RUNDIR=/gscratch/scrubbed/$USER/seedvar/rampnet_s2 sbatch stage_two/run_train_seed.slurm
+#
+# RUNDIR IS ON A VOLUME THAT PURGES. best_model.pth -- the only artifact this campaign
+# produces -- lands in RUNDIR, and /gscratch/scrubbed purges on a ~21-day idle window
+# (docs/stage2_epoch_curve_84.md). Campaign B's calendar is unbounded (ckpt-all duty
+# cycle 3.9%), so a replicate can finish and then sit unscored past that window while
+# its siblings are still pending. COPY IT OUT as soon as the job completes:
+#   cp "$RUNDIR/best_model.pth" /gscratch/makelab/$USER/seedvar/rampnet_s${SEED}_best.pth
+# /gscratch/makelab is purchased and never purged, which is where #84 put Run A's
+# checkpoints for the same reason. The default is deliberately left on scrubbed so it
+# matches the replicates already queued; changing it is a decision, not a cleanup.
+#
+# COST. klone ckpt is FREE and preemptable. One replicate is 1 epoch = ~3.5 h on 16
+# GPUs (~56 GPU-h, measured -- docs/stage2_training_cost.md). Preemption is handled by
+# --requeue plus train.py's own latest_checkpoint.pth resume, so a requeued replicate
+# continues rather than restarting. Calendar, not money, is the risk here: ckpt-all's
+# duty cycle was 3.9% in 2026-08 (#135), so a replicate can sit pending for days.
+#SBATCH -p ckpt-all
+#SBATCH --job-name=rampnet_seedvar
+#SBATCH --nodes=4
+#SBATCH --ntasks-per-node=1
+#SBATCH --gpus-per-node=4
+#SBATCH --cpus-per-task=12
+#SBATCH --mem=48G
+#SBATCH --time=24:00:00
+#SBATCH --output=logs/seedvar_%j.out
+#SBATCH --error=logs/seedvar_%j.err
+#SBATCH --requeue
+#SBATCH --constraint='l40s|l40|a40|a100'
+
+set -euo pipefail
+
+# The seed IS the experiment, so it is required rather than defaulted -- a replicate
+# that silently ran at 42 would be a duplicate of the published run wearing a new name.
+# NO APOSTROPHE in this message. Inside ${VAR:?...} bash parses the word for quoting
+# even within double quotes, so a lone ' opens a quote and the closing } is never found:
+# "unexpected EOF while looking for matching `}'". The script then dies at submit time
+# with exit 2 in about one second, having printed nothing -- which is how jobs
+# 39515025/26/27 failed on 2026-09-03 and then sat unnoticed for a day.
+SEED="${SEED:?set SEED to the seed for this replicate, e.g. SEED=1}"
+
+REPO="${REPO:-$HOME/RampNet}"
+DATA_ROOT="${DATA_ROOT:-/gscratch/scrubbed/$USER/rampnet_dataset}"
+# NOT $HOME: klone home is a separate 10 GB quota that gscratch cleanup does not touch,
+# and one replicate's per-epoch checkpoints alone would blow it.
+RUNDIR="${RUNDIR:-/gscratch/scrubbed/$USER/seedvar/rampnet_s${SEED}}"
+EPOCHS="${EPOCHS:-1}"        # the published recipe is 1 epoch / 9,378 steps (#84)
+
+mkdir -p "$RUNDIR/checkpoints" "$REPO/logs"
+
+# Interpreter. Default behaviour is unchanged -- `source activate sidewalkcv2`, then
+# torchrun off PATH -- but every other klone launcher here carries an escape hatch
+# (PYTHON= in run_yolo_train.slurm and run_gold_bundle.slurm, RAMPNET_ENV in
+# run_train_epoch_curve.slurm) and this one did not. It matters because #84's env was
+# built at a PREFIX, /gscratch/scrubbed/$USER/envs/sidewalkcv2, which `source activate
+# <name>` cannot resolve: under `set -e` the job would then die here having printed one
+# line. Set RAMPNET_ENV to a conda prefix and the env's own torchrun is called by
+# absolute path instead -- the pattern run_train_epoch_curve.slurm documents as the one
+# proven on this cluster.
+RAMPNET_ENV="${RAMPNET_ENV:-}"
+if [ -n "$RAMPNET_ENV" ]; then
+    TORCHRUN="$RAMPNET_ENV/bin/torchrun"
+    if [ ! -x "$TORCHRUN" ]; then
+        echo "FATAL: RAMPNET_ENV=$RAMPNET_ENV has no executable bin/torchrun" >&2
+        exit 1
+    fi
+    echo "Using conda prefix ${RAMPNET_ENV} (no activation)"
+else
+    echo "Loading Conda environment..."
+    source activate sidewalkcv2
+    TORCHRUN=torchrun
+fi
+
+export MASTER_ADDR=$(scontrol show hostnames $SLURM_JOB_NODELIST | head -n 1)
+export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK
+export MASTER_PORT=$(expr 10000 + $(echo -n $SLURM_JOBID | tail -c 4))
+# train.py lives in stage_two/ but imports the `rampnet` package from the repo root, and
+# we run from RUNDIR, so neither is on sys.path by default.
+export PYTHONPATH="$REPO:${PYTHONPATH:-}"
+
+NPROC_PER_NODE=${SLURM_GPUS_PER_NODE:-4}
+WORLD_SIZE=$(($SLURM_NNODES * $NPROC_PER_NODE))
+
+echo "--- Stage 2 seed replicate (#51 / #135) ---"
+echo "Job ID:      ${SLURM_JOBID}"
+echo "Seed:        ${SEED}"
+echo "Run dir:     ${RUNDIR}   (cwd -- best_model.pth and latest_checkpoint.pth land here)"
+echo "Data root:   ${DATA_ROOT}"
+echo "Epochs:      ${EPOCHS}"
+echo "Nodes:       ${SLURM_NNODES} x ${NPROC_PER_NODE} GPU  (world size ${WORLD_SIZE})"
+echo "Node list:   ${SLURM_NODELIST}"
+echo "Restarts:    ${SLURM_RESTART_COUNT:-0}   (requeue resumes from latest_checkpoint.pth)"
+echo "-------------------------------------------"
+
+# World size IS the global batch: train.py uses batch_size=1 per rank, so a replicate
+# that lands on any other node/GPU count is a different optimisation regime, not a seed
+# replicate of the published recipe. Same guard as run_train_epoch_curve.slurm.
+if [ "${WORLD_SIZE}" -ne 16 ]; then
+    echo "WARNING: world size ${WORLD_SIZE} != 16. The published recipe's global batch" >&2
+    echo "         was 16; this is NOT a seed replicate of it at any other world size." >&2
+fi
+
+cd "$RUNDIR"
+
+srun --export=ALL \
+    "$TORCHRUN" --nnodes $SLURM_NNODES \
+             --nproc_per_node $NPROC_PER_NODE \
+             --rdzv_id $SLURM_JOB_ID \
+             --rdzv_backend c10d \
+             --rdzv_endpoint $MASTER_ADDR:$MASTER_PORT \
+             "$REPO/stage_two/train.py" \
+             --seed "$SEED" \
+             --epochs "$EPOCHS" \
+             --data-root "$DATA_ROOT" \
+             --checkpoint-dir "$RUNDIR/checkpoints"
+
+echo "--- Slurm job finished ---"

stage_two/train.py

@@ -22,6 +22,7 @@
 
 from rampnet.model import KeypointModel
 from rampnet.loading import load_checkpoint
+from rampnet.seeding import HISTORICAL_SEED, sampler_seed_for
 
 # Learning-rate defaults per preset: training from ImageNet initialization
 # uses the paper's 1e-5; fine-tuning released/earlier RampNet weights wants a
@@ -130,6 +131,11 @@ def parse_args():
                              "run always takes precedence, and warm-starting applies at step 0 only.")
     parser.add_argument('--checkpoint-dir', default='checkpoints',
                         help="Directory for per-epoch checkpoints (default: checkpoints)")
+    parser.add_argument('--seed', type=int, default=HISTORICAL_SEED,
+                        help=f"Seed for torch/numpy/random AND the DistributedSampler shuffle "
+                             f"(default: {HISTORICAL_SEED}, the value every published run used). "
+                             f"Change it only to measure run-to-run variance -- see "
+                             f"docs/seed_variance_51_135.md")
     parser.add_argument('--lr-schedule', choices=LR_SCHEDULES, default='constant',
                         help="'constant' is the paper recipe and the default -- every "
                              "existing invocation is unaffected. 'cosine' decays --lr to "
@@ -172,9 +178,18 @@ def cleanup_distributed():
 
 rank, local_rank, world_size = setup_distributed()
 
-torch.manual_seed(42)
-random.seed(42)
-np.random.seed(42)
+# These were hardcoded to 42, so every published run shares one seed and the recipe's
+# run-to-run spread is unmeasured (n=1). --seed defaults to 42, so nothing about an
+# existing run changes; it exists so a seed sweep is possible at all. See
+# docs/seed_variance_51_135.md for why that spread is now the binding number.
+#
+# The sampler seed has to move WITH this one. DistributedSampler takes its own `seed`
+# (default 0) and derives the shuffle from seed + epoch via set_epoch(), so leaving it
+# alone would give every "different" seed the identical data order -- a sweep that
+# varies initialization only, silently understating the true spread.
+torch.manual_seed(args.seed)
+random.seed(args.seed)
+np.random.seed(args.seed)
 
 new_root_dir = args.data_root
 
@@ -356,8 +371,21 @@ def __getitem__(self, idx):
 if len(val_dataset) == 0 and rank == 0:
     print("Warning: Validation dataset is empty.")
 
+# DistributedSampler has its OWN seed (default 0) and derives the shuffle from
+# seed + epoch in set_epoch(), so it is independent of torch.manual_seed above. Every
+# published run therefore paired manual_seed(42) with sampler seed 0, and
+# sampler_seed_for() preserves that pairing exactly at the default: passing --seed 42
+# gives those runs' initialization AND their data order, while any other --seed moves
+# the data order too. Not bit-identical, and no claim to be: cuDNN autotuning, AMP loss
+# scaling and DDP allreduce ordering are not seeded and torch.use_deterministic_algorithms
+# is not set. What is preserved is every source of randomness this script controls.
+#
+# Both halves have to move together. A sweep that varied initialization but reused one
+# data order would understate the true run-to-run spread -- and would do it silently,
+# since nothing in the logs distinguishes the two.
 train_sampler = ResumeSkipSampler(
-    DistributedSampler(train_dataset, num_replicas=world_size, rank=rank, shuffle=True, drop_last=True))
+    DistributedSampler(train_dataset, num_replicas=world_size, rank=rank, shuffle=True, drop_last=True,
+                       seed=sampler_seed_for(args.seed)))
 val_sampler = DistributedSampler(val_dataset, num_replicas=world_size, rank=rank, shuffle=False, drop_last=False) if len(val_dataset) > 0 else None
 
 train_loader = DataLoader(train_dataset, batch_size=1, sampler=train_sampler, num_workers=4, pin_memory=True)
@@ -384,6 +412,11 @@ def __getitem__(self, idx):
     os.makedirs(args.checkpoint_dir, exist_ok=True)
     writer = SummaryWriter(log_dir='runs/experiment_1')
     print(f"Preset: {args.preset}, lr: {args.lr}, epochs: {args.epochs}, data root: {new_root_dir}")
+    # Both seeds in the log, always. A seed sweep whose arms cannot be told apart from
+    # their own logs is not reproducible, and the sampler half is the one that is easy
+    # to leave unset without noticing.
+    print(f"Seed: {args.seed} (sampler seed: {sampler_seed_for(args.seed)}), "
+          f"checkpoint dir: {args.checkpoint_dir}")
     print(f"LR schedule: {args.lr_schedule}"
           + (f" -> {args.lr * args.lr_final_frac:.3g} over {total_train_steps} steps"
              if args.lr_schedule != 'constant' else " (no decay, as in the paper)"))

tests/test_resume_skip_sampler.py

@@ -29,6 +29,8 @@
 
 from check_lr_schedule_135 import load_from_train_py  # noqa: E402
 
+from rampnet.seeding import HISTORICAL_SEED
+
 import itertools  # noqa: E402
 
 LIFTED = load_from_train_py("ResumeSkipSampler", itertools=itertools, Sampler=Sampler)
@@ -151,7 +153,8 @@ def test_checkpoint_interval_default_is_still_the_paper_recipe():
     checkpointing granularity along with it.
     """
     import argparse
-    mod = load_from_train_py("parse_args", "PRESET_LR", "LR_SCHEDULES", argparse=argparse)
+    mod = load_from_train_py("parse_args", "PRESET_LR", "LR_SCHEDULES",
+                             argparse=argparse, HISTORICAL_SEED=HISTORICAL_SEED)
     argv = sys.argv
     try:
         sys.argv = ["train.py"]