diff --git a/Phase2/Lasana-Person4/README.md b/Phase2/Lasana-Person4/README.md new file mode 100644 index 0000000..a8b9996 --- /dev/null +++ b/Phase2/Lasana-Person4/README.md @@ -0,0 +1,92 @@ +# Person 4 — Evaluation Lead (Phase 2) + +Role: **Evaluation Lead**. Proposal §6.2.2, Weeks 7–10: implement AAMO +(done in Phase 1), run the complete ablation across all model configurations, +and report results with multiple seeds (mean ± std). + +| Weeks | Deliverable | Status | +|---|---|---| +| 7–9 | Fold full-scale U-Net / SegFormer-B0 / SegFormer-B0+L_att into one table | Done — `results/baseline_comparison_full_scale.md` | +| 7–9 | Attention-consistency row uses Dinura's λ2 sweep winner (`l2_1_mse`) | Done | +| 7–9 | Multi-seed mean±std reporting | Done for DeepLabV3+ (seeds 42/43/44); U-Net/SegFormer remain Seeds=1 pending GPU reruns | +| 10 | Boundary Loss ablation row | Pending — blocked on Person 5 integration | + +## Layout + +``` +Phase2/Lasana-Person4/ + README.md + fold_full_scale_results.py consolidates Table-1 rows (no model loading) + train_deeplab_multiseed.py DeepLabV3+ seeds 42/43/44 + ablation tables + checkpoints/ Deeplab seed 43/44 weights (gitignored *.pt) + results/ + baseline_comparison_full_scale.csv / .md + ablation_per_seed.csv + ablation_mean_std.csv / .md + deeplab_multiseed.json + tests/ + test_fold_results.py +``` + +Phase 1 code under `Phase1/Lasana-Person4_Evaluation/` is a **frozen short-paper +snapshot** (CONTRIBUTING.md). This Phase 2 folder imports its metrics / +adapters via `sys.path` and does not edit those files. Multi-seed aggregation +uses a local `aggregate_mean_std` (sample std, ddof=1) rather than the Phase 1 +helper. + +## Quick start / Reproducing + +```bash +cd Phase2/Lasana-Person4 + +# Unit tests (no GPU, no weights) +python tests/test_fold_results.py + +# Fold Chanupa + Kalana + Dinura winner + DeepLab into one table +python fold_full_scale_results.py + +# Train DeepLab seeds 43/44 (seed 42 reuses Phase 1 ckpt), evaluate all three, +# write ablation_per_seed + ablation_mean_std +python train_deeplab_multiseed.py + +# Or aggregate only, if seed 43/44 checkpoints already exist +python train_deeplab_multiseed.py --skip-train +``` + +Smoke defaults (override with env vars): `DEEPLAB_MAX_SAMPLES=400`, +`DEEPLAB_EPOCHS=5`, `DEEPLAB_BATCH=2`. + +## Dependencies on teammates + +| Direction | What | +|---|---| +| I need | Kalana's full-scale SegFormer vanilla metrics (`Phase2/Kalana-Person2/results/`) | +| I need | Dinura's λ2 sweep winner `l2_1_mse` (`Phase2/Dinura-Person3/results/`) | +| I need | Chanupa's U-Net full-scale row (already in Phase 1 Person 4 results) | +| I hand off | `results/baseline_comparison_full_scale.md` + `results/ablation_mean_std.md` for Person 5's Week 11–12 paper assembly | + +## Results + +U-Net / SegFormer / L_att rows share the 3576/766/766 seed-42 test set; DeepLabV3+ +is a 400-sample CPU-smoke subset (seed-42 Dice 0.7821 from `deeplab_multiseed.json`). + +| Model | Dice | IoU | AAMO | Seeds | +|---|---|---|---|---| +| U-Net (CNN baseline) | 0.8615 | 0.7568 | n/a | 1 | +| SegFormer-B0 (no attention loss) | 0.8743 | 0.7766 | 0.0334 | 1 | +| SegFormer-B0 + Attention Consistency (λ2=1.0 MSE) | 0.8577 | 0.7508 | 0.7476 | 1 | +| SegFormer-B0 + Attention + Boundary Loss | — | — | pending | 0 | +| DeepLabV3+ (MobileNetV3) — extra baseline | 0.7862 ± 0.0193 | 0.6481 ± 0.0264 | n/a | 3 | + +Selection rule for the attention row (Dinura): max test AAMO, then max Dice. +Winner run tag `l2_1_mse` supersedes Kalana's default-λ2=0.3 attention numbers. + +DeepLabV3+ is CPU smoke-scale (400 samples / 5 epochs) and exists to exercise +the multi-seed aggregation pipeline. U-Net / SegFormer full-scale weights live +on Drive (GitHub 100 MiB limit); extra seeds for those rows need Colab GPU +reruns from Person 1/2/3. + +## Cross-folder edits + +**None.** No file outside this folder was modified. Phase 1 Person 4 results +are left untouched on purpose. diff --git a/Phase2/Lasana-Person4/checkpoints/.gitignore b/Phase2/Lasana-Person4/checkpoints/.gitignore new file mode 100644 index 0000000..4b6ebe5 --- /dev/null +++ b/Phase2/Lasana-Person4/checkpoints/.gitignore @@ -0,0 +1 @@ +*.pt diff --git a/Phase2/Lasana-Person4/checkpoints/README.md b/Phase2/Lasana-Person4/checkpoints/README.md new file mode 100644 index 0000000..1d5aa20 --- /dev/null +++ b/Phase2/Lasana-Person4/checkpoints/README.md @@ -0,0 +1,12 @@ +# Checkpoints + +DeepLabV3+ multi-seed smoke weights land here as: + +- `deeplabv3_mobilenet_seed43_best.pt` +- `deeplabv3_mobilenet_seed44_best.pt` + +Seed 42 reuses the Phase 1 checkpoint at +`Phase1/Lasana-Person4_Evaluation/checkpoints/deeplabv3_mobilenet_best.pt` +(not re-copied here). + +Do **not** commit `.pt` files. Commit metrics under `results/` instead. diff --git a/Phase2/Lasana-Person4/fold_full_scale_results.py b/Phase2/Lasana-Person4/fold_full_scale_results.py new file mode 100644 index 0000000..e3a8ced --- /dev/null +++ b/Phase2/Lasana-Person4/fold_full_scale_results.py @@ -0,0 +1,250 @@ +""" +Fold full-scale Table-1 rows into Phase 2 Person 4 results. + +Sources (no model loading — table joins only): + - U-Net: Phase1/Lasana-Person4_Evaluation/results/baseline_comparison.csv + - SegFormer-B0: Phase2/Kalana-Person2/results/baseline_comparison.csv (vanilla) + - SegFormer+Att: Phase2/Dinura-Person3/results/runs/l2_1_mse/ (sweep winner λ2=1.0 MSE) + - Boundary Loss: pending (Dhinanjaya still integrating) — see TODO below + - DeepLabV3+: results/deeplab_multiseed.json seed-42 entry (single eval path) + +Writes: + results/baseline_comparison_full_scale.csv + results/baseline_comparison_full_scale.md +""" +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any, Dict, List, Optional + +ROOT = Path(__file__).resolve().parent +PROJECT = ROOT.parent.parent # DNN_Project +RESULTS = ROOT / "results" +RESULTS.mkdir(parents=True, exist_ok=True) + +PHASE1_P4 = PROJECT / "Phase1" / "Lasana-Person4_Evaluation" / "results" +KALANA = PROJECT / "Phase2" / "Kalana-Person2" / "results" +DINURA_WIN = PROJECT / "Phase2" / "Dinura-Person3" / "results" / "runs" / "l2_1_mse" +WINNING_CFG = PROJECT / "Phase2" / "Dinura-Person3" / "results" / "winning_config.json" +DEEPLAB_MULTI = RESULTS / "deeplab_multiseed.json" + +# TODO(Person 5 / Dhinanjaya): once Boundary Refinement is wired into +# SegFormer+L_att training and a baseline_comparison.csv row exists, replace +# the pending boundary _row(...) below with a real source, e.g.: +# BOUNDARY = PROJECT / "Phase2" / "Dhinanjaya-Person5" / "results" / "baseline_comparison.csv" +# bound = _find_row(_read_csv_rows(BOUNDARY), "Boundary") +# (exact path TBD when Dhinanjaya hands off the checkpoint / CSV.) + +FIELDS = [ + "model", + "dice", + "iou", + "f1", + "precision", + "recall", + "pixel_acc", + "aamo", + "params", + "gflops", + "fps", + "ms_per_image", + "source", + "n_seeds", + "notes", +] + + +def _read_csv_rows(path: Path) -> List[Dict[str, str]]: + with open(path, newline="", encoding="utf-8") as f: + return list(csv.DictReader(f)) + + +def _find_row(rows: List[Dict[str, str]], substr: str) -> Optional[Dict[str, str]]: + for r in rows: + if substr in (r.get("model") or ""): + return r + return None + + +def _deeplab_seed42_row() -> Dict[str, Any]: + """Single evaluation path: seed-42 entry from train_deeplab_multiseed.py.""" + if not DEEPLAB_MULTI.exists(): + raise FileNotFoundError( + f"Missing {DEEPLAB_MULTI}\n" + "Run: python train_deeplab_multiseed.py --skip-train" + ) + entries = json.loads(DEEPLAB_MULTI.read_text(encoding="utf-8")) + for e in entries: + if int(e.get("seed", -1)) == 42: + return e + raise ValueError(f"No seed=42 entry in {DEEPLAB_MULTI}") + + +def _row( + model: str, + src: Optional[Dict[str, str]], + *, + source: str, + n_seeds: int = 1, + notes: str = "", + overrides: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + out: Dict[str, Any] = {k: "" for k in FIELDS} + out["model"] = model + out["source"] = source + out["n_seeds"] = n_seeds + out["notes"] = notes + if src: + for k in FIELDS: + if k in ("model", "source", "n_seeds", "notes"): + continue + if k in src and src[k] not in (None, ""): + out[k] = src[k] + if overrides: + out.update(overrides) + return out + + +def fold() -> List[Dict[str, Any]]: + p4 = _read_csv_rows(PHASE1_P4 / "baseline_comparison.csv") + kalana = _read_csv_rows(KALANA / "baseline_comparison.csv") + dinura = _read_csv_rows(DINURA_WIN / "baseline_comparison.csv") + + winner_meta = {} + if WINNING_CFG.exists(): + winner_meta = json.loads(WINNING_CFG.read_text(encoding="utf-8")).get("winner", {}) + + unet = _find_row(p4, "U-Net") + vanilla = _find_row(kalana, "no attention") + att = _find_row(dinura, "Attention Consistency") + dl42 = _deeplab_seed42_row() + + att_notes = ( + f"Dinura sweep winner {winner_meta.get('run_tag', 'l2_1_mse')} " + f"(λ2={winner_meta.get('lambda2', 1.0)}, {winner_meta.get('att_mode', 'mse')}); " + "selection: max test AAMO then max Dice. " + "Supersedes Kalana default-λ2=0.3 attention row." + ) + + rows = [ + _row( + "U-Net (CNN baseline)", + unet, + source="Phase1/Lasana-Person4_Evaluation (Chanupa PyTorch ckpt)", + notes="Full-scale 3576/766/766 seed 42; dataset-wide Dice/IoU.", + ), + _row( + "SegFormer-B0 (no attention loss)", + vanilla, + source="Phase2/Kalana-Person2 (full-scale Colab)", + notes="Full-scale 3576/766/766 seed 42; Person 4 metrics/aamo formulas.", + ), + _row( + "SegFormer-B0 + Attention Consistency Loss (λ2=1.0 MSE)", + att, + source="Phase2/Dinura-Person3/results/runs/l2_1_mse", + notes=att_notes, + ), + # TODO(Person 5): replace this pending stub when Boundary Refinement + # lands. Expected upstream CSV (path TBD at handoff): + # Phase2/Dhinanjaya-Person5/results/baseline_comparison.csv + # Then: bound = _find_row(_read_csv_rows(BOUNDARY), "Boundary") + # and drop overrides / n_seeds=0. + _row( + "SegFormer-B0 + Attention Consistency + Boundary Loss", + None, + source="pending", + n_seeds=0, + notes="Blocked until Dhinanjaya wires Boundary Refinement into training.", + overrides={ + "dice": "-", + "iou": "-", + "f1": "-", + "aamo": "pending", + "params": "-", + "gflops": "-", + "fps": "-", + }, + ), + _row( + "DeepLabV3+ (MobileNetV3) — extra baseline", + None, + source="Phase2/Lasana-Person4/train_deeplab_multiseed.py (seed 42, 400-sample smoke)", + notes=( + "CPU smoke (400 samples / 5 epochs), seed-42 eval from " + "deeplab_multiseed.json. Multi-seed mean±std in ablation_mean_std.md." + ), + overrides={ + "dice": dl42["dice"], + "iou": dl42["iou"], + "f1": dl42["f1"], + "precision": dl42.get("precision", ""), + "recall": dl42.get("recall", ""), + "pixel_acc": dl42.get("pixel_acc", ""), + "aamo": dl42.get("aamo", "n/a"), + "params": dl42.get("params", ""), + "gflops": dl42.get("gflops", "n/a"), + }, + ), + ] + return rows + + +def write_tables(rows: List[Dict[str, Any]]) -> None: + csv_path = RESULTS / "baseline_comparison_full_scale.csv" + md_path = RESULTS / "baseline_comparison_full_scale.md" + + with open(csv_path, "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=FIELDS) + w.writeheader() + w.writerows(rows) + + lines = [ + "# Full-scale baseline comparison (Phase 2 / Lasana-Person4)", + "", + "Folded from Chanupa (U-Net), Kalana (SegFormer-B0 vanilla), Dinura", + "(`l2_1_mse` attention winner), and Person 4's DeepLabV3+ extra baseline.", + "U-Net / SegFormer / L_att rows share the 3576/766/766 seed-42 test set;", + "the DeepLabV3+ row is a 400-sample CPU-smoke subset evaluated by", + "`train_deeplab_multiseed.py`.", + "", + "| Model | Dice | IoU | F1 | AAMO | Params | GFLOPs | FPS | Source |", + "|-------|------|-----|----|------|--------|--------|-----|--------|", + ] + for r in rows: + lines.append( + f"| {r['model']} | {r.get('dice', '-')} | {r.get('iou', '-')} | " + f"{r.get('f1', '-')} | {r.get('aamo', '-')} | {r.get('params', '-')} | " + f"{r.get('gflops', '-')} | {r.get('fps', '-')} | {r.get('source', '')} |" + ) + lines.extend( + [ + "", + "## Notes", + "", + "- Attention-consistency row uses Dinura's sweep winner " + "(`λ2=1.0`, MSE, run tag `l2_1_mse`: Dice 0.8577 / IoU 0.7508 / " + "AAMO 0.7476), not Kalana's default-λ2=0.3 attention run.", + "- Boundary Loss row stays pending until Person 5 finishes integration.", + "- DeepLabV3+ Dice/IoU come from `deeplab_multiseed.json` seed 42 " + "(same path as the multi-seed ablation); see `ablation_mean_std.md` " + "for seeds 42/43/44 mean±std.", + "- Phase 1 `Lasana-Person4_Evaluation/results/` is left untouched " + "(frozen short-paper snapshot per CONTRIBUTING.md).", + "", + ] + ) + md_path.write_text("\n".join(lines), encoding="utf-8") + print(f"Wrote {csv_path}") + print(f"Wrote {md_path}") + + +def main() -> None: + rows = fold() + write_tables(rows) + + +if __name__ == "__main__": + main() diff --git a/Phase2/Lasana-Person4/results/ablation_mean_std.csv b/Phase2/Lasana-Person4/results/ablation_mean_std.csv new file mode 100644 index 0000000..23eb0e4 --- /dev/null +++ b/Phase2/Lasana-Person4/results/ablation_mean_std.csv @@ -0,0 +1,6 @@ +model,n_seeds,dice,iou,f1,aamo,params,gflops +U-Net (CNN baseline),1,0.8615,0.7568,0.8615,n/a,31037698,109.48 +SegFormer-B0 (no attention loss),1,0.8743,0.7766,0.8743,0.0334,3714658,1.692 +SegFormer-B0 + Attention Consistency Loss (λ2=1.0 MSE),1,0.8577,0.7508,0.8577,0.7476,3714658,1.692 +SegFormer-B0 + Attention Consistency + Boundary Loss,0,-,-,-,pending,-,- +DeepLabV3+ (MobileNetV3) — extra baseline,3,0.7862 ± 0.0193,0.6481 ± 0.0264,0.7862 ± 0.0193,n/a,11020594,n/a diff --git a/Phase2/Lasana-Person4/results/ablation_mean_std.md b/Phase2/Lasana-Person4/results/ablation_mean_std.md new file mode 100644 index 0000000..b1760d9 --- /dev/null +++ b/Phase2/Lasana-Person4/results/ablation_mean_std.md @@ -0,0 +1,25 @@ +# Ablation results (mean ± std) — Phase 2 / Lasana-Person4 + +Std is **sample** standard deviation (ddof=1). Single-seed rows +(U-Net, SegFormer-B0, SegFormer-B0+L_att λ2=1.0) show the bare value +pending additional GPU-trained seeds from Person 1/2/3. DeepLabV3+ +extra baseline has genuine 3-seed mean±std (seeds 42/43/44, CPU smoke: +400 samples / 5 epochs). + +| Model | Seeds | Dice | IoU | F1 | AAMO | +|-------|-------|------|-----|----|------| +| U-Net (CNN baseline) | 1 | 0.8615 | 0.7568 | 0.8615 | n/a | +| SegFormer-B0 (no attention loss) | 1 | 0.8743 | 0.7766 | 0.8743 | 0.0334 | +| SegFormer-B0 + Attention Consistency Loss (λ2=1.0 MSE) | 1 | 0.8577 | 0.7508 | 0.8577 | 0.7476 | +| SegFormer-B0 + Attention Consistency + Boundary Loss | 0 | - | - | - | pending | +| DeepLabV3+ (MobileNetV3) — extra baseline | 3 | 0.7862 ± 0.0193 | 0.6481 ± 0.0264 | 0.7862 ± 0.0193 | n/a | + +## Notes + +- Attention-consistency config = Dinura `l2_1_mse` (λ2=1.0, MSE). +- Boundary Loss row pending Person 5 integration. +- DeepLabV3+ smoke numbers are not paper-scale; they demonstrate + the multi-seed aggregation pipeline Person 4 owns. +- U-Net / SegFormer full-scale checkpoints live on Drive (not in git); + re-running extra seeds requires Colab GPU access from teammates. +- Teammate GPU multi-seed reporting must also use sample std (ddof=1). diff --git a/Phase2/Lasana-Person4/results/ablation_per_seed.csv b/Phase2/Lasana-Person4/results/ablation_per_seed.csv new file mode 100644 index 0000000..9c21679 --- /dev/null +++ b/Phase2/Lasana-Person4/results/ablation_per_seed.csv @@ -0,0 +1,8 @@ +model,seed,dice,iou,f1,precision,recall,pixel_acc,aamo,params,gflops,status,checkpoint,max_samples +U-Net (CNN baseline),42,0.8615,0.7568,0.8615,,,,n/a,31037698,109.48,single_seed_full_scale,, +SegFormer-B0 (no attention loss),42,0.8743,0.7766,0.8743,,,,0.0334,3714658,1.692,single_seed_full_scale,, +SegFormer-B0 + Attention Consistency Loss (λ2=1.0 MSE),42,0.8577,0.7508,0.8577,,,,0.7476,3714658,1.692,single_seed_full_scale,, +SegFormer-B0 + Attention Consistency + Boundary Loss,-,-,-,-,,,,pending,-,-,pending_checkpoint,, +DeepLabV3+ (MobileNetV3) — extra baseline,42,0.7821,0.6422,0.7821,0.6591,0.9616,0.6941,n/a,11020594,n/a,ok,Phase1/Lasana-Person4_Evaluation/checkpoints/deeplabv3_mobilenet_best.pt,400 +DeepLabV3+ (MobileNetV3) — extra baseline,43,0.8073,0.6769,0.8073,0.766,0.8533,0.7674,n/a,11020594,n/a,ok,Phase2/Lasana-Person4/checkpoints/deeplabv3_mobilenet_seed43_best.pt,400 +DeepLabV3+ (MobileNetV3) — extra baseline,44,0.7693,0.6251,0.7693,0.6489,0.9444,0.6765,n/a,11020594,n/a,ok,Phase2/Lasana-Person4/checkpoints/deeplabv3_mobilenet_seed44_best.pt,400 diff --git a/Phase2/Lasana-Person4/results/baseline_comparison_full_scale.csv b/Phase2/Lasana-Person4/results/baseline_comparison_full_scale.csv new file mode 100644 index 0000000..fd3b735 --- /dev/null +++ b/Phase2/Lasana-Person4/results/baseline_comparison_full_scale.csv @@ -0,0 +1,6 @@ +model,dice,iou,f1,precision,recall,pixel_acc,aamo,params,gflops,fps,ms_per_image,source,n_seeds,notes +U-Net (CNN baseline),0.8615,0.7568,0.8615,0.8009,0.9321,0.8174,n/a,31037698,109.48,1.47,680.55,Phase1/Lasana-Person4_Evaluation (Chanupa PyTorch ckpt),1,Full-scale 3576/766/766 seed 42; dataset-wide Dice/IoU. +SegFormer-B0 (no attention loss),0.8743,0.7766,0.8743,0.8517,0.898,0.8425,0.0334,3714658,1.692,84.52,11.83,Phase2/Kalana-Person2 (full-scale Colab),1,Full-scale 3576/766/766 seed 42; Person 4 metrics/aamo formulas. +SegFormer-B0 + Attention Consistency Loss (λ2=1.0 MSE),0.8577,0.7508,0.8577,0.7885,0.9402,0.8098,0.7476,3714658,1.692,103.36,9.67,Phase2/Dinura-Person3/results/runs/l2_1_mse,1,"Dinura sweep winner l2_1_mse (λ2=1.0, mse); selection: max test AAMO then max Dice. Supersedes Kalana default-λ2=0.3 attention row." +SegFormer-B0 + Attention Consistency + Boundary Loss,-,-,-,,,,pending,-,-,-,,pending,0,Blocked until Dhinanjaya wires Boundary Refinement into training. +DeepLabV3+ (MobileNetV3) — extra baseline,0.7821,0.6422,0.7821,0.6591,0.9616,0.6941,n/a,11020594,n/a,,,"Phase2/Lasana-Person4/train_deeplab_multiseed.py (seed 42, 400-sample smoke)",1,"CPU smoke (400 samples / 5 epochs), seed-42 eval from deeplab_multiseed.json. Multi-seed mean±std in ablation_mean_std.md." diff --git a/Phase2/Lasana-Person4/results/baseline_comparison_full_scale.md b/Phase2/Lasana-Person4/results/baseline_comparison_full_scale.md new file mode 100644 index 0000000..c06eacc --- /dev/null +++ b/Phase2/Lasana-Person4/results/baseline_comparison_full_scale.md @@ -0,0 +1,22 @@ +# Full-scale baseline comparison (Phase 2 / Lasana-Person4) + +Folded from Chanupa (U-Net), Kalana (SegFormer-B0 vanilla), Dinura +(`l2_1_mse` attention winner), and Person 4's DeepLabV3+ extra baseline. +U-Net / SegFormer / L_att rows share the 3576/766/766 seed-42 test set; +the DeepLabV3+ row is a 400-sample CPU-smoke subset evaluated by +`train_deeplab_multiseed.py`. + +| Model | Dice | IoU | F1 | AAMO | Params | GFLOPs | FPS | Source | +|-------|------|-----|----|------|--------|--------|-----|--------| +| U-Net (CNN baseline) | 0.8615 | 0.7568 | 0.8615 | n/a | 31037698 | 109.48 | 1.47 | Phase1/Lasana-Person4_Evaluation (Chanupa PyTorch ckpt) | +| SegFormer-B0 (no attention loss) | 0.8743 | 0.7766 | 0.8743 | 0.0334 | 3714658 | 1.692 | 84.52 | Phase2/Kalana-Person2 (full-scale Colab) | +| SegFormer-B0 + Attention Consistency Loss (λ2=1.0 MSE) | 0.8577 | 0.7508 | 0.8577 | 0.7476 | 3714658 | 1.692 | 103.36 | Phase2/Dinura-Person3/results/runs/l2_1_mse | +| SegFormer-B0 + Attention Consistency + Boundary Loss | - | - | - | pending | - | - | - | pending | +| DeepLabV3+ (MobileNetV3) — extra baseline | 0.7821 | 0.6422 | 0.7821 | n/a | 11020594 | n/a | | Phase2/Lasana-Person4/train_deeplab_multiseed.py (seed 42, 400-sample smoke) | + +## Notes + +- Attention-consistency row uses Dinura's sweep winner (`λ2=1.0`, MSE, run tag `l2_1_mse`: Dice 0.8577 / IoU 0.7508 / AAMO 0.7476), not Kalana's default-λ2=0.3 attention run. +- Boundary Loss row stays pending until Person 5 finishes integration. +- DeepLabV3+ Dice/IoU come from `deeplab_multiseed.json` seed 42 (same path as the multi-seed ablation); see `ablation_mean_std.md` for seeds 42/43/44 mean±std. +- Phase 1 `Lasana-Person4_Evaluation/results/` is left untouched (frozen short-paper snapshot per CONTRIBUTING.md). diff --git a/Phase2/Lasana-Person4/results/deeplab_multiseed.json b/Phase2/Lasana-Person4/results/deeplab_multiseed.json new file mode 100644 index 0000000..14a892b --- /dev/null +++ b/Phase2/Lasana-Person4/results/deeplab_multiseed.json @@ -0,0 +1,50 @@ +[ + { + "model": "DeepLabV3+ (MobileNetV3) \u2014 extra baseline", + "seed": 42, + "dice": 0.7821, + "iou": 0.6422, + "f1": 0.7821, + "precision": 0.6591, + "recall": 0.9616, + "pixel_acc": 0.6941, + "aamo": "n/a", + "params": 11020594, + "gflops": "n/a", + "checkpoint": "Phase1/Lasana-Person4_Evaluation/checkpoints/deeplabv3_mobilenet_best.pt", + "max_samples": 400, + "status": "ok" + }, + { + "model": "DeepLabV3+ (MobileNetV3) \u2014 extra baseline", + "seed": 43, + "dice": 0.8073, + "iou": 0.6769, + "f1": 0.8073, + "precision": 0.766, + "recall": 0.8533, + "pixel_acc": 0.7674, + "aamo": "n/a", + "params": 11020594, + "gflops": "n/a", + "checkpoint": "Phase2/Lasana-Person4/checkpoints/deeplabv3_mobilenet_seed43_best.pt", + "max_samples": 400, + "status": "ok" + }, + { + "model": "DeepLabV3+ (MobileNetV3) \u2014 extra baseline", + "seed": 44, + "dice": 0.7693, + "iou": 0.6251, + "f1": 0.7693, + "precision": 0.6489, + "recall": 0.9444, + "pixel_acc": 0.6765, + "aamo": "n/a", + "params": 11020594, + "gflops": "n/a", + "checkpoint": "Phase2/Lasana-Person4/checkpoints/deeplabv3_mobilenet_seed44_best.pt", + "max_samples": 400, + "status": "ok" + } +] \ No newline at end of file diff --git a/Phase2/Lasana-Person4/tests/test_fold_results.py b/Phase2/Lasana-Person4/tests/test_fold_results.py new file mode 100644 index 0000000..769f65a --- /dev/null +++ b/Phase2/Lasana-Person4/tests/test_fold_results.py @@ -0,0 +1,57 @@ +"""Unit tests for fold_full_scale_results (no GPU / no model weights).""" +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from fold_full_scale_results import fold # noqa: E402 + + +class TestFoldFullScale(unittest.TestCase): + def test_fold_returns_five_rows(self): + rows = fold() + self.assertEqual(len(rows), 5) + + def test_unet_row_present(self): + rows = fold() + unet = next(r for r in rows if "U-Net" in r["model"]) + self.assertNotIn(unet["dice"], ("-", "", None)) + float(unet["dice"]) # must be numeric + + def test_vanilla_segformer_from_kalana(self): + rows = fold() + vanilla = next(r for r in rows if "no attention" in r["model"]) + self.assertAlmostEqual(float(vanilla["dice"]), 0.8743, places=4) + self.assertAlmostEqual(float(vanilla["aamo"]), 0.0334, places=4) + self.assertIn("Kalana", vanilla["source"]) + + def test_attention_row_is_dinura_winner(self): + rows = fold() + att = next(r for r in rows if "Attention Consistency Loss" in r["model"] and "Boundary" not in r["model"]) + self.assertAlmostEqual(float(att["dice"]), 0.8577, places=4) + self.assertAlmostEqual(float(att["iou"]), 0.7508, places=4) + self.assertAlmostEqual(float(att["aamo"]), 0.7476, places=4) + self.assertIn("l2_1_mse", att["source"] + att["notes"]) + + def test_boundary_row_pending(self): + rows = fold() + bound = next(r for r in rows if "Boundary" in r["model"]) + self.assertEqual(bound["aamo"], "pending") + self.assertEqual(bound["n_seeds"], 0) + + def test_deeplab_row_present(self): + rows = fold() + dl = next(r for r in rows if "DeepLab" in r["model"]) + self.assertAlmostEqual(float(dl["dice"]), 0.7821, places=4) + self.assertAlmostEqual(float(dl["iou"]), 0.6422, places=4) + self.assertIn("train_deeplab_multiseed", dl["source"]) + self.assertNotIn("0.7369", str(dl["dice"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/Phase2/Lasana-Person4/train_deeplab_multiseed.py b/Phase2/Lasana-Person4/train_deeplab_multiseed.py new file mode 100644 index 0000000..4b9806b --- /dev/null +++ b/Phase2/Lasana-Person4/train_deeplab_multiseed.py @@ -0,0 +1,472 @@ +""" +DeepLabV3+ multi-seed training + ablation aggregation (Person 4, Weeks 7–9). + +Seeds 42 / 43 / 44: + - seed 42: evaluate the existing Phase 1 smoke checkpoint (no retrain) + - seeds 43, 44: train fresh with the same smoke settings, save under + Phase2/Lasana-Person4/checkpoints/ + +Also writes the Phase 2 ablation tables by combining: + - single-seed U-Net / SegFormer-vanilla / SegFormer+L_att (from fold) + - 3-seed DeepLabV3+ mean±std (this script) + - pending Boundary Loss row + +Usage +----- + cd Phase2/Lasana-Person4 + python train_deeplab_multiseed.py + python train_deeplab_multiseed.py --skip-train # eval + aggregate only +""" +from __future__ import annotations + +import argparse +import csv +import json +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +import numpy as np +import torch +import torch.nn.functional as F +from torch.utils.data import DataLoader, TensorDataset + +ROOT = Path(__file__).resolve().parent +PROJECT = ROOT.parent.parent +PHASE1_P4 = PROJECT / "Phase1" / "Lasana-Person4_Evaluation" +RESULTS = ROOT / "results" +CKPT_DIR = ROOT / "checkpoints" +RESULTS.mkdir(parents=True, exist_ok=True) +CKPT_DIR.mkdir(parents=True, exist_ok=True) + +if str(PHASE1_P4) not in sys.path: + sys.path.insert(0, str(PHASE1_P4)) + +import config # noqa: E402 (Phase1 Person4 config) +from adapters.data import load_pairs, split_dataset # noqa: E402 +from adapters.deeplab_model import build_deeplabv3, forest_prob_from_logits # noqa: E402 +from metrics import ConfusionCounts, binarize, metrics_from_counts # noqa: E402 + +SEEDS = [42, 43, 44] +SEED42_CKPT = PHASE1_P4 / "checkpoints" / "deeplabv3_mobilenet_best.pt" +DEEPLAB_LABEL = "DeepLabV3+ (MobileNetV3) — extra baseline" + +# Smoke defaults match Phase1/train_deeplab_extra.py +MAX_SAMPLES = int(os.environ.get("DEEPLAB_MAX_SAMPLES", "400")) +EPOCHS = int(os.environ.get("DEEPLAB_EPOCHS", "5")) +BATCH_SIZE = int(os.environ.get("DEEPLAB_BATCH", "2")) +LR = float(os.environ.get("DEEPLAB_LR", "1e-4")) + + +def aggregate_mean_std(rows: List[Dict[str, Any]]) -> Dict[str, Any]: + """Sample std (ddof=1) across seeds; bare mean when n<2. + + Local override of Phase1 ablation_runner.aggregate_mean_std (frozen snapshot). + Persons 1/2/3 GPU multi-seed reporting must also use ddof=1. + """ + numeric = ["dice", "iou", "f1", "precision", "recall", "pixel_acc"] + out: Dict[str, Any] = {"model": rows[0]["model"], "n_seeds": len(rows)} + + def fmt(vals: List[float]) -> str: + if not vals: + return "n/a" + m = float(np.mean(vals)) + if len(vals) < 2: + return f"{m:.4f}" + return f"{m:.4f} ± {float(np.std(vals, ddof=1)):.4f}" + + for k in numeric: + vals: List[float] = [] + for r in rows: + try: + vals.append(float(r[k])) + except (TypeError, ValueError, KeyError): + pass + out[k] = fmt(vals) + + avals: List[float] = [] + for r in rows: + v = r.get("aamo") + if v not in (None, "n/a", ""): + try: + avals.append(float(v)) + except (TypeError, ValueError): + pass + out["aamo"] = fmt(avals) + out["params"] = rows[0].get("params", "n/a") + out["gflops"] = rows[0].get("gflops", "n/a") + return out + + +def _to_tensor_images(images: np.ndarray) -> torch.Tensor: + x = torch.from_numpy(images).permute(0, 3, 1, 2).float() + mean = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1) + std = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1) + return (x - mean) / std + + +def dice_loss_from_probs(probs: torch.Tensor, targets: torch.Tensor, eps: float = 1.0) -> torch.Tensor: + p = probs.reshape(probs.size(0), -1) + t = targets.reshape(targets.size(0), -1) + inter = (p * t).sum(dim=1) + return (1 - (2 * inter + eps) / (p.sum(dim=1) + t.sum(dim=1) + eps)).mean() + + +@torch.no_grad() +def eval_split(model, images, masks, device, batch_size=4) -> dict: + model.eval() + counts = ConfusionCounts() + for i in range(0, len(images), batch_size): + xb = _to_tensor_images(images[i : i + batch_size]).to(device) + out = model(xb)["out"] + if out.shape[-2:] != (config.IMG_SIZE, config.IMG_SIZE): + out = F.interpolate( + out, size=(config.IMG_SIZE, config.IMG_SIZE), mode="bilinear", align_corners=False + ) + probs = forest_prob_from_logits(out).cpu().numpy() + preds = binarize(probs, 0.5) + counts.update(preds, masks[i : i + batch_size]) + return metrics_from_counts(counts) + + +def train_one_seed(seed: int, device: torch.device) -> Path: + """Train DeepLabV3+ for one seed; return best-checkpoint path. + + Data split is always seed 42 (same held-out test set as Phase 1). + Only the training RNG (weight init / shuffle) varies with ``seed``. + """ + torch.manual_seed(seed) + np.random.seed(seed) + + images, masks = load_pairs(max_samples=MAX_SAMPLES) + # Fixed split seed 42 — multi-seed compares training init, not data folds. + splits = split_dataset(images, masks, seed=42) + + X_train, y_train = splits["train"] + X_val, y_val = splits["val"] + + train_x = _to_tensor_images(X_train) + train_y_f = torch.from_numpy(y_train).float() + loader = DataLoader( + TensorDataset(train_x, train_y_f), + batch_size=BATCH_SIZE, + shuffle=True, + drop_last=True, + ) + + model = build_deeplabv3(num_classes=2, pretrained_backbone=True).to(device) + opt = torch.optim.AdamW(model.parameters(), lr=LR, weight_decay=1e-4) + + best_iou = -1.0 + best_path = CKPT_DIR / f"deeplabv3_mobilenet_seed{seed}_best.pt" + + for epoch in range(1, EPOCHS + 1): + model.train() + losses = [] + for xb, yb in loader: + xb = xb.to(device) + yb = yb.to(device) + out = model(xb)["out"] + if out.shape[-2:] != (config.IMG_SIZE, config.IMG_SIZE): + out = F.interpolate( + out, size=(config.IMG_SIZE, config.IMG_SIZE), mode="bilinear", align_corners=False + ) + ce = F.cross_entropy(out, yb.long()) + probs = forest_prob_from_logits(out) + dsc = dice_loss_from_probs(probs, yb) + loss = 0.5 * ce + 0.5 * dsc + opt.zero_grad() + loss.backward() + opt.step() + losses.append(float(loss.item())) + + val = eval_split(model, X_val, y_val, device, batch_size=BATCH_SIZE) + print( + f" seed={seed} epoch {epoch}/{EPOCHS} loss={np.mean(losses):.4f} " + f"val_dice={val['dice']:.4f} val_iou={val['iou']:.4f}" + ) + ckpt = { + "model_state": model.state_dict(), + "epoch": epoch, + "val_dice": val["dice"], + "val_iou": val["iou"], + "variant": "deeplabv3_mobilenet", + "seed": seed, + "max_samples": MAX_SAMPLES, + } + if val["iou"] > best_iou: + best_iou = val["iou"] + torch.save(ckpt, best_path) + print(f" saved best -> {best_path} (iou={best_iou:.4f})") + + return best_path + + +def evaluate_checkpoint(ckpt_path: Path, seed: int, device: torch.device) -> Dict[str, Any]: + """Evaluate a DeepLab checkpoint on the fixed seed-42 smoke test split.""" + images, masks = load_pairs(max_samples=MAX_SAMPLES) + splits = split_dataset(images, masks, seed=42) + X_test, y_test = splits["test"] + + model = build_deeplabv3(num_classes=2, pretrained_backbone=False).to(device) + ckpt = torch.load(ckpt_path, map_location="cpu") + state = ckpt["model_state"] if isinstance(ckpt, dict) and "model_state" in ckpt else ckpt + model.load_state_dict(state) + model.eval() + + metrics = eval_split(model, X_test, y_test, device, batch_size=BATCH_SIZE) + n_params = sum(p.numel() for p in model.parameters() if p.requires_grad) + + try: + ckpt_str = ckpt_path.resolve().relative_to(PROJECT).as_posix() + except ValueError: + ckpt_str = ckpt_path.name + + row = { + "model": DEEPLAB_LABEL, + "seed": seed, + "dice": round(float(metrics["dice"]), 4), + "iou": round(float(metrics["iou"]), 4), + "f1": round(float(metrics["f1"]), 4), + "precision": round(float(metrics["precision"]), 4), + "recall": round(float(metrics["recall"]), 4), + "pixel_acc": round(float(metrics["pixel_acc"]), 4), + "aamo": "n/a", + "params": n_params, + "gflops": "n/a", + "checkpoint": ckpt_str, + "max_samples": MAX_SAMPLES, + "status": "ok", + } + return row + + +def _ckpt_for_seed(seed: int) -> Path: + if seed == 42: + return SEED42_CKPT + return CKPT_DIR / f"deeplabv3_mobilenet_seed{seed}_best.pt" + + +def load_folded_full_scale() -> List[Dict[str, str]]: + path = RESULTS / "baseline_comparison_full_scale.csv" + if not path.exists(): + from fold_full_scale_results import fold, write_tables + + write_tables(fold()) + with open(path, newline="", encoding="utf-8") as f: + return list(csv.DictReader(f)) + + +def write_ablation_tables( + deeplab_per_seed: List[Dict[str, Any]], + folded: List[Dict[str, str]], +) -> None: + """Combine folded single-seed rows + DeepLab multi-seed into ablation tables.""" + per_seed_rows: List[Dict[str, Any]] = [] + + # Single-seed rows from the folded full-scale table (skip DeepLab — replaced below) + for r in folded: + model = r.get("model", "") + if "DeepLab" in model: + continue + if r.get("dice") in ("-", "", None) and "Boundary" in model: + per_seed_rows.append( + { + "model": model, + "seed": "-", + "dice": "-", + "iou": "-", + "f1": "-", + "aamo": "pending", + "params": "-", + "gflops": "-", + "status": "pending_checkpoint", + } + ) + continue + per_seed_rows.append( + { + "model": model, + "seed": 42, + "dice": r.get("dice", "-"), + "iou": r.get("iou", "-"), + "f1": r.get("f1", "-"), + "aamo": r.get("aamo", "n/a"), + "params": r.get("params", "n/a"), + "gflops": r.get("gflops", "n/a"), + "status": "single_seed_full_scale", + } + ) + + for r in deeplab_per_seed: + per_seed_rows.append(r) + + # Write per-seed CSV + per_path = RESULTS / "ablation_per_seed.csv" + fields = [ + "model", + "seed", + "dice", + "iou", + "f1", + "precision", + "recall", + "pixel_acc", + "aamo", + "params", + "gflops", + "status", + "checkpoint", + "max_samples", + ] + with open(per_path, "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore") + w.writeheader() + for r in per_seed_rows: + w.writerow({k: r.get(k, "") for k in fields}) + print(f"Wrote {per_path}") + + # Aggregate mean±std per model + by_model: Dict[str, List[Dict[str, Any]]] = {} + for r in per_seed_rows: + if r.get("status") == "pending_checkpoint": + continue + if r.get("dice") in ("-", "", None): + continue + by_model.setdefault(r["model"], []).append(r) + + summary: List[Dict[str, Any]] = [] + # Preserve table order from folded rows, then DeepLab + order = [r["model"] for r in folded] + seen = set() + for name in order: + if name in seen: + continue + seen.add(name) + if "Boundary" in name and name not in by_model: + summary.append( + { + "model": name, + "n_seeds": 0, + "dice": "-", + "iou": "-", + "f1": "-", + "aamo": "pending", + "params": "-", + "gflops": "-", + } + ) + continue + if name not in by_model: + continue + summary.append(aggregate_mean_std(by_model[name])) + + sum_csv = RESULTS / "ablation_mean_std.csv" + sum_md = RESULTS / "ablation_mean_std.md" + sum_fields = ["model", "n_seeds", "dice", "iou", "f1", "aamo", "params", "gflops"] + with open(sum_csv, "w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=sum_fields, extrasaction="ignore") + w.writeheader() + for r in summary: + w.writerow({k: r.get(k, "") for k in sum_fields}) + + lines = [ + "# Ablation results (mean ± std) — Phase 2 / Lasana-Person4", + "", + "Std is **sample** standard deviation (ddof=1). Single-seed rows", + "(U-Net, SegFormer-B0, SegFormer-B0+L_att λ2=1.0) show the bare value", + "pending additional GPU-trained seeds from Person 1/2/3. DeepLabV3+", + "extra baseline has genuine 3-seed mean±std (seeds 42/43/44, CPU smoke:", + "400 samples / 5 epochs).", + "", + "| Model | Seeds | Dice | IoU | F1 | AAMO |", + "|-------|-------|------|-----|----|------|", + ] + for r in summary: + lines.append( + f"| {r.get('model')} | {r.get('n_seeds')} | {r.get('dice')} | " + f"{r.get('iou')} | {r.get('f1')} | {r.get('aamo')} |" + ) + lines.extend( + [ + "", + "## Notes", + "", + "- Attention-consistency config = Dinura `l2_1_mse` (λ2=1.0, MSE).", + "- Boundary Loss row pending Person 5 integration.", + "- DeepLabV3+ smoke numbers are not paper-scale; they demonstrate", + " the multi-seed aggregation pipeline Person 4 owns.", + "- U-Net / SegFormer full-scale checkpoints live on Drive (not in git);", + " re-running extra seeds requires Colab GPU access from teammates.", + "- Teammate GPU multi-seed reporting must also use sample std (ddof=1).", + "", + ] + ) + sum_md.write_text("\n".join(lines), encoding="utf-8") + print(f"Wrote {sum_csv}") + print(f"Wrote {sum_md}") + + # Also dump DeepLab per-seed JSON for easy inspection + dl_json = RESULTS / "deeplab_multiseed.json" + dl_json.write_text(json.dumps(deeplab_per_seed, indent=2), encoding="utf-8") + print(f"Wrote {dl_json}") + + +def main() -> None: + p = argparse.ArgumentParser(description="DeepLabV3+ multi-seed + ablation tables") + p.add_argument( + "--skip-train", + action="store_true", + help="Skip training seeds 43/44; only evaluate existing checkpoints + aggregate", + ) + p.add_argument( + "--seeds", + type=int, + nargs="+", + default=SEEDS, + help="Seeds to run (default: 42 43 44)", + ) + args = p.parse_args() + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + print( + f"device={device} | max_samples={MAX_SAMPLES} | epochs={EPOCHS} | " + f"batch={BATCH_SIZE} | seeds={args.seeds}" + ) + + deeplab_rows: List[Dict[str, Any]] = [] + for seed in args.seeds: + ckpt = _ckpt_for_seed(seed) + if seed != 42 and not args.skip_train: + if not ckpt.exists(): + print(f"\n=== Training DeepLabV3+ seed={seed} ===") + ckpt = train_one_seed(seed, device) + else: + print(f"\n=== Reusing existing checkpoint for seed={seed}: {ckpt} ===") + elif seed != 42 and args.skip_train and not ckpt.exists(): + print(f"[skip] seed={seed}: no checkpoint at {ckpt}") + continue + + if seed == 42 and not ckpt.exists(): + raise FileNotFoundError( + f"Seed-42 DeepLab checkpoint missing: {ckpt}\n" + "Train it first in Phase1:\n" + " cd Phase1/Lasana-Person4_Evaluation && python train_deeplab_extra.py" + ) + + print(f"\n=== Evaluating DeepLabV3+ seed={seed} | {ckpt} ===") + row = evaluate_checkpoint(ckpt, seed, device) + print( + f" seed={seed} test dice={row['dice']:.4f} iou={row['iou']:.4f} " + f"f1={row['f1']:.4f}" + ) + deeplab_rows.append(row) + + folded = load_folded_full_scale() + write_ablation_tables(deeplab_rows, folded) + print("\nMulti-seed DeepLab + ablation tables done.") + + +if __name__ == "__main__": + main()