diff --git a/src/gpu/distill_specs.yaml b/src/gpu/distill_specs.yaml index 0e5e734..68ca28a 100644 --- a/src/gpu/distill_specs.yaml +++ b/src/gpu/distill_specs.yaml @@ -164,6 +164,58 @@ ara-diac-small-layerdrop: verbatim (both width-cut rungs failed); int4 target ~75MB; Muon optimizer per E3 optimizer: muon +ara-diac-small-2-6ep: + # G2a: the residual-attribution disambiguation - same as the 2.0 rung + # (r7 labels, Muon, vanilla ByT5-small) at 6 epochs instead of 3. + # If no gain, the 2.25pp residual is NOT undertraining -> domain. + teacher: rababa_arabic_byt5/run-007-news/best + teacher_volume: rababa + out_volume: rababa + student_init: google/byt5-small + train: r5-units/domain.txt + train_extra: + - r5-units/replay.txt + unit_limits: + - 24000 + - 6000 + max_len: 1450 + label_beams: '1' + out: rababa_arabic_distill_small/run-007-r7-muon-6ep + labels_file: teacher_labels_r7.jsonl + labels_complete: 'true' + mode: sequence + optimizer: muon + note: G2a epochs-rung; gate vs 4.8218 - flat means domain-limited +ara-diac-small-d768: + # G2b companion + frontier point: the UNTESTED gentle width cut + # (2x, not 3.8x) with SVD stitch + Muon. ~105M params, int8 ~105MB. + # Closes the "you never tried a gentle stitch" reviewer attack. + teacher: rababa_arabic_byt5/run-007-news/best + teacher_volume: rababa + out_volume: rababa + student_init: google/byt5-small + student_config: + d_model: 768 + d_kv: 64 + d_ff: 1856 + num_heads: 12 + enc_layers: 12 + dec_layers: 4 + feed_forward_proj: gated-gelu + train: r5-units/domain.txt + train_extra: + - r5-units/replay.txt + unit_limits: + - 24000 + - 6000 + max_len: 1450 + label_beams: '1' + out: rababa_arabic_distill_small/run-008-d768-stitch + labels_file: teacher_labels_r7.jsonl + labels_complete: 'true' + mode: sequence + optimizer: muon + note: gentle-stitch rung (2x cut); frontier point at ~105M ara-diac-tiny-stitched: teacher: rababa_arabic_byt5/run-006-morph/best teacher_volume: rababa