diff --git a/.gitignore b/.gitignore index 3eca07aa..9684c894 100644 --- a/.gitignore +++ b/.gitignore @@ -117,3 +117,31 @@ logs/ pytorch_*.err pytorch_*.out scratch/ + +# Python cache +__pycache__/ +*.pyc + +# Local accidental CLI files +--config +--log-dir + +# Local backups +*.bak +*.bak_* + +# Large MFX/generated outputs +runs/ +outputs/ +mfx_shoebox*/ +mfx_shoebox_chunks*/ +diagnostics_qi_25files/ +diagnostics_qi_meeting_25files/ +diagnostics_qp_mean_profiles/ +diagnostics_qp_mean_three_models_25files/ +*.npy +*.parquet +*.ckpt +*.mtz +*.png +*.log diff --git a/README.md b/README.md index 6155f085..38b8e04f 100644 --- a/README.md +++ b/README.md @@ -37,5 +37,33 @@ micromamba activate integrator-dev micromamba env create -f environment-cuda-dev.yml micromamba activate integrator-cuda-dev ``` +## MFX Workflow -Each installs DIALS, laue-DIALS, PyTorch, the integrator package (editable), and the upstream reciprocalspaceship build required to read DIALS `.refl` files. +This repository includes an end-to-end workflow for applying the variational-inference Integrator to experimental MFX serial femtosecond crystallography data from LCLS. + +The MFX workflow extends the base Integrator framework with: + +- scalable Jungfrau shoebox extraction from cctbx.xfel `.refl/.expt` files +- chunked preprocessing and GPU training for datasets containing more than 100 million reflections +- alternative intensity posteriors including FoldedNormal and LogNormal +- Normal and Student-t observation likelihoods for floating-point and negative-valued detector data +- Monte Carlo KL fallback for posterior/prior combinations without a built-in analytic KL divergence +- MFX prediction write-back from Integrator intensities and uncertainties to the original reflection tables +- downstream cctbx.xfel scaling/merging and PHENIX refinement for direct crystallographic comparison +- diagnostic tools for shoeboxes, learned spot profiles, training behavior, and resolution-dependent merging statistics + +### Current MFX study + +The current production study uses experiment `mfx101555026`, covering runs 269–289 excluding 275: + +- **124,975** indexed MFX `.refl/.expt` pairs +- **~115 million** extracted 25 × 25 Jungfrau shoeboxes +- two production Integrator models using **ASINH** and **sqrt-squareplus** preprocessing with a **FoldedNormal intensity posterior** +- matched cctbx.xfel baselines for apples-to-apples comparison +- evaluation through **CC½, Rint, Rsplit, R-work, and R-free** + +The current results establish an end-to-end proof of concept: Integrator can scale to a full experimental MFX dataset, produce probabilistic intensity estimates, write them back into the crystallographic workflow, and generate refinable structures. The cctbx.xfel baseline currently performs better overall, while the elevated high-resolution Integrator CC½ behavior remains an active validation question. + +See the full workflow, commands, configs, Slurm jobs, PHIL files, diagnostics, and results here: + +- [`workflows/mfx101555026`](workflows/mfx101555026/README.md) — complete MFX cctbx.xfel → Integrator → scaling/merging → PHENIX workflow \ No newline at end of file diff --git a/inspect_d_script.py b/inspect_d_script.py new file mode 100644 index 00000000..e69de29b diff --git a/pyproject.toml b/pyproject.toml index c0b48c11..d64417a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ laue = ['laue-dials'] "integrator.predict" = "integrator.cli.predict:main" "integrator.make_shoeboxes" = "integrator.cli.make_shoeboxes:main" "integrator.resolve" = "integrator.cli.resolve:main" +"integrator.preprocess" = "integrator.cli.preprocess:main" [project.urls] Homepage = "https://github.com/Hekstra-Lab/integrator/tree/integrator_mvn" diff --git a/src/integrator/callbacks/plots.py b/src/integrator/callbacks/plots.py index 6bdf3f43..dfe2017e 100644 --- a/src/integrator/callbacks/plots.py +++ b/src/integrator/callbacks/plots.py @@ -31,22 +31,34 @@ def _collect(self, split, outputs): ) if not lc: return + for term in _TERMS: - v = lc.get(term) - if v is not None: - self._acc[split][term].append(float(v)) + value = lc.get(term) + if value is not None: + self._acc[split][term].append(float(value)) def on_train_epoch_start(self, trainer, pl_module): self._reset_epoch() def on_train_batch_end( - self, trainer, pl_module, outputs, batch, batch_idx + self, + trainer, + pl_module, + outputs, + batch, + batch_idx, ): if not trainer.sanity_checking: self._collect("train", outputs) def on_validation_batch_end( - self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx=0 + self, + trainer, + pl_module, + outputs, + batch, + batch_idx, + dataloader_idx=0, ): if not trainer.sanity_checking: self._collect("val", outputs) @@ -54,14 +66,19 @@ def on_validation_batch_end( def on_train_epoch_end(self, trainer, pl_module): if trainer.sanity_checking: return + row: dict = {"epoch": int(trainer.current_epoch)} + for split in _SPLITS: for term in _TERMS: - vals = self._acc[split].get(term) - if vals: - row[f"{split}_{term}"] = sum(vals) / len(vals) + values = self._acc[split].get(term) + + if values: + row[f"{split}_{term}"] = sum(values) / len(values) + if len(row) == 1: return + self._hist.append(row) self._plot(trainer) @@ -69,37 +86,68 @@ def _plot(self, trainer): import matplotlib.pyplot as plt import polars as pl - rl = get_run_logger(self, trainer) - epochs = [r["epoch"] for r in self._hist] + run_logger = get_run_logger(self, trainer) + epochs = [row["epoch"] for row in self._hist] def series(key): - return [r.get(key) for r in self._hist] + return [row.get(key) for row in self._hist] fig, ax = plt.subplots(figsize=(4, 3), dpi=90) + for split, style in (("train", "-"), ("val", "--")): - ys = series(f"{split}_loss") - if any(y is not None for y in ys): - ax.plot(epochs, ys, style, label=split) + values = series(f"{split}_loss") + + if any(value is not None for value in values): + ax.plot( + epochs, + values, + style, + label=split, + ) + ax.set_xlabel("epoch") ax.set_ylabel("total loss (ELBO)") ax.set_title("loss") ax.legend() - rl.log_figure("loss_total", fig) + + run_logger.log_figure("loss_total", fig) panel = ("nll", "kl_prf", "kl_i", "kl_bg") - fig2, axes = plt.subplots(2, 2, figsize=(7, 5), dpi=90) - for ax_, term in zip(axes.ravel(), panel, strict=False): + + fig2, axes = plt.subplots( + 2, + 2, + figsize=(7, 5), + dpi=90, + ) + + for axis, term in zip( + axes.ravel(), + panel, + strict=False, + ): for split, style in (("train", "-"), ("val", "--")): - ys = series(f"{split}_{term}") - if any(y is not None for y in ys): - ax_.plot(epochs, ys, style, label=split) - ax_.set_title(term) - ax_.set_xlabel("epoch") - ax_.legend(fontsize=7) + values = series(f"{split}_{term}") + + if any(value is not None for value in values): + axis.plot( + epochs, + values, + style, + label=split, + ) + + axis.set_title(term) + axis.set_xlabel("epoch") + axis.legend(fontsize=7) + fig2.tight_layout() - rl.log_figure("loss_terms", fig2) + run_logger.log_figure("loss_terms", fig2) - rl.log_table("loss_history", pl.DataFrame(self._hist)) + run_logger.log_table( + "loss_history", + pl.DataFrame(self._hist), + ) class PredictionScatterLogger(Callback): @@ -111,10 +159,18 @@ class PredictionScatterLogger(Callback): """ def __init__( - self, out_dir=None, max_points: int = 2000, every_n_epochs: int = 1 + self, + out_dir=None, + max_points: int = 2000, + every_n_epochs: int = 1, ): super().__init__() - self.out_dir = Path(out_dir) if out_dir is not None else None + + self.out_dir = ( + Path(out_dir) + if out_dir is not None + else None + ) self.max_points = max_points self.every_n_epochs = every_n_epochs self._buf: dict[str, list] = defaultdict(list) @@ -123,59 +179,110 @@ def on_validation_epoch_start(self, trainer, pl_module): self._buf = defaultdict(list) def on_validation_batch_end( - self, trainer, pl_module, outputs, batch, batch_idx, dataloader_idx=0 + self, + trainer, + pl_module, + outputs, + batch, + batch_idx, + dataloader_idx=0, ): - if trainer.sanity_checking or not isinstance(outputs, dict): + if trainer.sanity_checking: + return + + if not isinstance(outputs, dict): + return + + output = outputs.get("forward_out") + + if output is None: return - out = outputs.get("forward_out") - if out is None or "qi_mean" not in out or "qbg_mean" not in out: + + if "qi_mean" not in output or "qbg_mean" not in output: return + if not isinstance(batch, (tuple, list)) or len(batch) < 4: return + metadata = batch[3] - di = metadata.get("intensity.prf.value") - if di is None: - di = metadata.get("intensity.sum.value") - dbg = metadata.get("background.mean") - if dbg is None: - dbg = metadata.get("background.sum.value") - if di is None or dbg is None: + + dials_intensity = metadata.get("intensity.prf.value") + + if dials_intensity is None: + dials_intensity = metadata.get("intensity.sum.value") + + dials_background = metadata.get("background.mean") + + if dials_background is None: + dials_background = metadata.get("background.sum.value") + + if dials_intensity is None or dials_background is None: return - self._buf["qi_mean"].append(out["qi_mean"].detach().flatten().cpu()) - self._buf["qbg_mean"].append(out["qbg_mean"].detach().flatten().cpu()) - self._buf["dials_intensity"].append(di.detach().flatten().cpu()) - self._buf["dials_background"].append(dbg.detach().flatten().cpu()) + self._buf["qi_mean"].append( + output["qi_mean"].detach().flatten().cpu() + ) + self._buf["qbg_mean"].append( + output["qbg_mean"].detach().flatten().cpu() + ) + self._buf["dials_intensity"].append( + dials_intensity.detach().flatten().cpu() + ) + self._buf["dials_background"].append( + dials_background.detach().flatten().cpu() + ) def on_validation_epoch_end(self, trainer, pl_module): - if trainer.sanity_checking or not self._buf.get("qi_mean"): + if trainer.sanity_checking: + return + + if not self._buf.get("qi_mean"): return + epoch = int(trainer.current_epoch) - if self.every_n_epochs and epoch % self.every_n_epochs != 0: + + if ( + self.every_n_epochs + and epoch % self.every_n_epochs != 0 + ): return import polars as pl import torch - cols = {k: torch.cat(v).numpy() for k, v in self._buf.items()} - n = len(cols["qi_mean"]) - if n > self.max_points: - idx = torch.randperm(n)[: self.max_points].numpy() - cols = {k: v[idx] for k, v in cols.items()} - df = pl.DataFrame(cols) + columns = { + key: torch.cat(values).numpy() + for key, values in self._buf.items() + } + + n_points = len(columns["qi_mean"]) + + if n_points > self.max_points: + indices = torch.randperm(n_points)[ + : self.max_points + ].numpy() + + columns = { + key: values[indices] + for key, values in columns.items() + } + + dataframe = pl.DataFrame(columns) + + run_logger = get_run_logger(self, trainer) - rl = get_run_logger(self, trainer) - rl.log_scatter( + run_logger.log_scatter( "val_intensity_model_vs_dials", - df, + dataframe, x="qi_mean", y="dials_intensity", step=epoch, loglog=True, ) - rl.log_scatter( + + run_logger.log_scatter( "val_background_model_vs_dials", - df, + dataframe, x="qbg_mean", y="dials_background", step=epoch, @@ -191,17 +298,28 @@ class WilsonParamLogger(Callback): polychromatic loss. No-op when the loss is not a Wilson loss. """ - def __init__(self, out_dir=None, n_lambda: int = 100): + def __init__( + self, + out_dir=None, + n_lambda: int = 100, + ): super().__init__() - self.out_dir = Path(out_dir) if out_dir is not None else None + + self.out_dir = ( + Path(out_dir) + if out_dir is not None + else None + ) self.n_lambda = n_lambda self._hist: list[dict] = [] - self._spectra: list[tuple] = [] # (epoch, lambdas, G) + self._spectra: list[tuple] = [] def on_train_epoch_end(self, trainer, pl_module): if trainer.sanity_checking: return + loss = getattr(pl_module, "loss", None) + if loss is None or not hasattr(loss, "get_B"): return # not a Wilson loss @@ -210,27 +328,63 @@ def on_train_epoch_end(self, trainer, pl_module): epoch = int(trainer.current_epoch) row: dict = {"epoch": epoch} + with torch.no_grad(): - row["B"] = float(loss.get_B()) + + # Per-image B/G: mean, std, min, max + + if hasattr(loss, "get_wilson_stats"): + stats = loss.get_wilson_stats() + + for key, value in stats.items(): + row[key] = float(value) + + # Existing global B/G behavior + else: + row["B"] = float(loss.get_B()) spectrum = getattr(loss, "spectrum", None) + if spectrum is not None: - lam_lo = float(spectrum.lam_mid - spectrum.lam_scale) - lam_hi = float(spectrum.lam_mid + spectrum.lam_scale) - lam = torch.linspace(lam_lo, lam_hi, self.n_lambda) + lam_lo = float( + spectrum.lam_mid - spectrum.lam_scale + ) + lam_hi = float( + spectrum.lam_mid + spectrum.lam_scale + ) + + lam = torch.linspace( + lam_lo, + lam_hi, + self.n_lambda, + ) + g = spectrum.get_log_G(lam).exp() + self._spectra.append( - (epoch, lam.cpu().numpy(), g.cpu().numpy()) + ( + epoch, + lam.cpu().numpy(), + g.cpu().numpy(), + ) ) + row["G_mean"] = float(g.mean()) - elif hasattr(loss, "get_G"): + + elif ( + not hasattr(loss, "get_wilson_stats") + and hasattr(loss, "get_G") + ): row["G"] = float(loss.get_G()) - if getattr(loss, "learn_concentration", False) and hasattr( - loss, "log_alpha_per_group" + if ( + getattr(loss, "learn_concentration", False) + and hasattr(loss, "log_alpha_per_group") ): row["alpha_mean"] = float( - F.softplus(loss.log_alpha_per_group).mean() + F.softplus( + loss.log_alpha_per_group + ).mean() ) self._hist.append(row) @@ -240,49 +394,241 @@ def _plot(self, trainer): import matplotlib.pyplot as plt import polars as pl - rl = get_run_logger(self, trainer) - epochs = [r["epoch"] for r in self._hist] + run_logger = get_run_logger(self, trainer) - rl.log_scalars( - {f"wilson/{k}": v for k, v in self._hist[-1].items() if k != "epoch"}, + epochs = [ + row["epoch"] + for row in self._hist + ] + + run_logger.log_scalars( + { + f"wilson/{key}": value + for key, value in self._hist[-1].items() + if key != "epoch" + }, step=self._hist[-1]["epoch"], ) def series(key): - return [r.get(key) for r in self._hist] + return [ + row.get(key) + for row in self._hist + ] + + # PLOTTING + """ + solid line: mean + shaded band: mean ± standard deviation + dashed lines: min and max + """ + + fig, axes = plt.subplots( + 1, + 2, + figsize=(7, 3), + dpi=90, + ) + + # Per-image B/G statistics + if any( + "B_mean" in row + for row in self._hist + ): + b_mean = series("B_mean") + b_std = series("B_std") + b_min = series("B_min") + b_max = series("B_max") + + g_mean = series("G_mean") + g_std = series("G_std") + g_min = series("G_min") + g_max = series("G_max") + + axes[0].plot( + epochs, + b_mean, + "-", + marker=".", + label="mean", + ) + + axes[0].fill_between( + epochs, + [ + mean - std + for mean, std in zip( + b_mean, + b_std, + ) + ], # lower y + [ + mean + std + for mean, std in zip( + b_mean, + b_std, + ) + ], # upper y + alpha=0.2, + label="mean ± std", + ) + + axes[0].plot( + epochs, + b_min, + "--", + label="min", + ) + + axes[0].plot( + epochs, + b_max, + "--", + label="max", + ) + + axes[1].plot( + epochs, + g_mean, + "-", + marker=".", + label="mean", + ) + + axes[1].fill_between( + epochs, + [ + mean - std + for mean, std in zip( + g_mean, + g_std, + ) + ], # lower y + [ + mean + std + for mean, std in zip( + g_mean, + g_std, + ) + ], # upper y + alpha=0.2, + label="mean ± std", + ) + + axes[1].plot( + epochs, + g_min, + "--", + label="min", + ) + + axes[1].plot( + epochs, + g_max, + "--", + label="max", + ) + + axes[0].legend(fontsize=7) + axes[1].legend(fontsize=7) + + else: + # Existing global B/G behavior + + g_key = ( + "G" + if any( + "G" in row + for row in self._hist + ) + else "G_mean" + ) + + axes[0].plot( + epochs, + series("B"), + "-", + marker=".", + ) + + if any( + g_key in row + for row in self._hist + ): + axes[1].plot( + epochs, + series(g_key), + "-", + marker=".", + ) - g_key = "G" if any("G" in r for r in self._hist) else "G_mean" - fig, axes = plt.subplots(1, 2, figsize=(7, 3), dpi=90) - axes[0].plot(epochs, series("B"), "-", marker=".") axes[0].set_xlabel("epoch") axes[0].set_ylabel("B (Ų)") axes[0].set_title("Wilson B-factor") - if any(g_key in r for r in self._hist): - axes[1].plot(epochs, series(g_key), "-", marker=".") + axes[1].set_xlabel("epoch") - axes[1].set_ylabel(g_key) + axes[1].set_ylabel("G") axes[1].set_title("Wilson scale") + fig.tight_layout() - rl.log_figure("wilson_params", fig) + + run_logger.log_figure( + "wilson_params", + fig, + ) # G(lambda) spectrum and its evolution (polychromatic only) if self._spectra: - n = len(self._spectra) - fig2, ax = plt.subplots(figsize=(4, 3), dpi=90) - for i, (ep, lam, g) in enumerate(self._spectra): - label = f"ep {ep}" if i in (0, n - 1) else None + n_spectra = len(self._spectra) + + fig2, ax = plt.subplots( + figsize=(4, 3), + dpi=90, + ) + + for index, ( + spectrum_epoch, + wavelengths, + g_values, + ) in enumerate(self._spectra): + label = ( + f"ep {spectrum_epoch}" + if index in (0, n_spectra - 1) + else None + ) + ax.plot( - lam, g, color=plt.cm.viridis(i / max(n - 1, 1)), label=label + wavelengths, + g_values, + color=plt.cm.viridis( + index / max(n_spectra - 1, 1) + ), + label=label, ) + ax.set_xlabel("wavelength λ (Å)") ax.set_ylabel("G(λ)") ax.set_title("Learned spectrum G(λ)") ax.legend(fontsize=8) - rl.log_figure("wilson_spectrum", fig2) - ep, lam, g = self._spectra[-1] - rl.log_table( - "wilson_spectrum", pl.DataFrame({"wavelength": lam, "G": g}) + run_logger.log_figure( + "wilson_spectrum", + fig2, + ) + + _, wavelengths, g_values = self._spectra[-1] + + run_logger.log_table( + "wilson_spectrum", + pl.DataFrame( + { + "wavelength": wavelengths, + "G": g_values, + } + ), ) - rl.log_table("wilson_params", pl.DataFrame(self._hist)) + run_logger.log_table( + "wilson_params", + pl.DataFrame(self._hist), + ) \ No newline at end of file diff --git a/src/integrator/callbacks/prediction_writer.py b/src/integrator/callbacks/prediction_writer.py index 24d7c0c5..868d2d7e 100644 --- a/src/integrator/callbacks/prediction_writer.py +++ b/src/integrator/callbacks/prediction_writer.py @@ -13,6 +13,17 @@ def assign_labels( dataset, save_dir: str, ): + # assign_labels() receives the DataModule directly (train.py:424). + # ChunkedDataModule opts in via _CHUNKED_DATA_MODULE = True — its train/val + # split is already baked into chunk files; iterating dataloaders here would + # be prohibitively expensive at 100 M+ reflections. + if getattr(dataset, "_CHUNKED_DATA_MODULE", False): + logger.info( + "assign_labels: ChunkedDataModule detected - " + "train/val split is already baked into chunk files; skipping." + ) + return + train_id_df = pl.DataFrame(schema=[("train_ids", int)]) val_id_df = pl.DataFrame(schema=[("val_ids", int)]) diff --git a/src/integrator/cli/make_shoeboxes.py b/src/integrator/cli/make_shoeboxes.py index 2bf08e4c..91a4e9fb 100644 --- a/src/integrator/cli/make_shoeboxes.py +++ b/src/integrator/cli/make_shoeboxes.py @@ -8,7 +8,7 @@ 1. Default (rotation / sequence): one imageset with a rotation scan. 2. --laue: many single-frame stills from laue-dials. --d must be 1; -Examples: +Examples: Run (rotation mode): integrator.make_shoeboxes \ @@ -18,7 +18,7 @@ --out-dir /n/.../pytorch_data \ --w 21 --h 21 --d 3 -Run (laue mode): +Run (laue mode): integrator.make_shoeboxes --laue \ --data-dir /n/.../laue-dials \ --refl integrated.refl \ @@ -27,7 +27,7 @@ --w 21 --h 21 --d 1 \ --max-images 1000 """ - +import time import argparse import os import re @@ -35,10 +35,15 @@ from pathlib import Path import numpy as np -import torch from numpy.lib.format import open_memmap +try: + import torch +except ModuleNotFoundError: + torch = None + from integrator.io import refl_as_pt, save_data +from dials.array_family import flex # re to parse image numbers from laue-dials filenames _TRAILING_INT_RE = re.compile(r"_(\d+)\.[A-Za-z0-9]+$") @@ -61,6 +66,62 @@ def parse_args(): "rotation sequence", ) + + #Thao: added MFX mode to process cctbx.xfel integrated refl/expt pairs for stills. + parser.add_argument( + "--mfx", + action="store_true", + help="Process MFX/cctbx.xfel serial stills output using *_integrated.refl/expt pairs.", + ) + + # Thao: added MFX combined mode to process one combined .expt/.refl pair + # created by dials.combine_experiments. + parser.add_argument( + "--mfx-combined", + action="store_true", + help=( + "Process one combined MFX/cctbx.xfel .expt/.refl pair. " + "The combined .refl should contain an id column linking each " + "reflection row to experiments[id] in the combined .expt file." + ), + ) + + parser.add_argument( + "--mfx-pattern", + default="idx-data_*_integrated.refl", + help="Glob pattern for MFX integrated reflection files inside data_dir.", + ) + + #Thao: Added start and max file arguments to allow batching of MFX integrated files for large datasets. + parser.add_argument( + "--start-file", + type=int, + default=0, + help="Starting index in the sorted MFX integrated file list.", + ) + + #Thao: Added max-files argument to allow batching of MFX integrated files for large datasets. + parser.add_argument( + "--max-files", + type=int, + default=None, + help="Maximum number of MFX integrated files to process.", + ) + + #Thao: Added detector-mask argument to allow users to provide a cctbx/DIALS detector mask file for MFX mode. + parser.add_argument( + "--detector-mask", + "--mask", + dest="detector_mask", + default=None, + help=( + "Optional cctbx/DIALS detector mask file for MFX mode. " + "Example: hot_lines_combined5.mask. The file is loaded with " + "libtbx.easy_pickle and should contain one boolean mask per panel, " + "where True means good pixel and False means bad/masked pixel." + ), + ) + common = parser.add_argument_group("common options") common.add_argument( "--data-dir", @@ -162,6 +223,17 @@ def parse_args(): type=int, default=10_000, ) + + #Thao: Added write-chunk-size argument to control how many shoeboxes are kept in memory before writing a chunk folder to disk in MFX mode. + common.add_argument( + "--write-chunk-size", + type=int, + default=10_000, + help=( + "MFX mode: number of shoeboxes to keep in memory before " + "writing one chunk folder to disk." + ), + ) common.add_argument( "--test-fraction", type=float, @@ -193,21 +265,1676 @@ def _require(args, names): def main(): args = parse_args() - if args.laue: + + if args.mfx_combined: + run_mfx_combined(args) + elif args.mfx: + run_mfx(args) + elif args.laue: run_laue(args) else: run_dials(args) +# Example MFX test command used during development. +# +# cd /sdf/home/t/thaoh/s3df_practice/integrator +# export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH +# +# dials.python src/integrator/cli/make_shoeboxes.py \ +# --mfx \ +# --data-dir /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/outputs/r0269/012_rg058/out \ +# --out-dir /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_test_masked \ +# --mfx-pattern "idx-data_36001_integrated.refl" \ +# --detector-mask /sdf/data/lcls/ds/mfx/mfx101555026/results/pam/hot_lines_combined5.mask \ +# --w 25 \ +# --h 25 \ +# --d 1 \ +# --counts-dtype float32 \ +# --no-mask-overlap \ +# --no-stats +# +# Why dials.python? It runs Python with the DIALS/cctbx environment loaded. +# Why PYTHONPATH? It lets Python find this local integrator source tree. + +def run_mfx(args): + """Create an integrator-style shoebox dataset from MFX/cctbx.xfel outputs. + + Expected input folder: + args.data_dir should point to a cctbx.xfel `out/` folder, for example: + + /.../outputs/r0269/012_rg058/out + + Expected files inside data_dir: + idx-data_XXXXX_integrated.refl + idx-data_XXXXX_integrated.expt + + Main idea: + - The .refl file gives reflection metadata: panel, bbox, Miller index, + intensity, background, etc. + - The .expt file lets dxtbx load the actual detector pixels. + - For a fixed-size integrator dataset, we crop a fixed window centered + on xyzcal.px from the correct detector panel. + + Notes for this first prototype: + - MFX still images should use --d 1. + - Existing MFX integrated bbox values can have different sizes, so this + function uses --w/--h fixed windows around xyzcal.px instead of + stacking variable-size integration bboxes. + - Edge reflections are padded with zeros and masked False where pixels + fall outside the panel boundary. + + Streaming/chunking update: + - Instead of storing every shoebox in Python lists until the end, this + version writes chunk folders such as chunk_00000, chunk_00001, ... + during extraction. + - After all files are processed, the chunk folders are merged into final + counts.npy, masks.npy, and metadata.npy. + - This keeps memory lower during the MFX extraction loop. + + Skip-bad-file update: + - Some .refl/.expt pairs can exist on disk but fail when dxtbx/psana2 + tries to open the image data referenced by the .expt file. + - Instead of crashing the whole Slurm array task, this version prints a + warning, records the bad pair, skips it, and continues to the next pair. + + """ + # To read cctbx.xfel integrated .refl/.expt files, we need DIALS and dxtbx. + # flex provides the DIALS array objects used by reflection tables. + from dials.array_family import flex + + # ExperimentListFactory loads the .expt experiment list and lets us access + # the underlying imageset/raw pixel data for the MFX stills. + from dxtbx.model.experiment_list import ExperimentListFactory + + from integrator.io import write_dataset_yaml + + # MFX/cctbx.xfel output files are still images, so depth should be 1. + if args.d != 1: + raise ValueError(f"--d must be 1 for MFX still-image extraction (got {args.d})") + + # Fixed-size windows need a clear center pixel. + if args.w % 2 == 0 or args.h % 2 == 0: + raise ValueError(f"--w and --h must be odd (got w={args.w}, h={args.h})") + + # MFX mode only needs a folder of integrated refl/expt pairs. + _require(args, ["data_dir"]) + + # Convert input/output paths from strings to Path objects. + data_dir = Path(args.data_dir) + out_dir = Path(args.out_dir or "out_dir") + + # Create output folder if needed. + out_dir.mkdir(parents=True, exist_ok=True) + + # Find all integrated reflection tables in the cctbx.xfel out/ folder. + # Example exact match: "idx-data_36001_integrated.refl" + # Example wildcard match: "idx-data_*_integrated.refl" + # + # Path.glob(...) searches only inside data_dir for names matching the pattern. + # sorted(...) makes the order deterministic. + refl_paths = sorted(data_dir.glob(args.mfx_pattern)) + if not refl_paths: + raise SystemExit(f"No MFX integrated reflection files found in {data_dir}") + + # Process only a slice of the sorted file list when batching MFX jobs. + # Example: --start-file 25 --max-files 25 processes files 25 through 49. + start = args.start_file + stop = None if args.max_files is None else start + args.max_files + refl_paths = refl_paths[start:stop] + if not refl_paths: + raise SystemExit( + f"No MFX integrated reflection files selected after slicing " + f"start={args.start_file}, max_files={args.max_files}" + ) + + # Storage dtype. Use float32 by default because Jungfrau/cctbx raw pixels can + # be floating point and may include negative values. + counts_dtype = np.dtype(args.counts_dtype or "float32") + + # Optional real detector mask for MFX data. + # + # Pam's/cctbx mask file loads as a tuple of 32 flex.bool arrays, one per + # Jungfrau panel. Each panel mask has the same shape as the corresponding + # raw detector panel, e.g. (514, 1030). In the inspected file, about 98.7% + # of pixels were True, so we treat True as good/usable and False as + # bad/hot/masked. + # + # If no detector mask is supplied, the MFX path falls back to the simpler + # finite/padding mask: True for finite pixels inside the panel, False for + # outside-panel padding or NaN/inf. + detector_masks = None + if args.detector_mask is not None: + from libtbx import easy_pickle + + detector_masks = easy_pickle.load(args.detector_mask) + print(f"loaded detector mask with {len(detector_masks)} panel mask(s): {args.detector_mask}") + + + # Chunk buffers. + # These replace the old full-run lists: + # counts = [] + # masks = [] + # metadata_rows = [] + # + # Each buffer stores only up to args.write_chunk_size shoeboxes. When the + # chunk is full, flush_chunk() writes it to disk and clears memory. + chunk_counts = [] + chunk_masks = [] + chunk_metadata_rows = [] + + # List of chunk folders written to disk. Used later by merge_mfx_chunks(). + chunk_dirs = [] + + # Chunk counter: chunk_00000, chunk_00001, ... + chunk_id = 0 + + # Global row counter across the final merged dataset. + # This keeps refl_ids unique across chunks. + global_row = 0 + + # Number of shoeboxes to hold in memory before writing one chunk. + write_chunk_size = int(getattr(args, "write_chunk_size", 10_000)) + + def flush_chunk(): + """Write current chunk buffers to disk and clear them from memory.""" + nonlocal chunk_id, global_row + nonlocal chunk_counts, chunk_masks, chunk_metadata_rows + + # Nothing to write. + if not chunk_counts: + return + + # Create folder like chunk_00000. + chunk_dir = out_dir / f"chunk_{chunk_id:05d}" + chunk_dir.mkdir(parents=True, exist_ok=True) + + # Stack chunk lists into arrays. + # counts_chunk shape: (n_chunk, d*h*w), usually (n_chunk, 625) + # masks_chunk shape: (n_chunk, d*h*w), usually (n_chunk, 625) + counts_chunk = np.stack(chunk_counts) + masks_chunk = np.stack(chunk_masks).astype(np.bool_) + + # Convert this chunk's row metadata into Luis-style dict. + # global_start tells the metadata helper where this chunk begins in the + # final merged dataset, so refl_ids stay unique after merging. + metadata_chunk = _mfx_metadata_rows_to_luis_dict( + chunk_metadata_rows, + args.test_fraction, + global_start=global_row, + ) + + # This helps long MFX jobs avoid "Too many open files" during np.save. + import gc + gc.collect() + + # Save this chunk to disk. + np.save(chunk_dir / args.counts_fname, counts_chunk) + np.save(chunk_dir / args.masks_fname, masks_chunk) + np.save(chunk_dir / "metadata.npy", metadata_chunk) + + # Remember this chunk for final merge. + chunk_dirs.append(chunk_dir) + + n_chunk = counts_chunk.shape[0] + print(f"wrote chunk {chunk_id}: {n_chunk} shoeboxes") + + # Advance global row counter. + global_row += n_chunk + + # Move to next chunk ID. + chunk_id += 1 + + # Clear memory for next chunk. + chunk_counts = [] + chunk_masks = [] + chunk_metadata_rows = [] + + n_files = 0 + n_reflections_seen = 0 + n_reflections_saved = 0 + + # New: keep a list of skipped bad file pairs for the final summary. + skipped_files = [] + + for refl_path in refl_paths: + # Each integrated .refl should have a matching integrated .expt. + expt_path = None + + # New: skip-bad-file protection. + # + # Some .refl/.expt pairs exist on disk, but dxtbx/psana2 may fail when + # opening the image data referenced by the .expt file. + # + # Without this try/except: + # one bad .expt crashes the whole Slurm array task. + # + # With this try/except: + # print the bad file pair, record it, and continue to the next file. + try: + expt_path = _matching_mfx_expt(refl_path) + + if expt_path is None: + print("WARNING: skipping MFX file pair with missing .expt") + print(f" refl: {refl_path}") + skipped_files.append((str(refl_path), "missing", "missing .expt")) + continue + + # Load reflection table metadata. + # + # A single integrated .refl file can contain many reflection rows. + # In the first test file, idx-data_36001_integrated.refl had 415 rows. + # Example row information looked like: + # row 0: panel=31, xyzcal.px=(777.30, 384.75, 0.0), + # intensity.sum.value=16.07, ... + reflections = flex.reflection_table.from_file(str(refl_path)) + + # Check required columns early so errors are easier to understand. + required_cols = [ + "panel", + "xyzcal.px", + "miller_index", + "intensity.sum.value", + "intensity.sum.variance", + "background.sum.value", + "background.sum.variance", + ] + missing = [col for col in required_cols if col not in reflections] + + if missing: + print("WARNING: skipping MFX file pair with missing columns") + print(f" refl: {refl_path}") + print(f" expt: {expt_path}") + print(f" missing: {missing}") + skipped_files.append( + (str(refl_path), str(expt_path), f"missing columns: {missing}") + ) + continue + + # Try to record the event/image index stored in the ImageSet JSON, if present. + # For example, idx-data_36001_integrated.expt had single_file_indices=[36001]. + image_index = _mfx_image_index_from_expt_json(expt_path) + + experiments = None + wavelength = np.nan + + # Load experiment and connect to underlying image source. + # Fast MFX/Jungfrau path: + # check_format=False avoids expensive psana/dxtbx validation. + # Then we explicitly force the lazy reader to use FormatXTCJungfrau. + # This preserves the working Jungfrau panel reader without the slow + # full format-check step. + experiments = ExperimentListFactory.from_json_file( + str(expt_path), + check_format=False, + ) + _mfx_force_xtc_jungfrau_for_experiments(experiments) + + # New: do not crash if this .expt has an unexpected number of experiments. + # For this MFX prototype, we expect exactly one experiment per file. + if len(experiments) != 1: + print("WARNING: skipping MFX file pair with unexpected experiment count") + print(f" refl: {refl_path}") + print(f" expt: {expt_path}") + print(f" n_experiments: {len(experiments)}") + skipped_files.append( + (str(refl_path), str(expt_path), f"n_experiments={len(experiments)}") + ) + continue + + # raw is a tuple of panel images for one event/image: + # raw[0], raw[1], ..., raw[31] + # + # experiments[0].imageset is a loader/access object, not pixels yet. + # get_raw_data(0) loads the actual pixels for image index 0 inside this + # one-image imageset. For this MFX Jungfrau data, it returns 32 panel + # flex arrays, each with shape about (514, 1030). + raw = experiments[0].imageset.get_raw_data(0) + + # Retrieve wavelength from the beam object in the experiment. + # If it fails, set wavelength to NaN. + try: + wavelength = float(experiments[0].beam.get_wavelength()) + except Exception: + wavelength = np.nan + + # If a detector mask was provided, it should have one mask per raw + # detector panel. For this MFX Jungfrau case, both should have length 32. + # + # New: skip instead of crashing if the mask panel count does not match. + if detector_masks is not None and len(detector_masks) != len(raw): + print("WARNING: skipping MFX file pair with detector mask mismatch") + print(f" refl: {refl_path}") + print(f" expt: {expt_path}") + print(f" detector mask panels: {len(detector_masks)}") + print(f" raw panels: {len(raw)}") + skipped_files.append( + (str(refl_path), str(expt_path), "detector mask panel mismatch") + ) + continue + + except Exception as e: + print("WARNING: skipping bad MFX file pair") + print(f" refl: {refl_path}") + print(f" expt: {expt_path if expt_path is not None else 'unknown'}") + print(f" error: {type(e).__name__}: {e}") + skipped_files.append( + ( + str(refl_path), + str(expt_path) if expt_path is not None else "unknown", + f"{type(e).__name__}: {e}", + ) + ) + continue + + n_files += 1 + + # Count reflection rows, not files. One file may add hundreds of rows. + n_reflections_seen += len(reflections) + + # image_index and wavelength were already set above. + # In normal mode, wavelength came from experiments[0].beam. + # In raw-panel-cache mode, wavelength came from the .expt JSON. + + # Loop over reflections in this event. + for i in range(len(reflections)): + panel_id = int(reflections["panel"][i]) + + # xyzcal.px is the DIALS/cctbx calculated pixel position from the + # refined geometry/model. It is not an ML prediction from integrator. + # We use it as the center of the fixed shoebox crop. + xyz = reflections["xyzcal.px"][i] + x_center = float(xyz[0]) + y_center = float(xyz[1]) + + if panel_id < 0 or panel_id >= len(raw): + # Bad panel ID; skip this reflection. + continue + + panel_image = raw[panel_id] + + # If a detector mask was supplied, select the mask for this same + # panel. The raw pixel panel and mask panel should have matching + # shapes, e.g. both (514, 1030). + panel_mask_np = None + if detector_masks is not None: + panel_mask_np = detector_masks[panel_id].as_numpy_array().astype( + bool, + copy=False, + ) + + # Crop a fixed-size window centered on xyzcal.px. + # Returns a 2D counts array with shape (args.h, args.w) and a mask. + # If panel_mask_np is provided, the mask combines: + # finite pixel values AND detector-good pixels. + # If panel_mask_np is None, the mask only checks finite pixels and + # outside-panel padding. + shoebox, mask, fixed_bbox = _crop_mfx_fixed_shoebox( + panel_image=panel_image, + x_center=x_center, + y_center=y_center, + width=args.w, + height=args.h, + panel_mask_np=panel_mask_np, + ) + + # Flatten the fixed-size 2D shoebox/mask into one 1D row. + # Pixel values stay the same; only the shape changes. + # Example: (25, 25) -> (625,). + # + # Streaming update: + # append to the current chunk buffer, not to a full-run list. + chunk_counts.append(shoebox.reshape(-1).astype(counts_dtype, copy=False)) + chunk_masks.append(mask.reshape(-1)) + n_reflections_saved += 1 + + # Existing integration bbox is useful metadata, but it may not have + # the fixed --w/--h shape. Keep it if the column exists. + integration_bbox = tuple(reflections["bbox"][i]) if "bbox" in reflections else None + + # MFX metadata is built manually, so compute d-spacing here from + # the experiment crystal/unit cell and the reflection Miller index. + miller_index = tuple(int(v) for v in reflections["miller_index"][i]) + d_spacing = _mfx_d_spacing(experiments[0], miller_index) + + # Add one metadata row for this shoebox. + # background.mean is optional in general, but your checked MFX + # integrated.refl files contain it, so copy it when present. + chunk_metadata_rows.append( + { + "source_refl": refl_path.name, + "source_expt": expt_path.name, + "image_index": image_index, + "reflection_id": i, + "panel": panel_id, + "fixed_bbox": tuple(fixed_bbox), + "integration_bbox": integration_bbox, + "detector_mask": str(args.detector_mask) if args.detector_mask else None, + "xyzcal_px": tuple(float(v) for v in reflections["xyzcal.px"][i]), + "xyzobs_px_value": tuple(float(v) for v in reflections["xyzobs.px.value"][i]) + if "xyzobs.px.value" in reflections + else None, + "miller_index": miller_index, + "d": d_spacing, + "intensity_sum_value": float(reflections["intensity.sum.value"][i]), + "intensity_sum_variance": float(reflections["intensity.sum.variance"][i]), + "background_sum_value": float(reflections["background.sum.value"][i]), + "background_sum_variance": float(reflections["background.sum.variance"][i]), + "background.mean": ( + float(reflections["background.mean"][i]) + if "background.mean" in reflections + else np.nan + ), + "wavelength": wavelength, + } + ) + + # If the current chunk is full, write it to disk and clear memory. + if len(chunk_counts) >= write_chunk_size: + flush_chunk() + + # imageset is the loader/cache object. clear_cache() tells it to forget + # loaded image pixels after we finish this event, so memory does not grow + # while processing many .refl/.expt pairs. + if experiments is not None: + try: + experiments[0].imageset.clear_cache() + except Exception as e: + print(f"WARNING: could not clear image cache for {expt_path}: {type(e).__name__}: {e}") + + # Write the final partial chunk, if any. + flush_chunk() + + if not chunk_dirs: + raise SystemExit("No MFX shoeboxes were extracted; check input files and bbox/panel data.") + + counts_path = out_dir / args.counts_fname + masks_path = out_dir / args.masks_fname + metadata_path = out_dir / "metadata.npy" + + # Merge all chunk folders into final counts.npy, masks.npy, metadata.npy. + # merge_mfx_chunks writes counts/masks through memmap so the final merge does + # not need to hold all pixel arrays in memory at once. + n_saved = merge_mfx_chunks( + chunk_dirs=chunk_dirs, + out_dir=out_dir, + counts_fname=args.counts_fname, + masks_fname=args.masks_fname, + ) + + # Add compact image IDs only after all chunk metadata has been merged. + # If image_id is computed inside each chunk, each chunk can restart at 0, + # which would make the same image_id refer to different images in the final dataset. + _add_image_id_metadata(metadata_path) + + stats = None + + # concentration.npy is created only when we do NOT use: --no-stats + if not args.no_stats: + stats = _raw_stats_from_memmap(counts_path, masks_path, chunk=args.stats_chunk) + + _save_concentration_from_memmap( + counts_path=counts_path, + out_dir=out_dir, + chunk=args.stats_chunk, + ) + print("wrote concentration.npy") + + # Write a simple dataset.yaml so the output folder has the expected dataset + # description. The metadata format may still need to be aligned with Luis's + # final data-loader expectations. + write_dataset_yaml( + out_dir, + geometry={"d": args.d, "h": args.h, "w": args.w}, + n_reflections=n_saved, + polychromatic=False, + anscombe=False, + files={ + "counts": args.counts_fname, + "masks": args.masks_fname, + "reference": "metadata.npy", + }, + crystal=None, + stats=stats, + refl_file=None, + ) + + print(f"processed {n_files} MFX integrated file(s)") + print(f"seen {n_reflections_seen} reflection(s)") + print(f"saved {n_reflections_saved} fixed-size shoebox(es)") + print(f"wrote {counts_path}") + print(f"wrote {masks_path}") + print(f"wrote {metadata_path}") + print(f"wrote dataset.yaml under {out_dir}") + + print(f"skipped {len(skipped_files)} bad MFX file pair(s)") + for refl_name, expt_name, reason in skipped_files[:50]: + print("SKIPPED:") + print(f" refl: {refl_name}") + print(f" expt: {expt_name}") + print(f" reason: {reason}") + if len(skipped_files) > 50: + print(f"... {len(skipped_files) - 50} more skipped file pair(s) not printed") + + +def run_mfx_combined(args): + """Create an integrator-style shoebox dataset from combined MFX files. + + Expected input: + args.data_dir should point to a folder containing one combined .expt + and one combined .refl, for example: + + combined_10/combined_10.expt + combined_10/combined_10.refl + + Run with: + --mfx-combined + --data-dir /path/to/combined_10 + --expt combined_10.expt + --refl combined_10.refl + + Main idea: + - dials.combine_experiments creates one combined experiment list and + one combined reflection table. + - The combined .refl has an "id" column. + - reflections["id"] tells which experiment each reflection belongs to. + - experiments[experiment_id] gives the matching image loader. + - The original image/event number is preserved in single_file_indices. + + Example mapping: + reflections with id == 0 -> experiments[0] -> original image 1418 + reflections with id == 1 -> experiments[1] -> original image 2203 + + This keeps the chunked writing approach from run_mfx: + - chunk_counts + - chunk_masks + - chunk_metadata_rows + - flush_chunk() + - merge_mfx_chunks() + + So the redesign changes how inputs are read, but keeps memory-safe output. + + Timing update: + - This version measures time spent loading combined files, selecting + reflection subsets, reading raw images, cropping/building metadata, + flushing chunks, merging chunks, and writing dataset.yaml/stats. + + Skip-bad-experiment update: + - If one experiment inside the combined .expt fails during raw-data + access, detector-mask validation, image-index lookup, or wavelength + lookup, record the failure and continue with the next experiment. + - This prevents one bad image/event from stopping the full combined run. + """ + from dials.array_family import flex + from dxtbx.model.experiment_list import ExperimentListFactory + from integrator.io import write_dataset_yaml + + # Total timer for the whole combined-file MFX approach. + t_total_start = time.perf_counter() + + # Timing counters. + t_path_setup = 0.0 + t_load_mask = 0.0 + t_load_refl = 0.0 + t_load_expt = 0.0 + t_get_ids = 0.0 + t_select_subset = 0.0 + t_get_raw = 0.0 + t_image_index_json = 0.0 + t_crop_loop = 0.0 + t_clear_cache = 0.0 + t_flush = 0.0 + t_merge = 0.0 + t_stats = 0.0 + t_write_yaml = 0.0 + + # MFX/cctbx.xfel output files are still images, so depth should be 1. + if args.d != 1: + raise ValueError(f"--d must be 1 for MFX still-image extraction (got {args.d})") + + # Fixed-size windows need a clear center pixel. + if args.w % 2 == 0 or args.h % 2 == 0: + raise ValueError(f"--w and --h must be odd (got w={args.w}, h={args.h})") + + # Combined mode needs one folder, one .refl, and one .expt. + _require(args, ["data_dir", "refl", "expt"]) + + t0 = time.perf_counter() + + data_dir = Path(args.data_dir) + out_dir = Path(args.out_dir or "out_dir") + out_dir.mkdir(parents=True, exist_ok=True) + + refl_path = data_dir / args.refl + expt_path = data_dir / args.expt + + if not refl_path.exists(): + raise SystemExit(f"combined reflection file not found: {refl_path}") + if not expt_path.exists(): + raise SystemExit(f"combined experiment file not found: {expt_path}") + + t_path_setup += time.perf_counter() - t0 + + # Storage dtype. Use float32 by default because Jungfrau/cctbx raw pixels can + # be floating point and may include negative values. + counts_dtype = np.dtype(args.counts_dtype or "float32") + + # Optional real detector mask for MFX data. + # + # Pam's/cctbx mask file loads as a tuple of 32 flex.bool arrays, one per + # Jungfrau panel. Each panel mask has the same shape as the corresponding + # raw detector panel, e.g. (514, 1030). In the inspected file, about 98.7% + # of pixels were True, so we treat True as good/usable and False as + # bad/hot/masked. + # + # If no detector mask is supplied, the MFX path falls back to the simpler + # finite/padding mask: True for finite pixels inside the panel, False for + # outside-panel padding or NaN/inf. + detector_masks = None + if args.detector_mask is not None: + t0 = time.perf_counter() + + from libtbx import easy_pickle + + detector_masks = easy_pickle.load(args.detector_mask) + print(f"loaded detector mask with {len(detector_masks)} panel mask(s): {args.detector_mask}") + + t_load_mask += time.perf_counter() - t0 + + # Load combined reflection table once. + t0 = time.perf_counter() + + reflections = flex.reflection_table.from_file(str(refl_path)) + t_load_refl += time.perf_counter() - t0 + + # Combined reflection table must have id so we can connect: + # reflections["id"] -> experiments[id] + required_cols = [ + "id", + "panel", + "xyzcal.px", + "miller_index", + "intensity.sum.value", + "intensity.sum.variance", + "background.sum.value", + "background.sum.variance", + ] + missing = [col for col in required_cols if col not in reflections] + if missing: + raise SystemExit(f"{refl_path} is missing required column(s): {missing}") + + # Load combined experiment list once. + # Fast MFX/Jungfrau path: + # check_format=False avoids expensive psana/dxtbx validation. + # Then explicitly force each lazy imageset reader to use FormatXTCJungfrau. + t0 = time.perf_counter() + experiments = ExperimentListFactory.from_json_file( + str(expt_path), + check_format=False, + ) + _mfx_force_xtc_jungfrau_for_experiments(experiments) + t_load_expt += time.perf_counter() - t0 + + print(f"loaded combined experiments: {len(experiments)}") + print(f"loaded combined reflections: {len(reflections)}") + + # Get actual experiment ids present in the reflection table. + # Usually this is 0, 1, 2, ..., N-1, but using unique ids is safer. + t0 = time.perf_counter() + ids_np = reflections["id"].as_numpy_array() + experiment_ids = sorted(int(v) for v in np.unique(ids_np)) + t_get_ids += time.perf_counter() - t0 + + print(f"experiment ids in combined refl: {experiment_ids[:10]}") + if len(experiment_ids) > 10: + print(f"... total experiment ids: {len(experiment_ids)}") + + # Chunk buffers. + # These replace the old full-run lists: + # counts = [] + # masks = [] + # metadata_rows = [] + # + # Each buffer stores only up to args.write_chunk_size shoeboxes. When the + # chunk is full, flush_chunk() writes it to disk and clears memory. + chunk_counts = [] + chunk_masks = [] + chunk_metadata_rows = [] + + # List of chunk folders written to disk. Used later by merge_mfx_chunks(). + chunk_dirs = [] + + # Chunk counter: chunk_00000, chunk_00001, ... + chunk_id = 0 + + # Global row counter across the final merged dataset. + # This keeps refl_ids unique across chunks. + global_row = 0 + + # Number of shoeboxes to hold in memory before writing one chunk. + write_chunk_size = int(getattr(args, "write_chunk_size", 10_000)) + + def flush_chunk(): + """Write current chunk buffers to disk and clear them from memory.""" + nonlocal chunk_id, global_row, t_flush + nonlocal chunk_counts, chunk_masks, chunk_metadata_rows + + # Nothing to write. + if not chunk_counts: + return + + t0 = time.perf_counter() + + # Create folder like chunk_00000. + chunk_dir = out_dir / f"chunk_{chunk_id:05d}" + chunk_dir.mkdir(parents=True, exist_ok=True) + + # Stack chunk lists into arrays. + # counts_chunk shape: (n_chunk, d*h*w), usually (n_chunk, 625) + # masks_chunk shape: (n_chunk, d*h*w), usually (n_chunk, 625) + counts_chunk = np.stack(chunk_counts) + masks_chunk = np.stack(chunk_masks).astype(np.bool_) + + # Convert this chunk's row metadata into Luis-style dict. + # global_start tells the metadata helper where this chunk begins in the + # final merged dataset, so refl_ids stay unique after merging. + metadata_chunk = _mfx_metadata_rows_to_luis_dict( + chunk_metadata_rows, + args.test_fraction, + global_start=global_row, + ) + + # This helps long MFX jobs avoid "Too many open files" during np.save. + import gc + gc.collect() + + # Save this chunk to disk. + np.save(chunk_dir / args.counts_fname, counts_chunk) + np.save(chunk_dir / args.masks_fname, masks_chunk) + np.save(chunk_dir / "metadata.npy", metadata_chunk) + + # Remember this chunk for final merge. + chunk_dirs.append(chunk_dir) + + n_chunk = counts_chunk.shape[0] + print(f"wrote chunk {chunk_id}: {n_chunk} shoeboxes") + + # Advance global row counter. + global_row += n_chunk + + # Move to next chunk ID. + chunk_id += 1 + + # Clear memory for next chunk. + chunk_counts = [] + chunk_masks = [] + chunk_metadata_rows = [] + + t_flush += time.perf_counter() - t0 + + n_experiments_attempted = 0 + n_experiments_seen = 0 + n_reflections_seen = 0 + n_reflections_saved = 0 + + # Keep a record of experiments that fail inside the combined file. + # This is the combined-mode equivalent of skip-bad-file logic in run_mfx(). + skipped_experiments = [] + + # Main combined-file loop. + # + # Old MFX mode: + # for each separate .refl/.expt pair: + # load one .refl + # load one .expt + # crop shoeboxes + # + # New combined MFX mode: + # load combined .refl once + # load combined .expt once + # for each experiment id: + # select reflections where reflections["id"] == experiment_id + # use experiments[experiment_id] + # crop shoeboxes + for experiment_id in experiment_ids: + n_experiments_attempted += 1 + + if experiment_id < 0 or experiment_id >= len(experiments): + skipped_experiments.append( + { + "experiment_id": experiment_id, + "image_index": -1, + "n_reflections": 0, + "error_type": "BAD_EXPERIMENT_ID", + "error_message": ( + f"reflection table has id={experiment_id}, but combined .expt " + f"contains only {len(experiments)} experiment(s)" + ), + } + ) + print( + f"WARNING: skipping combined experiment id {experiment_id}: " + f"outside experiment list range" + ) + continue + + # Select the rows in the combined reflection table that belong to this + # experiment/image. + t0 = time.perf_counter() + select_mask_np = ids_np == experiment_id + + # New: keep the original row numbers from the full combined .refl. + # reflection_id below is the row inside this experiment subset, while + # combined_refl_row is the row in the full combined reflection table. + # This makes future write-back to the combined .refl safer. + subset_indices_np = np.nonzero(select_mask_np)[0] + + select_mask = flex.bool(select_mask_np.tolist()) + refl_subset = reflections.select(select_mask) + t_select_subset += time.perf_counter() - t0 + + if len(refl_subset) == 0: + continue + + experiment = experiments[experiment_id] + + # raw is a tuple of panel images for one event/image: + # raw[0], raw[1], ..., raw[31] + # + # experiments[experiment_id].imageset is the loader/access object. + # get_raw_data(0) loads the actual pixels for image index 0 inside this + # one-image imageset. The original event/image index is preserved in + # single_file_indices inside the combined .expt file. + # + # New skip logic: + # If raw-data loading, detector-mask checking, or image-index lookup + # fails for this experiment, skip only this experiment and continue. + try: + t0 = time.perf_counter() + _mfx_force_xtc_jungfrau_reader(experiment.imageset) + raw = experiment.imageset.get_raw_data(0) + t_get_raw += time.perf_counter() - t0 + + # If a detector mask was provided, it should have one mask per raw + # detector panel. For this MFX Jungfrau case, both should have length 32. + if detector_masks is not None and len(detector_masks) != len(raw): + raise RuntimeError( + f"detector mask panel count ({len(detector_masks)}) does not " + f"match raw panel count ({len(raw)}) for experiment id {experiment_id}" + ) + + # Record original detector image/event number, e.g. 1418. + t0 = time.perf_counter() + image_index = _mfx_image_index_from_combined_expt_json( + expt_path, + experiment_id, + ) + t_image_index_json += time.perf_counter() - t0 + + # Retrieve wavelength from the beam object in the experiment. If it fails, + # set wavelength to NaN. + try: + wavelength = float(experiment.beam.get_wavelength()) + except Exception: + wavelength = np.nan + + except Exception as e: + skipped_experiments.append( + { + "experiment_id": experiment_id, + "image_index": -1, + "n_reflections": len(refl_subset), + "error_type": type(e).__name__, + "error_message": str(e), + } + ) + print( + f"WARNING: skipping combined experiment id {experiment_id} " + f"with {len(refl_subset)} reflection(s): {type(e).__name__}: {e}" + ) + + # Try to release any cached data before moving on. + try: + experiment.imageset.clear_cache() + except Exception: + pass + + continue + + n_experiments_seen += 1 + n_reflections_seen += len(refl_subset) + + # Loop over reflections for this one experiment/image. + t0 = time.perf_counter() + + for i in range(len(refl_subset)): + panel_id = int(refl_subset["panel"][i]) + + # xyzcal.px is the DIALS/cctbx calculated pixel position from the + # refined geometry/model. It is not an ML prediction from integrator. + # We use it as the center of the fixed shoebox crop. + xyz = refl_subset["xyzcal.px"][i] + x_center = float(xyz[0]) + y_center = float(xyz[1]) + + if panel_id < 0 or panel_id >= len(raw): + # Bad panel ID; skip this reflection. + continue + + panel_image = raw[panel_id] + + # If a detector mask was supplied, select the mask for this same + # panel. The raw pixel panel and mask panel should have matching + # shapes, e.g. both (514, 1030). + panel_mask_np = None + if detector_masks is not None: + panel_mask_np = detector_masks[panel_id].as_numpy_array().astype( + bool, + copy=False, + ) + + # Crop a fixed-size window centered on xyzcal.px. + # Returns a 2D counts array with shape (args.h, args.w) and a mask. + # If panel_mask_np is provided, the mask combines: + # finite pixel values AND detector-good pixels. + # If panel_mask_np is None, the mask only checks finite pixels and + # outside-panel padding. + shoebox, mask, fixed_bbox = _crop_mfx_fixed_shoebox( + panel_image=panel_image, + x_center=x_center, + y_center=y_center, + width=args.w, + height=args.h, + panel_mask_np=panel_mask_np, + ) + + # Flatten the fixed-size 2D shoebox/mask into one 1D row. + # Pixel values stay the same; only the shape changes. + # Example: (25, 25) -> (625,). + # + # Streaming update: + # append to the current chunk buffer, not to a full-run list. + chunk_counts.append(shoebox.reshape(-1).astype(counts_dtype, copy=False)) + chunk_masks.append(mask.reshape(-1)) + n_reflections_saved += 1 + + # Existing integration bbox is useful metadata, but it may not have + # the fixed --w/--h shape. Keep it if the column exists. + integration_bbox = tuple(refl_subset["bbox"][i]) if "bbox" in refl_subset else None + + # MFX metadata is built manually, so compute d-spacing here from + # the experiment crystal/unit cell and the reflection Miller index. + miller_index = tuple(int(v) for v in refl_subset["miller_index"][i]) + d_spacing = _mfx_d_spacing(experiment, miller_index) + + # Add one metadata row for this shoebox. + # background.mean is optional in general, but your checked MFX + # integrated.refl files contain it, so copy it when present. + # + # For combined mode: + # source_refl/source_expt are the combined files. + # experiment_id tells which combined experiment was used. + # image_index preserves the original detector image/event number. + chunk_metadata_rows.append( + { + "source_refl": refl_path.name, + "source_expt": expt_path.name, + "experiment_id": experiment_id, + "image_index": image_index, + "reflection_id": int(i), + "combined_refl_row": int(subset_indices_np[i]), + "panel": panel_id, + "fixed_bbox": tuple(fixed_bbox), + "integration_bbox": integration_bbox, + "detector_mask": str(args.detector_mask) if args.detector_mask else None, + "xyzcal_px": tuple(float(v) for v in refl_subset["xyzcal.px"][i]), + "xyzobs_px_value": tuple(float(v) for v in refl_subset["xyzobs.px.value"][i]) + if "xyzobs.px.value" in refl_subset + else None, + "miller_index": miller_index, + "d": d_spacing, + "intensity_sum_value": float(refl_subset["intensity.sum.value"][i]), + "intensity_sum_variance": float(refl_subset["intensity.sum.variance"][i]), + "background_sum_value": float(refl_subset["background.sum.value"][i]), + "background_sum_variance": float(refl_subset["background.sum.variance"][i]), + "background.mean": ( + float(refl_subset["background.mean"][i]) + if "background.mean" in refl_subset + else np.nan + ), + "wavelength": wavelength, + } + ) + + # If the current chunk is full, write it to disk and clear memory. + if len(chunk_counts) >= write_chunk_size: + flush_chunk() + + t_crop_loop += time.perf_counter() - t0 + + # imageset is the loader/cache object. clear_cache() tells it to forget + # loaded image pixels after we finish this event, so memory does not grow + # while processing many experiments inside the combined .expt file. + t0 = time.perf_counter() + experiment.imageset.clear_cache() + t_clear_cache += time.perf_counter() - t0 + + # Write the final partial chunk, if any. + flush_chunk() + + if not chunk_dirs: + raise SystemExit("No MFX shoeboxes were extracted; check input files and bbox/panel data.") + + counts_path = out_dir / args.counts_fname + masks_path = out_dir / args.masks_fname + metadata_path = out_dir / "metadata.npy" + + # Merge all chunk folders into final counts.npy, masks.npy, metadata.npy. + # merge_mfx_chunks writes counts/masks through memmap so the final merge does + # not need to hold all pixel arrays in memory at once. + t0 = time.perf_counter() + n_saved = merge_mfx_chunks( + chunk_dirs=chunk_dirs, + out_dir=out_dir, + counts_fname=args.counts_fname, + masks_fname=args.masks_fname, + ) + + # Add compact image IDs only after all chunk metadata has been merged. + # If image_id is computed inside each chunk, each chunk can restart at 0, + # which would make the same image_id refer to different images in the final dataset. + _add_image_id_metadata(metadata_path) + + t_merge += time.perf_counter() - t0 + + stats = None + + # concentration.npy is created only when we do NOT use: --no-stats + t0 = time.perf_counter() + if not args.no_stats: + stats = _raw_stats_from_memmap(counts_path, masks_path, chunk=args.stats_chunk) + + _save_concentration_from_memmap( + counts_path=counts_path, + out_dir=out_dir, + chunk=args.stats_chunk, + ) + print("wrote concentration.npy") + t_stats += time.perf_counter() - t0 + + # Write a simple dataset.yaml so the output folder has the expected dataset + # description. The metadata format may still need to be aligned with Luis's + # final data-loader expectations. + t0 = time.perf_counter() + write_dataset_yaml( + out_dir, + geometry={"d": args.d, "h": args.h, "w": args.w}, + n_reflections=n_saved, + polychromatic=False, + anscombe=False, + files={ + "counts": args.counts_fname, + "masks": args.masks_fname, + "reference": "metadata.npy", + }, + crystal=None, + stats=stats, + refl_file=None, + ) + t_write_yaml += time.perf_counter() - t0 + + # Write a small text report of skipped combined experiments. + # This makes it easy to inspect whether failures were raw-data access, + # detector-mask mismatch, bad experiment ids, or image-index lookup issues. + if skipped_experiments: + skipped_path = out_dir / "skipped_combined_experiments.txt" + with skipped_path.open("w") as f: + for row in skipped_experiments: + f.write( + f"experiment_id={row['experiment_id']} " + f"image_index={row['image_index']} " + f"n_reflections={row['n_reflections']} " + f"error_type={row['error_type']} " + f"error_message={row['error_message']}\n" + ) + print(f"wrote {skipped_path}") + + t_total = time.perf_counter() - t_total_start + + accounted = ( + t_path_setup + + t_load_mask + + t_load_refl + + t_load_expt + + t_get_ids + + t_select_subset + + t_get_raw + + t_image_index_json + + t_crop_loop + + t_clear_cache + + t_flush + + t_merge + + t_stats + + t_write_yaml + ) + + print("") + print("MFX combined-file timing summary:") + print(f" total wall time: {t_total:.2f} sec ({t_total / 60:.2f} min)") + print(f" path/setup checks: {t_path_setup:.2f} sec") + print(f" load detector mask: {t_load_mask:.2f} sec") + print(f" load combined .refl: {t_load_refl:.2f} sec") + print(f" load combined .expt: {t_load_expt:.2f} sec") + print(f" get unique ids: {t_get_ids:.2f} sec") + print(f" select refl subsets: {t_select_subset:.2f} sec") + print(f" get_raw_data: {t_get_raw:.2f} sec") + print(f" read image index json: {t_image_index_json:.2f} sec") + print(f" crop/metadata loop: {t_crop_loop:.2f} sec") + print(f" clear image cache: {t_clear_cache:.2f} sec") + print(f" flush chunks: {t_flush:.2f} sec") + print(f" merge chunks: {t_merge:.2f} sec") + print(f" stats/concentration: {t_stats:.2f} sec") + print(f" write dataset.yaml: {t_write_yaml:.2f} sec") + print(f" unaccounted/other: {t_total - accounted:.2f} sec") + print("") + print("Note:") + print(" crop/metadata loop includes any flush_chunk() calls that happen inside") + print(" the reflection loop, so crop/metadata and flush chunks overlap slightly.") + print("") + + print(f"attempted {n_experiments_attempted} combined experiment id(s)") + print(f"processed {n_experiments_seen} combined experiment(s)") + print(f"skipped {len(skipped_experiments)} combined experiment(s)") + print(f"seen {n_reflections_seen} reflection(s)") + print(f"saved {n_reflections_saved} fixed-size shoebox(es)") + print(f"wrote {counts_path}") + print(f"wrote {masks_path}") + print(f"wrote {metadata_path}") + print(f"wrote dataset.yaml under {out_dir}") + + +def _mfx_force_xtc_jungfrau_reader(imageset): + """Force a lazy MFX imageset reader to use the Jungfrau XTC format. + + Why this exists: + ExperimentListFactory.from_json_file(..., check_format=False) is much + faster for MFX .expt files, but dxtbx may otherwise leave the lazy + reader as generic FormatMultiImage. That generic reader fails later at + get_raw_data(0). Setting reader.format_class to FormatXTCJungfrau keeps + the fast JSON load while preserving the working Jungfrau raw-panel + reader. + + Working pattern tested on MFX run 0269: + experiments = ExperimentListFactory.from_json_file(path, check_format=False) + imageset = experiments.imagesets()[0] + imageset.reader().format_class = FormatXTCJungfrau + raw = imageset.get_raw_data(0) # returns 32 panels + """ + from dxtbx.format.FormatXTCJungfrau import FormatXTCJungfrau + + reader = imageset.reader() + reader.format_class = FormatXTCJungfrau + return reader + + +def _mfx_force_xtc_jungfrau_for_experiments(experiments): + """Apply the MFX Jungfrau reader override to every imageset once.""" + seen = set() + + # experiments.imagesets() is the most direct way to get all imagesets. + for imageset in experiments.imagesets(): + key = id(imageset) + if key in seen: + continue + _mfx_force_xtc_jungfrau_reader(imageset) + seen.add(key) + + # Some dxtbx ExperimentList objects may expose imagesets differently, so + # also walk experiments defensively. This is harmless if already handled. + for experiment in experiments: + imageset = experiment.imageset + key = id(imageset) + if key in seen: + continue + _mfx_force_xtc_jungfrau_reader(imageset) + seen.add(key) + + + +def _mfx_wavelength_from_expt_json(expt_path: Path): + """Read beam wavelength from an MFX .expt JSON without loading image data.""" + import json + + try: + with open(expt_path) as f: + data = json.load(f) + + exp = data["experiment"][0] + beam_id = int(exp.get("beam", 0)) + return float(data["beam"][beam_id]["wavelength"]) + except Exception: + return np.nan + + +def _angle_degrees(u, v): + """Return angle between two vectors in degrees.""" + import math + + u = np.asarray(u, dtype=float) + v = np.asarray(v, dtype=float) + denom = np.linalg.norm(u) * np.linalg.norm(v) + if denom == 0: + return np.nan + + cosang = float(np.dot(u, v) / denom) + cosang = max(-1.0, min(1.0, cosang)) + return float(math.degrees(math.acos(cosang))) + + +def _mfx_unit_cell_from_expt_json(expt_path: Path): + """Read crystal unit cell from an MFX .expt JSON without loading image data. + + The .expt file stores real-space basis vectors. We convert those vectors to + the six unit-cell parameters needed by cctbx. This avoids the expensive + ExperimentListFactory.from_json_file(..., check_format=True) call in + raw-panel-cache mode. + """ + import json + + try: + from cctbx import uctbx + + with open(expt_path) as f: + data = json.load(f) + + exp = data["experiment"][0] + crystal_id = int(exp.get("crystal", 0)) + crystal = data["crystal"][crystal_id] + + a_vec = np.asarray(crystal["real_space_a"], dtype=float) + b_vec = np.asarray(crystal["real_space_b"], dtype=float) + c_vec = np.asarray(crystal["real_space_c"], dtype=float) + + a = float(np.linalg.norm(a_vec)) + b = float(np.linalg.norm(b_vec)) + c = float(np.linalg.norm(c_vec)) + alpha = _angle_degrees(b_vec, c_vec) + beta = _angle_degrees(a_vec, c_vec) + gamma = _angle_degrees(a_vec, b_vec) + + return uctbx.unit_cell((a, b, c, alpha, beta, gamma)) + except Exception: + return None + + +def _mfx_d_spacing_from_unit_cell(unit_cell, miller_index): + """Compute d-spacing from a cached-mode unit cell, or NaN if unavailable.""" + if unit_cell is None: + return np.nan + + try: + hkl = tuple(int(v) for v in miller_index) + return float(unit_cell.d(hkl)) + except Exception: + return np.nan + + +#Thao: Added a helper function to find the matching .expt file for a given MFX integrated reflection table path. +def _matching_mfx_expt(refl_path: Path): + # Given an MFX integrated reflection table path, return the matching .expt path. + # Example: + # idx-data_36001_integrated.refl -> idx-data_36001_integrated.expt + expt_path = refl_path.with_suffix(".expt") + if not expt_path.exists(): + return None + + return expt_path + + +# Thao: added helper for combined MFX .expt files. +# In combined files, each experiment has a new internal id: 0, 1, 2, ... +# Each experiment points to an imageset, and that imageset preserves the +# original single_file_indices value, e.g. 1418, 2203, 2660. +def _mfx_image_index_from_combined_expt_json(expt_path: Path, experiment_id: int): + """Return original single_file_indices for one experiment in a combined .expt. + + Example: + combined experiment id 0 -> original image/event 1418 + combined experiment id 1 -> original image/event 2203 + + This is only metadata for tracking/debugging. Pixel loading still happens + through ExperimentListFactory and experiments[experiment_id].imageset. + """ + import json + + with open(expt_path) as f: + data = json.load(f) + + try: + exp = data["experiment"][experiment_id] + imageset_id = int(exp["imageset"]) + return int(data["imageset"][imageset_id]["single_file_indices"][0]) + except (KeyError, IndexError, TypeError, ValueError): + return None + + +#Thao: added a helper function to extract the first single_file_indices value from an MFX .expt file. +# This is only metadata for tracking/debugging. dxtbx can still load the pixels through ExperimentListFactory even if this value is absent. +def _mfx_image_index_from_expt_json(expt_path: Path): + """Return the first single_file_indices value from an MFX .expt file. + + This is only metadata for tracking/debugging. dxtbx can still load the + pixels through ExperimentListFactory even if this value is absent. + """ + import json + + # A DIALS .expt file is stored in a JSON-like text format. We only read it + # here to record metadata such as the event/image index; pixel loading still + # happens through ExperimentListFactory above. + with open(expt_path) as f: + data = json.load(f) + + # Example .expt structure: + # { + # "imageset": [ + # { + # "images": ["/.../data.loc"], + # "single_file_indices": [36001] + # } + # ] + # } + try: + # Example return value: 36001 from {"single_file_indices": [36001]}. + return int(data["imageset"][0]["single_file_indices"][0]) + except (KeyError, IndexError, TypeError): + return None + + +def _mfx_d_spacing(experiment, miller_index): + """Compute d-spacing for one MFX reflection from the experiment crystal.""" + + #Example expt file: + # { + # "experiment": [ + # { + # "crystal": 0, + # "detector": 0, + # "beam": 0, + # "imageset": 0 + # } + # ], + # + # "crystal": [ + # { + # "real_space_a": [...], + # "real_space_b": [...], + # "real_space_c": [...], + # "space_group_hall_symbol": "..." + # } + # ], + + + unit_cell = experiment.crystal.get_unit_cell() + + #converts miller_index into a clean tuple of three integers: (h,k,l) + hkl = tuple(int(v) for v in miller_index) + return float(unit_cell.d(hkl)) + + +def _crop_mfx_fixed_shoebox( + panel_image, + x_center, + y_center, + width, + height, + panel_mask_np=None, +): + """Crop/pad one fixed-size MFX shoebox from one detector panel. + + Parameters + ---------- + panel_image + scitbx/DIALS flex 2D array for one detector panel. Use + panel_image.all() to get shape because this is not a NumPy array. + + x_center, y_center + Predicted reflection center in pixel coordinates from xyzcal.px. + + width, height + Desired fixed output size from --w and --h. + + panel_mask_np + Optional NumPy boolean detector mask for the same panel as panel_image. + Shape should match panel_image.as_numpy_array().shape. + + Convention used here: + True = good/usable detector pixel + False = bad/hot/masked detector pixel + + If panel_mask_np is provided, the output shoebox mask is: + finite detector pixel && good detector-mask pixel + + If panel_mask_np is None, the output shoebox mask is only: + finite detector pixel inside the panel + + Returns + ------- + shoebox : np.ndarray + 2D pixel array with shape (height, width). Pixels outside the panel are + zero-padded. + mask : np.ndarray + Boolean array with shape (height, width). + + True means this output pixel is valid for training/integration. + False means this pixel is outside-panel padding, NaN/inf, or marked bad + by the detector mask. + fixed_bbox : tuple[int, int, int, int, int, int] + The full requested bbox in DIALS order: (x0, x1, y0, y1, z0, z1). + """ + + """ + Full detector panel image + example shape: 514 x 1030 + ↓ + + Predicted reflection center + from xyzcal.px: x_center, y_center + ↓ + + Request fixed 25 x 25 crop around center + ↓ + + Check if requested crop is fully inside detector panel + ↓ + + If fully inside: + copy real 25 x 25 detector pixels + mask = True for finite/good pixels + + If near detector edge: + requested 25 x 25 box partly falls outside panel + only copy available real detector pixels + missing outside-panel pixels stay zero + mask = False for missing/outside pixels + ↓ + + Output fixed shoebox + counts[i] shape: 25 x 25 or flattened 625 + mask[i] shape: 25 x 25 or flattened 625 + ↓ + + Encoder + reads the 25 x 25 shoebox + ↓ + + Feature vector + 32 learned numbers + ↓ + + Posterior / surrogate distributions applied here: + + qp branch: + learned_basis_profile + predicts spot/profile shape + + qi branch: + Gamma / LogNormal / FoldedNormal + predicts reflection intensity + + qbg branch: + Gamma / LogNormal / FoldedNormal + predicts background + ↓ + + Combine qp + qi + qbg + ↓ + + Predicted pixels / predicted shoebox + ↓ + + Observation likelihood applied here: + Normal / Poisson / StudentT + compares predicted pixels with real input pixels + ↓ + + Loss + + """ + # panel_image can be either: + # 1. a DIALS/scitbx flex array from get_raw_data(0), or + # 2. a NumPy array loaded from raw_panels_image_XXXXX.npz. + # + # Support both so the same crop code works for the normal psana/dxtbx path + # and the raw-panel-cache path. + if isinstance(panel_image, np.ndarray): + panel_np = panel_image + panel_h, panel_w = panel_np.shape + else: + # flex arrays use .all() for shape. For this data: (514, 1030). + panel_h, panel_w = panel_image.all() + + # Convert once to NumPy so ordinary slicing/assignment works. + panel_np = panel_image.as_numpy_array() + + # If a detector mask is supplied, make sure it lines up with this panel. + # This catches mistakes such as using a mask from a different detector size. + if panel_mask_np is not None and panel_mask_np.shape != panel_np.shape: + raise ValueError( + "panel mask shape does not match panel image shape: " + f"mask={panel_mask_np.shape}, image={panel_np.shape}" + ) + + # Example: for a 25x25 shoebox, nx=12 and ny=12. + nx = width // 2 + ny = height // 2 + + # Center the fixed window on the predicted centroid. + xc = int(np.floor(x_center)) + yc = int(np.floor(y_center)) + + x0 = xc - nx + x1 = x0 + width + y0 = yc - ny + y1 = y0 + height + + fixed_bbox = (x0, x1, y0, y1, 0, 1) + + # Clip source coordinates to the actual panel bounds. + # This avoids negative or out-of-range NumPy indexing. + # Example requested y range: -3:22 -> clipped source y range: 0:22. + xs0 = max(0, x0) + xs1 = min(panel_w, x1) + ys0 = max(0, y0) + ys1 = min(panel_h, y1) + + # Allocate full fixed-size output. Areas outside the detector stay zero and + # mask False. + shoebox = np.zeros((height, width), dtype=panel_np.dtype) + mask = np.zeros((height, width), dtype=bool) + + # If the requested box has no overlap with the panel at all, return all-zero data + # and an all-False mask. + if xs0 >= xs1 or ys0 >= ys1: + return shoebox, mask, fixed_bbox + + # Destination offsets inside the fixed-size shoebox. + # + # x0/y0 are the requested crop start coordinates. + # xs0/ys0 are the clipped crop start coordinates inside the real detector panel. + # + # Normal case: + # x0 = 100, xs0 = 100 + # xd0 = 100 - 100 = 0 + # Real pixels start at column 0 in the 25x25 shoebox. + # + # Edge case: + # x0 = -5, xs0 = 0 + # xd0 = 0 - (-5) = 5 + # The first 5 columns are outside-panel padding. + # Real pixels start at column 5 in the 25x25 shoebox. + + xd0 = xs0 - x0 + yd0 = ys0 - y0 + + # panel_np is the full detector panel image as a NumPy array. + # Example panel shape: (y, x) = (514, 1030). + # + # patch is the cropped region of the panel that fits within the fixed bbox. + # Example: panel_np[372:397, 765:790] -> patch.shape == (25, 25). + patch = panel_np[ys0:ys1, xs0:xs1] + + # Place the available real pixels into the fixed-size shoebox. Any area that + # was outside the panel remains zero padding. + # + # Edge example: + # shoebox[3:25, 0:25] = patch + # rows 0:3 stay zero padding; rows 3:25 contain real detector pixels. + + + #paste the available real detector pixels into the correct location inside the fixed 25×25 shoebox. + shoebox[yd0 : yd0 + patch.shape[0], xd0 : xd0 + patch.shape[1]] = patch + + # Basic validity mask: True only for normal finite numbers. This rejects + # NaN, +inf, and -inf values. + finite_patch = np.isfinite(patch) + + if panel_mask_np is None: + # No real detector mask was provided. The mask only means: + # finite pixel inside the detector panel. + mask_patch = finite_patch + else: + # Crop the detector mask using exactly the same source coordinates as + # the detector pixel patch. Because the slices are identical, patch and + # detector_mask_patch have the same shape. + detector_mask_patch = panel_mask_np[ys0:ys1, xs0:xs1] + + # Combine the two masks element-by-element: + # True only if the pixel is finite AND detector mask says good. + # False if the pixel is NaN/inf OR detector mask says bad. + mask_patch = finite_patch & detector_mask_patch + + # Place the mask patch into the same position as the shoebox pixel patch. + # Outside-panel padding remains False. + mask[yd0 : yd0 + patch.shape[0], xd0 : xd0 + patch.shape[1]] = mask_patch + + # returns: + # shoebox: np.ndarray, shape (height, width), dtype same as detector pixels + # mask: np.ndarray, shape (height, width), dtype bool + # fixed_bbox: tuple of ints, (x0, x1, y0, y1, 0, 1) + return shoebox, mask, fixed_bbox + def _stats_from_memmap( counts_path: Path, masks_path: Path, chunk: int = 10_000, ) -> dict: + """Return (mean, var) of masked counts and their Anscombe transform. Streams the on-disk memmap in chunks; values are stored in dataset.yaml. """ + + #r: means read-only. NumPy reads chunks from disk as needed. counts = np.load(counts_path, mmap_mode="r") masks = np.load(masks_path, mmap_mode="r") n, _ = counts.shape @@ -218,6 +1945,10 @@ def _stats_from_memmap( c = counts[i : i + chunk].astype(np.float64) m = masks[i : i + chunk] c = c * m + + #c >= -0.375 + #we have valid MFX counts include values like:-7, -115 + # so for MFX Anscombe becomes invalid and produces nan. a = 2.0 * np.sqrt(c + 0.375) sum_c += c.sum() sumsq_c += (c * c).sum() @@ -225,6 +1956,11 @@ def _stats_from_memmap( sumsq_a += (a * a).sum() nel += c.size + #raw mean + #raw variance + #anscombe mean + #anscombe variance + mean_c = sum_c / nel var_c = sumsq_c / nel - mean_c * mean_c mean_a = sum_a / nel @@ -236,6 +1972,337 @@ def _stats_from_memmap( } +def _raw_stats_from_memmap( + counts_path: Path, + masks_path: Path, + chunk: int = 10_000, +) -> dict: + """ + one image -> many reflections/shoeboxes: counts.npy = shoeboxes from many images. + + Return raw mean/variance of over the whole counts.npy dataset + + Ex: For 1500-file dataset: all shoeboxes from all 1500 files + + MFX data can contain negative valid pixel values, so Anscombe stats are + skipped because sqrt(c + 0.375) can become invalid. + """ + counts = np.load(counts_path, mmap_mode="r") + masks = np.load(masks_path, mmap_mode="r") + + # n = number of shoeboxes. _ = pixels per shoebox, usually 625. + n, _ = counts.shape + + sum_c = 0.0 + sumsq_c = 0.0 + nel = 0 + + for i in range(0, n, chunk): + c = counts[i : i + chunk].astype(np.float64) + m = masks[i : i + chunk] + c = c * m + + sum_c += c.sum() + sumsq_c += (c * c).sum() + nel += c.size + + mean_c = sum_c / nel + var_c = sumsq_c / nel - mean_c * mean_c + + return { + "raw": [float(mean_c), float(var_c)], + } + + +def merge_mfx_chunks(chunk_dirs, out_dir, counts_fname="counts.npy", masks_fname="masks.npy"): + """Merge MFX chunk folders into final counts.npy, masks.npy, metadata.npy. + + Each chunk folder contains: + counts.npy + masks.npy + metadata.npy + + Final output folder will contain: + counts.npy + masks.npy + metadata.npy + + This function writes final counts/masks through memmap so it does not need + to hold all shoebox pixels in RAM at once during the merge. + """ + chunk_dirs = list(chunk_dirs) + if not chunk_dirs: + raise RuntimeError("No chunks to merge.") + + # First pass: inspect chunk shapes and dtypes. + # Example chunk counts shape: (10000, 625). + total_rows = 0 + n_pixels = None + counts_dtype = None + metadata_list = [] + + for chunk_dir in chunk_dirs: + counts_chunk = np.load(chunk_dir / counts_fname, mmap_mode="r") + masks_chunk = np.load(chunk_dir / masks_fname, mmap_mode="r") + + if n_pixels is None: + n_pixels = counts_chunk.shape[1] + counts_dtype = counts_chunk.dtype + elif counts_chunk.shape[1] != n_pixels: + raise ValueError( + f"chunk {chunk_dir} has {counts_chunk.shape[1]} pixels per row, " + f"expected {n_pixels}" + ) + + if masks_chunk.shape != counts_chunk.shape: + raise ValueError( + f"mask/counts shape mismatch in {chunk_dir}: " + f"counts={counts_chunk.shape}, masks={masks_chunk.shape}" + ) + + total_rows += counts_chunk.shape[0] + metadata = np.load(chunk_dir / "metadata.npy", allow_pickle=True).item() + metadata_list.append(metadata) + + # Create final output arrays on disk. + counts_out = open_memmap( + out_dir / counts_fname, + mode="w+", + dtype=counts_dtype, + shape=(total_rows, n_pixels), + ) + masks_out = open_memmap( + out_dir / masks_fname, + mode="w+", + dtype=np.bool_, + shape=(total_rows, n_pixels), + ) + + # Second pass: copy chunk arrays into the final arrays. + row0 = 0 + for chunk_dir in chunk_dirs: + counts_chunk = np.load(chunk_dir / counts_fname, mmap_mode="r") + masks_chunk = np.load(chunk_dir / masks_fname, mmap_mode="r") + + row1 = row0 + counts_chunk.shape[0] + counts_out[row0:row1] = counts_chunk + masks_out[row0:row1] = masks_chunk + row0 = row1 + + counts_out.flush() + masks_out.flush() + del counts_out, masks_out + + # Merge metadata key-by-key. + # Metadata arrays are much smaller than counts/masks, so this part is fine + # to concatenate in memory for now. + merged_meta = {} + keys = metadata_list[0].keys() + for key in keys: + merged_meta[key] = np.concatenate([m[key] for m in metadata_list], axis=0) + + np.save(out_dir / "metadata.npy", merged_meta) + + return total_rows + + +def _add_image_id_metadata(metadata_path: Path): + """Add compact image_id and n_images to final merged metadata.npy. + + image_num is the original MFX/cctbx image number, e.g. 2660, 3017. + image_id is compact for ML embeddings, e.g. 0, 1, 2. + + This should run only after merge_mfx_chunks(...) has created the final + metadata.npy. Do not run it inside each chunk, because each chunk may + restart image IDs from 0 and create inconsistent IDs in the merged dataset. + """ + metadata = np.load(metadata_path, allow_pickle=True).item() + + if "image_num" not in metadata: + raise KeyError("metadata.npy does not contain image_num") + + image_num = np.asarray(metadata["image_num"]).astype(np.int64) + + unique = np.array(sorted(set(image_num.tolist())), dtype=np.int64) + image_to_id = {img: i for i, img in enumerate(unique)} + + metadata["image_id"] = np.array( + [image_to_id[x] for x in image_num], + dtype=np.int64, + ) + """ + Ex: + image_num = [2660, 2660, 3017, 3744, 3744] + unique = [2660, 3017, 3744] + len(unique) = 3 + + for every reflection row: + reflection 0: image_num=2660, image_id=0, n_images=3 + reflection 1: image_num=2660, image_id=0, n_images=3 + reflection 2: image_num=3017, image_id=1, n_images=3 + + we want to save n_images because later he model/loss needs to create one learnable value per image. + + nn.Embedding(n_images, 1) + + then PyTorch creates: + + image 0 -> learnable B/G value + image 1 -> learnable B/G value + + So each reflection uses: + + image_id = which image this reflection came from + n_images = how many total image-specific parameters to create + + """ + + metadata["n_images"] = np.full( + len(image_num), + len(unique), + dtype=np.int64, + ) + + np.save(metadata_path, metadata) + + print("added image_id to metadata.npy") + print(f"n_images = {len(unique)}") + + +def _mfx_metadata_rows_to_luis_dict(metadata_rows, test_fraction, global_start=0): + """Convert MFX per-reflection metadata rows into Luis-style metadata.npy. + + Luis's metadata.npy is a zero-dimensional object array containing one dict. + Each key maps to a NumPy array with one value per reflection. + + Example: + metadata["panel"].shape == (N,) + metadata["H"].shape == (N,) + metadata["K"].shape == (N,) + metadata["L"].shape == (N,) + + global_start: + Starting row index for this chunk in the full final dataset. + Example: + chunk 0 starts at 0 + chunk 1 starts at 10000 + chunk 2 starts at 20000 + """ + n = len(metadata_rows) + + meta = {} + + # Simple scalar columns. + # Use row.get(..., np.nan) so optional columns like background.mean do not + # crash if they are absent in some future MFX reflection table. + scalar_keys = [ + "reflection_id", + "combined_refl_row", + "experiment_id", + "panel", + "d", + "intensity_sum_value", + "intensity_sum_variance", + "background_sum_value", + "background_sum_variance", + "background.mean", + ] + + for key in scalar_keys: + meta[key] = np.array( + [row.get(key, np.nan) for row in metadata_rows], + dtype=np.float32, + ) + + # Match Luis-style names for intensity/background columns. + meta["intensity.sum.value"] = meta.pop("intensity_sum_value") + meta["intensity.sum.variance"] = meta.pop("intensity_sum_variance") + meta["background.sum.value"] = meta.pop("background_sum_value") + meta["background.sum.variance"] = meta.pop("background_sum_variance") + + # Miller indices: Luis uses H, K, L. + miller = np.array( + [row["miller_index"] for row in metadata_rows], + dtype=np.float32, + ) + meta["H"] = miller[:, 0] + meta["K"] = miller[:, 1] + meta["L"] = miller[:, 2] + + # Fixed bbox: use the fixed shoebox bbox, because that aligns with counts/masks. + bbox = np.array( + [row["fixed_bbox"] for row in metadata_rows], + dtype=np.float32, + ) + for j in range(6): + meta[f"bbox.{j}"] = bbox[:, j] + + # xyzcal.px + xyzcal = np.array( + [row["xyzcal_px"] for row in metadata_rows], + dtype=np.float32, + ) + for j in range(3): + meta[f"xyzcal.px.{j}"] = xyzcal[:, j] + + # xyzobs.px.value, if available. + xyzobs = np.array( + [ + row["xyzobs_px_value"] + if row["xyzobs_px_value"] is not None + else (np.nan, np.nan, np.nan) + for row in metadata_rows + ], + dtype=np.float32, + ) + for j in range(3): + meta[f"xyzobs.px.value.{j}"] = xyzobs[:, j] + + # this is_test follows Luis's convention: randomly flag a fraction of reflections as test. + # For example, if test_fraction=0.1, then about 10% of reflections will have is_test=True. + # global_start makes the split deterministic across chunks, matching the same + # RNG sequence as if all rows were processed together. + rng = np.random.default_rng(42) + is_test_all = rng.random(global_start + n) + meta["is_test"] = (is_test_all[global_start:] < test_fraction).astype(np.float32) + + meta["wavelength"] = np.array( + [row.get("wavelength", np.nan) for row in metadata_rows], + dtype=np.float32, + ) + + meta["image_num"] = np.array( + [ + row["image_index"] if row["image_index"] is not None else -1 + for row in metadata_rows + ], + dtype=np.float32, + ) + + # Global row ids across the final merged dataset. + # This keeps refl_ids unique across chunks. + meta["refl_ids"] = np.arange(global_start, global_start + n, dtype=np.float32) + + meta["source_refl"] = np.array( + [row.get("source_refl", "") for row in metadata_rows], + dtype=object, + ) + + meta["source_expt"] = np.array( + [row.get("source_expt", "") for row in metadata_rows], + dtype=object, + ) + + meta["detector_mask"] = np.array( + [row.get("detector_mask", "") for row in metadata_rows], + dtype=object, + ) + + + return meta + + + def _save_concentration_from_memmap( counts_path: Path, out_dir: Path, @@ -249,6 +2316,13 @@ def _save_concentration_from_memmap( for i in range(0, n, chunk): c = counts[i : i + chunk].astype(np.float32) conc[i : i + chunk] = c.mean(axis=1) + + if torch is None: + raise RuntimeError( + "torch is required to save concentration.npy; " + "rerun with --no-stats or use an environment with torch." + ) + save_data(torch.from_numpy(conc), out_dir / out_fname) diff --git a/src/integrator/cli/predict.py b/src/integrator/cli/predict.py index b0d365c1..3add1d3a 100644 --- a/src/integrator/cli/predict.py +++ b/src/integrator/cli/predict.py @@ -1,13 +1,68 @@ import argparse import logging import re +from pathlib import Path from integrator.cli.utils.logger import setup_logging from integrator.io import write_mtz_from_preds, write_refl_from_preds +from integrator.utils import resolve_source_data_dir logger = logging.getLogger(__name__) +def _resolve_and_check_metadata(config: dict) -> Path: + """Resolve source_data_dir and verify metadata.npy exists. + + Used by --mfx-writeback to obtain the metadata.npy path automatically + from manifest.yaml (chunked_rotation_data) or from data_loader.args.data_dir + (rotation_data). + + Raises: + FileNotFoundError: metadata.npy not found under the resolved directory, + with the expected path included in the message. + ValueError / KeyError: propagated from resolve_source_data_dir when the + config or manifest is malformed. + """ + source_data_dir = resolve_source_data_dir(config) + metadata_path = source_data_dir / "metadata.npy" + if not metadata_path.exists(): + raise FileNotFoundError( + f"metadata.npy not found at expected path:\n" + f" {metadata_path}\n" + "Verify that source_data_dir in manifest.yaml points to the correct " + "original shoebox dataset directory, or check that the file has not " + "been moved or deleted." + ) + return metadata_path + + +def _resolve_and_check_mtz_sources(config: dict) -> Path: + """Resolve source_data_dir and verify metadata.npy + dataset.yaml both exist. + + Used by --write-mtz. MTZ export requires both metadata.npy (wavelength data) + and dataset.yaml (crystal geometry). + + Raises: + FileNotFoundError: either required file not found under source_data_dir, + with the expected path included in the message. + ValueError / KeyError: propagated from resolve_source_data_dir when the + config or manifest is malformed. + """ + source_data_dir = resolve_source_data_dir(config) + for fname in ("metadata.npy", "dataset.yaml"): + fpath = source_data_dir / fname + if not fpath.exists(): + raise FileNotFoundError( + f"--write-mtz requires {fname} under source_data_dir but it was " + f"not found:\n" + f" {fpath}\n" + "Verify that source_data_dir in manifest.yaml points to the correct " + "original shoebox dataset directory, or check that the file has not " + "been moved or deleted." + ) + return source_data_dir + + def parse_args(): parser = argparse.ArgumentParser( prog="integrator.predict", @@ -44,6 +99,27 @@ def parse_args(): action="store_true", help="Write predictions as a .refl file", ) + # MFX write-back extension (Thao): keep Luis's --write-refl path, + # but allow writing predictions into many original MFX .refl files. + parser.add_argument( + "--mfx-writeback", + action="store_true", + help=( + "Use MFX many-file .refl write-back instead of the default " + "single-source .refl write-back. Requires --write-refl and " + "--original-refl-dir." + ), + ) + parser.add_argument( + "--original-refl-dir", + type=str, + default=None, + help=( + "Original MFX/cctbx out folder containing " + "idx-data_*_integrated.refl/.expt files. Used only with " + "--write-refl --mfx-writeback." + ), + ) parser.add_argument( "--write-mtz", action="store_true", @@ -166,12 +242,20 @@ def main(): print(f" {k}") return - # path to input refl file (only needed for --write-refl) + # Path to input refl file for Luis's original single-file write-back. + # MFX write-back extension (Thao): the many-file MFX path uses + # --original-refl-dir + metadata.npy instead, so it does not require + # output.refl_file in the YAML config. refl_file = config.get("output", {}).get("refl_file") - if args.write_refl and not refl_file: + if args.write_refl and not args.mfx_writeback and not refl_file: raise ValueError( "--write-refl requires 'output.refl_file' in the YAML config" ) + if args.mfx_writeback: + if not args.write_refl: + raise ValueError("--mfx-writeback must be used with --write-refl") + if args.original_refl_dir is None: + raise ValueError("--mfx-writeback requires --original-refl-dir") epoch_re = re.compile(r"epoch=(\d+)") for ckpt in checkpoints: @@ -227,23 +311,41 @@ def main(): if args.write_refl: logger.info("Writing .refl output for epoch %d", epoch) - write_refl_from_preds( - ckpt_dir=ckpt_dir, - refl_file=refl_file, - epoch=epoch, - filetype="parquet", - ) + + if args.mfx_writeback: + # MFX write-back: resolve and validate the original shoebox data + # directory automatically from manifest.yaml (chunked) or from + # data_loader.args.data_dir (rotation_data). + # --original-refl-dir remains separate: it points to the + # original .refl/.expt files, which may differ from source_data_dir. + from integrator.io.pred_io import write_mfx_refl_from_preds + + metadata_path = _resolve_and_check_metadata(config) + write_mfx_refl_from_preds( + ckpt_dir=ckpt_dir, + metadata_path=metadata_path, + original_refl_dir=Path(args.original_refl_dir), + out_dir=pred_dir / "mfx_refl_writeback", + filetype="parquet", + ) + else: + write_refl_from_preds( + ckpt_dir=ckpt_dir, + refl_file=refl_file, + epoch=epoch, + filetype="parquet", + ) if args.write_mtz: from integrator.io import get_pred_files logger.info("Writing .mtz output for epoch %d", epoch) pred_data = get_pred_files(ckpt_dir=ckpt_dir, filetype="parquet") - data_dir = Path(config["data_loader"]["args"]["data_dir"]) + source_data_dir = _resolve_and_check_mtz_sources(config) write_mtz_from_preds( pred_data=pred_data, - metadata_path=data_dir / "metadata.npy", - data_dir=data_dir, + metadata_path=source_data_dir / "metadata.npy", + data_dir=source_data_dir, out_path=ckpt_dir / f"preds_epoch_{epoch:04d}.mtz", ) diff --git a/src/integrator/cli/preprocess.py b/src/integrator/cli/preprocess.py new file mode 100644 index 00000000..c84ec998 --- /dev/null +++ b/src/integrator/cli/preprocess.py @@ -0,0 +1,1050 @@ +""" +integrator.preprocess — one-time preprocessing: filter, transform, save chunks. + +Two-pass design +─────────────── +Pass 1 (_pass1_scan) + • load raw mmaps and metadata lazily + • read dataset.yaml for stats; --transform is an explicit CLI arg + • compute valid_indices (same logic as RotationDataModule.setup) + • verify required metadata columns; warn about optional ones + • check for duplicate refl_ids (raises ValueError) + • split assignment — two mutually exclusive modes: + + Mode B (preferred for new datasets): + --validation-split FLOAT --split-seed INT + Generates a deterministic split from sorted refl_ids + seeded RNG. + Independent of input row order. Exports train_labels.csv and + val_labels.csv to for later reuse. + Every valid reflection is assigned to exactly one split. + + Mode A (backward compatible): + --train-labels PATH --val-labels PATH + Reads existing CSV split files. Rows absent from both CSVs + (e.g. test reflections) get is_train=False is_val=False and + appear in predict-only; they are NOT an error. + + • compute group_label for ALL n_bins >= 1 (required by _step()) + • append _EXTRA_SAVE_COLS (reflection_id, source_refl, source_expt, + wavelength) when present in reference + • n_images from image_id only — image_num is not suitable for embeddings + +Pass 2 (_pass2_write_chunks) + • iterate chunks of valid_indices + • copy rows from mmaps, apply vectorised transform (NO global clip — + only anscombe and log1p clamp to min=0, matching _transform_counts) + • write counts.npy, shoebox.npy, mask.npy (large pixel arrays, separate) + • write metadata.npz — all numeric + bool columns in one file + • write strings.parquet — all string columns (omitted when none exist) + • write manifest.yaml + +Chunk layout +──────────── +chunk_NNNNN/ + counts.npy float32 (N, pixels) raw pixel counts + shoebox.npy float32 (N, pixels) transform-applied (model input) + mask.npy bool (N, pixels) valid-pixel mask + metadata.npz all numeric + bool metadata columns: + row_id, is_train, is_val, image_id, d, lp, + group_label, profile_group_label, refl_ids, ... + strings.parquet all string columns (source_refl, source_expt, ...); + omitted entirely when no string columns exist + +Usage — Mode B (preferred) +────────────────────────── +integrator.preprocess \ + --data-dir /data/mfx \ + --validation-split 0.2 \ + --split-seed 42 \ + [--transform asinh] [--chunk-size 50000] [--n-bins 10] \ + [--out-dir /data/mfx/chunks] [--min-valid-pixels 10] + +Usage — Mode A (existing split labels) +─────────────────────────────────────── +integrator.preprocess \ + --data-dir /data/mfx \ + --train-labels /runs/run_abc/train_labels.csv \ + --val-labels /runs/run_abc/val_labels.csv \ + [--transform asinh] [--chunk-size 50000] [--n-bins 10] +""" + +from __future__ import annotations + +import argparse +import logging +from dataclasses import dataclass +from pathlib import Path + +import numpy as np +import polars as pl +import torch +import yaml + +logger = logging.getLogger(__name__) + +_VALID_TRANSFORMS = ( + "anscombe", + "log1p", + "standardization", + "asinh", + "log_softplus", + "sqrt_squareplus", +) + +# Columns that must be present for the pipeline to work correctly. +_REQUIRED_COLS = ["refl_ids", "d"] +# At least one image identifier must exist. +_REQUIRED_ONE_OF_IMAGE = ["image_id", "image_num"] +# Optional columns worth warning about when absent. +_VERIFIED_OPTIONAL = [ + "reflection_id", + "wavelength", + "lp", + "source_refl", + "source_expt", +] +# Columns critical for MFX write-back and polychromatic loss that are absent +# from DEFAULT_DS_COLS. Saved when present in reference. +# Note: lp IS in DEFAULT_DS_COLS (line 154) and does not appear here. +_EXTRA_SAVE_COLS = [ + "reflection_id", # MFX write-back: row index within source .refl + "source_refl", # MFX write-back: source .refl filename (string) + "source_expt", # MFX write-back: source .expt filename (string) + "wavelength", # polychromatic loss +] + +# Mode B defaults +_DEFAULT_VALIDATION_SPLIT = 0.2 +_DEFAULT_SPLIT_SEED = 42 + + +# ── CLI ─────────────────────────────────────────────────────────────────────── + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + prog="integrator.preprocess", + description=( + "Preprocess shoeboxes into fixed-size chunks.\n\n" + "Split modes (mutually exclusive):\n" + " Mode B (preferred): --validation-split / --split-seed\n" + " Mode A (compat): --train-labels / --val-labels" + ), + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "--data-dir", + required=True, + help="Directory with counts.npy, masks.npy, metadata.npy, dataset.yaml", + ) + # ── Mode A args ── + p.add_argument( + "--train-labels", + default=None, + metavar="PATH", + help=( + "Mode A: CSV with column 'train_ids' (refl_ids of training " + "reflections). Mutually exclusive with --validation-split." + ), + ) + p.add_argument( + "--val-labels", + default=None, + metavar="PATH", + help=( + "Mode A: CSV with column 'val_ids'. " + "Must be given together with --train-labels." + ), + ) + # ── Mode B args ── + p.add_argument( + "--validation-split", + type=float, + default=None, + metavar="FLOAT", + help=( + f"Mode B: fraction of valid reflections held out for validation " + f"(default {_DEFAULT_VALIDATION_SPLIT} when neither mode is " + "specified). Must be in (0, 1). " + "Mutually exclusive with --train-labels / --val-labels." + ), + ) + p.add_argument( + "--split-seed", + type=int, + default=_DEFAULT_SPLIT_SEED, + metavar="INT", + help=( + f"Mode B: RNG seed for the deterministic split " + f"(default {_DEFAULT_SPLIT_SEED})." + ), + ) + # ── Shared args ── + p.add_argument( + "--transform", + default=None, + choices=_VALID_TRANSFORMS, + help=( + "Pixel transform applied before saving shoeboxes. " + "Must match the transform used for training. " + "Default: inferred from dataset.yaml " + "('anscombe' if anscombe:true, else 'standardization')." + ), + ) + p.add_argument( + "--chunk-size", + type=int, + default=50_000, + help="Rows per chunk. Default 50 000; benchmark before locking in.", + ) + p.add_argument( + "--n-bins", + type=int, + default=1, + help=( + "Resolution bins for group_label. group_label is always saved " + "regardless of this value (required by the training step). " + "Must be >= 1." + ), + ) + p.add_argument( + "--out-dir", + default=None, + help="Chunk output directory (default: /chunks)", + ) + p.add_argument("--min-valid-pixels", type=int, default=10) + p.add_argument("--resolution-cutoff", type=float, default=None) + p.add_argument("-v", "--verbose", action="count", default=0) + return p.parse_args() + + +# ── Transform — vectorised numpy, one full chunk at a time ──────────────────── + + +def _apply_transform_np( + counts_f32: np.ndarray, # (N, pixels) float32 + masks_bool: np.ndarray, # (N, pixels) bool + mean: float, + var: float, + transform: str, +) -> np.ndarray: + """Vectorised equivalent of IntegratorDataset._transform_counts. + + Clamping policy (matches _transform_counts exactly): + anscombe, log1p → clamp input to min=0 before the formula + all others → use raw pixel values (no pre-clamp) + + Masking fix for standardisation: ((x-mean)/std)*m so that masked pixels + are 0.0, not -mean/std. All other transforms already multiply by m last. + """ + x = counts_f32.astype(np.float32, copy=False) # no global clip + m = masks_bool.astype(np.float32) + + if transform == "anscombe": + xc = np.clip(x, 0.0, None) # clamp only here + t = 2.0 * np.sqrt(xc + 0.375) + return ((t - mean) / np.sqrt(var)) * m + + if transform == "log1p": + return np.log1p(np.clip(x, 0.0, None)) * m # clamp only here + + if transform == "asinh": + # scale = sqrt(var).clamp(min=1e-8) — raw x, no clamp + scale = max(float(np.sqrt(var)), 1e-8) + return np.arcsinh(x / scale) * m + + if transform == "log_softplus": + # numerically stable softplus(x) = log(1+exp(x)) — raw x, no clamp + sp = np.log1p(np.exp(-np.abs(x))) + np.maximum(x, 0.0) + return np.log(sp + 1e-8) * m + + if transform == "sqrt_squareplus": + # raw x, no clamp — negative x gives a small positive squareplus value + b = 4.0 + sqp = 0.5 * (x + np.sqrt(x * x + b)) + return np.sqrt(sqp + 1e-8) * m + + # standardization — masking fix: multiply by m after normalisation + return ((x - mean) / np.sqrt(var)) * m + + +# ── Column helpers ──────────────────────────────────────────────────────────── + + +def _is_string_col(v) -> bool: + arr = ( + v.detach().cpu().numpy() + if isinstance(v, torch.Tensor) + else np.asarray(v) + ) + return arr.dtype.kind in ("O", "U", "S") + + +def _to_np(v) -> np.ndarray: + if isinstance(v, torch.Tensor): + return v.detach().cpu().numpy() + return np.asarray(v) + + +def _vectorized_isin(values: np.ndarray, sorted_ids: np.ndarray) -> np.ndarray: + """Binary-search membership test. O(N log M), no Python objects, no hash table.""" + pos = np.searchsorted(sorted_ids, values) + valid = pos < len(sorted_ids) + out = np.zeros(len(values), dtype=bool) + out[valid] = sorted_ids[pos[valid]] == values[valid] + return out + + +# ── Split helpers ───────────────────────────────────────────────────────────── + + +def _make_deterministic_split( + refl_ids: np.ndarray, # int64, (N,) — must be duplicate-free + validation_split: float, + seed: int, +) -> tuple[np.ndarray, np.ndarray]: + """Assign each refl_id to train or val deterministically. + + Assignment is based on position in sorted refl_id order combined with a + seeded RNG — independent of input row order. The same set of refl_ids + with the same seed always produces identical is_train / is_val arrays. + + Returns: + is_train: bool (N,) + is_val: bool (N,) + """ + # Sort to a canonical order, making assignment row-order-independent + sort_order = np.argsort(refl_ids, kind="stable") + + rng = np.random.default_rng(seed) + fracs = rng.random(len(refl_ids)) # one float per sorted position + + is_val_sorted = fracs < validation_split + is_train_sorted = ~is_val_sorted + + # Build inverse permutation: sorted-rank → original position + inverse = np.empty_like(sort_order) + inverse[sort_order] = np.arange(len(sort_order)) + + return is_train_sorted[inverse], is_val_sorted[inverse] + + +def _save_split_csvs( + out_dir: Path, + refl_ids_valid: np.ndarray, # int64 (N_valid,) + is_train: np.ndarray, # bool (N_valid,) + is_val: np.ndarray, # bool (N_valid,) +) -> None: + """Export train_labels.csv and val_labels.csv to out_dir. + + These files follow the same schema as those produced by assign_labels() + in prediction_writer.py and can be used as Mode A inputs on a future run. + """ + out_dir.mkdir(parents=True, exist_ok=True) + pl.DataFrame({"train_ids": refl_ids_valid[is_train]}).write_csv( + out_dir / "train_labels.csv" + ) + pl.DataFrame({"val_ids": refl_ids_valid[is_val]}).write_csv( + out_dir / "val_labels.csv" + ) + logger.info( + "Exported train_labels.csv (%d rows) and val_labels.csv (%d rows) to %s", + int(is_train.sum()), + int(is_val.sum()), + out_dir, + ) + + +# ── ScanResult ──────────────────────────────────────────────────────────────── + + +@dataclass +class ScanResult: + valid_indices: np.ndarray # int64, (N_valid,) + refl_ids_valid: np.ndarray # int64, (N_valid,) — for CSV export + is_train: np.ndarray # bool, (N_valid,) + is_val: np.ndarray # bool, (N_valid,) + group_label: np.ndarray # int64, (N_all_rows,) — always set + profile_group_label: np.ndarray | None # int64, (N_all_rows,) or None + stats: tuple[float, float] # (mean, variance) + transform: str + numeric_columns: list[str] # ordered list of numeric/bool column names + string_columns: list[str] # ordered list of string column names + n_images: int | None + n_valid: int + n_train: int + n_val: int + split_meta: dict # recorded in manifest.yaml + + +# ── Pass 1: scan, validate, plan ───────────────────────────────────────────── + + +def _pass1_scan( + data_dir: Path, + train_labels_path: str | None, # Mode A; None triggers Mode B + val_labels_path: str | None, # Mode A + validation_split: float | None, # Mode B + split_seed: int | None, # Mode B + min_valid_pixels: int, + resolution_cutoff: float | None, + n_bins: int, + transform: str, # resolved by caller before entering pass 1 + masks, # numpy mmap + reference: dict, + spec: dict, +) -> ScanResult: + from integrator.data_loaders.data_module import ( + DEFAULT_DS_COLS, + _compute_valid_indices, + ) + from integrator.io import data_path, load_data + from integrator.utils.prepare_priors import _bin_by_resolution, _nbins_path + + # ── Stats from dataset.yaml ─────────────────────────────────────────────── + stats_key = "anscombe" if transform == "anscombe" else "raw" + stats_arr = spec.get("stats", {}).get(stats_key) + if stats_arr is None: + raise KeyError( + f"dataset.yaml is missing 'stats.{stats_key}'. " + "Regenerate the dataset with integrator.make_shoeboxes." + ) + mean, var = float(stats_arr[0]), float(stats_arr[1]) + if var <= 0: + raise ValueError(f"stats.{stats_key} variance={var} is non-positive.") + + # ── Required-column verification ────────────────────────────────────────── + for col in _REQUIRED_COLS: + if col not in reference: + raise KeyError( + f"Required metadata column '{col}' is missing from metadata.npy. " + "Re-run integrator.make_shoeboxes." + ) + if not any(c in reference for c in _REQUIRED_ONE_OF_IMAGE): + raise KeyError( + f"At least one of {_REQUIRED_ONE_OF_IMAGE} must be present in metadata.npy." + ) + for col in _VERIFIED_OPTIONAL: + if col not in reference: + logger.warning( + "Optional metadata column '%s' not found; will not be saved.", + col, + ) + + # ── Valid indices ───────────────────────────────────────────────────────── + logger.info("Pass 1: computing valid indices...") + valid_indices = _compute_valid_indices( + masks=masks, + reference=reference, + min_valid_pixels=min_valid_pixels, + resolution_cutoff=resolution_cutoff, + ) + n_valid = len(valid_indices) + logger.info(" valid rows: %d / %d total", n_valid, len(masks)) + if n_valid == 0: + raise RuntimeError( + "No valid rows after filtering. " + "Check --min-valid-pixels and --resolution-cutoff." + ) + + # ── Duplicate refl_ids check (both modes) ───────────────────────────────── + refl_ids_valid = _to_np(reference["refl_ids"])[valid_indices].astype( + np.int64 + ) + unique_ids, id_counts = np.unique(refl_ids_valid, return_counts=True) + dups = id_counts > 1 + if dups.any(): + examples = unique_ids[dups][:3].tolist() + raise ValueError( + f"metadata.npy contains {int(dups.sum())} duplicate refl_id(s) " + f"in the valid set (examples: {examples}). " + "This indicates a data integrity issue in the make_shoeboxes output." + ) + + # ── Split assignment ─────────────────────────────────────────────────────── + if train_labels_path is not None: + # Mode A: read existing CSV split files. + # Rows absent from both CSVs (e.g. test reflections) receive + # is_train=False is_val=False and appear in predict-only; not an error. + logger.info("Pass 1: reading train/val split labels (Mode A — CSV)...") + train_ids_sorted = np.sort( + pl.read_csv(train_labels_path)["train_ids"] + .to_numpy() + .astype(np.int64) + ) + val_ids_sorted = np.sort( + pl.read_csv(val_labels_path)["val_ids"].to_numpy().astype(np.int64) + ) + overlap = np.intersect1d(train_ids_sorted, val_ids_sorted) + if len(overlap) > 0: + raise ValueError( + f"train_labels and val_labels share {len(overlap)} refl_ids. " + "They must not overlap. " + "Regenerate train_labels.csv / val_labels.csv." + ) + is_train = _vectorized_isin(refl_ids_valid, train_ids_sorted) + is_val = _vectorized_isin(refl_ids_valid, val_ids_sorted) + + n_train_missing = len(train_ids_sorted) - int(is_train.sum()) + n_val_missing = len(val_ids_sorted) - int(is_val.sum()) + if n_train_missing > 0: + logger.warning( + "%d train refl_ids absent from valid_indices " + "(filtered out by min_valid_pixels / resolution_cutoff).", + n_train_missing, + ) + if n_val_missing > 0: + logger.warning( + "%d val refl_ids absent from valid_indices " + "(filtered out by min_valid_pixels / resolution_cutoff).", + n_val_missing, + ) + split_meta: dict = { + "mode": "csv", + "train_labels": str(Path(train_labels_path).resolve()), + "val_labels": str(Path(val_labels_path).resolve()), + } + + else: + # Mode B: generate deterministic split. + # Every valid reflection is assigned to exactly one split by construction + # (is_train = ~is_val). + logger.info( + "Pass 1: generating deterministic split " + "(validation_split=%.3f seed=%d) (Mode B)...", + validation_split, + split_seed, + ) + is_train, is_val = _make_deterministic_split( + refl_ids_valid, + float(validation_split), + int(split_seed), + ) + split_meta = { + "mode": "generated", + "validation_split": float(validation_split), + "seed": int(split_seed), + } + + n_train = int(is_train.sum()) + n_val = int(is_val.sum()) + + # Overlap guard (both modes) + if (is_train & is_val).any(): + raise RuntimeError( + "Internal error: train and val sets overlap after split assignment." + ) + + # Coverage check — mode-specific behaviour + n_unassigned = int((~is_train & ~is_val).sum()) + if train_labels_path is not None: + # Mode A: unlabelled rows are permitted (test reflections etc.) + if n_unassigned > 0: + logger.info( + " %d reflection(s) in neither train nor val " + "(not present in CSV files — will appear in predict only).", + n_unassigned, + ) + else: + # Mode B: full coverage is mathematically guaranteed (is_train = ~is_val). + # Any non-zero count indicates a bug in _make_deterministic_split. + if n_unassigned != 0: + raise RuntimeError( + f"Generated split left {n_unassigned} valid reflections " + "unassigned. This should be impossible — please report a bug." + ) + + logger.info( + " train: %d val: %d other: %d", + n_train, + n_val, + n_unassigned, + ) + + # ── Resolution-bin labels — always computed (n_bins >= 1) ───────────────── + # group_label is required by _step() even when n_bins=1 (all-zeros array). + logger.info("Pass 1: computing group_labels (%d bin(s))...", n_bins) + group_label_all: np.ndarray + profile_group_label_all: np.ndarray | None = None + + gl_path = _nbins_path("group_labels.npy", n_bins, data_dir) + if data_path(gl_path) is not None: + logger.info(" loading existing group_labels from %s", gl_path.name) + group_label_all = _to_np(load_data(gl_path)).astype(np.int64) + else: + gl_tensor, _, actual = _bin_by_resolution( + torch.as_tensor(_to_np(reference["d"]), dtype=torch.float32), + n_bins, + ) + group_label_all = gl_tensor.numpy().astype(np.int64) + if actual != n_bins: + logger.warning( + "n_bins reduced %d -> %d (sparse resolution shells)", + n_bins, + actual, + ) + + pgl_path = _nbins_path("profile_group_labels.npy", n_bins, data_dir) + if data_path(pgl_path) is not None: + logger.info(" loading profile_group_labels from %s", pgl_path.name) + profile_group_label_all = _to_np(load_data(pgl_path)).astype(np.int64) + + # ── Classify metadata columns: numeric vs. string ───────────────────────── + # group_label and profile_group_label are handled separately (see below). + base_cols = [ + c + for c in DEFAULT_DS_COLS + if c in reference and c not in ("group_label", "profile_group_label") + ] + numeric_columns: list[str] = [] + string_columns: list[str] = [] + for col in base_cols: + if _is_string_col(reference[col]): + string_columns.append(col) + else: + numeric_columns.append(col) + + for col in _EXTRA_SAVE_COLS: + if ( + col in reference + and col not in numeric_columns + and col not in string_columns + ): + if _is_string_col(reference[col]): + string_columns.append(col) + else: + numeric_columns.append(col) + + # group_label always saved (required by training step even for n_bins=1) + numeric_columns.append("group_label") + if profile_group_label_all is not None: + numeric_columns.append("profile_group_label") + + # ── n_images: image_id only ─────────────────────────────────────────────── + # image_num is not suitable: it may have gaps or repeats across runs. + # nn.Embedding requires a compact 0-indexed integer (image_id). + # If image_id is absent n_images=None; train.py will raise a clear error + # at startup if image_level_wilson=True is configured. + n_images = None + if "image_id" in reference: + n_images = int(_to_np(reference["image_id"]).max()) + 1 + else: + logger.warning( + "image_id not found in metadata.npy; n_images will be None. " + "Training will fail at startup if image_level_wilson=True." + ) + + return ScanResult( + valid_indices=valid_indices, + refl_ids_valid=refl_ids_valid, + is_train=is_train, + is_val=is_val, + group_label=group_label_all, + profile_group_label=profile_group_label_all, + stats=(mean, var), + transform=transform, + numeric_columns=numeric_columns, + string_columns=string_columns, + n_images=n_images, + n_valid=n_valid, + n_train=n_train, + n_val=n_val, + split_meta=split_meta, + ) + + +# ── Pass 2: write chunks ────────────────────────────────────────────────────── + + +def _pass2_write_chunks( + scan: ScanResult, + counts, # numpy mmap + masks, # numpy mmap + reference: dict, + chunk_size: int, + out_dir: Path, +) -> list[dict]: + n_valid = scan.n_valid + n_chunks = (n_valid + chunk_size - 1) // chunk_size + mean, var = scan.stats + infos: list[dict] = [] + + has_image_id = "image_id" in reference + + for i in range(n_chunks): + start = i * chunk_size + end = min(start + chunk_size, n_valid) + rows = scan.valid_indices[start:end] + n = len(rows) + + logger.info("Pass 2: chunk %d / %d (%d rows)...", i + 1, n_chunks, n) + + counts_c = np.array(counts[rows], copy=True) + if counts_c.dtype == np.uint16: + counts_c = counts_c.astype(np.int32) + counts_f32 = counts_c.astype(np.float32) + masks_bool = np.array(masks[rows], copy=True).astype(bool) + shoebox_f32 = _apply_transform_np( + counts_f32, masks_bool, mean, var, scan.transform + ) + + d = out_dir / f"chunk_{i:05d}" + d.mkdir(parents=True, exist_ok=True) + + # ── Large pixel arrays — separate files (unchanged) ─────────────────── + np.save(d / "counts.npy", counts_f32) + np.save(d / "shoebox.npy", shoebox_f32) + np.save(d / "mask.npy", masks_bool) + + # ── All numeric + bool metadata — one consolidated file ─────────────── + meta_arrays: dict[str, np.ndarray] = { + "row_id": rows.astype(np.int64), + "is_train": scan.is_train[start:end], + "is_val": scan.is_val[start:end], + } + for col in scan.numeric_columns: + if col == "group_label": + meta_arrays[col] = scan.group_label[rows] + elif col == "profile_group_label": + meta_arrays[col] = scan.profile_group_label[rows] + else: + meta_arrays[col] = _to_np(reference[col])[rows] + np.savez(d / "metadata.npz", **meta_arrays) + + # ── String columns — one consolidated Parquet file ──────────────────── + if scan.string_columns: + str_data: dict[str, list[str]] = {} + for col in scan.string_columns: + raw = _to_np(reference[col]) + str_data[col] = [str(raw[r]) for r in rows] + pl.DataFrame(str_data).write_parquet(d / "strings.parquet") + + # ── Per-chunk image statistics ──────────────────────────────────────── + if has_image_id: + chunk_img_ids = _to_np(reference["image_id"])[rows] + unique_imgs = np.unique(chunk_img_ids) + n_unique_images: int | None = int(len(unique_imgs)) + min_image_id: int | None = int(unique_imgs.min()) + max_image_id: int | None = int(unique_imgs.max()) + else: + n_unique_images = None + min_image_id = None + max_image_id = None + + n_tr = int(scan.is_train[start:end].sum()) + n_va = int(scan.is_val[start:end].sum()) + infos.append( + { + "dir": f"chunk_{i:05d}", + "n_rows": n, + "n_train": n_tr, + "n_val": n_va, + "n_unique_images": n_unique_images, + "min_image_id": min_image_id, + "max_image_id": max_image_id, + } + ) + + return infos + + +# ── Manifest ────────────────────────────────────────────────────────────────── + + +def _write_manifest( + scan: ScanResult, + chunk_infos: list[dict], + chunk_size: int, + n_bins: int, + out_dir: Path, + data_dir: Path, +) -> Path: + """Write manifest.yaml to out_dir. + + data_dir is the original shoebox input directory (absolute path). + It is stored as source_data_dir so integrator.predict can locate + metadata.npy and dataset.yaml without a manual --data-dir argument. + """ + manifest = { + "version": 2, + "source_data_dir": str(data_dir.resolve()), + "chunk_size": chunk_size, + "n_chunks": len(chunk_infos), + "n_valid_rows": scan.n_valid, + "n_train": scan.n_train, + "n_val": scan.n_val, + "n_images": scan.n_images, + "transform": scan.transform, + "n_bins": n_bins, + "numeric_columns": scan.numeric_columns, + "string_columns": scan.string_columns, + "split": scan.split_meta, + "chunks": chunk_infos, + } + path = out_dir / "manifest.yaml" + with open(path, "w", encoding="utf-8") as f: + yaml.safe_dump(manifest, f, sort_keys=False) + return path + + +# ── Background prior ───────────────────────────────────────────────────────── + + +def _write_bg_prior( + data_dir: Path, + out_dir: Path, + scan: "ScanResult", + reference: dict, + n_bins: int, +) -> Path | None: + """Compute and save the empirical per-bin background Gamma prior. + + Uses the same fitting function (_fit_per_bin_gamma) and math as + prepare_per_bin_priors() in prepare_priors.py, ensuring that + chunked training uses an identical prior to the rotation_data workflow. + + Background column priority (matches prepare_per_bin_priors exactly): + 1. background.mean + 2. background.sum.value + + group_label indexing: uses scan.group_label (all rows, matching the + d-spacing arrays), consistent with how prepare_per_bin_priors operates + on all rows of metadata["d"] before valid-row filtering. + + Saves bg_prior_{n_bins}.npy to out_dir (= chunk_dir). + + Returns: + Path to the saved file, or None if no usable background column found. + """ + import torch as _torch + + from integrator.io import save_data + from integrator.utils.prepare_priors import _fit_per_bin_gamma, _nbins_path + + # Try background.mean first, then background.sum.value — same priority + # as prepare_per_bin_priors (prepare_priors.py). + bg_raw = None + for key in ("background.mean", "background.sum.value"): + v = reference.get(key) + if v is not None: + arr = _to_np(v).astype("float32") + if arr.ndim == 1 and len(arr) > 0: + bg_raw = arr + logger.info( + "bg_prior: using column '%s' (%d rows)", key, len(arr) + ) + break + + if bg_raw is None: + logger.warning( + "bg_prior: no usable background column found in metadata " + "(tried 'background.mean', 'background.sum.value'). " + "bg_prior_%d.npy will NOT be written. " + "Add bg_rate and bg_concentration explicitly to loss.args in your " + "config YAML to avoid a FileNotFoundError at training time.", + n_bins, + ) + return None + + bg_vals = _torch.as_tensor(bg_raw, dtype=_torch.float32) + + if int((bg_vals > 0).sum()) < 10: + logger.warning( + "bg_prior: fewer than 10 positive background values found. " + "bg_prior_%d.npy will NOT be written.", + n_bins, + ) + return None + + # actual n_bins after potential reduction in _pass1_scan + actual_n_bins = int(scan.group_label.max()) + 1 + group_labels_tensor = _torch.as_tensor( + scan.group_label.astype("int64"), dtype=_torch.long + ) + + alphas, rates = _fit_per_bin_gamma( + bg_vals, group_labels_tensor, actual_n_bins + ) + + payload: dict = ( + {"bg_concentration": alphas[0], "bg_rate": rates[0]} + if actual_n_bins == 1 + else {"bg_concentration": alphas, "bg_rate": rates} + ) + payload["n_bins"] = actual_n_bins + + prior_path = _nbins_path("bg_prior.npy", actual_n_bins, out_dir) + saved = save_data(payload, prior_path) + logger.info( + "bg_prior: saved bg_prior_%d.npy to %s " + "(concentration=%.4f rate=%.4f%s)", + actual_n_bins, + out_dir, + float(alphas[0]), + float(rates[0]), + f" [{actual_n_bins} bins]" if actual_n_bins > 1 else "", + ) + return saved + + +# ── Entry point ─────────────────────────────────────────────────────────────── + + +def main() -> None: + from integrator.data_loaders.data_module import ( + _load_shoebox_array, + _squeeze_last_axis, + ) + from integrator.io import load_metadata, read_dataset_spec + + args = parse_args() + + # ── Guard: n_bins ───────────────────────────────────────────────────────── + if args.n_bins < 1: + raise ValueError("--n-bins must be at least 1") + + # ── Determine split mode ────────────────────────────────────────────────── + has_csv = bool(args.train_labels) or bool(args.val_labels) + has_gen = args.validation_split is not None + + if has_csv and has_gen: + raise ValueError( + "Specify either (--train-labels + --val-labels) " + "OR --validation-split [--split-seed], not both." + ) + if bool(args.train_labels) != bool(args.val_labels): + raise ValueError( + "--train-labels and --val-labels must be provided together." + ) + + if has_csv: + split_mode = "csv" + else: + split_mode = "generated" + if args.validation_split is None: + args.validation_split = _DEFAULT_VALIDATION_SPLIT + if not (0.0 < args.validation_split < 1.0): + raise ValueError( + f"--validation-split must be in (0, 1), got {args.validation_split}" + ) + + logging.basicConfig( + level=[logging.WARNING, logging.INFO, logging.DEBUG][ + min(args.verbose, 2) + ], + format="%(levelname)s %(name)s %(message)s", + ) + + data_dir = Path(args.data_dir).resolve() + out_dir = ( + Path(args.out_dir).resolve() if args.out_dir else data_dir / "chunks" + ) + out_dir.mkdir(parents=True, exist_ok=True) + + logger.info("data_dir: %s", data_dir) + logger.info("out_dir: %s", out_dir) + logger.info("chunk_size: %d", args.chunk_size) + logger.info("split_mode: %s", split_mode) + + counts = _squeeze_last_axis(_load_shoebox_array(data_dir / "counts.npy")) + masks = _squeeze_last_axis(_load_shoebox_array(data_dir / "masks.npy")) + logger.info("mmaps: counts=%s masks=%s", counts.shape, masks.shape) + + spec = read_dataset_spec(data_dir) + if spec is None: + raise FileNotFoundError( + f"dataset.yaml not found in {data_dir}. " + "Run integrator.make_shoeboxes first." + ) + + transform = args.transform or ( + "anscombe" if spec.get("anscombe") else "standardization" + ) + logger.info("transform: %s", transform) + + reference = load_metadata(data_dir / "metadata.npy") + + # ── Pass 1 ─────────────────────────────────────────────────────────────── + scan = _pass1_scan( + data_dir=data_dir, + train_labels_path=args.train_labels, + val_labels_path=args.val_labels, + validation_split=args.validation_split, + split_seed=args.split_seed, + min_valid_pixels=args.min_valid_pixels, + resolution_cutoff=args.resolution_cutoff, + n_bins=args.n_bins, + transform=transform, + masks=masks, + reference=reference, + spec=spec, + ) + + # Mode B: export generated split labels alongside the chunks + if split_mode == "generated": + _save_split_csvs( + out_dir=out_dir, + refl_ids_valid=scan.refl_ids_valid, + is_train=scan.is_train, + is_val=scan.is_val, + ) + + n_chunks = (scan.n_valid + args.chunk_size - 1) // args.chunk_size + logger.info( + "Pass 1 complete: %d valid -> %d chunk(s) (train=%d val=%d other=%d)", + scan.n_valid, + n_chunks, + scan.n_train, + scan.n_val, + scan.n_valid - scan.n_train - scan.n_val, + ) + + # ── Pass 2 ─────────────────────────────────────────────────────────────── + chunk_infos = _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=args.chunk_size, + out_dir=out_dir, + ) + + manifest_path = _write_manifest( + scan=scan, + chunk_infos=chunk_infos, + chunk_size=args.chunk_size, + n_bins=args.n_bins, + out_dir=out_dir, + data_dir=data_dir, + ) + + # ── Empirical background Gamma prior ────────────────────────────────────── + # Fits the same Gamma MLE used by prepare_per_bin_priors() for rotation_data, + # preserving Luis's original Wilson ELBO background prior math. + # Saved as bg_prior_{n_bins}.npy in out_dir so the factory can load it at + # training time via _get_loss_module() in factory_utils.py. + _write_bg_prior( + data_dir=data_dir, + out_dir=out_dir, + scan=scan, + reference=reference, + n_bins=args.n_bins, + ) + + print(f"\nDone. {len(chunk_infos)} chunk(s) written to {out_dir}") + print( + f"Train: {scan.n_train:,} Val: {scan.n_val:,} " + f"Other: {scan.n_valid - scan.n_train - scan.n_val:,}" + ) + if split_mode == "generated": + print(f"Split labels: {out_dir}/train_labels.csv and val_labels.csv") + print(f"Manifest: {manifest_path}") + print(f"\nTo train, set in your config:") + print(f" data_loader:") + print(f" name: chunked_rotation_data") + print(f" args:") + print(f" chunk_dir: {out_dir}") + + +if __name__ == "__main__": + main() diff --git a/src/integrator/cli/train.py b/src/integrator/cli/train.py index d3049142..79f5fe21 100644 --- a/src/integrator/cli/train.py +++ b/src/integrator/cli/train.py @@ -308,6 +308,7 @@ def main(): construct_data_loader, construct_integrator, construct_trainer, + fit_wilson_bg_from_chunks, inject_binning_labels, load_config, prepare_per_bin_priors, @@ -332,10 +333,31 @@ def main(): # (prepare_per_bin_priors logs each file action itself) prepare_per_bin_priors(cfg) + # Fit per-image Wilson B/G from cctbx intensities (lazy, cached). + # No-op unless loss.args.image_level_wilson=true AND wilson_bg_init=cctbx. + fit_wilson_bg_from_chunks(cfg) + data_loader = construct_data_loader(cfg) + + # we already calculate n_images inside this RotationDataModule.setup() function + data_loader.setup() inject_binning_labels(data_loader, cfg) + loss_args = cfg.setdefault("loss", {}).setdefault("args", {}) + if loss_args.get("image_level_wilson", False): + if data_loader.n_images is None: + raise ValueError( + "image_level_wilson=True requires image_id metadata." + ) + + loss_args["n_images"] = data_loader.n_images + + logger.info( + "Detected n_image=%d from the data module", + data_loader.n_images, + ) + tags = [ cfg["integrator"]["name"], cfg["integrator"]["args"]["data_dim"], diff --git a/src/integrator/data_loaders/__init__.py b/src/integrator/data_loaders/__init__.py index 9cb2c301..f7122a16 100644 --- a/src/integrator/data_loaders/__init__.py +++ b/src/integrator/data_loaders/__init__.py @@ -1,7 +1,9 @@ from .data_module import RotationDataModule from .poly_data_module import PolychromaticDataModule +from .chunked_data_module import ChunkedDataModule __all__ = [ "RotationDataModule", "PolychromaticDataModule", + "ChunkedDataModule", ] diff --git a/src/integrator/data_loaders/chunked_data_module.py b/src/integrator/data_loaders/chunked_data_module.py new file mode 100644 index 00000000..0e645cb2 --- /dev/null +++ b/src/integrator/data_loaders/chunked_data_module.py @@ -0,0 +1,436 @@ +""" +ChunkedDataModule — loads one pre-processed chunk at a time. + +Chunk layout (produced by integrator.preprocess) +───────────────────────────────────────────────── +chunk_NNNNN/ + counts.npy float32 (N, pixels) raw pixel counts + shoebox.npy float32 (N, pixels) transform-applied (model input) + mask.npy bool (N, pixels) valid-pixel mask + metadata.npz all numeric + bool columns: + row_id, is_train, is_val, image_id, d, lp, + group_label, profile_group_label, refl_ids, ... + strings.parquet all string columns (source_refl, source_expt, ...); + omitted entirely when no string columns exist + +manifest.yaml (authoritative schema, version 2) + version: 2 + numeric_columns: [image_id, d, group_label, ...] # list of column names + string_columns: [source_refl, source_expt] # list of column names + chunks: + - dir: chunk_00000 + n_rows: 50000 + n_train: ... + n_val: ... + n_unique_images: 42 # unique image_id values in this chunk + min_image_id: 0 # inclusive (null when image_id absent) + max_image_id: 199 # inclusive (null when image_id absent) + +Batch design +──────────── +ChunkedDataset yields complete mini-batches (not individual reflections). +The DataLoader is created with batch_size=None so items from __iter__ are +returned as-is without additional collation overhead. This eliminates the +per-reflection Python loop that would otherwise dominate CPU time. + +Each mini-batch contains rows from a single chunk only. Shuffling is applied +at two levels during training: + 1. chunk order is randomised each epoch (shuffle_chunks=True) + 2. rows within each chunk are shuffled before batching (shuffle_within=True) +Cross-chunk batch mixing (requiring a multi-chunk buffer) is deferred. + +Worker distribution +─────────────────── +With num_workers > 0 each worker receives a disjoint subset of chunk +directories (round-robin by index) so no chunk is processed more than once +per epoch. persistent_workers=False is required: with IterableDataset, +worker iterators are not reliably reset between epochs when persistent. + +Prediction ordering +─────────────────── +predict_dataloader() forces num_workers=0. With IterableDataset and +num_workers > 0 the DataLoader interleaves batches from workers in an +undefined order, breaking ID-preserving write-back. +""" + +from __future__ import annotations + +import logging +import random +from pathlib import Path + +import numpy as np +import polars as pl +from pytorch_lightning import LightningDataModule +import torch +import yaml +from torch.utils.data import DataLoader, IterableDataset + +logger = logging.getLogger(__name__) + + +class ChunkedDataset(IterableDataset): + """Yields pre-formed mini-batches from chunk directories. + + Each call to __iter__ loads one chunk directory at a time into RAM, + applies optional row-filtering and shuffling, then slices into mini-batches + using numpy — one torch conversion per batch, not per reflection. + + Each mini-batch contains rows from a single chunk. Training shuffles + chunk order each epoch and rows within each chunk before batching. + Validation and prediction do not shuffle. + """ + + def __init__( + self, + chunk_dirs: list[Path], + numeric_columns: list[ + str + ], # ordered list of numeric/bool column names + string_columns: list[str], # ordered list of string column names + filter_key: str | None, # "is_train" | "is_val" | None (all rows) + shuffle_chunks: bool = False, + shuffle_within: bool = False, + batch_size: int = 32, + ): + self.chunk_dirs = list(chunk_dirs) + self.numeric_columns = list(numeric_columns) + self.string_columns = list(string_columns) + self.filter_key = filter_key + self.shuffle_chunks = shuffle_chunks + self.shuffle_within = shuffle_within + self.batch_size = batch_size + + @staticmethod + def _load_chunk( + chunk_dir: Path, + numeric_columns: list[str], + string_columns: list[str], + ) -> dict: + """Load all arrays for one chunk directory into RAM. + + Reads: + counts.npy — float32 (N, pixels) + shoebox.npy — float32 (N, pixels) + mask.npy — bool (N, pixels) + metadata.npz — all numeric + bool columns (row_id, is_train, is_val, + and every name in numeric_columns). Loaded via a + context manager; arrays are .copy()-ed before the + file handle is released. + strings.parquet — all string columns. Raises FileNotFoundError if + string_columns is non-empty but the file is absent. + + Raises: + FileNotFoundError: strings.parquet missing with non-empty string_columns. + KeyError: metadata.npz is missing a required key (is_train, is_val, + row_id, or any column in numeric_columns). + ValueError: shoebox.npy or mask.npy first-dimension does not match + counts.npy; any array length does not match counts.shape[0]; + or strings.parquet is missing a column listed in + string_columns. + """ + d = chunk_dir + chunk: dict = { + "counts": np.load(d / "counts.npy"), + "shoebox": np.load(d / "shoebox.npy"), + "mask": np.load(d / "mask.npy"), + } + + # Validate that shoebox.npy and mask.npy have the same first-dimension + # row count as counts.npy. Catches truncated pixel-array writes. + n_rows = chunk["counts"].shape[0] + for key in ("shoebox", "mask"): + if chunk[key].shape[0] != n_rows: + raise ValueError( + f"Shape mismatch in {d}: " + f"counts has {n_rows} rows but '{key}.npy' has " + f"{chunk[key].shape[0]} rows." + ) + + # Load all numeric + bool metadata. + # Context manager closes the NpzFile promptly; .copy() detaches each + # array from the underlying mmap before the file handle is released. + with np.load(d / "metadata.npz", allow_pickle=False) as meta: + # Verify all required keys are present before accessing any of them. + required_keys = {"is_train", "is_val", "row_id", *numeric_columns} + missing_keys = required_keys - set(meta.files) + if missing_keys: + raise KeyError( + f"metadata.npz in {d} is missing key(s): " + f"{sorted(missing_keys)}.\n" + "Re-run integrator.preprocess to regenerate chunks." + ) + chunk["is_train"] = meta["is_train"].copy() + chunk["is_val"] = meta["is_val"].copy() + chunk["row_id"] = meta["row_id"].copy() + for col in numeric_columns: + chunk[col] = meta[col].copy() + + # Load string columns. Raise immediately if the file is declared in + # the manifest but physically absent (e.g. truncated preprocess run). + strings_path = d / "strings.parquet" + if string_columns: + if not strings_path.exists(): + raise FileNotFoundError( + f"strings.parquet not found in {d} but " + f"string_columns={string_columns!r} are declared in " + "manifest.yaml.\n" + "Re-run integrator.preprocess to regenerate chunks." + ) + str_df = pl.read_parquet(strings_path) + # Verify every declared string column exists in the file. + missing_cols = [ + col for col in string_columns if col not in str_df.columns + ] + if missing_cols: + raise ValueError( + f"strings.parquet in {d} is missing column(s): " + f"{missing_cols}.\n" + "Re-run integrator.preprocess to regenerate chunks." + ) + for col in string_columns: + chunk[col] = str_df[col].to_list() + + # Validate all lengths against counts.shape[0]. + # Catches truncated writes (disk-full or interrupted preprocess run). + for key in ("is_train", "is_val", "row_id", *numeric_columns): + if len(chunk[key]) != n_rows: + raise ValueError( + f"Length mismatch in {d}: " + f"counts has {n_rows} rows but '{key}' has " + f"{len(chunk[key])} rows." + ) + for col in string_columns: + if len(chunk[col]) != n_rows: + raise ValueError( + f"Length mismatch in {d}: " + f"counts has {n_rows} rows but string column '{col}' has " + f"{len(chunk[col])} rows." + ) + + return chunk + + def __iter__(self): + worker_info = torch.utils.data.get_worker_info() + dirs = list(self.chunk_dirs) + + # Each chunk assigned to exactly one worker — no duplication. + if worker_info is not None: + dirs = [ + d + for j, d in enumerate(dirs) + if j % worker_info.num_workers == worker_info.id + ] + + if self.shuffle_chunks: + random.shuffle(dirs) + + nc = self.numeric_columns + sc = self.string_columns + bs = self.batch_size + + for chunk_dir in dirs: + chunk = self._load_chunk(chunk_dir, nc, sc) + n = len(chunk["counts"]) + + if self.filter_key is not None: + idx = np.flatnonzero(chunk[self.filter_key].astype(bool)) + else: + idx = np.arange(n) + + if self.shuffle_within: + idx = idx[np.random.permutation(len(idx))] + + # Yield mini-batches as numpy slices — one torch conversion per batch. + for b_start in range(0, len(idx), bs): + b_idx = idx[b_start : b_start + bs] + + counts_b = torch.from_numpy(chunk["counts"][b_idx]).float() + shoebox_b = torch.from_numpy(chunk["shoebox"][b_idx]).float() + mask_b = torch.from_numpy(chunk["mask"][b_idx]).bool() + + meta_b: dict = {} + for col in nc: + # dtype preserved: int64->torch.int64, float32->torch.float32 + meta_b[col] = torch.as_tensor(chunk[col][b_idx]) + for col in sc: + meta_b[col] = [chunk[col][i] for i in b_idx] # list[str] + + # row_id always included for traceability + meta_b["row_id"] = torch.as_tensor(chunk["row_id"][b_idx]) + + yield counts_b, shoebox_b, mask_b, meta_b + + +class ChunkedDataModule(LightningDataModule): + """LightningDataModule that loads pre-processed chunks one at a time. + + Requires a chunk directory produced by ``integrator.preprocess`` (v6 layout, + manifest version 2). The train/val split is baked into each chunk's + metadata.npz arrays (is_train, is_val) and is never re-randomised at + training time. + + YAML config example:: + + data_loader: + name: chunked_rotation_data + args: + chunk_dir: /data/mfx/chunks + batch_size: 256 + num_workers: 4 + """ + + # Explicit opt-in sentinel checked by assign_labels() in prediction_writer.py. + # assign_labels() is always called with the DataModule directly (train.py:424). + # ChunkedDataModule handles its own train/val split at preprocess time, so the + # expensive refl_id scan in assign_labels() must be skipped entirely. + _CHUNKED_DATA_MODULE: bool = True + + def __init__( + self, + chunk_dir: str | Path, + batch_size: int = 10, + num_workers: int = 0, + ): + super().__init__() + self.chunk_dir = Path(chunk_dir) + self.batch_size = batch_size + self.num_workers = num_workers + + # Set in setup(); numeric_columns / string_columns exposed so + # inject_binning_labels() can verify group_label is present before + # training starts. + self.chunk_dirs: list[Path] = [] + self.numeric_columns: list[str] = [] + self.string_columns: list[str] = [] + self.n_images: int | None = None + + def setup(self, stage: str | None = None) -> None: + manifest_path = self.chunk_dir / "manifest.yaml" + if not manifest_path.exists(): + raise FileNotFoundError( + f"No manifest.yaml found in {self.chunk_dir}.\n" + "Run integrator.preprocess first." + ) + m = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + + # Require manifest version 2 (v6 layout: metadata.npz + strings.parquet). + # Version 1 uses per-column col_*.npy / str_*.txt files which are no + # longer supported. Fail fast rather than crashing mid-epoch with an + # obscure FileNotFoundError. + version = m.get("version", 1) + if version != 2: + raise ValueError( + f"manifest.yaml declares version {version}; version 2 is " + "required.\nRe-run integrator.preprocess to regenerate " + "chunks with the v6 layout." + ) + + string_columns = list(m.get("string_columns", [])) + required_files = [ + "counts.npy", + "shoebox.npy", + "mask.npy", + "metadata.npz", + ] + if string_columns: + required_files.append("strings.parquet") + + logger.info( + "ChunkedDataModule.setup: verifying %d chunk(s) in %s ...", + len(m["chunks"]), + self.chunk_dir, + ) + for entry in m["chunks"]: + cdir = self.chunk_dir / entry["dir"] + for fname in required_files: + if not (cdir / fname).exists(): + raise FileNotFoundError( + f"Required file '{fname}' missing from {cdir}.\n" + "Re-run integrator.preprocess to regenerate chunks." + ) + + self.chunk_dirs = [self.chunk_dir / e["dir"] for e in m["chunks"]] + self.numeric_columns = list(m.get("numeric_columns", [])) + self.string_columns = string_columns + self.n_images = m.get("n_images") + + logger.info( + "ChunkedDataModule: %d chunk(s) total=%s train=%s val=%s n_images=%s", + len(self.chunk_dirs), + m.get("n_valid_rows", "?"), + m.get("n_train", "?"), + m.get("n_val", "?"), + self.n_images, + ) + + def _make_loader( + self, + filter_key: str | None, + shuffle_chunks: bool, + shuffle_within: bool, + num_workers: int, + string_columns: list[str] | None = None, + ) -> DataLoader: + # Allow callers to override which string columns are loaded. + # Train and val pass [] to skip strings.parquet entirely (those columns + # are only needed during prediction for write-back). Predict passes + # self.string_columns to get source_refl / source_expt. + sc = self.string_columns if string_columns is None else string_columns + ds = ChunkedDataset( + chunk_dirs=sorted(self.chunk_dirs), + numeric_columns=self.numeric_columns, + string_columns=sc, + filter_key=filter_key, + shuffle_chunks=shuffle_chunks, + shuffle_within=shuffle_within, + batch_size=self.batch_size, + ) + return DataLoader( + ds, + batch_size=None, # ChunkedDataset yields complete batches + num_workers=num_workers, + pin_memory=False, # pin_memory=True causes _PinMemoryThread deadlock + # at epoch transitions with IterableDataset; + # host→GPU transfer time is negligible for this + # workload (disk-I/O + GPU-compute bound). + # persistent_workers=True for num_workers>0: workers live for the full + # training run and reset their iterator at each epoch, avoiding the + # fork-after-CUDA deadlock that occurs when workers are killed and + # respawned at every epoch boundary with persistent_workers=False. + # Must be False for num_workers=0 (no workers to persist). + persistent_workers=num_workers > 0, + ) + + def train_dataloader(self) -> DataLoader: + return self._make_loader( + filter_key="is_train", + shuffle_chunks=True, + shuffle_within=True, + num_workers=self.num_workers, + string_columns=[], # source_refl/source_expt unused during training + ) + + def val_dataloader(self) -> DataLoader: + return self._make_loader( + filter_key="is_val", + shuffle_chunks=False, + shuffle_within=False, + num_workers=self.num_workers, + string_columns=[], # source_refl/source_expt unused during validation + ) + + def predict_dataloader(self) -> DataLoader: + """All valid rows in deterministic sorted-chunk order. + + num_workers is forced to 0: with IterableDataset and num_workers > 0 + the DataLoader interleaves batches from workers in an undefined order, + breaking the ordering guarantee required for ID-preserving write-back. + """ + return self._make_loader( + filter_key=None, + shuffle_chunks=False, + shuffle_within=False, + num_workers=0, + string_columns=self.string_columns, # needed for write-back + ) diff --git a/src/integrator/data_loaders/data_module.py b/src/integrator/data_loaders/data_module.py index d63a7d6f..7ec6f7e5 100644 --- a/src/integrator/data_loaders/data_module.py +++ b/src/integrator/data_loaders/data_module.py @@ -5,30 +5,124 @@ import numpy as np import pytorch_lightning as pl import torch -from torch.utils.data import ( - DataLoader, - Dataset, - Subset, -) +from torch.utils.data import DataLoader, Dataset logger = logging.getLogger(__name__) +# LAZY-READING EDIT: read .npy files with mmap_mode='r' so the full arrays +# stay on disk instead of being copied into CPU RAM at setup time. def _load_shoebox_array(path, weights_only=True): - """Load counts/masks from either new .npy or .pt. Returns torch.Tensor.""" + """Load counts/masks from .npy lazily, or fall back to eager .pt loading.""" p = Path(path) npy = p.with_suffix(".npy") if npy.exists(): - arr = np.load(npy) - if arr.dtype == np.uint16: - arr = arr.astype(np.int32) - return torch.from_numpy(arr) + return np.load(npy, mmap_mode="r") + + logger.warning( + "Lazy loading is only available for .npy files; loading %s eagerly.", p + ) try: return torch.load(p, weights_only=weights_only) except TypeError: return torch.load(p) +def _squeeze_last_axis(array): + """Remove a trailing singleton axis without materializing the whole array.""" + if array.ndim > 0 and array.shape[-1] == 1: + return array[..., 0] + return array + + +def _metadata_value_at(value, idx): + """Read one metadata value while preserving tensors when possible.""" + return value[idx] + + +def _metadata_numpy(value): + """Return a NumPy view/copy suitable for indexing and boolean tests.""" + if isinstance(value, torch.Tensor): + return value.detach().cpu().numpy() + return np.asarray(value) + + +def _compute_valid_indices( + masks, + reference, + min_valid_pixels, + resolution_cutoff, + chunk_size=250_000, +): + """Build valid row indices in chunks without loading all masks into RAM.""" + n_rows = len(masks) + variance_key = None + if "intensity.prf.variance" in reference: + variance_key = "intensity.prf.variance" + elif "intensity.sum.variance" in reference: + variance_key = "intensity.sum.variance" + + variance = ( + _metadata_numpy(reference[variance_key]) if variance_key is not None else None + ) + d_values = ( + _metadata_numpy(reference["d"]) + if resolution_cutoff is not None + else None + ) + + valid_chunks = [] + n_dead = 0 + n_bad_variance = 0 + n_cut = 0 + + # LAZY-READING EDIT: inspect only a manageable slice of masks at a time. + for start in range(0, n_rows, chunk_size): + stop = min(start + chunk_size, n_rows) + mask_chunk = np.asarray(masks[start:stop]) + keep = mask_chunk.sum(axis=-1) >= min_valid_pixels + n_dead += int((~keep).sum()) + + if variance is not None: + variance_ok = variance[start:stop] >= 0 + n_bad_variance += int((~variance_ok).sum()) + keep &= variance_ok + + if d_values is not None: + resolution_ok = d_values[start:stop] < resolution_cutoff + n_cut += int((~resolution_ok).sum()) + keep &= resolution_ok + + local = np.flatnonzero(keep).astype(np.int64, copy=False) + if local.size: + valid_chunks.append(local + start) + + if n_dead: + logger.info( + "Removed %d reflections with < %d valid pixels", + n_dead, + min_valid_pixels, + ) + if variance_key is not None and n_bad_variance: + logger.info( + "Removed %d reflections with %s < 0", + n_bad_variance, + variance_key, + ) + if resolution_cutoff is not None and n_cut: + logger.info( + "Removed %d reflections with d >= %.2f", + n_cut, + resolution_cutoff, + ) + if variance_key is None: + logger.info("No intensity variance key found; skipping variance filtering") + + if not valid_chunks: + return np.empty(0, dtype=np.int64) + return np.concatenate(valid_chunks) + + # Default columns from rs.io.read_dials_stills DEFAULT_DS_COLS = [ "zeta", @@ -88,6 +182,9 @@ def _load_shoebox_array(path, weights_only=True): "is_coset", "group_label", "profile_group_label", + "image_num", + "image_id", + "n_images", ] @@ -95,60 +192,102 @@ class IntegratorDataset(Dataset): def __init__( self, counts, - standardized_counts, masks, reference, + valid_indices, + stats, + transform, column_names: list = DEFAULT_DS_COLS, ): self.counts = counts - self.standardized_counts = standardized_counts self.masks = masks self.reference = reference + self.valid_indices = np.asarray(valid_indices, dtype=np.int64) + self.stats = torch.as_tensor(stats, dtype=torch.float32) + self.transform = transform self.column_names = column_names def __len__(self): - return len(self.counts) + return len(self.valid_indices) + + def _transform_counts(self, counts, masks): + """Apply the configured transform to one reflection only.""" + stats = self.stats + + if counts.dim() == 1: + if self.transform == "anscombe": + transformed = 2 * (counts.clamp(min=0) + 0.375).sqrt() + return ((transformed - stats[0]) / stats[1].sqrt()) * masks + + if self.transform == "log1p": + return torch.log1p(counts.clamp(min=0)) * masks + + if self.transform == "asinh": + scale = stats[1].sqrt().clamp(min=1e-8) + return torch.asinh(counts / scale) * masks + + if self.transform == "log_softplus": + return torch.log(torch.nn.functional.softplus(counts) + 1e-8) * masks + + if self.transform == "sqrt_squareplus": + b = 4.0 + squareplus = 0.5 * (counts + torch.sqrt(counts * counts + b)) + return torch.sqrt(squareplus + 1e-8) * masks + + return ((counts * masks) - stats[0]) / stats[1].sqrt() + + # Preserve the original behavior for arrays with extra feature channels. + standardized = ((counts[..., -1] * masks) - stats[0]) / stats[1].sqrt() + counts = counts.clone() + if counts.dim() >= 2 and counts.size(-1) >= 3: + for channel in range(3): + channel_max = counts[..., channel].max().clamp(min=1e-8) + counts[..., channel] = 2 * (counts[..., channel] / channel_max) - 1 + return standardized def __getitem__(self, idx): if torch.is_tensor(idx): idx = idx.item() - counts = self.counts[idx] - standardized_counts = self.standardized_counts[idx] - masks = self.masks[idx] + source_idx = int(self.valid_indices[idx]) + + # LAZY-READING EDIT: copy only this one row from the disk-backed arrays. + # The DataLoader later combines individual rows into a batch. + counts_np = np.array(self.counts[source_idx], copy=True) + masks_np = np.array(self.masks[source_idx], copy=True) + + if counts_np.dtype == np.uint16: + counts_np = counts_np.astype(np.int32, copy=False) + + counts = torch.from_numpy(counts_np).to(torch.float32) + masks = torch.from_numpy(masks_np).bool() + standardized_counts = self._transform_counts(counts, masks) meta = { - k: self.reference[k][idx] - for k in self.column_names - if k in self.reference + key: _metadata_value_at(self.reference[key], source_idx) + for key in self.column_names + if key in self.reference } return counts, standardized_counts, masks, meta -# Filter to remove reflections with variance = -1 -def _remove_flagged_variance( - counts: torch.Tensor, - masks: torch.Tensor, - metadata: dict, - filter_key: str = "intensity.prf.variance", -) -> tuple[torch.Tensor, torch.Tensor, dict]: - bad = metadata[filter_key] < 0 - n_bad = bad.sum().item() - if n_bad > 0: - logger.info("Removed %d reflections with %s < 0", n_bad, filter_key) - counts = counts[~bad] - masks = masks[~bad] - metadata = {k: v[~bad] for k, v in metadata.items()} - return counts, masks, metadata +class IndexedDataset(Dataset): + """Memory-efficient subset that stores NumPy indices instead of Python lists.""" + def __init__(self, dataset, indices): + self.dataset = dataset + self.indices = np.asarray(indices, dtype=np.int64) -class RotationDataModule(pl.LightningDataModule): - """LightningDataModule for rotation-geometry shoebox data. + def __len__(self): + return len(self.indices) + + def __getitem__(self, idx): + return self.dataset[int(self.indices[idx])] - Attributes: - transform: Count transform `anscombe`, `log1p`, or `standardization`. - """ + +class RotationDataModule(pl.LightningDataModule): + """LightningDataModule for rotation-geometry shoebox data.""" def __init__( self, @@ -173,6 +312,8 @@ def __init__( self.resolution_cutoff = resolution_cutoff self.min_valid_pixels = min_valid_pixels self.full_dataset = None + self.n_images = None + if shoebox_file_names is None: shoebox_file_names = { "counts": "counts.npy", @@ -180,21 +321,36 @@ def __init__( "reference": "metadata.npy", } self.shoebox_file_names = shoebox_file_names + transform = transform or "standardization" - if transform not in ("anscombe", "log1p", "standardization"): + if transform not in ( + "anscombe", + "log1p", + "standardization", + "asinh", + "log_softplus", + "sqrt_squareplus", + ): raise ValueError( - f"transform must be 'anscombe', 'log1p', or 'standardization'; " + f"transform must be 'anscombe', 'log1p', 'standardization', " + f"'asinh', 'log_softplus', or 'sqrt_squareplus'; " f"got {transform!r}" ) self.transform = transform def setup(self, stage=None): - counts = _load_shoebox_array( - os.path.join(self.data_dir, self.shoebox_file_names["counts"]) - ).squeeze(-1) - masks = _load_shoebox_array( - os.path.join(self.data_dir, self.shoebox_file_names["masks"]) - ).squeeze(-1) + # LAZY-READING EDIT: these are NumPy memory maps, not full RAM copies. + counts = _squeeze_last_axis( + _load_shoebox_array( + os.path.join(self.data_dir, self.shoebox_file_names["counts"]) + ) + ) + masks = _squeeze_last_axis( + _load_shoebox_array( + os.path.join(self.data_dir, self.shoebox_file_names["masks"]) + ) + ) + from integrator.io import load_metadata, read_dataset_spec spec = read_dataset_spec(self.data_dir) @@ -203,105 +359,73 @@ def setup(self, stage=None): f"dataset.yaml not found in {self.data_dir}; " "regenerate the dataset with mksbox" ) + stats_key = "anscombe" if self.transform == "anscombe" else "raw" stats = torch.tensor(spec["stats"][stats_key], dtype=torch.float32) + reference = load_metadata( os.path.join(self.data_dir, self.shoebox_file_names["reference"]) ) - # Filter out reflections with too few valid pixels - all_dead = masks.sum(-1) < self.min_valid_pixels - n_dead = all_dead.sum().item() - if n_dead > 0: - logger.info( - "Removed %d reflections with < %d valid pixels", - n_dead, - self.min_valid_pixels, + if "image_id" in reference: + reference["image_id"] = torch.as_tensor( + reference["image_id"], dtype=torch.long ) - counts = counts[~all_dead] - masks = masks[~all_dead] - reference = {k: v[~all_dead] for k, v in reference.items()} - - counts, masks, reference = _remove_flagged_variance( - counts, masks, reference - ) - - # Apply resolution cutoff before standardization - if self.resolution_cutoff is not None: - selection = reference["d"] < self.resolution_cutoff - n_cut = (~selection).sum().item() - if n_cut > 0: - logger.info( - "Removed %d reflections with d >= %.2f", - n_cut, - self.resolution_cutoff, - ) - counts = counts[selection] - masks = masks[selection] - reference = {k: v[selection] for k, v in reference.items()} - - if counts.dim() == 2: - if self.transform == "anscombe": - anscombe_transformed = 2 * (counts.clamp(min=0) + 0.375).sqrt() - standardized_counts = ( - (anscombe_transformed - stats[0]) / stats[1].sqrt() - ) * masks - elif self.transform == "log1p": - standardized_counts = torch.log1p(counts.clamp(min=0)) * masks - else: - standardized_counts = ((counts * masks) - stats[0]) / stats[ - 1 - ].sqrt() + self.n_images = int(reference["image_id"].max().item()) + 1 else: - standardized_counts = ( - (counts[..., -1] * masks) - stats[0] - ) / stats[1].sqrt() - if counts.dim() >= 3 and counts.size(-1) >= 3: - counts[:, :, 0] = ( - 2 * (counts[:, :, 0] / (counts[:, :, 0].max() + 1e-8)) - 1 - ) - counts[:, :, 1] = ( - 2 * (counts[:, :, 1] / (counts[:, :, 1].max() + 1e-8)) - 1 - ) - counts[:, :, 2] = ( - 2 * (counts[:, :, 2] / (counts[:, :, 2].max() + 1e-8)) - 1 - ) + self.n_images = None + + # LAZY-READING EDIT: keep the original arrays untouched and retain only + # integer row indices for reflections that pass the filters. + valid_indices = _compute_valid_indices( + masks=masks, + reference=reference, + min_valid_pixels=self.min_valid_pixels, + resolution_cutoff=self.resolution_cutoff, + ) self.full_dataset = IntegratorDataset( - counts, - standardized_counts, - masks, - reference, + counts=counts, + masks=masks, + reference=reference, + valid_indices=valid_indices, + stats=stats, + transform=self.transform, ) - # Split using is_test if available + all_indices = np.arange(len(self.full_dataset), dtype=np.int64) + + if self.subset_size is not None and self.subset_size < len(all_indices): + all_indices = np.random.default_rng().choice( + all_indices, + size=self.subset_size, + replace=False, + ) + is_test = reference.get("is_test") - all_indices = torch.arange(len(self.full_dataset)) + if is_test is not None: + source_indices = valid_indices[all_indices] + is_test_values = _metadata_numpy(is_test)[source_indices].astype(bool) + else: + is_test_values = None - if self.subset_size is not None and self.subset_size < len( - self.full_dataset - ): - all_indices = all_indices[ - torch.randperm(len(all_indices))[: self.subset_size] - ] - - if is_test is not None and is_test.any(): - test_mask = is_test[all_indices].bool() - test_idx = all_indices[test_mask] - train_val_idx = all_indices[~test_mask] + if is_test_values is not None and is_test_values.any(): + test_idx = all_indices[is_test_values] + train_val_idx = all_indices[~is_test_values] else: - test_idx = torch.tensor([], dtype=torch.long) + test_idx = np.empty(0, dtype=np.int64) train_val_idx = all_indices - self.test_dataset = Subset(self.full_dataset, test_idx.tolist()) + self.test_dataset = IndexedDataset(self.full_dataset, test_idx) - perm = torch.randperm(len(train_val_idx)) + # This permutation is only an int64 index array, not a copy of counts. + perm = np.random.default_rng().permutation(len(train_val_idx)) val_size = int(len(train_val_idx) * self.validation_split) val_idx = train_val_idx[perm[:val_size]] train_idx = train_val_idx[perm[val_size:]] - self.val_dataset = Subset(self.full_dataset, val_idx.tolist()) - self.train_dataset = Subset(self.full_dataset, train_idx.tolist()) + self.val_dataset = IndexedDataset(self.full_dataset, val_idx) + self.train_dataset = IndexedDataset(self.full_dataset, train_idx) def train_dataloader(self): return DataLoader( @@ -310,6 +434,7 @@ def train_dataloader(self): shuffle=True, num_workers=self.num_workers, pin_memory=True, + persistent_workers=self.num_workers > 0, ) def val_dataloader(self): @@ -319,6 +444,7 @@ def val_dataloader(self): shuffle=False, num_workers=self.num_workers, pin_memory=True, + persistent_workers=self.num_workers > 0, ) def test_dataloader(self): @@ -329,9 +455,9 @@ def test_dataloader(self): shuffle=False, num_workers=self.num_workers, pin_memory=True, + persistent_workers=self.num_workers > 0, ) - else: - return None + return None def predict_dataloader(self): return DataLoader( @@ -340,4 +466,5 @@ def predict_dataloader(self): shuffle=False, num_workers=self.num_workers, pin_memory=True, + persistent_workers=self.num_workers > 0, ) diff --git a/src/integrator/io/__init__.py b/src/integrator/io/__init__.py index f22dd5ce..0bbb2cf9 100644 --- a/src/integrator/io/__init__.py +++ b/src/integrator/io/__init__.py @@ -17,8 +17,26 @@ save_data, ) from .mtz_io import write_mtz_from_preds -from .pred_io import get_pred_files, write_refl_from_preds -from .refl_io import unstack_preds, write_refl_with_predictions + + +#needed for writing predictions back into .refl files. +try: + from .refl_io import unstack_preds, write_refl_with_predictions +except ModuleNotFoundError as e: + if e.name != "dials": + raise + + unstack_preds = None + write_refl_with_predictions = None + +try: + from .pred_io import get_pred_files, write_refl_from_preds +except ModuleNotFoundError as e: + if e.name != "dials": + raise + + get_pred_files = None + write_refl_from_preds = None __all__ = [ "DEFAULT_REFL_COLS", @@ -38,4 +56,4 @@ "read_dataset_spec", "write_dataset_yaml", "data_dim_for", -] +] \ No newline at end of file diff --git a/src/integrator/io/metadata.py b/src/integrator/io/metadata.py index 86fd2649..a01c1088 100644 --- a/src/integrator/io/metadata.py +++ b/src/integrator/io/metadata.py @@ -43,24 +43,38 @@ def data_path(path) -> Path | None: return p if p.exists() else None +def is_numeric_array(v): + arr = np.asarray(v) + #it is numeric if this is true + return arr.dtype.kind not in {"O", "U", "S"} + + def load_data(path, map_location="cpu"): - """Load a tensor or dict-of-tensors""" + """Load a tensor or dict-of-tensors/string arrays.""" target = data_path(path) or Path(path) + if target.suffix == ".npy": arr = np.load(target, allow_pickle=True) + if arr.dtype == object: obj = arr.item() + if isinstance(obj, dict): - return {k: torch.as_tensor(v) for k, v in obj.items()} + return { + k: torch.as_tensor(np.asarray(v)) + if is_numeric_array(v) + else np.asarray(v) + for k, v in obj.items() + } + return torch.as_tensor(obj) + return torch.as_tensor(arr) + try: return torch.load(target, weights_only=True, map_location=map_location) except Exception: - return torch.load( - target, weights_only=False, map_location=map_location - ) - + return torch.load(target, weights_only=False, map_location=map_location) def load_metadata(path, map_location="cpu") -> dict: """Load a per-reflection metadata dict.""" diff --git a/src/integrator/io/mtz_io.py b/src/integrator/io/mtz_io.py index e90b48d9..ef2ec38c 100644 --- a/src/integrator/io/mtz_io.py +++ b/src/integrator/io/mtz_io.py @@ -36,8 +36,10 @@ def write_mtz_from_preds( crystal = spec["crystal"] refl_ids = pred_data["refl_ids"] + if isinstance(refl_ids, torch.Tensor): - refl_ids = refl_ids.numpy() + refl_ids = refl_ids.detach().cpu().numpy() + refl_ids = refl_ids.astype(int) idx = np.argsort(refl_ids) @@ -46,15 +48,26 @@ def write_mtz_from_preds( def _get(key): v = pred_data[key] if isinstance(v, torch.Tensor): - v = v.numpy() + v = v.detach().cpu().numpy() return v[idx] qi_mean = _get("qi_mean") qi_var = _get("qi_var") qbg_mean = _get("qbg_mean") + + def _as_numpy(v): + """ + if v is torch tensor -> convert to NumPy + if v is already NumPy/list -> make sure it is NumPy + """ + if isinstance(v, torch.Tensor): + return v.detach().cpu().numpy() + return np.asarray(v) + def _meta(key): - return meta[key].numpy()[refl_ids] + return _as_numpy(meta[key])[refl_ids] + H = _meta("H").astype(np.int32) K = _meta("K").astype(np.int32) diff --git a/src/integrator/io/pred_io.py b/src/integrator/io/pred_io.py index 95321e98..8adae795 100644 --- a/src/integrator/io/pred_io.py +++ b/src/integrator/io/pred_io.py @@ -4,7 +4,11 @@ import polars as pl import torch -from .refl_io import unstack_preds, write_refl_with_predictions +from .refl_io import ( + unstack_preds, + write_mfx_refl_with_predictions, + write_refl_with_predictions, +) # Load integrator predictions from disk and write them back to a DIALS .refl @@ -30,11 +34,14 @@ def get_pred_files( ) if not pred_files: raise RuntimeError(f"No prediction files found in {ckpt_dir}") + # Single lazy scan over all parquet files — one pass, four columns. + # Previously: four separate .collect() calls = four full scans. lf = pl.scan_parquet(pred_files) - refl_ids = lf.select("refl_ids").collect().to_numpy().ravel() - qi_mean = lf.select("qi_mean").collect().to_numpy().ravel() - qi_var = lf.select("qi_var").collect().to_numpy().ravel() - qbg_mean = lf.select("qbg_mean").collect().to_numpy().ravel() + df = lf.select(["refl_ids", "qi_mean", "qi_var", "qbg_mean"]).collect() + refl_ids = df["refl_ids"].to_numpy() + qi_mean = df["qi_mean"].to_numpy() + qi_var = df["qi_var"].to_numpy() + qbg_mean = df["qbg_mean"].to_numpy() data = { "refl_ids": refl_ids, @@ -77,3 +84,31 @@ def write_refl_from_preds( bg_mean=pred_df["qbg_mean"].to_numpy(), ) return fname + + +def write_mfx_refl_from_preds( + ckpt_dir, + metadata_path, + original_refl_dir, + out_dir, + filetype: Literal["pt", "parquet"], + variance_floor: float = 1.0e-6, + copy_expt: bool = False, +): + """Write prediction intensities back into many MFX .refl files. + + MFX write-back extension (Thao): Luis's original write_refl_from_preds() + handles one source .refl with a refl_ids column. This wrapper keeps the + same prediction-loading code but passes the predictions plus metadata.npy + to the many-file MFX writer. + """ + data = get_pred_files(ckpt_dir=ckpt_dir, filetype=filetype) + + return write_mfx_refl_with_predictions( + pred_data=data, + metadata_path=metadata_path, + original_refl_dir=original_refl_dir, + out_dir=out_dir, + variance_floor=variance_floor, + copy_expt=copy_expt, + ) diff --git a/src/integrator/io/refl_io.py b/src/integrator/io/refl_io.py index 9a10cb53..27fdf3aa 100644 --- a/src/integrator/io/refl_io.py +++ b/src/integrator/io/refl_io.py @@ -1,4 +1,6 @@ import numpy as np +# dials.array_family.flex is imported locally inside functions that need it. +# Do NOT import it at module level: integrator-cuda-dev does not have dials. def unstack_preds(preds: dict[str, list[np.ndarray]]) -> dict[str, np.ndarray]: @@ -64,3 +66,218 @@ def write_refl_with_predictions( rt.as_file(str(out_file)) return rt + + +def _as_numpy(v): + """Return tensor/list/array metadata as a NumPy array.""" + try: + import torch + + if isinstance(v, torch.Tensor): + return v.detach().cpu().numpy() + except ImportError: + pass + return np.asarray(v) + + +def _as_text(v): + """Normalize NumPy scalar/bytes/path-like values to a Python string.""" + if hasattr(v, "item"): + v = v.item() + if isinstance(v, bytes): + return v.decode() + return str(v) + + +def _mfx_source_name(metadata, key, row, image_number, suffix): + """Return source file name from metadata, with image-number fallback.""" + if key in metadata: + name = _as_text(_as_numpy(metadata[key])[row]) + if name: + return name + return f"idx-data_{int(image_number):05d}_integrated{suffix}" + + +def _copy_or_link_expt(src, dst, copy_expt=False): + """Copy or symlink a matching .expt file into the write-back output.""" + import os + import shutil + + if dst.exists() or dst.is_symlink(): + return + if copy_expt: + shutil.copy2(src, dst) + else: + os.symlink(src, dst) + + +def write_mfx_refl_with_predictions( + pred_data, + metadata_path, + original_refl_dir, + out_dir, + variance_floor=1.0e-6, + copy_expt=False, +): + """Write Integrator predictions into many original MFX .refl files. + + MFX write-back extension (Thao): Luis's original write_refl_with_predictions() + assumes one source .refl with a refl_ids column. This function supports the + MFX/cctbx case where one prediction table maps back to many files: + + prediction refl_ids -> metadata row -> source_refl/source_expt or + image_num/image_index -> reflection_id row inside that source .refl + + Required prediction columns: refl_ids, qi_mean, qi_var, qbg_mean. + Required metadata columns: refl_ids, reflection_id, plus either + source_refl/source_expt or image_num/image_index. + """ + from pathlib import Path + + from dials.array_family import flex + from dxtbx import flumpy + + from .metadata import load_metadata + + metadata_path = Path(metadata_path) + original_refl_dir = Path(original_refl_dir) + out_dir = Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + metadata = load_metadata(metadata_path) + + pred_ids = np.asarray(pred_data["refl_ids"], dtype=np.int64) + pred_order = np.argsort(pred_ids) + pred_ids = pred_ids[pred_order] + qi_mean = np.asarray(pred_data["qi_mean"], dtype=np.float64)[pred_order] + qi_var = np.asarray(pred_data["qi_var"], dtype=np.float64)[pred_order] + qbg_mean = np.asarray(pred_data["qbg_mean"], dtype=np.float64)[pred_order] + qi_var = np.clip(qi_var, variance_floor, None) + + metadata_ids = _as_numpy(metadata["refl_ids"]).astype(np.int64) + metadata_order = np.argsort(metadata_ids) + metadata_ids_sorted = metadata_ids[metadata_order] + + # BatchPredWriter stores refl_ids as float32, which loses precision for + # values > 2^23 (~8.4 M). Large IDs get rounded; at the boundary this + # can push the maximum value one step beyond the metadata range. + # Clamp before searchsorted so edge over-counts map to the nearest valid + # metadata row instead of raising an out-of-bounds error. + meta_lo = int(metadata_ids_sorted[0]) + meta_hi = int(metadata_ids_sorted[-1]) + n_oob = int(np.sum((pred_ids < meta_lo) | (pred_ids > meta_hi))) + if n_oob > 0: + import warnings + + warnings.warn( + f"{n_oob} pred refl_ids are outside metadata range " + f"[{meta_lo}, {meta_hi}] (float32 boundary rounding). " + "Clamping to nearest valid ID.", + UserWarning, + stacklevel=2, + ) + pred_ids = np.clip(pred_ids, meta_lo, meta_hi) + + found = np.searchsorted(metadata_ids_sorted, pred_ids) + found = np.clip(found, 0, len(metadata_ids_sorted) - 1) + if not np.array_equal(metadata_ids_sorted[found], pred_ids): + n_bad = int((metadata_ids_sorted[found] != pred_ids).sum()) + raise ValueError( + f"{n_bad:,} prediction refl_ids do not match any metadata refl_id " + "after clamping. Check that the correct metadata.npy is being used." + ) + metadata_rows = metadata_order[found] + + if "reflection_id" not in metadata: + raise KeyError( + "metadata.npy must contain reflection_id for MFX write-back" + ) + reflection_id = _as_numpy(metadata["reflection_id"]).astype(np.int64) + + if "image_num" in metadata: + image_numbers = _as_numpy(metadata["image_num"]).astype(np.int64) + elif "image_index" in metadata: + image_numbers = _as_numpy(metadata["image_index"]).astype(np.int64) + elif "source_refl" in metadata: + image_numbers = np.full(len(metadata_ids), -1, dtype=np.int64) + else: + raise KeyError( + "metadata.npy must contain source_refl/source_expt or image_num/image_index" + ) + + row_in_source = reflection_id[metadata_rows] + + # Pull source_refl / source_expt via vectorized fancy-index — one C call, + # no per-row Python loop. metadata["source_refl"] is the source of truth + # for filenames; run-prefixed names (e.g. r0269_idx-data_01418_integrated.refl) + # are NOT reconstructible from image_num alone. + source_refl_all = _as_numpy(metadata["source_refl"]) + source_expt_all = _as_numpy(metadata["source_expt"]) + source_refl_subset = source_refl_all[metadata_rows] + source_expt_subset = source_expt_all[metadata_rows] + + # Encode unique source_refl strings to sorted int IDs — one C pass. + # Groupby via argsort on encoded IDs: O(N log N), no O(N x n_files) scan. + unique_refl, encoded = np.unique(source_refl_subset, return_inverse=True) + order = np.argsort(encoded, kind="stable") + sorted_encoded = encoded[order] + boundaries = np.flatnonzero(sorted_encoded[1:] != sorted_encoded[:-1]) + 1 + starts = np.concatenate(([0], boundaries)) + ends = np.concatenate((boundaries, [len(order)])) + + written = [] + for i, (start, end) in enumerate(zip(starts, ends)): + group = order[start:end] + refl_name = _as_text(unique_refl[i]) + + # Validate exactly one .expt per .refl group. + expt_names = np.unique(source_expt_subset[group]) + if len(expt_names) != 1: + raise ValueError( + f"Expected one source_expt for {refl_name}, " + f"got {expt_names.tolist()}" + ) + expt_name = _as_text(expt_names[0]) + + src_refl = original_refl_dir / refl_name + if not src_refl.exists(): + raise FileNotFoundError(f"source .refl not found: {src_refl}") + + rt = flex.reflection_table.from_file(str(src_refl)) + n_rows = len(rt) + rows = row_in_source[group].astype(np.int64) + if np.any(rows < 0) or np.any(rows >= n_rows): + raise IndexError( + f"reflection_id out of range for {src_refl}: " + f"table has {n_rows} rows" + ) + + def _set_column(name, values): + if name in rt: + arr = flumpy.to_numpy(rt[name]).astype(np.float64, copy=True) + else: + arr = np.zeros(n_rows, dtype=np.float64) + arr[rows] = values + rt[name] = flumpy.from_numpy(arr) + + _set_column("intensity.prf.value", qi_mean[group]) + _set_column("intensity.prf.variance", qi_var[group]) + _set_column("intensity.sum.value", qi_mean[group]) + _set_column("intensity.sum.variance", qi_var[group]) + _set_column("background.mean", qbg_mean[group]) + + out_refl = out_dir / refl_name + rt.as_file(str(out_refl)) + written.append(out_refl) + + src_expt = original_refl_dir / expt_name + if not src_expt.exists(): + raise FileNotFoundError(f"source .expt not found: {src_expt}") + _copy_or_link_expt(src_expt, out_dir / expt_name, copy_expt=copy_expt) + + return { + "out_dir": out_dir, + "n_prediction_rows": int(len(pred_ids)), + "n_refl_files": int(len(written)), + "written_refl_files": written, + } diff --git a/src/integrator/model/distributions/folded_normal.py b/src/integrator/model/distributions/folded_normal.py new file mode 100644 index 00000000..92def5c4 --- /dev/null +++ b/src/integrator/model/distributions/folded_normal.py @@ -0,0 +1,166 @@ +import math + +import torch +import torch.nn as nn +from torch.distributions import Distribution, Normal +from torch.distributions import constraints + +from .utils import get_positive_constraint + + +class FoldedNormal(Distribution): + """ + FoldedNormal distribution. + + If: + X ~ Normal(loc, scale) + Y = |X| + + then Y follows a folded normal distribution. + + We implement this manually instead of using AbsTransform because + PyTorch's AbsTransform does not provide everything needed for log_prob. + """ + + arg_constraints = { + "loc": constraints.real, + "scale": constraints.positive, + } + support = constraints.nonnegative + has_rsample = True + + def __init__(self, loc: torch.Tensor, scale: torch.Tensor, validate_args=None): + self.loc = loc + self.scale = scale + self.base_dist = Normal(loc=loc, scale=scale) + super().__init__(batch_shape=loc.shape, validate_args=validate_args) + + def rsample(self, sample_shape=torch.Size()): + """ + Reparameterized sample. + + Draw X from Normal(loc, scale), then return |X|. + """ + return self.base_dist.rsample(sample_shape).abs() + + def sample(self, sample_shape=torch.Size()): + """ + Non-gradient sample. + + Draw X from Normal(loc, scale), then return |X|. + """ + with torch.no_grad(): + return self.rsample(sample_shape) + + def log_prob(self, value: torch.Tensor): + """ + Log probability of FoldedNormal. + + For y >= 0: + + p_Y(y) = Normal(y | loc, scale) + Normal(-y | loc, scale) + + because both +y and -y fold into the same absolute value y. + """ + value = value.clamp_min(0.0) + + log_p_pos = self.base_dist.log_prob(value) + log_p_neg = self.base_dist.log_prob(-value) + + return torch.logaddexp(log_p_pos, log_p_neg) + + @property + def mean(self): + """ + E[|X|] where X ~ Normal(loc, scale). + """ + z = self.loc / (math.sqrt(2.0) * self.scale) + + return ( + self.scale * math.sqrt(2.0 / math.pi) * torch.exp(-z.pow(2)) + + self.loc * torch.erf(z) + ) + + @property + def variance(self): + """ + Var(|X|) = E[X^2] - E[|X|]^2 + + For X ~ Normal(loc, scale): + E[X^2] = loc^2 + scale^2 + """ + return self.loc.pow(2) + self.scale.pow(2) - self.mean.pow(2) + + +class FoldedNormalDistribution(nn.Module): + """ + FoldedNormal posterior distribution. + + The network predicts: + loc = mean of the underlying Normal distribution + scale = standard deviation of the underlying Normal distribution + + Then the distribution samples positive intensity values by folding: + y = abs(z) + """ + + def __init__( + self, + in_features: int = 64, + eps: float = 1e-6, + scale_min: float = 0.01, + positive_constraint: str = "softplus", + **kwargs, + ): + super().__init__() + + self.eps = eps + self.scale_min = scale_min + + self.scale_constrain = get_positive_constraint(positive_constraint) + + self.linear_loc = nn.Linear(in_features, 1) + self.linear_scale = nn.Linear(in_features, 1) + + def forward(self, x: torch.Tensor, x_: torch.Tensor): + #x is input features used to predict the mean (loc): x-> network -> loc + #x_ is input features used to predict the standard deviation (scale): x_ ->nn -> scle + + loc = self.linear_loc(x) + scale = self.scale_constrain(self.linear_scale(x_)) + scale = scale + self.scale_min + self.eps + + loc = loc.flatten() + scale = scale.flatten() + + return FoldedNormal(loc=loc, scale=scale) + + +def build_folded_normal(**kwargs) -> nn.Module: + """ + Factory function used by the surrogate registry. + """ + _reject_unknown_folded_normal_args(kwargs) + return FoldedNormalDistribution(**kwargs) + + +def _folded_normal_valid_args() -> set[str]: + return { + "in_features", + "eps", + "scale_min", + "positive_constraint", + } + + +def _reject_unknown_folded_normal_args(kwargs: dict) -> None: + valid = _folded_normal_valid_args() + unknown = set(kwargs) - valid + if unknown: + raise ValueError( + f"Unknown folded_normal surrogate arg(s): {sorted(unknown)}. " + f"Valid args: {sorted(valid)}." + ) + + +build_folded_normal.arg_names = _folded_normal_valid_args() # type: ignore[attr-defined] \ No newline at end of file diff --git a/src/integrator/model/distributions/lognormal.py b/src/integrator/model/distributions/lognormal.py new file mode 100644 index 00000000..3d6d0286 --- /dev/null +++ b/src/integrator/model/distributions/lognormal.py @@ -0,0 +1,155 @@ +import torch +import torch.nn as nn +from torch.distributions import LogNormal + +from .utils import get_positive_constraint + + +""" +Pattern: +class SomeDistribution(nn.Module) + -> define linear heads + -> constrain scale/positive params + -> forward(...) returns a torch.distributions object + -> build_* factory function + -> arg_names for YAML/factory +""" + + +""" +YAML/config or registry + | + v +build_lognormal(**kwargs) + | + v +_reject_unknown_lognormal_args(kwargs) + | + v +LogNormalDistribution(**kwargs) + | + v +__init__(...) + | + v +model is created with: + - linear_loc + - linear_scale + - scale_constrain + | + v +forward(x, x_) + | + v +loc = linear_loc(x) +scale = linear_scale(x_) +scale = positive constraint(scale) +scale = scale + scale_min + eps + | + v +return LogNormal(loc, scale) +""" + + +class LogNormalDistribution(nn.Module): + """ + LogNormal posterior distribution. + + A LogNormal random variable is always positive. + + This can be useful for intensity-like quantities because intensities are + often positive and skewed. + + The network predicts two values: + loc = mean of the underlying Normal distribution + scale = standard deviation of the underlying Normal distribution + + Then PyTorch defines: + + z ~ Normal(loc, scale) + y = exp(z) > 0 + + so: + + y ~ LogNormal(loc, scale) + """ + + def __init__( + self, + in_features: int = 64, + eps: float = 1e-6, + scale_min: float = 0.01, + positive_constraint: str = "softplus", + **kwargs, + ): + super().__init__() + + self.eps = eps + self.scale_min = scale_min + + # Returns a function that makes scale positive. + self.scale_constrain = get_positive_constraint(positive_constraint) + + # Predicts loc. Loc can be negative, zero, or positive. + self.linear_loc = nn.Linear(in_features, 1) + + # Predicts scale. Scale must be positive. + self.linear_scale = nn.Linear(in_features, 1) + + def forward(self, x: torch.Tensor, x_: torch.Tensor): + """ + x -> linear_loc -> loc + x_ -> linear_scale -> positive scale + """ + + # Predict loc from one encoder feature vector. + loc = self.linear_loc(x) + + # Predict positive scale from another encoder feature vector. + scale = self.scale_constrain(self.linear_scale(x_)) + + # Keep scale away from zero. + scale = scale + self.scale_min + self.eps + + # Each sample needs one scalar loc and one scalar scale. + + """ + LogNormal: + z ~ Normal(loc, scale) + y = exp(z) + """ + return LogNormal(loc=loc.flatten(), scale=scale.flatten()) + + +def build_lognormal(**kwargs) -> nn.Module: + """ + Helper function that creates the surrogate object for the registry. + """ + _reject_unknown_lognormal_args(kwargs) + return LogNormalDistribution(**kwargs) + + +def _lognormal_valid_args() -> set[str]: + return { + "in_features", + "eps", + "scale_min", + "positive_constraint", + } + + +def _reject_unknown_lognormal_args(kwargs: dict) -> None: + valid = _lognormal_valid_args() + + # set(kwargs) converts the dictionary's keys into a set. + # unknown = keys passed in kwargs but not allowed by valid. + unknown = set(kwargs) - valid + + if unknown: + raise ValueError( + f"Unknown lognormal surrogate arg(s): {sorted(unknown)}. " + f"Valid args: {sorted(valid)}." + ) + + +build_lognormal.arg_names = _lognormal_valid_args() # type: ignore[attr-defined] diff --git a/src/integrator/model/distributions/profile_surrogates.py b/src/integrator/model/distributions/profile_surrogates.py index 83f8080d..54987b65 100644 --- a/src/integrator/model/distributions/profile_surrogates.py +++ b/src/integrator/model/distributions/profile_surrogates.py @@ -8,8 +8,11 @@ def _softplus_inverse(x: float) -> float: - """Inverse of softplus: log(exp(x) - 1).""" - return math.log(math.exp(x) - 1.0) + """Inverse of softplus: log(exp(x) - 1). + + Uses math.expm1(x) = exp(x) - 1 to avoid overflow for x > 709. + """ + return math.log(math.expm1(x)) def _sample_and_decode( diff --git a/src/integrator/model/integrators/hierarchical_integrator.py b/src/integrator/model/integrators/hierarchical_integrator.py index 0ab9d0ca..e56ac916 100644 --- a/src/integrator/model/integrators/hierarchical_integrator.py +++ b/src/integrator/model/integrators/hierarchical_integrator.py @@ -48,7 +48,9 @@ def _forward_impl( mask: Tensor, metadata: dict, ) -> dict[str, Any]: - counts = torch.clamp(counts, min=0) + obs_name = getattr(self.loss.observation_model, "name", None) + if obs_name == "poisson": + counts = torch.clamp(counts, min=0) b = shoebox.shape[0] shoebox_masked = shoebox * mask @@ -122,7 +124,9 @@ def _forward_impl( mask: Tensor, metadata: dict, ) -> dict[str, Any]: - counts = torch.clamp(counts, min=0) + obs_name = getattr(self.loss.observation_model, "name", None) + if obs_name == "poisson": + counts = torch.clamp(counts, min=0) b = shoebox.shape[0] shoebox_masked = shoebox * mask diff --git a/src/integrator/model/loss/learned_spectrum.py b/src/integrator/model/loss/learned_spectrum.py index 05d0ca14..5f06a780 100644 --- a/src/integrator/model/loss/learned_spectrum.py +++ b/src/integrator/model/loss/learned_spectrum.py @@ -52,10 +52,14 @@ def _chebyshev(x: Tensor, degree: int) -> list[Tensor]: def design_matrix(self, wavelength: Tensor) -> Tensor: """(B,) -> (B, degree+1)""" - x = ((wavelength - self.lam_mid) / self.lam_scale).clamp(-1.0, 1.0) + + lam_mid = self.lam_mid.to(device=wavelength.device, dtype=wavelength.dtype) + lam_scale = self.lam_scale.to(device=wavelength.device, dtype=wavelength.dtype) + x = ((wavelength - lam_mid) / lam_scale).clamp(-1.0, 1.0) return torch.stack(self._chebyshev(x, self.degree), dim=-1) def get_log_G(self, wavelength: Tensor) -> Tensor: """(B,) -> (B,)""" phi = self.design_matrix(wavelength) - return phi @ self.c + c = self.c.to(device=phi.device, dtype=phi.dtype) + return phi @ c diff --git a/src/integrator/model/loss/monochromatic_wilson_loss.py b/src/integrator/model/loss/monochromatic_wilson_loss.py index 4b82ed74..1badc569 100644 --- a/src/integrator/model/loss/monochromatic_wilson_loss.py +++ b/src/integrator/model/loss/monochromatic_wilson_loss.py @@ -1,3 +1,6 @@ +import logging + +import numpy as np import torch import torch.nn as nn import torch.nn.functional as F @@ -5,33 +8,188 @@ from integrator.model.loss.wilson_loss import WilsonLoss +logger = logging.getLogger(__name__) + + +def _softplus_inverse(x: Tensor) -> Tensor: + """Numerically stable inverse of softplus. + + Uses the algebraic identity: + log(exp(x) - 1) = x + log(1 - exp(-x)) + implemented as x + log(-expm1(-x)). + + For large x (e.g. G~1000): exp(-x) → 0, so log(-expm1(-x)) → log(1) = 0 + and the result is simply x. This avoids the overflow in the naive + log(exp(x)-1) formula, which produces inf for x~1000. + """ + return x + torch.log(-torch.expm1(-x)) + class MonochromaticWilsonLoss(WilsonLoss): - """Wilson loss for monochromatic data with scalar G.""" + """Wilson loss for monochromatic data with optional per-image B/G.""" def __init__( self, *, - init_log_G: float = 0.0, + # init_log_G is captured directly. + init_log_G: float = 1000.0, lp_correction: bool = False, + # Per-image B/G initialization from cctbx (factory injects this path). + wilson_bg_init_path: str | None = None, + # Whether to keep per-image B/G embeddings trainable (default) or frozen. + wilson_bg_trainable: bool = True, + # init_log_B goes inside kwargs **kwargs, ): super().__init__(**kwargs) + + self.init_log_G = init_log_G self._apply_lp = lp_correction + + # Global G parameter used when image_level_wilson=False. self.raw_G = nn.Parameter(torch.tensor(float(init_log_G))) + # Optional per-image B/G parameters. + # Requires WilsonLoss.__init__ to define: + # self.image_level_wilson + # self.n_images + # self.init_log_B + if self.image_level_wilson: + if self.n_images is None: + raise ValueError("image_level_wilson=True requires n_images.") + + self.raw_B_by_image = nn.Embedding(self.n_images, 1) + self.raw_G_by_image = nn.Embedding(self.n_images, 1) + + nn.init.constant_( + self.raw_B_by_image.weight, + float(self.init_log_B), + ) + nn.init.constant_( + self.raw_G_by_image.weight, + float(self.init_log_G), + ) + + # ADDITION A — override with per-image cctbx initialization + if wilson_bg_init_path is not None: + data = np.load(wilson_bg_init_path, allow_pickle=True).item() + B_arr = torch.as_tensor(data["B_array"], dtype=torch.float32) + G_arr = torch.as_tensor(data["G_array"], dtype=torch.float32) + + # Length check — protects against stale .npy from a different + # dataset silently misaligning image_id → embedding rows. + if len(B_arr) != self.n_images or len(G_arr) != self.n_images: + raise ValueError( + "Loaded Wilson B/G arrays do not match n_images: " + f"len(B)={len(B_arr)}, len(G)={len(G_arr)}, " + f"n_images={self.n_images}" + ) + + # Convert fitted B, G to raw (unconstrained) embedding values. + # softplus_inverse is applied after clamping to avoid log(0). + raw_B = _softplus_inverse( + torch.clamp(B_arr - self.b_min, min=1e-4) + ) + raw_G = _softplus_inverse(torch.clamp(G_arr, min=1e-4)) + + with torch.no_grad(): + self.raw_B_by_image.weight.copy_(raw_B.unsqueeze(1)) + self.raw_G_by_image.weight.copy_(raw_G.unsqueeze(1)) + + logger.info( + "Per-image B/G initialized from %s " + "(B: [%.2f, %.2f], G: [%.2f, %.2f])", + wilson_bg_init_path, + float(B_arr.min()), + float(B_arr.max()), + float(G_arr.min()), + float(G_arr.max()), + ) + + # ADDITION B — freeze embeddings if requested + if not wilson_bg_trainable: + self.raw_B_by_image.requires_grad_(False) + self.raw_G_by_image.requires_grad_(False) + logger.info( + "raw_B_by_image and raw_G_by_image frozen " + "(wilson_bg_trainable=false)" + ) + + def get_wilson_stats(self) -> dict[str, Tensor]: + """Return summary statistics for the learned Wilson B/G values + it reads the current (trained) values and summarizes them + + So the structure is: + monochromatic_wilson_loss.py -> calculates B/G statistics + WilsonParamLogger callback -> calls get_wilson_stats() every epoch + train.py -> already registers the callback + """ + with torch.no_grad(): + if self.image_level_wilson: + # torch.arange: creates a tensor of integers from 0 to n - 1. + image_id = torch.arange( + self.n_images, + device=self.raw_B_by_image.weight.device, + ) + + B, G = self._get_image_B_G(image_id) + + else: + B = self.get_B().reshape(1) + G = self.get_G().reshape(1) + return { + "B_mean": B.mean(), + # population standard deviation formula: divide by N, + # if = True means: sample standard deviation formula: divide by N - 1 + "B_std": B.std(unbiased=False), + "B_min": B.min(), + "B_max": B.max(), + "G_mean": G.mean(), + "G_std": G.std(unbiased=False), + "G_min": G.min(), + "G_max": G.max(), + } + def get_G(self) -> Tensor: return F.softplus(self.raw_G) def _get_tau( - self, metadata: dict, s_sq: Tensor, device: torch.device + self, + metadata: dict, + s_sq: Tensor, + device: torch.device, ) -> Tensor: - G = self.get_G() - B = self.get_B() - tau = (1.0 / G) * torch.exp(2.0 * B * s_sq) + if self.image_level_wilson: + if "image_id" not in metadata: + raise ValueError( + "image_level_wilson=True requires metadata['image_id']." + ) + + image_id = metadata["image_id"].to(device).long() + + B, G = self._get_image_B_G(image_id) + else: + B = self.get_B() + G = self.get_G() + + tau = (1.0 / G.clamp(min=self.eps)) * torch.exp(2.0 * B * s_sq) if self._apply_lp: + if "lp" not in metadata: + raise ValueError("lp_correction=True requires metadata['lp'].") lp = metadata["lp"].to(device).clamp(min=1e-8) tau = tau * lp return tau + + def _get_image_B_G( + self, + image_id: Tensor, + ) -> tuple[Tensor, Tensor]: + raw_B = self.raw_B_by_image(image_id).squeeze(-1) + raw_G = self.raw_G_by_image(image_id).squeeze(-1) + + B = F.softplus(raw_B) + self.b_min + G = F.softplus(raw_G) + + return B, G diff --git a/src/integrator/model/loss/wilson_loss.py b/src/integrator/model/loss/wilson_loss.py index bbfdcd65..0353a273 100644 --- a/src/integrator/model/loss/wilson_loss.py +++ b/src/integrator/model/loss/wilson_loss.py @@ -4,7 +4,14 @@ import torch.nn as nn import torch.nn.functional as F from torch import Tensor -from torch.distributions import Distribution, Gamma, Poisson, kl_divergence +from torch.distributions import ( + Distribution, + Gamma, + Normal, + Poisson, + StudentT, + kl_divergence, +) from integrator.model.distributions.profile_surrogates import ( ProfileSurrogateOutput, @@ -14,36 +21,194 @@ _DEFAULT_PROFILE_PRIOR_SCALE = 3.0 +def _kl_divergence_with_mc_fallback( + q: Distribution, + p: Distribution, + n_samples: int = 8, + eps: float = 1.0e-8, +) -> Tensor: + """Compute KL(q || p), with a sampling fallback for unsupported pairs. + + PyTorch has exact KL formulas for some distribution pairs, such as + Gamma vs Gamma. For newer MFX tests, qi may be LogNormal or FoldedNormal + while the Wilson intensity prior p_i is still Gamma. PyTorch does not + always have a built-in exact KL formula for those pairs. + + This fallback estimates: + + KL(q || p) = E_q[log q(x) - log p(x)] + + by sampling x from q. + """ + try: + return kl_divergence(q, p) + except NotImplementedError: + if getattr(q, "has_rsample", False): + x = q.rsample((n_samples,)) + else: + x = q.sample((n_samples,)) + + x = x.clamp_min(eps) + + log_q = q.log_prob(x) + log_p = p.log_prob(x) + + return (log_q - log_p).mean(dim=0) + + +class ObservationLikelihood(nn.Module): + """Pixel observation likelihood helper. + + Options: + poisson: + Original behavior. Use for nonnegative count-like data. + + normal: + Continuous likelihood. Handles float and negative values. + + student_t: + Continuous likelihood like Normal, but more robust to outliers. + """ + + def __init__( + self, + name: str = "poisson", + init_scale: float = 1.0, + student_t_df: float = 4.0, + eps: float = 1e-6, + ): + super().__init__() + + valid_names = {"poisson", "normal", "student_t"} + if name not in valid_names: + raise ValueError( + f"Unknown observation_likelihood={name!r}. " + f"Valid options are {sorted(valid_names)}." + ) + + self.name = name + self.student_t_df = float(student_t_df) + self.eps = eps + + # NOTE 2026-10-03 (Thao): the old Student-t branch used a learned + # global scale (self.raw_scale / get_scale()). That has been + # removed: Student-t now shares the SAME brightness-dependent + # scale as Normal (see forward() below), so this controlled + # experiment isolates the likelihood family (Normal vs + # Student-t) without introducing an extra learned parameter. + # init_scale is kept as a constructor arg for backward + # compatibility with existing configs/call sites but is unused. + + def forward(self, rate: Tensor, counts: Tensor) -> Tensor: + """Compute pixel log probability. + + Args: + rate: + Shape: (batch, mc_samples, pixels) + + counts: + Shape: (batch, pixels) + + Returns: + Tensor with shape: (batch, mc_samples, pixels) + """ + y = counts.unsqueeze(1) + + if self.name == "poisson": + return Poisson(rate.clamp(min=1e-12)).log_prob(y) + + if self.name == "normal": + # CHANGED 2026-09-04 (Thao): use observed counts to set sigma, + # with correct Poisson + gain noise model: Var(y) = g * E[y]. + # + # Previously: scale = sqrt(rate) + eps + # - used predicted rate instead of observed counts + # - missing gain factor g → underestimated sigma by sqrt(g) ~ 3× + # + # Now: scale = sqrt(g * counts.clamp(min=g)) + eps + # - g = 9.5 ADU/photon, measured from Jungfrau single-photon + # peaks in mfx101555026 pixel histogram (beam ~9.70 keV) + # - uses observed counts so gradient flows only through the + # (y - rate) residual, not through scale (Kevin's suggestion) + # - floor at g: sigma >= g = 9.5 ADU (1-photon noise level), + # handles negative ADU pixels from Jungfrau pedestal correction + g = 9.5 + scale = torch.sqrt(g * y.clamp(min=g)) + self.eps + return Normal(loc=rate, scale=scale).log_prob(y) + + if self.name == "student_t": + # CHANGED 2026-10-03 (Thao): use the SAME brightness-dependent + # scale as the "normal" branch above, instead of the old + # learned global scale (self.get_scale()). This isolates the + # likelihood-family comparison (Normal vs Student-t) while + # holding the noise model fixed. df is NOT rescaled to + # variance-match Normal; see wilson_loss.py docstring / + # experiment notes for rationale. + g = 9.5 + scale = torch.sqrt(g * y.clamp(min=g)) + self.eps + df = torch.tensor( + self.student_t_df, + device=rate.device, + dtype=rate.dtype, + ) + return StudentT(df=df, loc=rate, scale=scale).log_prob(y) + + raise RuntimeError(f"Unsupported likelihood: {self.name}") + + class WilsonLoss(nn.Module): """Base ELBO loss with Wilson intensity prior. Subclasses implement `_get_tau` to define how the Wilson prior rate - is computed (scalar G for monochromatic, G(lambda) for polychromatic). + is computed. + + Examples: + MonochromaticWilsonLoss: + scalar G, optionally per-image B/G. + + PolychromaticWilsonLoss: + wavelength-dependent G(lambda). """ def __init__( self, *, - # Background Gamma prior: scalar, or per-resolution-bin prior + # Background Gamma prior: scalar, or per-resolution-bin prior. bg_rate: float | list[float] = 1.0, bg_concentration: float | list[float] = 1.0, - # B factor - init_log_B: float = 3.0, + # B factor. + init_log_B: float = 20.0, b_min: float = 0.0, - # Resolution bins for per-bin background prior + # Resolution bins for per-bin background prior. n_bins: int = 1, - # Prior configs from yaml + # Prior configs from YAML. pi_cfg=None, pbg_cfg=None, pprf_cfg=None, - # KL weights + # KL weights. profile_kl_weight: float = 1.0, background_kl_weight: float = 1.0, intensity_kl_weight: float = 1.0, + # MFX/new observation likelihood options. + observation_likelihood: str = "poisson", + init_obs_scale: float = 1.0, + student_t_df: float = 4.0, + # Optional image-level Wilson options. + image_level_wilson: bool = False, + n_images: int | None = None, + eps: float = 1e-6, ): super().__init__() - self.b_min = b_min # minimum B-factor + + self.eps = eps + self.b_min = b_min self.n_bins = n_bins + + # Store image-level Wilson settings for subclasses. + self.image_level_wilson = image_level_wilson + self.n_images = n_images + self.init_log_B = init_log_B + self.register_buffer( "bg_concentration", torch.as_tensor(bg_concentration, dtype=torch.float32), @@ -54,7 +219,8 @@ def __init__( torch.as_tensor(bg_rate, dtype=torch.float32), persistent=False, ) - # keep the prior cfgs so run artifacts can record them + + # Keep the prior cfgs so run artifacts can record them. self.pprf_cfg = pprf_cfg self.pbg_cfg = pbg_cfg self.pi_cfg = pi_cfg @@ -62,19 +228,33 @@ def __init__( self.profile_kl_weight = ( pprf_cfg.weight if pprf_cfg is not None else profile_kl_weight ) - self.background_kl_weight = pbg_cfg.weight if pbg_cfg is not None else background_kl_weight - self.intensity_kl_weight = pi_cfg.weight if pi_cfg is not None else intensity_kl_weight + self.background_kl_weight = ( + pbg_cfg.weight if pbg_cfg is not None else background_kl_weight + ) + self.intensity_kl_weight = ( + pi_cfg.weight if pi_cfg is not None else intensity_kl_weight + ) - # Point-estimate B factor - # used by polychromatic and monochromatic loss classes + # Global B factor. + # MonochromaticWilsonLoss may also create per-image B embeddings. self.raw_B = nn.Parameter(torch.tensor(float(init_log_B))) + self.observation_model = ObservationLikelihood( + name=observation_likelihood, + init_scale=init_obs_scale, + student_t_df=student_t_df, + eps=self.eps, + ) + def get_B(self) -> Tensor: return F.softplus(self.raw_B) + self.b_min @abstractmethod def _get_tau( - self, metadata: dict, s_sq: Tensor, device: torch.device + self, + metadata: dict, + s_sq: Tensor, + device: torch.device, ) -> Tensor: """Compute Wilson prior rate tau per reflection.""" @@ -91,38 +271,49 @@ def forward( ) -> dict[str, Tensor]: device = rate.device batch_size = rate.shape[0] + counts = counts.to(device) mask = mask.to(device) kl = torch.zeros(batch_size, device=device) - # get metadata - + # Metadata is needed for Wilson prior calculation. metadata = kwargs.get("metadata") if metadata is None or "d" not in metadata: raise ValueError("Wilson loss requires metadata['d'].") - # profile kl-divergence + # Profile KL-divergence. prf_prior_scale = getattr( - qp, "prior_scale", _DEFAULT_PROFILE_PRIOR_SCALE + qp, + "prior_scale", + _DEFAULT_PROFILE_PRIOR_SCALE, ) kl_prf = compute_profile_kl( - qp, prf_prior_scale, self.profile_kl_weight, device + qp, + prf_prior_scale, + self.profile_kl_weight, + device, ) kl = kl + kl_prf - # Wilson intensity KL + # Wilson intensity KL. d = metadata["d"].to(device) s_sq = 1.0 / (4.0 * d.clamp(min=1e-6).pow(2)) tau = self._get_tau(metadata, s_sq, device) - p_i = Gamma(concentration=torch.ones_like(tau), rate=tau) + p_i = Gamma( + concentration=torch.ones_like(tau), + rate=tau.clamp(min=self.eps), + ) - kl_i = kl_divergence(qi, p_i) * self.intensity_kl_weight + kl_i = ( + _kl_divergence_with_mc_fallback(qi, p_i, eps=self.eps) + * self.intensity_kl_weight + ) kl = kl + kl_i - # background prior: shared Gamma, or per-resolution-bin + # Background prior: shared Gamma, or per-resolution-bin. if self.bg_concentration.ndim == 1: if group_labels is None: raise ValueError( @@ -134,13 +325,21 @@ def forward( else: bg_conc = self.bg_concentration bg_rate = self.bg_rate - p_bg = Gamma(concentration=bg_conc, rate=bg_rate) + + p_bg = Gamma( + concentration=bg_conc, + rate=bg_rate, + ) kl_bg = kl_divergence(qbg, p_bg) * self.background_kl_weight kl = kl + kl_bg - # Poisson NLL - ll = Poisson(rate.clamp(min=1e-12)).log_prob(counts.unsqueeze(1)) + # Pixel negative log likelihood. + ll = self.observation_model(rate, counts) + + # Average over Monte Carlo samples, then apply valid-pixel mask. ll_mean = torch.mean(ll, dim=1) * mask.squeeze(-1) + + # Negative log likelihood per reflection. neg_ll = (-ll_mean).sum(1) loss = (neg_ll + kl).mean() diff --git a/src/integrator/registry.py b/src/integrator/registry.py index ec658e31..a3683eab 100644 --- a/src/integrator/registry.py +++ b/src/integrator/registry.py @@ -2,12 +2,17 @@ from integrator.data_loaders import ( PolychromaticDataModule, RotationDataModule, + ChunkedDataModule, ) from integrator.model.distributions import ( DirichletDistribution, ProfileSurrogate, build_gamma, ) + +from integrator.model.distributions.lognormal import build_lognormal +from integrator.model.distributions.folded_normal import build_folded_normal + from integrator.model.encoders import ( IntensityEncoder, ProfileEncoder, @@ -34,14 +39,18 @@ "hierarchical": HierarchicalIntegrator, "hierarchical_3enc": HierarchicalIntegrator3Enc, }, + # Normal and StudentT observation likelihoods added for MFX data "surrogates": { "gamma": build_gamma, "dirichlet": DirichletDistribution, "learned_basis_profile": ProfileSurrogate, + "lognormal": build_lognormal, + "foldednormal": build_folded_normal, }, "data_loader": { "rotation_data": RotationDataModule, "polychromatic_data": PolychromaticDataModule, + "chunked_rotation_data": ChunkedDataModule, }, "priors": { "gamma": (GammaParams, ()), diff --git a/src/integrator/utils/__init__.py b/src/integrator/utils/__init__.py index 164e6658..35566c21 100644 --- a/src/integrator/utils/__init__.py +++ b/src/integrator/utils/__init__.py @@ -5,12 +5,14 @@ construct_trainer, load_config, resolve_config, + resolve_source_data_dir, save_run_artifacts, ) from .prepare_priors import ( inject_binning_labels, prepare_per_bin_priors, ) +from .prepare_wilson_bg import fit_wilson_bg_from_chunks __all__ = [ "apply_dataset_defaults", @@ -19,7 +21,9 @@ "construct_trainer", "load_config", "resolve_config", + "resolve_source_data_dir", "save_run_artifacts", "inject_binning_labels", "prepare_per_bin_priors", + "fit_wilson_bg_from_chunks", ] diff --git a/src/integrator/utils/factory_utils.py b/src/integrator/utils/factory_utils.py index d02147bb..035d026e 100644 --- a/src/integrator/utils/factory_utils.py +++ b/src/integrator/utils/factory_utils.py @@ -1,9 +1,12 @@ +import logging import os from dataclasses import asdict from importlib.resources import as_file from pathlib import Path from typing import Any +logger = logging.getLogger(__name__) + import pytorch_lightning as pl import torch.nn as nn import yaml @@ -215,7 +218,88 @@ def _require(cfg: dict, *path: str) -> Any: def _get_data_dir(cfg: dict) -> str: - return str(_require(cfg, "data_loader", "args", "data_dir")) + """Return the data directory for the current config. + + For rotation_data: returns data_loader.args.data_dir (unchanged). + For chunked_rotation_data: returns data_loader.args.chunk_dir. + chunk_dir does not contain metadata.npy — callers that look for + optional files (bg_prior.npy, prior configs) will not find them + in chunk_dir unless integrator.preprocess has been run. + Raises KeyError if neither key is present. + """ + args = cfg.get("data_loader", {}).get("args", {}) + if "data_dir" in args: + return str(args["data_dir"]) + if "chunk_dir" in args: + return str(args["chunk_dir"]) + raise KeyError( + "Config missing 'data_dir' (or 'chunk_dir' for chunked_rotation_data) " + "under 'data_loader.args'" + ) + + +def resolve_source_data_dir(cfg: dict) -> Path: + """Return the original shoebox data directory for the given config. + + This is the single authoritative resolver for paths that need the original + shoebox dataset directory — metadata.npy for MFX write-back, dataset.yaml + for MTZ export, bg_prior.npy for the Wilson loss, etc. + + --original-refl-dir remains separate: it points to the original .refl/.expt + files which may be in a different location from the shoebox dataset. + + For rotation_data: + Returns Path(data_loader.args.data_dir). + No manifest access — safe to call with no chunk directory present. + + For chunked_rotation_data: + Reads source_data_dir from chunk_dir/manifest.yaml. + source_data_dir is written by integrator.preprocess and points to the + original shoebox dataset directory (containing metadata.npy, dataset.yaml, + counts.npy, masks.npy). + + Raises: + KeyError: + rotation_data config is missing data_loader.args.data_dir. + FileNotFoundError: + chunked_rotation_data: manifest.yaml not found in chunk_dir. + Solution: run integrator.preprocess first. + ValueError: + chunked_rotation_data: manifest.yaml exists but lacks source_data_dir. + The chunk directory was produced by an older integrator.preprocess. + Solution: re-run integrator.preprocess. + """ + import yaml as _yaml + + name = cfg.get("data_loader", {}).get("name", "") + args = cfg.get("data_loader", {}).get("args", {}) + + if name == "chunked_rotation_data": + chunk_dir = Path(args["chunk_dir"]) + manifest_path = chunk_dir / "manifest.yaml" + if not manifest_path.exists(): + raise FileNotFoundError( + f"No manifest.yaml found in {chunk_dir}.\n" + "Run integrator.preprocess first." + ) + m = _yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + if "source_data_dir" not in m: + raise ValueError( + f"manifest.yaml in {chunk_dir} does not contain 'source_data_dir'.\n" + "This chunk directory was produced by an older version of " + "integrator.preprocess.\n" + "Re-run integrator.preprocess to regenerate the chunks with " + "source_data_dir recorded in manifest.yaml." + ) + return Path(m["source_data_dir"]) + + # rotation_data and all other loaders: require explicit data_dir. + if "data_dir" not in args: + raise KeyError( + "Config missing 'data_dir' under 'data_loader.args'.\n" + "Set data_loader.args.data_dir in your config YAML." + ) + return Path(args["data_dir"]) def _get_n_bins(cfg: dict) -> int | None: @@ -359,11 +443,26 @@ def _get_loss_module( kwargs["spectrum_init_from"], data_dir, None ) - # Inject the empirical background prior + # Inject the empirical background prior. + # + # For rotation_data: bg_prior_{n}.npy is looked up in data_dir (the raw + # shoebox dataset directory). If absent the loss falls back to default + # bg_rate=1.0, bg_concentration=1.0 — existing behaviour, unchanged. + # + # For chunked_rotation_data: bg_prior_{n}.npy is written into chunk_dir + # by integrator.preprocess (_write_bg_prior), so data_dir == chunk_dir + # here (set by the updated _get_data_dir above). If the file is missing + # we raise a clear error rather than silently using Gamma(1,1) defaults, + # because that would silently change Luis's empirical Wilson prior math. + # The user can suppress the error by setting bg_rate and bg_concentration + # explicitly in loss.args. if "bg_rate" not in kwargs or "bg_concentration" not in kwargs: from integrator.io import data_path, load_data from integrator.utils.prepare_priors import _nbins_path + is_chunked = ( + cfg.get("data_loader", {}).get("name") == "chunked_rotation_data" + ) n_bins = int(kwargs.get("n_bins", 1)) bg_path = _nbins_path("bg_prior.npy", n_bins, Path(data_dir)) if data_path(bg_path) is not None: @@ -373,6 +472,42 @@ def _get_loss_module( kwargs.setdefault( "bg_concentration", bg_prior["bg_concentration"].tolist() ) + elif is_chunked: + raise FileNotFoundError( + f"bg_prior_{n_bins}.npy not found in chunk_dir {data_dir}.\n" + "integrator.preprocess must be run (or re-run) to generate it.\n" + "Alternatively, set bg_rate and bg_concentration explicitly " + "under loss.args in your config YAML to override the prior." + ) + + # ── Per-image Wilson B/G initialization ────────────────────────────────── + # wilson_bg_init is a factory routing key — it tells us where to find the + # cached .npy file. It is NOT a loss constructor argument, so it must be + # popped before loss_cls(**kwargs) is called, otherwise the loss will see + # an unexpected keyword argument and raise TypeError. + wilson_bg_init = kwargs.pop("wilson_bg_init", None) + + wilson_bg_init_path = None + if kwargs.get("image_level_wilson", False) and wilson_bg_init == "cctbx": + wilson_bg_path = Path(data_dir) / "wilson_bg_running_sums.npy" + if not wilson_bg_path.exists(): + raise FileNotFoundError( + f"wilson_bg_init=cctbx but {wilson_bg_path} does not exist.\n" + "fit_wilson_bg_from_chunks() is called before this point in " + "train.py, so a missing file means something went wrong.\n" + "Check that image_id and intensity.sum.value are present in " + "the chunk metadata.npz (see Open Questions Q1, Q2 in the " + "plan notes)." + ) + wilson_bg_init_path = str(wilson_bg_path) + logger.info( + "Per-image Wilson B/G init: loading from %s", wilson_bg_path + ) + + # Inject the resolved path so the loss constructor receives it. + # wilson_bg_trainable is already in kwargs from the YAML (defaults to True) + # and flows through to loss_cls(**kwargs) naturally — no manual injection. + kwargs["wilson_bg_init_path"] = wilson_bg_init_path valid_keys = _valid_loss_keys(loss_cls) unknown = set(kwargs) - valid_keys diff --git a/src/integrator/utils/prepare_priors.py b/src/integrator/utils/prepare_priors.py index 0f0e2848..9797cb0d 100644 --- a/src/integrator/utils/prepare_priors.py +++ b/src/integrator/utils/prepare_priors.py @@ -18,6 +18,87 @@ def _nbins_path(filename: str, n_bins: int, data_dir: Path) -> Path: return data_dir / suffixed +def _validate_chunked_priors(cfg: dict, n_bins: int) -> None: + """Validate manifest.yaml for a chunked_rotation_data training config. + + Checks performed (all raise with actionable messages on failure): + - manifest.yaml exists in chunk_dir + - version == 2 (v6 chunk layout) + - 'group_label' is in numeric_columns (baked by integrator.preprocess) + - manifest n_bins matches loss.args.n_bins + + Does NOT regenerate group_labels — integrator.preprocess already writes + group_label into every chunk's metadata.npz. + + Does NOT check for bg_prior.npy here; that is the factory's responsibility + at loss-construction time (_get_loss_module in factory_utils.py). + + If concentration_cfg is present but d_min/d_max are missing, raises + ValueError — raw metadata.npy is not available in chunk_dir so auto-fill + is impossible; the user must set them explicitly in the YAML. + """ + import yaml as _yaml + + chunk_dir = Path(cfg["data_loader"]["args"]["chunk_dir"]) + manifest_path = chunk_dir / "manifest.yaml" + if not manifest_path.exists(): + raise FileNotFoundError( + f"No manifest.yaml found in {chunk_dir}.\n" + "Run integrator.preprocess first." + ) + + m = _yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + + version = m.get("version", 1) + if version != 2: + raise ValueError( + f"manifest.yaml in {chunk_dir} declares version {version}; " + "version 2 is required.\n" + "Re-run integrator.preprocess to regenerate chunks with the " + "v6 layout." + ) + + numeric_cols = m.get("numeric_columns", []) + if "group_label" not in numeric_cols: + raise RuntimeError( + f"manifest.yaml in {chunk_dir} does not list 'group_label' in " + "numeric_columns.\n" + "group_label is required even for n_bins=1 (training calls " + "metadata['group_label'].long() in every step).\n" + f"Re-run: integrator.preprocess --n-bins {n_bins}" + ) + + manifest_n_bins = int(m.get("n_bins", 1)) + if manifest_n_bins != n_bins: + raise ValueError( + f"manifest.yaml was built with n_bins={manifest_n_bins} but " + f"loss.args.n_bins={n_bins}.\n" + "The resolution binning must match between preprocess and training.\n" + f"Re-run: integrator.preprocess --n-bins {n_bins} " + f"(or set loss.args.n_bins: {manifest_n_bins} in your config)." + ) + + loss_args = cfg.get("loss", {}).get("args", {}) + conc_cfg = loss_args.get("concentration_cfg") + if isinstance(conc_cfg, dict) and ( + "d_min" not in conc_cfg or "d_max" not in conc_cfg + ): + raise ValueError( + "loss.args.concentration_cfg requires explicit d_min and d_max " + "when using chunked_rotation_data.\n" + "Raw metadata.npy is not available in chunk_dir for auto-fill.\n" + "Set d_min and d_max explicitly in your config YAML under " + "loss.args.concentration_cfg." + ) + + logger.info( + "prepare_per_bin_priors: chunked_rotation_data — " + "manifest v2 validated, group_label present (n_bins=%d); " + "skipping group_label regeneration (baked into chunks by preprocess).", + n_bins, + ) + + def prepare_per_bin_priors( cfg: dict, *, @@ -34,6 +115,17 @@ def prepare_per_bin_priors( if loss_name not in ("monochromatic_wilson", "polychromatic_wilson"): return + # ── chunked_rotation_data: validate manifest; skip group_label regen ────── + # group_label is already baked into every chunk's metadata.npz by + # integrator.preprocess. bg_prior.npy is written to chunk_dir by + # integrator.preprocess and loaded by the factory at loss-construction time. + if cfg.get("data_loader", {}).get("name") == "chunked_rotation_data": + loss_args = cfg.get("loss", {}).get("args", {}) + _n = n_bins if n_bins > 0 else int(loss_args.get("n_bins", 1)) + _validate_chunked_priors(cfg, n_bins=_n) + return + + # ── Standard rotation_data path (unchanged) ─────────────────────────────── data_dir = Path(cfg["data_loader"]["args"]["data_dir"]) loss_args = cfg["loss"].get("args", {}) @@ -129,6 +221,31 @@ def prepare_per_bin_priors( def inject_binning_labels(data_loader, cfg: dict) -> None: """Load binning label files and inject into the dataset's metadata.""" + # ChunkedDataModule: group_label must be baked into chunk files at preprocess + # time via integrator.preprocess --n-bins N. + # group_label is required even when n_bins=1 because _step() always calls + # metadata["group_label"].long(). Raise loudly rather than letting training + # crash with a confusing KeyError inside the forward pass. + # Detection uses _CHUNKED_DATA_MODULE sentinel (set on ChunkedDataModule class) + # for robustness — same sentinel used in prediction_writer.py (v8). + if getattr(data_loader, "_CHUNKED_DATA_MODULE", False): + loss_args = cfg.get("loss", {}).get("args", {}) + n_bins = int(loss_args.get("n_bins", 1)) + numeric_cols = getattr(data_loader, "numeric_columns", {}) + if "group_label" not in numeric_cols: + raise RuntimeError( + "ChunkedDataModule chunks are missing 'group_label'. " + "group_label is required even for n_bins=1 " + "(training will crash without it).\n" + f"Re-run: integrator.preprocess --n-bins {n_bins}" + ) + logger.info( + "inject_binning_labels: group_label verified in chunk manifest " + "(%d bin(s)); skipping re-injection.", + n_bins, + ) + return + loss_args = cfg.get("loss", {}).get("args", {}) n_bins = int(loss_args.get("n_bins", 1)) data_dir = Path(cfg["data_loader"]["args"]["data_dir"]) diff --git a/src/integrator/utils/prepare_wilson_bg.py b/src/integrator/utils/prepare_wilson_bg.py new file mode 100644 index 00000000..dc674a6e --- /dev/null +++ b/src/integrator/utils/prepare_wilson_bg.py @@ -0,0 +1,202 @@ +"""Per-image Wilson B/G initialization from cctbx summation intensities. + +Fits one (B, G) pair per image via ordinary least squares on the Wilson +plot linearization: + + log(I) = log(G) - B / (2 * d^2) + y = a + m*x where x = 1/d^2, y = log(I) + B = -2 * m + G = exp(a) + +The result is cached as chunk_dir/wilson_bg_running_sums.npy and loaded by +MonochromaticWilsonLoss to initialize per-image nn.Embedding weights. +""" + +import logging +from pathlib import Path + +import numpy as np +import yaml + +logger = logging.getLogger(__name__) + +# Fallback values used for images with too few reflections or invalid fits. +# These are NOT controlled by init_log_B / init_log_G in the YAML — those +# govern the scalar init path (wilson_bg_init: null). +GLOBAL_B: float = 20.0 +GLOBAL_G: float = 1000.0 +MIN_REFLECTIONS: int = 10 + + +def fit_wilson_bg_from_chunks(cfg: dict, *, force: bool = False) -> None: + """Fit per-image Wilson B and G from cctbx summation intensities. + + Reads chunk metadata.npz files, accumulates running sums for OLS across + all chunks (vectorized per chunk with np.add.at), solves the regression + for all images simultaneously, and saves the result. + + Only runs when ALL of: + - loss.name == "monochromatic_wilson" + - loss.args.image_level_wilson == True + - loss.args.wilson_bg_init == "cctbx" + + The result is cached; subsequent calls with force=False are no-ops. + + Args: + cfg: Full YAML config dict. + force: Recompute even if the output file already exists. + """ + from integrator.utils.factory_utils import _get_data_dir + + # ── Early-return for configs that don't request cctbx init ─────────────── + loss_name = cfg.get("loss", {}).get("name", "") + if loss_name != "monochromatic_wilson": + return + loss_args = cfg.get("loss", {}).get("args", {}) + if not loss_args.get("image_level_wilson", False): + return + if loss_args.get("wilson_bg_init") != "cctbx": + return + + chunk_dir = Path(_get_data_dir(cfg)) + out_path = chunk_dir / "wilson_bg_running_sums.npy" + + if out_path.exists() and not force: + logger.info( + "fit_wilson_bg_from_chunks: %s already exists, skipping.", + out_path, + ) + return + + # ── Read n_images from manifest.yaml ───────────────────────────────────── + manifest_path = chunk_dir / "manifest.yaml" + if not manifest_path.exists(): + raise FileNotFoundError( + f"No manifest.yaml found in {chunk_dir}.\n" + "Run integrator.preprocess first." + ) + manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) + n_images = int(manifest["n_images"]) + + # ── Verify intensity column is present ─────────────────────────────────── + intensity_col = "intensity.sum.value" + numeric_cols = manifest.get("numeric_columns", []) + if isinstance(numeric_cols, dict): + numeric_cols = list(numeric_cols.keys()) + if intensity_col not in numeric_cols: + raise RuntimeError( + f"Column '{intensity_col}' not found in manifest numeric_columns: " + f"{numeric_cols}.\n" + "Re-run integrator.preprocess with intensity.sum.value kept as a " + "numeric column (add it to the --numeric-columns list)." + ) + + # ── Pre-allocate running sum arrays (float64 for numerical precision) ──── + count = np.zeros(n_images, dtype=np.float64) + sum_x = np.zeros(n_images, dtype=np.float64) + sum_y = np.zeros(n_images, dtype=np.float64) + sum_x2 = np.zeros(n_images, dtype=np.float64) + sum_xy = np.zeros(n_images, dtype=np.float64) + + # ── Scan chunks ────────────────────────────────────────────────────────── + n_chunks_processed = 0 + for chunk_path in sorted(chunk_dir.glob("chunk_*")): + meta_path = chunk_path / "metadata.npz" + if not meta_path.exists(): + continue + + meta = np.load(meta_path, allow_pickle=False) + + image_id = meta["image_id"].astype(np.int64) + d = meta["d"].astype(np.float64) + I = meta[intensity_col].astype(np.float64) + + # Vectorized validity filter + valid = np.isfinite(d) & np.isfinite(I) & (d > 0.0) & (I > 0.0) + image_id = image_id[valid] + d = d[valid] + I = I[valid] + + if len(d) == 0: + continue + + # Wilson linearization + x = 1.0 / (d**2) + y = np.log(I) + + # Accumulate running sums — np.add.at handles repeated image_ids + # within a chunk without buffering (unlike +=). + np.add.at(count, image_id, 1.0) + np.add.at(sum_x, image_id, x) + np.add.at(sum_y, image_id, y) + np.add.at(sum_x2, image_id, x * x) + np.add.at(sum_xy, image_id, x * y) + + n_chunks_processed += 1 + + logger.info( + "fit_wilson_bg_from_chunks: scanned %d chunks for %d images", + n_chunks_processed, + n_images, + ) + + # ── Batch OLS for all images simultaneously ─────────────────────────────── + denom = count * sum_x2 - sum_x**2 + valid_fit = (count >= MIN_REFLECTIONS) & (denom != 0.0) + safe_denom = np.where(denom != 0.0, denom, 1.0) + safe_count = np.where(count > 0.0, count, 1.0) + + m = np.where( + valid_fit, + (count * sum_xy - sum_x * sum_y) / safe_denom, + 0.0, + ) + a = np.where( + valid_fit, + (sum_y - m * sum_x) / safe_count, + 0.0, + ) + + B_array = np.where(valid_fit, -2.0 * m, GLOBAL_B) + G_array = np.where(valid_fit, np.exp(np.clip(a, -20.0, 20.0)), GLOBAL_G) + + # Fallback any non-finite or non-positive result to globals. + # B > 0 check is intentional: the fitted Wilson B parameter is required + # to be positive for this fixed-prior experiment. + B_array = np.where( + np.isfinite(B_array) & (B_array > 0.0), B_array, GLOBAL_B + ) + G_array = np.where( + np.isfinite(G_array) & (G_array > 0.0), G_array, GLOBAL_G + ) + + n_fallback = int((~valid_fit).sum()) + logger.info( + "fit_wilson_bg_from_chunks: %d/%d images used fallback " + "(B=%.1f, G=%.1f)", + n_fallback, + n_images, + GLOBAL_B, + GLOBAL_G, + ) + logger.info( + "B: min=%.3f median=%.3f max=%.3f", + float(B_array.min()), + float(np.median(B_array)), + float(B_array.max()), + ) + logger.info( + "G: min=%.3f median=%.3f max=%.3f", + float(G_array.min()), + float(np.median(G_array)), + float(G_array.max()), + ) + + # ── Save ────────────────────────────────────────────────────────────────── + payload = { + "B_array": B_array.astype(np.float32), + "G_array": G_array.astype(np.float32), + "n_images": n_images, + } + np.save(out_path, payload) + logger.info("fit_wilson_bg_from_chunks: saved %s", out_path) diff --git a/tests/test_prepare_priors_chunked.py b/tests/test_prepare_priors_chunked.py new file mode 100644 index 00000000..9b671411 --- /dev/null +++ b/tests/test_prepare_priors_chunked.py @@ -0,0 +1,513 @@ +""" +Tests for the chunked_rotation_data path in prepare_per_bin_priors and related +factory / preprocess functions. + +Coverage: + prepare_priors.py + - _validate_chunked_priors: happy path and all error branches + - prepare_per_bin_priors: early-return for chunked, untouched for rotation_data + - inject_binning_labels: _CHUNKED_DATA_MODULE sentinel detection + + factory_utils.py + - _get_data_dir: returns chunk_dir when data_dir absent + - _get_loss_module (via _get_data_dir): loads bg_prior from chunk_dir + - _get_loss_module: raises FileNotFoundError when chunked bg_prior missing + - _get_loss_module: explicit YAML bg_rate/bg_concentration overrides file + + preprocess.py + - _write_bg_prior: writes correct bg_prior_{n}.npy into out_dir + - _write_bg_prior: skips gracefully when no background column present +""" + +from __future__ import annotations + +import numpy as np +import pytest +import yaml +from pathlib import Path + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _make_manifest( + tmp_path: Path, + *, + version: int = 2, + n_bins: int = 1, + numeric_columns: list | None = None, + string_columns: list | None = None, +) -> Path: + """Write a minimal manifest.yaml to tmp_path.""" + if numeric_columns is None: + numeric_columns = ["image_id", "d", "refl_ids", "group_label"] + if string_columns is None: + string_columns = [] + manifest = { + "version": version, + "chunk_size": 100, + "n_chunks": 1, + "n_valid_rows": 100, + "n_train": 80, + "n_val": 20, + "n_images": 5, + "transform": "asinh", + "n_bins": n_bins, + "numeric_columns": numeric_columns, + "string_columns": string_columns, + "split": {"mode": "generated", "validation_split": 0.2, "seed": 42}, + "chunks": [ + { + "dir": "chunk_00000", + "n_rows": 100, + "n_train": 80, + "n_val": 20, + "n_unique_images": 5, + "min_image_id": 0, + "max_image_id": 4, + } + ], + } + p = tmp_path / "manifest.yaml" + p.write_text(yaml.safe_dump(manifest)) + return p + + +def _chunked_cfg(chunk_dir: Path, n_bins: int = 1) -> dict: + """Minimal config dict for chunked_rotation_data.""" + return { + "data_loader": { + "name": "chunked_rotation_data", + "args": { + "chunk_dir": str(chunk_dir), + "batch_size": 32, + "num_workers": 0, + }, + }, + "loss": { + "name": "monochromatic_wilson", + "args": { + "observation_likelihood": "normal", + "init_obs_scale": 10.0, + "n_bins": n_bins, + "lp_correction": False, + }, + }, + } + + +def _rotation_cfg(data_dir: Path, n_bins: int = 1) -> dict: + """Minimal config dict for rotation_data.""" + return { + "data_loader": { + "name": "rotation_data", + "args": {"data_dir": str(data_dir), "batch_size": 32}, + }, + "loss": { + "name": "monochromatic_wilson", + "args": {"n_bins": n_bins, "lp_correction": False}, + }, + } + + +# ── _validate_chunked_priors ────────────────────────────────────────────────── + + +def test_validate_chunked_priors_happy_path(tmp_path): + """Valid manifest, group_label present, n_bins match → no exception.""" + from integrator.utils.prepare_priors import _validate_chunked_priors + + _make_manifest(tmp_path, version=2, n_bins=1) + cfg = _chunked_cfg(tmp_path, n_bins=1) + # Should not raise + _validate_chunked_priors(cfg, n_bins=1) + + +def test_validate_chunked_priors_raises_if_manifest_missing(tmp_path): + """FileNotFoundError when manifest.yaml is absent.""" + from integrator.utils.prepare_priors import _validate_chunked_priors + + cfg = _chunked_cfg(tmp_path, n_bins=1) + with pytest.raises(FileNotFoundError, match="manifest.yaml"): + _validate_chunked_priors(cfg, n_bins=1) + + +def test_validate_chunked_priors_raises_if_version_not_2(tmp_path): + """ValueError when manifest version != 2.""" + from integrator.utils.prepare_priors import _validate_chunked_priors + + _make_manifest(tmp_path, version=1, n_bins=1) + cfg = _chunked_cfg(tmp_path, n_bins=1) + with pytest.raises(ValueError, match="version 2 is required"): + _validate_chunked_priors(cfg, n_bins=1) + + +def test_validate_chunked_priors_raises_if_group_label_missing(tmp_path): + """RuntimeError when group_label absent from numeric_columns.""" + from integrator.utils.prepare_priors import _validate_chunked_priors + + _make_manifest( + tmp_path, + version=2, + n_bins=1, + numeric_columns=["image_id", "d", "refl_ids"], + ) # no group_label + cfg = _chunked_cfg(tmp_path, n_bins=1) + with pytest.raises(RuntimeError, match="group_label"): + _validate_chunked_priors(cfg, n_bins=1) + + +def test_validate_chunked_priors_raises_if_n_bins_mismatch(tmp_path): + """ValueError when manifest n_bins != loss.args.n_bins.""" + from integrator.utils.prepare_priors import _validate_chunked_priors + + _make_manifest( + tmp_path, version=2, n_bins=1 + ) # manifest built with n_bins=1 + cfg = _chunked_cfg(tmp_path, n_bins=10) # config wants 10 + with pytest.raises(ValueError, match="n_bins=1"): + _validate_chunked_priors(cfg, n_bins=10) + + +def test_validate_chunked_priors_raises_if_concentration_cfg_missing_d_range( + tmp_path, +): + """ValueError when concentration_cfg present but d_min/d_max absent.""" + from integrator.utils.prepare_priors import _validate_chunked_priors + + _make_manifest(tmp_path, version=2, n_bins=1) + cfg = _chunked_cfg(tmp_path, n_bins=1) + cfg["loss"]["args"]["concentration_cfg"] = { + "some_param": 1.0 + } # no d_min/d_max + with pytest.raises(ValueError, match="d_min"): + _validate_chunked_priors(cfg, n_bins=1) + + +# ── prepare_per_bin_priors ──────────────────────────────────────────────────── + + +def test_prepare_per_bin_priors_chunked_returns_early(tmp_path): + """prepare_per_bin_priors returns without touching data_dir for chunked.""" + from integrator.utils.prepare_priors import prepare_per_bin_priors + + _make_manifest(tmp_path, version=2, n_bins=1) + cfg = _chunked_cfg(tmp_path, n_bins=1) + # If this tries to access data_dir it would KeyError. Should not raise. + prepare_per_bin_priors(cfg) + + +def test_prepare_per_bin_priors_non_wilson_skips(tmp_path): + """Non-Wilson loss name → returns immediately, no manifest access needed.""" + from integrator.utils.prepare_priors import prepare_per_bin_priors + + cfg = { + "data_loader": { + "name": "chunked_rotation_data", + "args": {"chunk_dir": str(tmp_path)}, + }, + "loss": {"name": "some_other_loss", "args": {}}, + } + # No manifest.yaml in tmp_path → if it tried to validate it would raise + prepare_per_bin_priors(cfg) + + +def test_prepare_per_bin_priors_rotation_data_unchanged(tmp_path): + """rotation_data with missing data_dir still raises KeyError (unchanged).""" + from integrator.utils.prepare_priors import prepare_per_bin_priors + + # rotation_data config with no data_dir key + cfg = { + "data_loader": {"name": "rotation_data", "args": {}}, + "loss": {"name": "monochromatic_wilson", "args": {"n_bins": 1}}, + } + with pytest.raises(KeyError): + prepare_per_bin_priors(cfg) + + +# ── _get_data_dir ───────────────────────────────────────────────────────────── + + +def test_get_data_dir_returns_data_dir_for_rotation(tmp_path): + """_get_data_dir returns data_dir for rotation_data config.""" + from integrator.utils.factory_utils import _get_data_dir + + cfg = _rotation_cfg(tmp_path) + assert _get_data_dir(cfg) == str(tmp_path) + + +def test_get_data_dir_returns_chunk_dir_for_chunked(tmp_path): + """_get_data_dir returns chunk_dir for chunked_rotation_data config.""" + from integrator.utils.factory_utils import _get_data_dir + + cfg = _chunked_cfg(tmp_path) + assert _get_data_dir(cfg) == str(tmp_path) + + +def test_get_data_dir_raises_if_neither_key_present(): + """_get_data_dir raises KeyError if neither data_dir nor chunk_dir is present.""" + from integrator.utils.factory_utils import _get_data_dir + + cfg = { + "data_loader": {"name": "rotation_data", "args": {}}, + "loss": {"name": "monochromatic_wilson", "args": {}}, + } + with pytest.raises(KeyError): + _get_data_dir(cfg) + + +# ── _write_bg_prior ─────────────────────────────────────────────────────────── + + +def _make_scan_and_reference(n_rows: int = 200, rng_seed: int = 0): + """Build a minimal ScanResult and reference dict for _write_bg_prior tests.""" + from dataclasses import replace + from integrator.cli.preprocess import ScanResult + + rng = np.random.default_rng(rng_seed) + valid_indices = np.arange(n_rows, dtype=np.int64) + refl_ids = rng.choice( + np.arange(1, 1_000_001, dtype=np.int64), size=n_rows, replace=False + ) + is_train = rng.random(n_rows) > 0.2 + is_val = ~is_train + group_label = np.zeros(n_rows, dtype=np.int64) # single bin + d_vals = rng.uniform(1.5, 4.0, size=n_rows).astype(np.float32) + bg_mean = rng.exponential(scale=2.0, size=n_rows).astype(np.float32) + + scan = ScanResult( + valid_indices=valid_indices, + refl_ids_valid=refl_ids, + is_train=is_train, + is_val=is_val, + group_label=group_label, + profile_group_label=None, + stats=(30.0, 200.0), + transform="asinh", + numeric_columns=["d", "refl_ids", "group_label"], + string_columns=[], + n_images=5, + n_valid=n_rows, + n_train=int(is_train.sum()), + n_val=int(is_val.sum()), + split_meta={"mode": "generated", "validation_split": 0.2, "seed": 0}, + ) + reference = { + "d": d_vals, + "refl_ids": refl_ids, + "background.mean": bg_mean, + } + return scan, reference + + +def test_write_bg_prior_creates_file(tmp_path): + """_write_bg_prior saves bg_prior_1.npy into out_dir.""" + from integrator.cli.preprocess import _write_bg_prior + + scan, reference = _make_scan_and_reference() + result = _write_bg_prior( + data_dir=tmp_path, + out_dir=tmp_path, + scan=scan, + reference=reference, + n_bins=1, + ) + assert result is not None + assert (tmp_path / "bg_prior_1.npy").exists(), ( + "bg_prior_1.npy should be written into out_dir" + ) + + +def test_write_bg_prior_correct_keys(tmp_path): + """bg_prior_1.npy contains bg_concentration, bg_rate, n_bins.""" + from integrator.cli.preprocess import _write_bg_prior + from integrator.io import load_data + + scan, reference = _make_scan_and_reference() + _write_bg_prior( + data_dir=tmp_path, + out_dir=tmp_path, + scan=scan, + reference=reference, + n_bins=1, + ) + prior = load_data(tmp_path / "bg_prior_1.npy") + assert "bg_concentration" in prior, "bg_concentration key missing" + assert "bg_rate" in prior, "bg_rate key missing" + assert int(prior["n_bins"]) == 1, "n_bins should be 1" + assert float(prior["bg_concentration"]) > 0, ( + "bg_concentration must be positive" + ) + assert float(prior["bg_rate"]) > 0, "bg_rate must be positive" + + +def test_write_bg_prior_skips_without_background_column(tmp_path): + """_write_bg_prior returns None and writes no file when no bg column exists.""" + from integrator.cli.preprocess import _write_bg_prior + + scan, reference = _make_scan_and_reference() + ref_no_bg = { + k: v + for k, v in reference.items() + if k not in ("background.mean", "background.sum.value") + } + result = _write_bg_prior( + data_dir=tmp_path, + out_dir=tmp_path, + scan=scan, + reference=ref_no_bg, + n_bins=1, + ) + assert result is None + assert not (tmp_path / "bg_prior_1.npy").exists() + + +def test_write_bg_prior_values_match_prepare_priors_math(tmp_path): + """bg_prior values match what prepare_per_bin_priors would produce + for the same data — confirming Luis's prior math is preserved.""" + import torch + from integrator.cli.preprocess import _write_bg_prior + from integrator.io import load_data + from integrator.utils.prepare_priors import _fit_per_bin_gamma + + scan, reference = _make_scan_and_reference(n_rows=500) + + _write_bg_prior( + data_dir=tmp_path, + out_dir=tmp_path, + scan=scan, + reference=reference, + n_bins=1, + ) + prior = load_data(tmp_path / "bg_prior_1.npy") + + # Replicate the exact math from prepare_per_bin_priors + bg_vals = torch.as_tensor( + reference["background.mean"], dtype=torch.float32 + ) + group_labels = torch.as_tensor(scan.group_label, dtype=torch.long) + alphas, rates = _fit_per_bin_gamma(bg_vals, group_labels, n_bins=1) + + np.testing.assert_almost_equal( + float(prior["bg_concentration"]), + float(alphas[0]), + decimal=5, + err_msg="bg_concentration must match _fit_per_bin_gamma output exactly", + ) + np.testing.assert_almost_equal( + float(prior["bg_rate"]), + float(rates[0]), + decimal=5, + err_msg="bg_rate must match _fit_per_bin_gamma output exactly", + ) + + +# ── factory: loads bg_prior from chunk_dir, raises if missing ───────────────── + + +def _write_minimal_bg_prior(chunk_dir: Path, n_bins: int = 1) -> None: + """Write a plausible bg_prior_{n}.npy into chunk_dir for factory tests.""" + from integrator.io import save_data + from integrator.utils.prepare_priors import _nbins_path + + payload = {"bg_concentration": 2.5, "bg_rate": 1.2, "n_bins": n_bins} + save_data(payload, _nbins_path("bg_prior.npy", n_bins, chunk_dir)) + + +def test_factory_get_data_dir_uses_chunk_dir(tmp_path): + """_get_data_dir returns chunk_dir for chunked config.""" + from integrator.utils.factory_utils import _get_data_dir + + cfg = _chunked_cfg(tmp_path) + assert Path(_get_data_dir(cfg)) == tmp_path + + +def test_factory_raises_if_chunked_bg_prior_missing(tmp_path): + """_get_loss_module raises FileNotFoundError when bg_prior missing for chunked.""" + from integrator.utils.factory_utils import _get_loss_module + + _make_manifest(tmp_path, version=2, n_bins=1) + cfg = _chunked_cfg(tmp_path, n_bins=1) + # No bg_prior_1.npy in tmp_path → must raise, not silently use defaults + + # _get_loss_module also needs surrogates etc.; we only care it raises + # FileNotFoundError before reaching loss construction. To avoid needing + # full surrogates config, we can call it directly and catch the expected error. + with pytest.raises(FileNotFoundError, match="bg_prior_1.npy"): + _get_loss_module(cfg) + + +def test_factory_explicit_bg_rate_overrides_file_requirement(tmp_path): + """Explicit bg_rate + bg_concentration in loss.args bypasses bg_prior.npy.""" + from integrator.utils.factory_utils import _get_loss_module + + _make_manifest(tmp_path, version=2, n_bins=1) + cfg = _chunked_cfg(tmp_path, n_bins=1) + # Set explicit overrides — factory should not look for bg_prior.npy + cfg["loss"]["args"]["bg_rate"] = 1.5 + cfg["loss"]["args"]["bg_concentration"] = 3.0 + + # Still needs full loss + surrogate config to actually construct — but the + # bg_prior FileNotFoundError must NOT be raised. + # We catch any other exception (missing surrogates etc.) as a sign that + # the bg_prior guard was passed successfully. + try: + _get_loss_module(cfg) + except FileNotFoundError as e: + if "bg_prior" in str(e): + pytest.fail( + "FileNotFoundError for bg_prior raised even though bg_rate " + "and bg_concentration were set explicitly in loss.args" + ) + except Exception: + pass # any other error is fine — bg_prior check was bypassed + + +def test_factory_loads_bg_prior_from_chunk_dir(tmp_path): + """When bg_prior_1.npy exists in chunk_dir, factory loads bg_rate/bg_concentration.""" + from integrator.utils.factory_utils import _get_data_dir + from integrator.io import data_path, load_data + from integrator.utils.prepare_priors import _nbins_path + + _make_manifest(tmp_path, version=2, n_bins=1) + _write_minimal_bg_prior(tmp_path, n_bins=1) + cfg = _chunked_cfg(tmp_path, n_bins=1) + + # Verify that _get_data_dir resolves to chunk_dir and file is found there + resolved_dir = Path(_get_data_dir(cfg)) + bg_path = _nbins_path("bg_prior.npy", 1, resolved_dir) + assert data_path(bg_path) is not None, ( + "bg_prior_1.npy should be found via _get_data_dir → chunk_dir" + ) + + prior = load_data(bg_path) + assert float(prior["bg_concentration"]) == pytest.approx(2.5) + assert float(prior["bg_rate"]) == pytest.approx(1.2) + + +# ── rotation_data behavior unchanged ───────────────────────────────────────── + + +def test_rotation_data_prepare_per_bin_priors_unchanged(tmp_path): + """rotation_data with existing group_labels file is untouched.""" + import torch + from integrator.io import save_data + from integrator.utils.prepare_priors import ( + prepare_per_bin_priors, + _nbins_path, + ) + + # Write a minimal metadata.npy (rotation_data format) + d_vals = torch.linspace(1.5, 4.0, 200) + bg_vals = torch.abs(torch.randn(200)) + 0.5 + metadata = {"d": d_vals, "background.mean": bg_vals} + np.save(tmp_path / "metadata.npy", metadata) + + # Write pre-existing group_labels_1.npy so prepare_per_bin_priors returns early + gl = torch.zeros(200, dtype=torch.long) + gl_path = _nbins_path("group_labels.npy", 1, tmp_path) + save_data(gl, gl_path) + + cfg = _rotation_cfg(tmp_path, n_bins=1) + # Should return without error and without regenerating group_labels + prepare_per_bin_priors(cfg) diff --git a/tests/test_preprocess_chunk_layout.py b/tests/test_preprocess_chunk_layout.py new file mode 100644 index 00000000..6e983766 --- /dev/null +++ b/tests/test_preprocess_chunk_layout.py @@ -0,0 +1,851 @@ +""" +Tests for the consolidated chunk layout introduced in v6. + +Covers: + metadata.npz + - contains all expected numeric/bool column names (no extras, no missing) + - row_id dtype is int64; is_train / is_val dtype is bool + - values round-trip correctly to source reference data + - old-style col_*.npy / str_*.txt files are NOT written + + strings.parquet + - contains all expected string column names + - values round-trip correctly to source reference data + - file is omitted entirely when string_columns is empty + + per-chunk image statistics in manifest.yaml + - n_unique_images, min_image_id, max_image_id present in every chunk entry + - values match np.unique of image_id for those rows + - values are null (None) when image_id is absent from reference + + manifest schema + - version field is 2 + - numeric_columns / string_columns are YAML sequences (lists), not mappings + + ChunkedDataset._load_chunk round-trips + - numeric columns loaded correctly from metadata.npz + - string columns loaded correctly from strings.parquet + - works without error when strings.parquet is absent +""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import numpy as np +import polars as pl +import pytest +import yaml + + +# ── Synthetic dataset ───────────────────────────────────────────────────────── + +N_ROWS = 120 # total valid reflections across all chunks +N_PIXELS = 25 # pixels per shoebox +CHUNK_SIZE = 50 # reflections per chunk → 3 chunks total + + +def _build_scan_and_data(): + """Return (ScanResult, counts, masks, reference) built from numpy only. + + No integrator imports needed: ScanResult is constructed directly so the + tests can run without the full package installed. + """ + from integrator.cli.preprocess import ScanResult + + rng = np.random.default_rng(0) + n = N_ROWS + + valid_indices = np.arange(n, dtype=np.int64) + refl_ids = rng.choice( + np.arange(1, 1_000_001, dtype=np.int64), size=n, replace=False + ) + + # ~20 % val, rest train — no overlap guaranteed + val_mask = rng.random(n) < 0.2 + is_train = ~val_mask + is_val = val_mask + + group_label = rng.integers(0, 3, size=n, dtype=np.int64) + image_ids = rng.integers(0, 10, size=n, dtype=np.int64) + d_vals = rng.uniform(1.0, 5.0, size=n).astype(np.float32) + + counts = rng.uniform(0, 100, size=(n, N_PIXELS)).astype(np.float32) + masks = (rng.uniform(0, 1, size=(n, N_PIXELS)) > 0.1).astype(bool) + + reference = { + "refl_ids": refl_ids, + "image_id": image_ids, + "d": d_vals, + "source_refl": np.array( + [f"run{i % 3}.refl" for i in range(n)], dtype=object + ), + "source_expt": np.array( + [f"run{i % 3}.expt" for i in range(n)], dtype=object + ), + } + + scan = ScanResult( + valid_indices=valid_indices, + refl_ids_valid=refl_ids, + is_train=is_train, + is_val=is_val, + group_label=group_label, + profile_group_label=None, + stats=(30.0, 200.0), + transform="anscombe", + numeric_columns=["image_id", "d", "refl_ids", "group_label"], + string_columns=["source_refl", "source_expt"], + n_images=10, + n_valid=n, + n_train=int(is_train.sum()), + n_val=int(is_val.sum()), + split_meta={"mode": "generated", "validation_split": 0.2, "seed": 0}, + ) + return scan, counts, masks, reference + + +# ── metadata.npz ────────────────────────────────────────────────────────────── + + +def test_metadata_npz_contains_expected_columns(tmp_path): + """metadata.npz must hold exactly numeric_columns + row_id + is_train + is_val.""" + from integrator.cli.preprocess import _pass2_write_chunks + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + meta = np.load( + tmp_path / "chunk_00000" / "metadata.npz", allow_pickle=False + ) + expected = set(scan.numeric_columns) | {"row_id", "is_train", "is_val"} + assert set(meta.files) == expected, ( + f"metadata.npz keys mismatch.\n" + f" expected: {sorted(expected)}\n" + f" got: {sorted(meta.files)}" + ) + + +def test_metadata_npz_dtypes(tmp_path): + """row_id must be int64; is_train / is_val must be bool.""" + from integrator.cli.preprocess import _pass2_write_chunks + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + meta = np.load( + tmp_path / "chunk_00000" / "metadata.npz", allow_pickle=False + ) + assert meta["row_id"].dtype == np.int64, ( + f"row_id dtype: expected int64, got {meta['row_id'].dtype}" + ) + assert meta["is_train"].dtype == bool, ( + f"is_train dtype: expected bool, got {meta['is_train'].dtype}" + ) + assert meta["is_val"].dtype == bool, ( + f"is_val dtype: expected bool, got {meta['is_val'].dtype}" + ) + + +def test_metadata_npz_values_match_source(tmp_path): + """Numeric column values in metadata.npz must match the source reference arrays.""" + from integrator.cli.preprocess import _pass2_write_chunks + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + rows = scan.valid_indices[:CHUNK_SIZE] + meta = np.load( + tmp_path / "chunk_00000" / "metadata.npz", allow_pickle=False + ) + + np.testing.assert_array_equal( + meta["row_id"], rows, err_msg="row_id mismatch" + ) + np.testing.assert_array_equal( + meta["is_train"], + scan.is_train[:CHUNK_SIZE], + err_msg="is_train mismatch", + ) + np.testing.assert_array_equal( + meta["is_val"], scan.is_val[:CHUNK_SIZE], err_msg="is_val mismatch" + ) + np.testing.assert_array_equal( + meta["image_id"], + reference["image_id"][rows], + err_msg="image_id mismatch", + ) + np.testing.assert_array_almost_equal( + meta["d"], reference["d"][rows], err_msg="d mismatch" + ) + np.testing.assert_array_equal( + meta["group_label"], + scan.group_label[rows], + err_msg="group_label mismatch", + ) + + +def test_no_col_npy_or_str_txt_files_written(tmp_path): + """Old-style col_*.npy and str_*.txt files must NOT appear in v6 chunks.""" + from integrator.cli.preprocess import _pass2_write_chunks + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + chunk_dir = tmp_path / "chunk_00000" + old_style = list(chunk_dir.glob("col_*.npy")) + list( + chunk_dir.glob("str_*.txt") + ) + assert len(old_style) == 0, ( + f"Old-style per-column files found: {[f.name for f in old_style]}" + ) + + +# ── strings.parquet ─────────────────────────────────────────────────────────── + + +def test_strings_parquet_contains_expected_columns(tmp_path): + """strings.parquet must contain every column in string_columns.""" + from integrator.cli.preprocess import _pass2_write_chunks + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + pq_path = tmp_path / "chunk_00000" / "strings.parquet" + assert pq_path.exists(), "strings.parquet not found in chunk_00000" + df = pl.read_parquet(pq_path) + assert set(df.columns) == set(scan.string_columns), ( + f"strings.parquet columns mismatch.\n" + f" expected: {sorted(scan.string_columns)}\n" + f" got: {sorted(df.columns)}" + ) + + +def test_strings_parquet_values_match_source(tmp_path): + """String values in strings.parquet must match the source reference arrays.""" + from integrator.cli.preprocess import _pass2_write_chunks + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + rows = scan.valid_indices[:CHUNK_SIZE] + df = pl.read_parquet(tmp_path / "chunk_00000" / "strings.parquet") + + expected_refl = [str(reference["source_refl"][r]) for r in rows] + assert df["source_refl"].to_list() == expected_refl, ( + "source_refl values mismatch" + ) + + expected_expt = [str(reference["source_expt"][r]) for r in rows] + assert df["source_expt"].to_list() == expected_expt, ( + "source_expt values mismatch" + ) + + +def test_strings_parquet_omitted_when_no_string_cols(tmp_path): + """strings.parquet must not be written when string_columns is empty.""" + from integrator.cli.preprocess import _pass2_write_chunks + + scan, counts, masks, reference = _build_scan_and_data() + scan_no_str = replace(scan, string_columns=[]) + _pass2_write_chunks( + scan=scan_no_str, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + pq_path = tmp_path / "chunk_00000" / "strings.parquet" + assert not pq_path.exists(), ( + "strings.parquet must not exist when string_columns is empty" + ) + + +# ── Per-chunk image statistics ──────────────────────────────────────────────── + + +def test_manifest_chunk_entries_have_image_stat_keys(tmp_path): + """Each chunk entry in manifest.yaml must have n_unique_images, min/max image_id.""" + from integrator.cli.preprocess import _pass2_write_chunks, _write_manifest + + scan, counts, masks, reference = _build_scan_and_data() + chunk_infos = _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + data_dir = tmp_path / "data" + data_dir.mkdir() + _write_manifest( + scan=scan, + chunk_infos=chunk_infos, + chunk_size=CHUNK_SIZE, + n_bins=1, + out_dir=tmp_path, + data_dir=data_dir, + ) + + m = yaml.safe_load((tmp_path / "manifest.yaml").read_text()) + for entry in m["chunks"]: + for key in ("n_unique_images", "min_image_id", "max_image_id"): + assert key in entry, ( + f"key '{key}' missing from chunk entry: {entry}" + ) + + +def test_image_stats_values_correct(tmp_path): + """Per-chunk image stats must match np.unique of image_id for those rows.""" + from integrator.cli.preprocess import _pass2_write_chunks, _write_manifest + + scan, counts, masks, reference = _build_scan_and_data() + chunk_infos = _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + data_dir = tmp_path / "data" + data_dir.mkdir() + _write_manifest( + scan=scan, + chunk_infos=chunk_infos, + chunk_size=CHUNK_SIZE, + n_bins=1, + out_dir=tmp_path, + data_dir=data_dir, + ) + + m = yaml.safe_load((tmp_path / "manifest.yaml").read_text()) + for i, entry in enumerate(m["chunks"]): + start = i * CHUNK_SIZE + end = min(start + CHUNK_SIZE, scan.n_valid) + rows = scan.valid_indices[start:end] + img_ids = reference["image_id"][rows] + unique = np.unique(img_ids) + + assert entry["n_unique_images"] == len(unique), ( + f"chunk {i}: n_unique_images {entry['n_unique_images']} != {len(unique)}" + ) + assert entry["min_image_id"] == int(unique.min()), ( + f"chunk {i}: min_image_id {entry['min_image_id']} != {int(unique.min())}" + ) + assert entry["max_image_id"] == int(unique.max()), ( + f"chunk {i}: max_image_id {entry['max_image_id']} != {int(unique.max())}" + ) + + +def test_image_stats_null_when_image_id_absent(tmp_path): + """n_unique_images / min/max image_id must be null when image_id is absent.""" + from integrator.cli.preprocess import _pass2_write_chunks, _write_manifest + + scan, counts, masks, reference = _build_scan_and_data() + ref_no_img = {k: v for k, v in reference.items() if k != "image_id"} + scan_no_img = replace( + scan, + n_images=None, + numeric_columns=[c for c in scan.numeric_columns if c != "image_id"], + ) + chunk_infos = _pass2_write_chunks( + scan=scan_no_img, + counts=counts, + masks=masks, + reference=ref_no_img, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + data_dir = tmp_path / "data" + data_dir.mkdir() + _write_manifest( + scan=scan_no_img, + chunk_infos=chunk_infos, + chunk_size=CHUNK_SIZE, + n_bins=1, + out_dir=tmp_path, + data_dir=data_dir, + ) + + m = yaml.safe_load((tmp_path / "manifest.yaml").read_text()) + for entry in m["chunks"]: + assert entry["n_unique_images"] is None, ( + f"n_unique_images should be null, got {entry['n_unique_images']}" + ) + assert entry["min_image_id"] is None, ( + f"min_image_id should be null, got {entry['min_image_id']}" + ) + assert entry["max_image_id"] is None, ( + f"max_image_id should be null, got {entry['max_image_id']}" + ) + + +# ── Manifest version and schema ─────────────────────────────────────────────── + + +def test_manifest_version_is_2(tmp_path): + """manifest.yaml must declare version: 2.""" + from integrator.cli.preprocess import _pass2_write_chunks, _write_manifest + + scan, counts, masks, reference = _build_scan_and_data() + chunk_infos = _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + data_dir = tmp_path / "data" + data_dir.mkdir() + _write_manifest( + scan=scan, + chunk_infos=chunk_infos, + chunk_size=CHUNK_SIZE, + n_bins=1, + out_dir=tmp_path, + data_dir=data_dir, + ) + + m = yaml.safe_load((tmp_path / "manifest.yaml").read_text()) + assert m["version"] == 2, f"Expected version 2, got {m['version']}" + + +def test_manifest_columns_are_lists_not_dicts(tmp_path): + """numeric_columns and string_columns in manifest.yaml must be lists.""" + from integrator.cli.preprocess import _pass2_write_chunks, _write_manifest + + scan, counts, masks, reference = _build_scan_and_data() + chunk_infos = _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + data_dir = tmp_path / "data" + data_dir.mkdir() + _write_manifest( + scan=scan, + chunk_infos=chunk_infos, + chunk_size=CHUNK_SIZE, + n_bins=1, + out_dir=tmp_path, + data_dir=data_dir, + ) + + m = yaml.safe_load((tmp_path / "manifest.yaml").read_text()) + assert isinstance(m["numeric_columns"], list), ( + f"numeric_columns should be list, got {type(m['numeric_columns']).__name__}" + ) + assert isinstance(m["string_columns"], list), ( + f"string_columns should be list, got {type(m['string_columns']).__name__}" + ) + # Sanity: expected names are present + assert "group_label" in m["numeric_columns"] + assert "source_refl" in m["string_columns"] + + +# ── ChunkedDataset._load_chunk round-trips ──────────────────────────────────── + + +def test_load_chunk_round_trips_numeric_columns(tmp_path): + """_load_chunk must restore numeric column values from metadata.npz.""" + from integrator.cli.preprocess import _pass2_write_chunks + from integrator.data_loaders.chunked_data_module import ChunkedDataset + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + chunk_dir = tmp_path / "chunk_00000" + chunk = ChunkedDataset._load_chunk( + chunk_dir, + numeric_columns=scan.numeric_columns, + string_columns=scan.string_columns, + ) + + rows = scan.valid_indices[:CHUNK_SIZE] + np.testing.assert_array_equal( + chunk["row_id"], rows, err_msg="_load_chunk: row_id mismatch" + ) + np.testing.assert_array_equal( + chunk["is_train"], + scan.is_train[:CHUNK_SIZE], + err_msg="_load_chunk: is_train mismatch", + ) + np.testing.assert_array_equal( + chunk["image_id"], + reference["image_id"][rows], + err_msg="_load_chunk: image_id mismatch", + ) + + +def test_load_chunk_round_trips_string_columns(tmp_path): + """_load_chunk must restore string column values from strings.parquet.""" + from integrator.cli.preprocess import _pass2_write_chunks + from integrator.data_loaders.chunked_data_module import ChunkedDataset + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + chunk_dir = tmp_path / "chunk_00000" + chunk = ChunkedDataset._load_chunk( + chunk_dir, + numeric_columns=scan.numeric_columns, + string_columns=scan.string_columns, + ) + + rows = scan.valid_indices[:CHUNK_SIZE] + expected_refl = [str(reference["source_refl"][r]) for r in rows] + assert chunk["source_refl"] == expected_refl, ( + "_load_chunk: source_refl values mismatch" + ) + + expected_expt = [str(reference["source_expt"][r]) for r in rows] + assert chunk["source_expt"] == expected_expt, ( + "_load_chunk: source_expt values mismatch" + ) + + +def test_load_chunk_works_without_strings_parquet(tmp_path): + """_load_chunk must not raise when strings.parquet is absent.""" + from integrator.cli.preprocess import _pass2_write_chunks + from integrator.data_loaders.chunked_data_module import ChunkedDataset + + scan, counts, masks, reference = _build_scan_and_data() + scan_no_str = replace(scan, string_columns=[]) + _pass2_write_chunks( + scan=scan_no_str, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + chunk_dir = tmp_path / "chunk_00000" + assert not (chunk_dir / "strings.parquet").exists() + + # Must load without raising even though strings.parquet is absent + chunk = ChunkedDataset._load_chunk( + chunk_dir, + numeric_columns=scan.numeric_columns, + string_columns=[], + ) + assert "source_refl" not in chunk + assert "counts" in chunk + assert "row_id" in chunk + + +# ── New tests for v7 robustness changes ─────────────────────────────────────── + + +def test_load_chunk_raises_if_strings_parquet_missing(tmp_path): + """FileNotFoundError when strings.parquet is absent but string_columns is set.""" + from integrator.cli.preprocess import _pass2_write_chunks + from integrator.data_loaders.chunked_data_module import ChunkedDataset + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + # Simulate a corrupted / partially-written chunk by removing the file. + (tmp_path / "chunk_00000" / "strings.parquet").unlink() + + with pytest.raises(FileNotFoundError, match="strings.parquet"): + ChunkedDataset._load_chunk( + tmp_path / "chunk_00000", + numeric_columns=scan.numeric_columns, + string_columns=scan.string_columns, # non-empty — must raise + ) + + +def test_setup_raises_on_version_1_manifest(tmp_path): + """setup() must raise ValueError when manifest.yaml declares version != 2.""" + import yaml + from integrator.data_loaders.chunked_data_module import ChunkedDataModule + + # Write a minimal version-1 manifest (old col_*.npy layout). + v1_manifest = { + "version": 1, + "chunk_size": CHUNK_SIZE, + "n_chunks": 0, + "n_valid_rows": 0, + "n_train": 0, + "n_val": 0, + "n_images": None, + "transform": "anscombe", + "n_bins": 1, + "numeric_columns": {}, # v1 used a dict col_name -> filename + "string_columns": {}, + "split": {"mode": "generated"}, + "chunks": [], + } + (tmp_path / "manifest.yaml").write_text(yaml.safe_dump(v1_manifest)) + + dm = ChunkedDataModule(chunk_dir=tmp_path, batch_size=10) + with pytest.raises(ValueError, match="version 2 is required"): + dm.setup() + + +def test_setup_raises_if_chunk_file_missing(tmp_path): + """setup() must raise FileNotFoundError when a required file is missing.""" + from integrator.cli.preprocess import _pass2_write_chunks, _write_manifest + from integrator.data_loaders.chunked_data_module import ChunkedDataModule + + scan, counts, masks, reference = _build_scan_and_data() + chunk_infos = _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + data_dir = tmp_path / "data" + data_dir.mkdir() + _write_manifest( + scan=scan, + chunk_infos=chunk_infos, + chunk_size=CHUNK_SIZE, + n_bins=1, + out_dir=tmp_path, + data_dir=data_dir, + ) + + # Simulate a partially-written chunk. + (tmp_path / "chunk_00000" / "metadata.npz").unlink() + + dm = ChunkedDataModule(chunk_dir=tmp_path, batch_size=10) + with pytest.raises(FileNotFoundError, match="metadata.npz"): + dm.setup() + + +def test_load_chunk_raises_on_length_mismatch(tmp_path): + """_load_chunk must raise ValueError when an array length mismatches counts.""" + from integrator.cli.preprocess import _pass2_write_chunks + from integrator.data_loaders.chunked_data_module import ChunkedDataset + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + chunk_dir = tmp_path / "chunk_00000" + + # Overwrite metadata.npz with a truncated row_id to simulate a disk-full + # write that left the array short by 5 rows. + with np.load(chunk_dir / "metadata.npz", allow_pickle=False) as meta_orig: + meta_bad = {k: meta_orig[k].copy() for k in meta_orig.files} + meta_bad["row_id"] = meta_bad["row_id"][:-5] # truncate + np.savez(chunk_dir / "metadata.npz", **meta_bad) + + with pytest.raises(ValueError, match="Length mismatch"): + ChunkedDataset._load_chunk( + chunk_dir, + numeric_columns=scan.numeric_columns, + string_columns=scan.string_columns, + ) + + +# ── New tests for v8 robustness changes ───────────────────────────────────────────── + + +@pytest.mark.parametrize("key", ["shoebox", "mask"]) +def test_load_chunk_raises_on_shoebox_mask_row_mismatch(tmp_path, key): + """_load_chunk must raise ValueError when shoebox.npy or mask.npy first + dimension does not match counts.npy.""" + from integrator.cli.preprocess import _pass2_write_chunks + from integrator.data_loaders.chunked_data_module import ChunkedDataset + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + chunk_dir = tmp_path / "chunk_00000" + + # Truncate the target array by 5 rows to simulate a partial write. + arr = np.load(chunk_dir / f"{key}.npy") + np.save(chunk_dir / f"{key}.npy", arr[:-5]) + + with pytest.raises(ValueError, match=key): + ChunkedDataset._load_chunk( + chunk_dir, + numeric_columns=scan.numeric_columns, + string_columns=scan.string_columns, + ) + + +def test_load_chunk_raises_on_missing_metadata_key(tmp_path): + """_load_chunk must raise KeyError naming the missing key and chunk + directory when metadata.npz is missing a required key.""" + from integrator.cli.preprocess import _pass2_write_chunks + from integrator.data_loaders.chunked_data_module import ChunkedDataset + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + chunk_dir = tmp_path / "chunk_00000" + + # Re-save metadata.npz without 'is_val' to simulate a missing key. + with np.load(chunk_dir / "metadata.npz", allow_pickle=False) as meta_orig: + meta_bad = { + k: meta_orig[k].copy() for k in meta_orig.files if k != "is_val" + } + np.savez(chunk_dir / "metadata.npz", **meta_bad) + + with pytest.raises(KeyError, match="is_val"): + ChunkedDataset._load_chunk( + chunk_dir, + numeric_columns=scan.numeric_columns, + string_columns=scan.string_columns, + ) + + +def test_load_chunk_raises_on_missing_string_column(tmp_path): + """_load_chunk must raise ValueError naming the missing column and chunk + directory when strings.parquet is missing a declared column.""" + from integrator.cli.preprocess import _pass2_write_chunks + from integrator.data_loaders.chunked_data_module import ChunkedDataset + + scan, counts, masks, reference = _build_scan_and_data() + _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + + chunk_dir = tmp_path / "chunk_00000" + + # Overwrite strings.parquet without 'source_refl'. + df_orig = pl.read_parquet(chunk_dir / "strings.parquet") + df_orig.drop("source_refl").write_parquet(chunk_dir / "strings.parquet") + + with pytest.raises(ValueError, match="source_refl"): + ChunkedDataset._load_chunk( + chunk_dir, + numeric_columns=scan.numeric_columns, + string_columns=scan.string_columns, + ) + + +def test_manifest_has_source_data_dir(tmp_path): + """_write_manifest records source_data_dir as an absolute resolved path.""" + from integrator.cli.preprocess import _pass2_write_chunks, _write_manifest + + scan, counts, masks, reference = _build_scan_and_data() + data_dir = tmp_path / "data" + data_dir.mkdir() + chunk_infos = _pass2_write_chunks( + scan=scan, + counts=counts, + masks=masks, + reference=reference, + chunk_size=CHUNK_SIZE, + out_dir=tmp_path, + ) + _write_manifest( + scan=scan, + chunk_infos=chunk_infos, + chunk_size=CHUNK_SIZE, + n_bins=1, + out_dir=tmp_path, + data_dir=data_dir, + ) + + m = yaml.safe_load((tmp_path / "manifest.yaml").read_text()) + assert "source_data_dir" in m, "manifest.yaml must contain source_data_dir" + assert m["source_data_dir"] == str(data_dir.resolve()), ( + f"source_data_dir mismatch:\n" + f" expected: {data_dir.resolve()}\n" + f" got: {m['source_data_dir']}" + ) diff --git a/tests/test_preprocess_split.py b/tests/test_preprocess_split.py new file mode 100644 index 00000000..c30a8ba7 --- /dev/null +++ b/tests/test_preprocess_split.py @@ -0,0 +1,133 @@ +""" +Tests for the deterministic train/val split in integrator.preprocess. + +Covers: + - same seed → same split + - different seeds → different splits + - row-order changes do not change membership + - no train/val overlap + - every reflection in exactly one split (Mode B full coverage) + - exported CSV IDs match chunk is_train / is_val arrays + - duplicate refl_ids detected + - val fraction is approximately correct +""" + +from __future__ import annotations + +import numpy as np +import pytest +import polars as pl + + +def _fake_ids(n: int = 1000, seed: int = 0) -> np.ndarray: + """Return n guaranteed-unique int64 refl_ids sampled without replacement.""" + rng = np.random.default_rng(seed) + return rng.choice( + np.arange(1, 10_000_001, dtype=np.int64), + size=n, + replace=False, + ) + + +def test_same_seed_same_split(): + """Identical inputs + seed must produce identical is_train / is_val.""" + from integrator.cli.preprocess import _make_deterministic_split + + ids = _fake_ids(1000) + a_tr, a_va = _make_deterministic_split(ids, 0.2, seed=42) + b_tr, b_va = _make_deterministic_split(ids, 0.2, seed=42) + np.testing.assert_array_equal(a_tr, b_tr) + np.testing.assert_array_equal(a_va, b_va) + + +def test_different_seeds_different_splits(): + """Different seeds must produce different splits.""" + from integrator.cli.preprocess import _make_deterministic_split + + ids = _fake_ids(1000) + tr_42, _ = _make_deterministic_split(ids, 0.2, seed=42) + tr_99, _ = _make_deterministic_split(ids, 0.2, seed=99) + assert not np.array_equal(tr_42, tr_99), ( + "Different seeds should produce different splits" + ) + + +def test_row_order_independent(): + """Shuffling input rows must not change any refl_id's train/val assignment.""" + from integrator.cli.preprocess import _make_deterministic_split + + rng = np.random.default_rng(7) + ids = _fake_ids(500) + perm = rng.permutation(len(ids)) + ids_shuffled = ids[perm] + + tr, va = _make_deterministic_split(ids, 0.2, seed=42) + tr_s, va_s = _make_deterministic_split(ids_shuffled, 0.2, seed=42) + + # Undo the shuffle to compare element-by-element + inv = np.argsort(perm) + np.testing.assert_array_equal(tr, tr_s[inv]) + np.testing.assert_array_equal(va, va_s[inv]) + + +def test_no_overlap(): + """No refl_id may appear in both train and val.""" + from integrator.cli.preprocess import _make_deterministic_split + + ids = _fake_ids(1000) + is_train, is_val = _make_deterministic_split(ids, 0.2, seed=42) + assert not np.any(is_train & is_val), "Train and val must not overlap" + + +def test_full_coverage(): + """Every reflection must be in exactly one of train or val (Mode B).""" + from integrator.cli.preprocess import _make_deterministic_split + + ids = _fake_ids(1000) + is_train, is_val = _make_deterministic_split(ids, 0.2, seed=42) + assert np.all(is_train | is_val), ( + "Every reflection must belong to train or val" + ) + + +def test_exported_csv_matches_chunks(tmp_path): + """IDs written to CSV must exactly match the is_train / is_val arrays.""" + from integrator.cli.preprocess import ( + _make_deterministic_split, + _save_split_csvs, + ) + + ids = _fake_ids(200, seed=5) + is_train, is_val = _make_deterministic_split(ids, 0.2, seed=42) + _save_split_csvs( + out_dir=tmp_path, + refl_ids_valid=ids, + is_train=is_train, + is_val=is_val, + ) + train_csv = pl.read_csv(tmp_path / "train_labels.csv")[ + "train_ids" + ].to_numpy() + val_csv = pl.read_csv(tmp_path / "val_labels.csv")["val_ids"].to_numpy() + np.testing.assert_array_equal(np.sort(train_csv), np.sort(ids[is_train])) + np.testing.assert_array_equal(np.sort(val_csv), np.sort(ids[is_val])) + + +def test_duplicate_refl_ids_detected(): + """np.unique must surface duplicated IDs (logic used in _pass1_scan).""" + ids = np.array([1, 2, 3, 2, 5], dtype=np.int64) + unique, counts = np.unique(ids, return_counts=True) + dups = unique[counts > 1] + assert len(dups) > 0 and 2 in dups + + +def test_val_fraction_approximately_correct(): + """Actual val fraction should be within 2 pp of the requested split.""" + from integrator.cli.preprocess import _make_deterministic_split + + ids = _fake_ids(10_000) + is_train, is_val = _make_deterministic_split(ids, 0.2, seed=0) + actual = is_val.sum() / len(ids) + assert abs(actual - 0.2) < 0.02, ( + f"Val fraction {actual:.3f} too far from 0.20" + ) diff --git a/tests/test_preprocess_transform.py b/tests/test_preprocess_transform.py new file mode 100644 index 00000000..0583c086 --- /dev/null +++ b/tests/test_preprocess_transform.py @@ -0,0 +1,116 @@ +""" +Verify that _apply_transform_np (vectorised numpy, used by integrator.preprocess) +produces numerically identical output to IntegratorDataset._transform_counts +(the per-reflection torch reference used during live training). + +Five transforms (anscombe, log1p, asinh, log_softplus, sqrt_squareplus) are +cross-checked against _transform_counts. The input deliberately includes +negative pixel values to expose any wrong pre-clamping. + +Standardisation is tested separately: its masking was intentionally corrected +in the preprocess path (masked pixels -> 0.0) relative to the original loader +which produces -mean/std for masked pixels. The corrected formula is verified +directly rather than compared against the original. +""" + +from __future__ import annotations + +import numpy as np +import pytest +import torch + +# Transforms verified numerically against _transform_counts +CROSS_CHECKED_TRANSFORMS = [ + "anscombe", + "log1p", + "asinh", + "log_softplus", + "sqrt_squareplus", +] + +MEAN = 30.0 +VAR = 200.0 + + +@pytest.fixture(scope="module") +def sample_data(): + """8 reflections x 441 pixels with negative values to expose wrong clamping.""" + rng = np.random.default_rng(42) + counts = rng.uniform(-5.0, 100.0, size=(8, 441)).astype(np.float32) + masks = rng.uniform(0.0, 1.0, size=(8, 441)) > 0.2 # ~80 % valid + return counts, masks + + +class _TransformStub: + """Minimal stand-in for IntegratorDataset exposing .transform and .stats.""" + + def __init__(self, transform: str, mean: float, var: float): + self.transform = transform + self.stats = torch.tensor([mean, var], dtype=torch.float32) + + +@pytest.mark.parametrize("transform", CROSS_CHECKED_TRANSFORMS) +def test_numpy_matches_torch(sample_data, transform): + """_apply_transform_np must be numerically identical to _transform_counts.""" + from integrator.cli.preprocess import _apply_transform_np + from integrator.data_loaders.data_module import IntegratorDataset + + counts_np, masks_np = sample_data + stub = _TransformStub(transform, MEAN, VAR) + + # Vectorised numpy path (full batch) + out_np = _apply_transform_np(counts_np, masks_np, MEAN, VAR, transform) + + # Reference: original per-row torch path called row by row + out_ref = np.stack( + [ + IntegratorDataset._transform_counts( + stub, + torch.from_numpy(counts_np[i].copy()).float(), + torch.from_numpy(masks_np[i].copy()).bool(), + ).numpy() + for i in range(len(counts_np)) + ] + ) + + np.testing.assert_allclose( + out_np, + out_ref, + rtol=1e-5, + atol=1e-6, + err_msg=( + f"Transform '{transform}': _apply_transform_np output differs " + "from _transform_counts reference." + ), + ) + + +def test_standardization_corrected_masking(sample_data): + """Standardisation: masked pixels must be 0.0 (corrected from original loader). + + The original _transform_counts returns ((counts*masks)-mean)/std, which + gives -mean/std for masked pixels. The preprocess path uses + ((x-mean)/std)*m so masked pixels are exactly 0.0. + """ + from integrator.cli.preprocess import _apply_transform_np + + counts_np, masks_np = sample_data + out = _apply_transform_np( + counts_np, masks_np, MEAN, VAR, "standardization" + ) + + # Masked pixels must be exactly zero + assert np.all(out[~masks_np] == 0.0), ( + "Masked pixels are not 0.0 for standardization. " + "Check that masking is applied after normalisation: ((x-mean)/std)*m" + ) + + # Unmasked pixels must match the corrected formula + expected = (counts_np - MEAN) / np.sqrt(VAR) + np.testing.assert_allclose( + out[masks_np], + expected[masks_np], + rtol=1e-5, + atol=1e-6, + err_msg="Unmasked pixels do not match (x-mean)/std for standardization.", + ) diff --git a/tests/test_source_data_dir.py b/tests/test_source_data_dir.py new file mode 100644 index 00000000..7df3f50e --- /dev/null +++ b/tests/test_source_data_dir.py @@ -0,0 +1,284 @@ +""" +Tests for resolve_source_data_dir and the prediction-level validation helpers +_resolve_and_check_metadata and _resolve_and_check_mtz_sources. + +Resolver tests (resolve_source_data_dir): + - rotation_data: returns data_dir without manifest access + - rotation_data: no manifest.yaml present — no raise + - rotation_data: missing data_dir key → KeyError + - chunked: one-chunk manifest → correct path + - chunked: multi-chunk manifest (5 chunks) → correct path + - chunked: automatic metadata resolution (metadata.npy reachable) + - chunked: manifest missing source_data_dir → ValueError with re-run hint + - chunked: manifest.yaml absent → FileNotFoundError + +Prediction validation helper tests: + - chunked MFX write-back: _resolve_and_check_metadata returns correct path + - chunked MTZ export: _resolve_and_check_mtz_sources returns source_data_dir + - missing metadata.npy: _resolve_and_check_metadata raises FileNotFoundError + with the expected path in the message + - missing dataset.yaml: _resolve_and_check_mtz_sources raises FileNotFoundError + naming dataset.yaml +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _write_manifest( + chunk_dir: Path, + *, + source_data_dir: Path, + n_chunks: int = 1, + include_source: bool = True, +) -> None: + """Write a minimal v2 manifest.yaml to chunk_dir.""" + m: dict = { + "version": 2, + "chunk_size": 100, + "n_chunks": n_chunks, + "n_valid_rows": n_chunks * 100, + "n_train": n_chunks * 80, + "n_val": n_chunks * 20, + "n_images": 5, + "transform": "asinh", + "n_bins": 1, + "numeric_columns": ["group_label"], + "string_columns": [], + "split": {"mode": "generated", "validation_split": 0.2, "seed": 42}, + "chunks": [ + { + "dir": f"chunk_{i:05d}", + "n_rows": 100, + "n_train": 80, + "n_val": 20, + "n_unique_images": 5, + "min_image_id": 0, + "max_image_id": 4, + } + for i in range(n_chunks) + ], + } + if include_source: + m["source_data_dir"] = str(Path(source_data_dir).resolve()) + (chunk_dir / "manifest.yaml").write_text(yaml.safe_dump(m)) + + +def _chunked_cfg(chunk_dir: Path) -> dict: + return { + "data_loader": { + "name": "chunked_rotation_data", + "args": {"chunk_dir": str(chunk_dir)}, + } + } + + +def _rotation_cfg(data_dir: Path) -> dict: + return { + "data_loader": { + "name": "rotation_data", + "args": {"data_dir": str(data_dir)}, + } + } + + +# ── resolve_source_data_dir — rotation_data ─────────────────────────────────── + + +def test_rotation_data_returns_data_dir(tmp_path): + """rotation_data: resolver returns data_dir directly.""" + from integrator.utils import resolve_source_data_dir + + data_dir = tmp_path / "data" + data_dir.mkdir() + assert resolve_source_data_dir(_rotation_cfg(data_dir)) == data_dir + + +def test_rotation_data_no_manifest_access(tmp_path): + """rotation_data: no manifest.yaml present — resolver must not raise.""" + from integrator.utils import resolve_source_data_dir + + data_dir = tmp_path / "data" + data_dir.mkdir() + # No manifest.yaml written anywhere — rotation_data must not read one. + resolve_source_data_dir(_rotation_cfg(data_dir)) + + +def test_rotation_data_missing_data_dir_key_raises(): + """rotation_data: missing data_dir key → KeyError.""" + from integrator.utils import resolve_source_data_dir + + cfg = {"data_loader": {"name": "rotation_data", "args": {}}} + with pytest.raises(KeyError): + resolve_source_data_dir(cfg) + + +# ── resolve_source_data_dir — chunked_rotation_data ─────────────────────────── + + +def test_chunked_one_chunk_manifest(tmp_path): + """chunked: single-chunk manifest — returns source_data_dir as absolute Path.""" + from integrator.utils import resolve_source_data_dir + + chunk_dir = tmp_path / "chunks" + chunk_dir.mkdir() + data_dir = tmp_path / "data" + data_dir.mkdir() + _write_manifest(chunk_dir, source_data_dir=data_dir, n_chunks=1) + + assert ( + resolve_source_data_dir(_chunked_cfg(chunk_dir)) == data_dir.resolve() + ) + + +def test_chunked_multi_chunk_manifest(tmp_path): + """chunked: 5-chunk manifest — returns source_data_dir correctly.""" + from integrator.utils import resolve_source_data_dir + + chunk_dir = tmp_path / "chunks" + chunk_dir.mkdir() + data_dir = tmp_path / "data" + data_dir.mkdir() + _write_manifest(chunk_dir, source_data_dir=data_dir, n_chunks=5) + + assert ( + resolve_source_data_dir(_chunked_cfg(chunk_dir)) == data_dir.resolve() + ) + + +def test_chunked_automatic_metadata_resolution(tmp_path): + """chunked: resolved source_data_dir / metadata.npy gives correct path.""" + from integrator.utils import resolve_source_data_dir + + chunk_dir = tmp_path / "chunks" + chunk_dir.mkdir() + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "metadata.npy").write_bytes(b"") # simulate file exists + + _write_manifest(chunk_dir, source_data_dir=data_dir, n_chunks=1) + result = resolve_source_data_dir(_chunked_cfg(chunk_dir)) + + assert (result / "metadata.npy").exists(), ( + "metadata.npy must be reachable via the resolved source_data_dir" + ) + + +def test_chunked_old_manifest_raises_valueerror(tmp_path): + """chunked: manifest without source_data_dir raises ValueError with re-run hint.""" + from integrator.utils import resolve_source_data_dir + + chunk_dir = tmp_path / "chunks" + chunk_dir.mkdir() + _write_manifest( + chunk_dir, source_data_dir=tmp_path / "data", include_source=False + ) + + with pytest.raises(ValueError, match="source_data_dir"): + resolve_source_data_dir(_chunked_cfg(chunk_dir)) + + +def test_chunked_missing_manifest_raises_filenotfounderror(tmp_path): + """chunked: no manifest.yaml → FileNotFoundError with preprocess reminder.""" + from integrator.utils import resolve_source_data_dir + + chunk_dir = tmp_path / "chunks" + chunk_dir.mkdir() + + with pytest.raises(FileNotFoundError, match="manifest.yaml"): + resolve_source_data_dir(_chunked_cfg(chunk_dir)) + + +# ── Prediction validation helpers ───────────────────────────────────────────── + + +def test_resolve_and_check_metadata_chunked_returns_correct_path(tmp_path): + """chunked MFX write-back: _resolve_and_check_metadata returns + source_data_dir/metadata.npy when the file exists.""" + from integrator.cli.predict import _resolve_and_check_metadata + + chunk_dir = tmp_path / "chunks" + chunk_dir.mkdir() + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "metadata.npy").write_bytes(b"") # file exists + + _write_manifest(chunk_dir, source_data_dir=data_dir, n_chunks=1) + cfg = _chunked_cfg(chunk_dir) + + result = _resolve_and_check_metadata(cfg) + assert result == (data_dir.resolve() / "metadata.npy") + + +def test_resolve_and_check_mtz_sources_chunked_returns_source_data_dir( + tmp_path, +): + """chunked MTZ export: _resolve_and_check_mtz_sources returns source_data_dir + when both metadata.npy and dataset.yaml exist.""" + from integrator.cli.predict import _resolve_and_check_mtz_sources + + chunk_dir = tmp_path / "chunks" + chunk_dir.mkdir() + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "metadata.npy").write_bytes(b"") + (data_dir / "dataset.yaml").write_bytes(b"") + + _write_manifest(chunk_dir, source_data_dir=data_dir, n_chunks=1) + cfg = _chunked_cfg(chunk_dir) + + result = _resolve_and_check_mtz_sources(cfg) + assert result == data_dir.resolve() + + +def test_resolve_and_check_metadata_raises_when_metadata_missing(tmp_path): + """Missing metadata.npy: _resolve_and_check_metadata raises FileNotFoundError + with the expected path in the message.""" + from integrator.cli.predict import _resolve_and_check_metadata + + chunk_dir = tmp_path / "chunks" + chunk_dir.mkdir() + data_dir = tmp_path / "data" + data_dir.mkdir() + # metadata.npy intentionally NOT created + + _write_manifest(chunk_dir, source_data_dir=data_dir, n_chunks=1) + cfg = _chunked_cfg(chunk_dir) + + expected_path = str(data_dir.resolve() / "metadata.npy") + with pytest.raises(FileNotFoundError, match="metadata.npy"): + _resolve_and_check_metadata(cfg) + + # Confirm the expected path appears in the error message + try: + _resolve_and_check_metadata(cfg) + except FileNotFoundError as e: + assert expected_path in str(e), ( + f"Expected path '{expected_path}' not found in error message:\n{e}" + ) + + +def test_resolve_and_check_mtz_raises_when_dataset_yaml_missing(tmp_path): + """Missing dataset.yaml: _resolve_and_check_mtz_sources raises FileNotFoundError + naming dataset.yaml (metadata.npy present, dataset.yaml absent).""" + from integrator.cli.predict import _resolve_and_check_mtz_sources + + chunk_dir = tmp_path / "chunks" + chunk_dir.mkdir() + data_dir = tmp_path / "data" + data_dir.mkdir() + (data_dir / "metadata.npy").write_bytes(b"") # present + # dataset.yaml intentionally NOT created + + _write_manifest(chunk_dir, source_data_dir=data_dir, n_chunks=1) + cfg = _chunked_cfg(chunk_dir) + + with pytest.raises(FileNotFoundError, match="dataset.yaml"): + _resolve_and_check_mtz_sources(cfg) diff --git a/workflows/mfx101555026/README.md b/workflows/mfx101555026/README.md new file mode 100644 index 00000000..b88016e2 --- /dev/null +++ b/workflows/mfx101555026/README.md @@ -0,0 +1,620 @@ +# MFX101555026 — Integrator Workflow + +End-to-end workflow for applying the variational-inference Integrator to +experimental MFX (Macromolecular Femtosecond Crystallography) data at LCLS, +then evaluating downstream crystallographic performance against the +cctbx.xfel baseline. + +--- + +## What this branch adds + +Extensions built on top of the base Integrator framework (Luis Aldama): + +| Extension | Description | +|---|---| +| **MFX/Jungfrau shoebox extraction** | `make_shoeboxes --mfx`: reads cctbx.xfel `.refl/.expt` pairs, handles Jungfrau 32-panel XTC format, clips shoeboxes at panel boundaries, applies detector mask. Parallel extraction via 16-job Slurm array. | +| **Chunked preprocessing** | `integrator.preprocess`: two-pass CLI that filters, splits, transforms, and writes fixed-size chunks from a raw shoebox dataset. Enables training on datasets too large for RAM. Produces `manifest.yaml` (version 2) with deterministic train/val split. | +| **ChunkedDataModule** | `chunked_rotation_data` loader: IterableDataset that loads one chunk at a time, yields complete mini-batches, shuffles at chunk and row level, preserves row order during prediction for write-back. | +| **LogNormal / FoldedNormal qi** | Two new intensity posteriors (`lognormal`, `foldednormal`) registered as surrogate options. FoldedNormal required a custom `Distribution` subclass to provide `log_prob` and `mean`. | +| **MC-KL fallback** | `_kl_divergence_with_mc_fallback`: automatically falls back to Monte Carlo KL estimation when PyTorch lacks an exact formula (e.g. FoldedNormal vs. Gamma Wilson prior). | +| **Normal / Student-t likelihoods** | `observation_likelihood: normal` or `student_t` in the Wilson loss. Added to support MFX/Jungfrau pixel values that can be floating-point and negative. | +| **MFX write-back** | `refl_io.py`: maps predicted I and σ(I) back to the original 124,975 `.refl` files via `refl_ids` + `reflection_id` metadata. Vectorised `np.unique` groupby (was O(N × n_files), now O(N log N)). Float32 clamping fix for refl_ids > 2²³. | +| **Image-level Wilson B/G** *(prototype)* | `image_level_wilson: true` in the loss: per-image B and G via `nn.Embedding` tables indexed by `image_id`. Implemented and tested; not yet evaluated downstream. | + +--- + +## Dataset + +| Property | Value | +|---|---| +| Experiment | mfx101555026 | +| Source files | 124,975 indexed MFX `.refl/.expt` pairs | +| Runs used | r0269–r0289 (excluding r0275) | +| Shoeboxes extracted | ~115 million reflections | +| Shoebox size | 1 × 25 × 25 pixels | +| Detector | Jungfrau (32 panels, XTC format) | + +--- + +## Pipeline + +``` +MFX raw data (cctbx.xfel indexed + integrated) + r*_idx-data_*_integrated.refl / .expt + │ + ▼ [cctbx env] + make_shoeboxes (MFX/Jungfrau path) + │ + ▼ + counts.npy masks.npy metadata.npy dataset.yaml + │ + ▼ [integrator-cuda-dev] + integrator.preprocess + (filter → split → transform → write chunks) + │ + ▼ + chunk_00000/ … chunk_NNNNN/ + manifest.yaml (version 2) + │ + ▼ [integrator-cuda-dev, GPU] + integrator.train (ChunkedDataModule + Wilson ELBO) + │ + ▼ + run_*/files/checkpoints/epoch=XXXX.ckpt + │ + ▼ [integrator-cuda-dev, GPU] + integrator.predict + │ + ▼ + epoch_XXXX/*.parquet (qi_mean, qi_var, qbg_mean, refl_ids, …) + │ + ▼ [integrator-cuda-dev] + export_preds_for_dials.py (parquet → .npz) + │ + ▼ [cctbx env] + run_writeback.py (dials.python) + │ + ▼ + mfx_refl_writeback/ (124,975 .refl + .expt) + │ + ▼ [cctbx env] + cxi.xfel.merge (scale PHIL) + cxi.xfel.merge (merge PHIL) + │ + ▼ + merged MTZ + CC½ / Rint / Rsplit diagnostics + │ + ▼ [cctbx env] + phenix.refine + │ + ▼ + R-work / R-free + electron density maps +``` + +--- + +## Environment rule + +Three environments — never mix them. + +| Environment | Activate | Use for | +|---|---|---| +| **cctbx / psana2** | `source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh` | make_shoeboxes, write-back, scale, merge, refine | +| **integrator-cuda-dev** | `micromamba activate integrator-cuda-dev` | preprocess, train, predict, NPZ export | +| **CCP4 / visualization** | `source /sdf/group/lcls/ds/tools/ccp4-8.0/bin/ccp4.setup-sh` | COOT structure and map viewing | + +> **Note:** `integrator-cuda-dev` has no DIALS/dxtbx. Do not run write-back there. +> `cctbx` has no polars/torch. Do not run training there. + +--- + +## Step-by-step walkthrough + +Set paths once at the top of your session: + +```bash +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx +DATA_DIR=$BASE/mfx_shoebox_allruns_269_289_no275_024_rg101 +CHUNK_DIR=$DATA_DIR/chunks +REFL_DIR=$BASE/all_runs_269_289_no275_024_rg101_integrated +MASK=/sdf/data/lcls/ds/mfx/mfx101555026/results/pam/hot_lines_combined5.mask +``` + +--- + +### Step 1 — Extract shoeboxes *(cctbx env)* + +Skip if `DATA_DIR` already contains `counts.npy`, `masks.npy`, `metadata.npy`, +and `dataset.yaml`. + +```bash +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh +export PYTHONPATH=$REPO/src:$PYTHONPATH + +dials.python src/integrator/cli/make_shoeboxes.py \ + --mfx \ + --data-dir "$REFL_DIR" \ + --out-dir "$DATA_DIR" \ + --mfx-pattern "r*_idx-data_*_integrated.refl" \ + --detector-mask "$MASK" \ + --w 25 --h 25 --d 1 \ + --counts-dtype float32 \ + --write-chunk-size 10000 \ + --no-mask-overlap +``` + +For the full dataset (124,975 files) use the 16-job Slurm array script: + +```bash +sbatch $REPO/workflows/mfx101555026/scripts/extraction/make_shoeboxes_allruns_024_rg101_16array.sh +# then merge chunks: +python $REPO/workflows/mfx101555026/scripts/extraction/merge_allruns_024_rg101_shoebox_chunks.py +``` + +Expected output under `DATA_DIR/`: +``` +counts.npy masks.npy metadata.npy dataset.yaml +``` + +--- + +### Step 2 — Activate the integrator environment + +```bash +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root +eval "$(/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba shell hook --shell bash)" +micromamba activate integrator-cuda-dev +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH=$REPO/src +cd $REPO +``` + +Verify: +```bash +which integrator.preprocess integrator.train integrator.predict +# if missing: pip install -e . +``` + +--- + +### Step 3 — Preprocess into chunks *(integrator-cuda-dev)* + +Run once per dataset + preprocessing configuration. + +```bash +integrator.preprocess \ + --data-dir "$DATA_DIR" \ + --validation-split 0.2 \ + --split-seed 42 \ + --transform asinh \ + --chunk-size 50000 \ + --n-bins 1 \ + --out-dir "$CHUNK_DIR" \ + -v +``` + +> `--n-bins` must match `loss.args.n_bins` in the training YAML. +> `--transform` must match the transform expected by the config. + +Verify `manifest.yaml`: +```bash +head -40 "$CHUNK_DIR/manifest.yaml" +# expect: version: 2, transform: asinh, n_bins: 1, source_data_dir: $DATA_DIR +``` + +For the `sqrt_squareplus` transform variant: +```bash +integrator.preprocess \ + --data-dir "$DATA_DIR" \ + --validation-split 0.2 \ + --split-seed 42 \ + --transform sqrt_squareplus \ + --chunk-size 50000 \ + --n-bins 1 \ + --out-dir "$DATA_DIR/chunks_sqrt_squareplus" \ + -v +``` + +--- + +### Step 4 — Request a GPU node + +```bash +srun \ + --account=lcls:prjlumine22 \ + --partition=ampere \ + --qos=normal \ + --constraint=OS_VER:8.6 \ + --nodes=1 \ + --cpus-per-task=24 \ + --gpus=1 \ + --mem=256G \ + --time=0-03:00 \ + --job-name=mfx_chunked \ + --pty bash -i +``` + +After entering the node, repeat Step 2 to activate `integrator-cuda-dev`. Verify: +```bash +nvidia-smi +python -c "import torch; print(torch.cuda.is_available(), torch.cuda.get_device_name(0))" +``` + +--- + +### Step 5 — Train *(integrator-cuda-dev, GPU)* + +**Interactive:** +```bash +RUNS_DIR=$BASE/runs +CONFIG=$REPO/workflows/mfx101555026/configs/mfx_chunked_5000.yaml + +/usr/bin/time -v integrator.train \ + --config "$CONFIG" \ + --log-dir "$RUNS_DIR" \ + 2>&1 | tee "$BASE/logs/train_$(date +%Y%m%d-%H%M%S).log" + +# Find the new run directory: +RUN_DIR=$(find "$RUNS_DIR" -maxdepth 1 -type d -name 'run_*' \ + -printf '%T@ %p\n' | sort -nr | head -n1 | cut -d' ' -f2-) +echo "$RUN_DIR" +``` + +**Batch (Slurm):** edit paths in the sbatch script, then submit: +```bash +sbatch $REPO/workflows/mfx101555026/sbatch/training/auto_train_predict_chunked.sbatch +# Hopper (preemptable) variant: +sbatch $REPO/workflows/mfx101555026/sbatch/training/auto_train_predict_chunked_HOPPER.sbatch +``` + +Checkpoints land in `$RUN_DIR/files/checkpoints/`. + +--- + +### Step 6 — Predict *(integrator-cuda-dev, GPU)* + +```bash +FINAL_CKPT="$RUN_DIR/files/checkpoints/epoch=0024.ckpt" + +/usr/bin/time -v integrator.predict \ + --run-dir "$RUN_DIR" \ + --ckpt "$FINAL_CKPT" \ + --batch-size 2048 \ + 2>&1 | tee "$BASE/logs/predict_$(date +%Y%m%d-%H%M%S).log" +``` + +Output: `$RUN_DIR/files/predictions/epoch_0024/` (~5,758 parquet files). +Estimated time: ~1.5 hours on Ampere A100. + +--- + +### Step 7a — Export predictions to NPZ *(integrator-cuda-dev)* + +Converts parquet predictions to `.npz` for the cctbx write-back environment +(cctbx has no polars; `.npz` is readable everywhere). + +```bash +PRED_DIR="$RUN_DIR/files/predictions/epoch_0024" +PRED_NPZ="$RUN_DIR/files/predictions/preds_epoch_0024.npz" + +python $REPO/workflows/mfx101555026/scripts/writeback/export_preds_for_dials.py \ + --ckpt-dir "$PRED_DIR" \ + --out "$PRED_NPZ" +# ~29 seconds on Ampere; ~115M rows, ~3.5 GB +``` + +**Batch (Slurm):** +```bash +# edit paths inside the script first, then: +sbatch $REPO/workflows/mfx101555026/sbatch/writeback/asinh/export_asinh_ep24_npz.sbatch +sbatch $REPO/workflows/mfx101555026/sbatch/writeback/sqrt/export_sqrt_ep24_npz.sbatch +``` + +--- + +### Step 7b — Write back to .refl files *(cctbx env)* + +```bash +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh +export PYTHONPATH=$REPO/src:$PYTHONPATH + +WB_OUT="$RUN_DIR/files/predictions/mfx_refl_writeback" + +dials.python $REPO/workflows/mfx101555026/scripts/writeback/run_writeback.py \ + --npz "$PRED_NPZ" \ + --metadata "$DATA_DIR/metadata.npy" \ + --refl-dir "$REFL_DIR" \ + --out-dir "$WB_OUT" +# ~17 minutes for 124,975 files + 115M prediction rows +``` + +Verify: +```bash +ls "$WB_OUT" | wc -l # ~249,950 (refl + expt symlink pairs) +``` + +**Batch (Slurm):** +```bash +sbatch $REPO/workflows/mfx101555026/sbatch/writeback/asinh/writeback_asinh_ep24.sbatch +sbatch $REPO/workflows/mfx101555026/sbatch/writeback/sqrt/writeback_sqrt_ep24.sbatch +``` + +#### Smoke test before a full run + +```bash +# Step 1 — integrator-cuda-dev +python $REPO/workflows/mfx101555026/scripts/writeback/test_predict_small.py \ + --config "$CONFIG" \ + --ckpt "$FINAL_CKPT" \ + --metadata "$DATA_DIR/metadata.npy" \ + --out-dir /tmp/test_smoke \ + --n-images 5 + +# Step 2 — dials.python +dials.python $REPO/workflows/mfx101555026/scripts/writeback/test_writeback_small.py \ + --npz /tmp/test_smoke/preds_small.npz \ + --metadata "$DATA_DIR/metadata.npy" \ + --refl-dir "$REFL_DIR" \ + --out-dir /tmp/test_smoke/writeback +# Expected: ALL CHECKS PASSED +``` + +--- + +### Step 8 — Scale and merge *(cctbx env)* + +Both scaling and merging use `cxi.xfel.merge` — the command is the same, +the PHIL file determines which operation runs. + +PHIL files are in `$REPO/workflows/mfx101555026/combinedmask_scale_merge_phil/`. + +```bash +PHIL_DIR=$REPO/workflows/mfx101555026/combinedmask_scale_merge_phil + +# Step 1 — scale (run the scaling PHIL with cxi.xfel.merge): +cxi.xfel.merge $PHIL_DIR/scale_matched_cctbx_asinh_dmin2.phil + +# Step 2 — merge (run the merge PHIL with cxi.xfel.merge): +cxi.xfel.merge $PHIL_DIR/merge_matched_cctbx_asinh_dmin2.phil +``` + +For the Integrator (asinh / sqrt-squareplus) variants: +```bash +cxi.xfel.merge $PHIL_DIR/PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil +cxi.xfel.merge $PHIL_DIR/PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil + +cxi.xfel.merge $PHIL_DIR/PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil +cxi.xfel.merge $PHIL_DIR/PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil +``` + +**Batch (Slurm):** +```bash +sbatch $REPO/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_matched_cctbx_asinh_dmin2.sbatch +sbatch $REPO/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_matched_cctbx_sqrt_dmin2.sbatch +``` + +For matched comparison (build the same-image subset used by both cctbx and Integrator): +```bash +sbatch $REPO/workflows/mfx101555026/sbatch/matching/match_cctbx_asinh.sbatch +sbatch $REPO/workflows/mfx101555026/sbatch/matching/match_cctbx_sqrt.sbatch +``` + +--- + +### Step 9 — PHENIX refinement *(cctbx env)* + +```bash +BASE_CCTBX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx +REF_PDB=/sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/\ +ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb + +phenix.refine \ + "$BASE_CCTBX/phenix_results/asinh_dmin2/input.mtz" \ + "$REF_PDB" \ + refinement.main.number_of_macro_cycles=5 +``` + +--- + +## Training configs + +| Config | Transform | qi posterior | Notes | +|---|---|---|---| +| `mfx_chunked_5000.yaml` | asinh | FoldedNormal | Production run (asinh) | +| `mfx_chunked_sqrt_squareplus.yaml` | sqrt_squareplus | FoldedNormal | Production run (sqrt) | +| `mfx_chunked_test10.yaml` | asinh | FoldedNormal | Smoke test (10 files) | +| `mfx_test_10_files.yaml` | asinh | Gamma | Legacy small test | + +All chunked configs use `data_loader.name: chunked_rotation_data` and +`loss.args.image_level_wilson: false` (global B/G). + +Key loss settings: +```yaml +loss: + name: monochromatic_wilson + args: + observation_likelihood: normal # MFX/Jungfrau pixels can be float/negative + init_log_B: 20.0 # softplus → B ≈ 20 + init_log_G: 1000.0 # softplus → G ≈ 1000 + image_level_wilson: false + n_bins: 1 + lp_correction: false +``` + +--- + +## Results + +### Matched merging statistics (apples-to-apples comparison) + +*Both cctbx and Integrator merged from the same image subset.* + +| Method | CC½ | Rint | Rsplit | Experiments | Reflections | +|---|---|---|---|---|---| +| cctbx matched (asinh subset) | 0.998 | 6.9% | 5.2% | 113,420 | 36,326,432 | +| Integrator — asinh | 0.990 | 15.8% | 11.1% | 113,804 | 32,066,173 | +| cctbx matched (sqrt subset) | 0.998 | 7.0% | 5.3% | 113,525 | 36,351,064 | +| Integrator — sqrt-squareplus | 0.990 | 15.8% | 11.1% | 113,910 | 32,094,591 | + +### PHENIX refinement (5 macro-cycles, 2.0 Å) + +| Method | R-work | R-free | N reflections | +|---|---|---|---| +| cctbx matched (asinh subset) | 0.179 | 0.213 | 44,236 | +| Integrator — asinh | 0.261 | 0.277 | 45,923 | +| cctbx matched (sqrt subset) | 0.179 | 0.214 | 44,276 | +| Integrator — sqrt-squareplus | 0.262 | 0.275 | 45,891 | + +### CC½ per resolution bin (selected bins) + +| Bin | d_min (Å) | cctbx (matched) | Integrator asinh | Integrator sqrt | +|---|---|---|---|---| +| 1 | 5.43 | 0.999 | 0.993 | 0.993 | +| 7 | 2.84 | 0.978 | 0.675 | 0.657 | +| 11 | 2.44 | 0.793 | 0.512 | 0.538 | +| 17 | 2.11 | 0.128 | 0.545 | 0.574 | +| 20 | 2.00 | 0.055 | 0.473 | 0.551 | + +Full 20-bin table: see `refinement_comparison.txt` (local, not committed). + +**Key observation:** cctbx CC½ drops steeply above ~2.4 Å. Both Integrator +variants remain elevated at high resolution — behavior that requires further +validation before interpreting as recovered signal. + +--- + +## Workflow files reference + +``` +workflows/mfx101555026/ + configs/ + mfx_chunked_5000.yaml production config — asinh + FoldedNormal + mfx_chunked_sqrt_squareplus.yaml production config — sqrt_squareplus + FoldedNormal + mfx_chunked_test10.yaml smoke-test config (10 files) + mfx_test_10_files.yaml legacy small-test (rotation_data loader) + mfx_allruns_asinh_foldednormal_imagewilson_global_B_G.yaml image-level Wilson variant + mfx_allruns_sqrt_squareplus_foldednormal_imagewilson_global_B_G.yaml + (+ other experimental configs) + + combinedmask_scale_merge_phil/ + scale_matched_cctbx_asinh_dmin2.phil scale — matched cctbx (asinh subset) + merge_matched_cctbx_asinh_dmin2.phil merge — matched cctbx (asinh subset) + scale_matched_cctbx_sqrt_dmin2.phil scale — matched cctbx (sqrt subset) + merge_matched_cctbx_sqrt_dmin2.phil merge — matched cctbx (sqrt subset) + PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil scale — Integrator asinh predictions + PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil merge — Integrator asinh predictions + PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil scale — Integrator sqrt predictions + PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil merge — Integrator sqrt predictions + scale_all_r269_r289__dmin2.phil scale — full cctbx allruns + merge_all_r269_r289__dmin2.phil merge — full cctbx allruns + (+ older d_min variants: dmin15, dmin21, dmin26, dmin28) + + sbatch/ + training/ + auto_preprocess.sbatch preprocess dataset into chunks + auto_train_predict_chunked.sbatch train + predict on Ampere (sequential) + auto_train_predict_chunked_HOPPER.sbatch train + predict on Hopper (preemptable) + resume_training.batch resume a paused training run + resume_training_HOPPER.batch resume on Hopper + prediction/ + prediction_only_ampere_sqrt.sbatch predict only — sqrt run, Ampere + prediction_only_hopper_asinh.sbatch predict only — asinh run, Hopper + writeback/ + asinh/ + export_asinh_ep24_npz.sbatch parquet → .npz for asinh epoch 24 + writeback_asinh_ep24.sbatch .npz → .refl write-back for asinh + sqrt/ + export_sqrt_ep24_npz.sbatch parquet → .npz for sqrt epoch 24 + writeback_sqrt_ep24.sbatch .npz → .refl write-back for sqrt + scale_merge/ + run_scale_merge_matched_cctbx_asinh_dmin2.sbatch + run_scale_merge_matched_cctbx_sqrt_dmin2.sbatch + run_scale_merge_all_dmin2_ASINH.sbatch + run_scale_merge_all_dmin2_SQRT.sbatch + matching/ + match_cctbx_asinh.sbatch build same-image subset (asinh) + match_cctbx_sqrt.sbatch build same-image subset (sqrt) + + scripts/ + extraction/ + make_shoeboxes_allruns_024_rg101_16array.sh 16-job Slurm array for shoebox extraction + merge_allruns_024_rg101_shoebox_chunks.py merge chunk outputs into final dataset + merge_allruns_024_rg101_shoebox_chunks.sbatch + + writeback/ + export_preds_for_dials.py parquet → .npz (integrator-cuda-dev) + run_writeback.py .npz → .refl write-back (dials.python, production) + test_predict_small.py smoke test: prediction on 5 files + test_writeback_small.py smoke test: write-back + 4 validation checks + test_predict_writeback_small.py end-to-end smoke test + diagnose_refl_ids.py debug refl_id mismatches / float32 precision + + diagnostics/ + inspect_mfx_shoeboxes.py visualise extracted shoeboxes + inspect_qp_mean_profiles.py inspect learned spot profile qp.mean + inspect_cctbx_refl_expt.py inspect cctbx .refl/.expt structure + inspect_integrator_parquet.py inspect prediction parquet contents + plot_loss_history.py plot training loss curves + plot_cc12_vs_resolution.py CC½ vs resolution (cctbx vs Integrator) + plot_cctbx_selected_profiles.py plot cctbx selected spot profiles + plot_mfx_negative_pixels.py visualise negative pixel distribution + plot_qi_by_strength.py qi distribution + mean profiles stratified by + weak/medium/strong (cctbx intensity percentiles + p10 / p50 / p90) + + archive/ old scripts kept for reference; not part of + the active workflow +``` + +--- + +## Data and scratch paths + +``` +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +$BASE/ + mfx_shoebox_allruns_269_289_no275_024_rg101/ shoebox dataset + counts.npy masks.npy metadata.npy dataset.yaml + chunks/ preprocessed chunks + manifest.yaml + chunk_00000/ … chunk_NNNNN/ + all_runs_269_289_no275_024_rg101_integrated/ original .refl/.expt files + runs/ training run directories + run_20260805-011728_9704/ asinh run (epoch 0-24) + run_20260805-183129_a8b9/ sqrt_squareplus run (epoch 0-24) + phenix_results/ + matched_cctbx_asinh_dmin2/2026_refine_001.{mtz,cif} + matched_cctbx_sqrt_dmin2/2026_refine_001.{mtz,cif} + asinh_dmin2/2026_refine_001.{mtz,cif} + sqrt_dmin2/2026_refine_001.{mtz,cif} + +$REPO/workflows/mfx101555026/combinedmask_scale_merge_phil/ + PHIL files for cxi.xfel.merge — scale and merge PHILs for matched and Integrator variants +``` + +> Large data (shoeboxes, training runs, prediction parquets, write-back .refl +> files) lives in scratch and is **not** committed to Git. + +--- + +## Viewing the structure in COOT + +```bash +# SSH to sdfiana003 with X11 forwarding: +ssh -X -J s3dflogin.slac.stanford.edu thaoh@sdfiana003 + +# Source CCP4: +source /sdf/group/lcls/ds/tools/ccp4-8.0/bin/ccp4.setup-sh + +# Open matched cctbx refinement result: +coot \ + --pdb $BASE/phenix_results/matched_cctbx_asinh_dmin2/2026_refine_001.cif \ + --auto $BASE/phenix_results/matched_cctbx_asinh_dmin2/2026_refine_001.mtz +``` + +Use NoMachine (`https://s3dfnx.slac.stanford.edu:4443/`) for smoother 3D rendering. diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil new file mode 100644 index 00000000..41f969e7 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_scale_all_r269_r289_asinh_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = PREDICTED_merge_all_r269_r289_asinh_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_merge_all_r269_r289_asinh_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil new file mode 100644 index 00000000..74571626 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_scale_all_r269_r289_sqrt_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = PREDICTED_merge_all_r269_r289_sqrt_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_merge_all_r269_r289_sqrt_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil new file mode 100644 index 00000000..ef409864 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/runs/run_20260805-011728_9704/predictions/mfx_refl_writeback +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = PREDICTED_scale_all_r269_r289_asinh_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_scale_all_r269_r289_asinh_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil new file mode 100644 index 00000000..5ffca98a --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path =/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/runs/run_20260805-183129_a8b9/predictions/mfx_refl_writeback +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = PREDICTED_scale_all_r269_r289_sqrt_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_scale_all_r269_r289_sqrt_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin15.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin15.phil new file mode 100644 index 00000000..8c5524cb --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin15.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin15 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 1.5 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_all_r269_r289_no275_024_rg101_mm24_dmin15 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_all_r269_r289_no275_024_rg101_dmin15 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin2.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin2.phil new file mode 100644 index 00000000..e62a118d --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin2.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_all_r269_r289_no275_024_rg101_mm24_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_all_r269_r289_no275_024_rg101_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin21.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin21.phil new file mode 100644 index 00000000..5568d84f --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin21.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin21 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.1 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_all_r269_r289_no275_024_rg101_mm24_dmin21 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_all_r269_r289_no275_024_rg101_dmin21 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin26.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin26.phil new file mode 100644 index 00000000..aac23ca3 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin26.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin26 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.6 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_all_r269_r289_no275_024_rg101_mm24_dmin26 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_all_r269_r289_no275_024_rg101_dmin26 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin28.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin28.phil new file mode 100644 index 00000000..ab2294f3 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_all_r269_r289__dmin28.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin28 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.8 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_all_r269_r289_no275_024_rg101_mm24_dmin28 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_all_r269_r289_no275_024_rg101_dmin28 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_matched_cctbx_asinh_dmin2.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_matched_cctbx_asinh_dmin2.phil new file mode 100644 index 00000000..16e2688f --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_matched_cctbx_asinh_dmin2.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_matched_cctbx_asinh_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_matched_cctbx_asinh_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_matched_cctbx_asinh_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_matched_cctbx_sqrt_dmin2.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_matched_cctbx_sqrt_dmin2.phil new file mode 100644 index 00000000..7551a916 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_matched_cctbx_sqrt_dmin2.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_matched_cctbx_sqrt_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_matched_cctbx_sqrt_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_matched_cctbx_sqrt_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_r0269_018_dmin26.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_r0269_018_dmin26.phil new file mode 100644 index 00000000..2bada848 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_r0269_018_dmin26.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_r0269_022_rg073_dmin26_combined_mask +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.6 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_r0269_022_rg073_dmin26_combined_mask +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_r0269_022_rg073_dmin26_combined_mask diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_r0269_018_dmin28.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_r0269_018_dmin28.phil new file mode 100644 index 00000000..33bd5a28 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/merge_r0269_018_dmin28.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_r0269_018_mincorr0p1_mm24_dmin28 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.8 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_r0269_018_mincorr0p1_mm24_dmin28 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_r0269_018_mincorr0p1_mm24_dmin28 \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin0_dmin15.sbatch b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin0_dmin15.sbatch new file mode 100644 index 00000000..e57290e5 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin0_dmin15.sbatch @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +#SBATCH --job-name=scale_merge_0_15 +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=08:00:00 +#SBATCH --output=scale_merge_all_dmin0_dmin15_%j.out +#SBATCH --error=scale_merge_all_dmin0_dmin15_%j.err + +set -eo pipefail + +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting scale dmin 0" +mpirun cctbx.xfel.merge scale_all_r269_r289__dmin0.phil + +echo "Starting merge dmin 0" +mpirun cctbx.xfel.merge merge_all_r269_r289__dmin0.phil + +echo "Starting scale dmin 1.5" +mpirun cctbx.xfel.merge scale_all_r269_r289__dmin15.phil + +echo "Starting merge dmin 1.5" +mpirun cctbx.xfel.merge merge_all_r269_r289__dmin15.phil + +echo "All done" diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin15_only.sbatch b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin15_only.sbatch new file mode 100644 index 00000000..225c3e5b --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin15_only.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --job-name=scale_merge_15 +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=08:00:00 +#SBATCH --output=scale_merge_all_dmin15_%j.out +#SBATCH --error=scale_merge_all_dmin15_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting scale dmin 1.5" +mpirun cctbx.xfel.merge scale_all_r269_r289__dmin15.phil + +echo "Starting merge dmin 1.5" +mpirun cctbx.xfel.merge merge_all_r269_r289__dmin15.phil + +echo "All done" diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin21_only.sbatch b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin21_only.sbatch new file mode 100644 index 00000000..791fb801 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin21_only.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --job-name=scale_merge_15 +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=08:00:00 +#SBATCH --output=scale_merge_all_dmin15_%j.out +#SBATCH --error=scale_merge_all_dmin15_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting scale dmin 2.1" +mpirun cctbx.xfel.merge scale_all_r269_r289__dmin21.phil + +echo "Starting merge dmin 2.1" +mpirun cctbx.xfel.merge merge_all_r269_r289__dmin21.phil + +echo "All done" diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin2_ASINH.sbatch b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin2_ASINH.sbatch new file mode 100644 index 00000000..f492239c --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin2_ASINH.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=08:00:00 +#SBATCH --job-name=predicted_asinh_dmin2 +#SBATCH --output=pred_asinh_dmin2_%j.out +#SBATCH --error=pred_asinh_dmin2_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting predicted ASINH scale dmin 2.0" +mpirun cctbx.xfel.merge PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil + +echo "Starting predicted ASINH merge dmin 2.0" +mpirun cctbx.xfel.merge PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil + +echo "All done for predicted ASINH dmin 2.0" \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin2_SQRT.sbatch b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin2_SQRT.sbatch new file mode 100644 index 00000000..0b45e346 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_all_dmin2_SQRT.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=08:00:00 +#SBATCH --job-name=predicted_sqrt_dmin2 +#SBATCH --output=pred_asinh_dmin2_%j.out +#SBATCH --error=pred_asinh_dmin2_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting predicted SQRT scale dmin 2.0" +mpirun cctbx.xfel.merge PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil + +echo "Starting predicted SQRT merge dmin 2.0" +mpirun cctbx.xfel.merge PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil + +echo "All done for predicted SQRT dmin 2.0" \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_matched_cctbx_asinh_dmin2.sbatch b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_matched_cctbx_asinh_dmin2.sbatch new file mode 100644 index 00000000..57932a34 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_matched_cctbx_asinh_dmin2.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=03:00:00 +#SBATCH --job-name=cctbx_asinh_dmin2 +#SBATCH --output=cctbx_asinh_dmin2_%j.out +#SBATCH --error=cctbx_asinh_dmin2_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting matched cctbx ASINH scale dmin 2.0" +mpirun cctbx.xfel.merge scale_matched_cctbx_asinh_dmin2.phil + +echo "Starting matched cctbx ASINH merge dmin 2.0" +mpirun cctbx.xfel.merge merge_matched_cctbx_asinh_dmin2.phil + +echo "All done for matched cctbx ASINH dmin 2.0" \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_matched_cctbx_sqrt_dmin2.sbatch b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_matched_cctbx_sqrt_dmin2.sbatch new file mode 100644 index 00000000..150c5060 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/run_scale_merge_matched_cctbx_sqrt_dmin2.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=08:00:00 +#SBATCH --job-name=cctbx_sqrt_dmin2 +#SBATCH --output=cctbx_sqrt_dmin2_%j.out +#SBATCH --error=cctbx_sqrt_dmin2_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting predicted SQRT scale dmin 2.0" +mpirun cctbx.xfel.merge scale_matched_cctbx_sqrt_dmin2.phil + +echo "Starting predicted SQRT merge dmin 2.0" +mpirun cctbx.xfel.merge merge_matched_cctbx_sqrt_dmin2.phil + +echo "All done for matched cctbx SQRT dmin 2.0" \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin15.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin15.phil new file mode 100644 index 00000000..5b61d236 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin15.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/all_runs_269_289_no275_024_rg101_integrated +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 1.5 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_all_r269_r289_no275_024_rg101_mincorr0p1_dmin15 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin15 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin2.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin2.phil new file mode 100644 index 00000000..f9a5ae6a --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin2.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/all_runs_269_289_no275_024_rg101_integrated +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_all_r269_r289_no275_024_rg101_mincorr0p1_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin21.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin21.phil new file mode 100644 index 00000000..b41293d4 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin21.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/all_runs_269_289_no275_024_rg101_integrated +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.1 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_all_r269_r289_no275_024_rg101_mincorr0p1_dmin21 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin21 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin26.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin26.phil new file mode 100644 index 00000000..4bd8f563 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin26.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/all_runs_269_289_no275_024_rg101_integrated +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.6 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_all_r269_r289_no275_024_rg101_mincorr0p1_dmin26 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin26 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin28.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin28.phil new file mode 100644 index 00000000..7b085608 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_all_r269_r289__dmin28.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/all_runs_269_289_no275_024_rg101_integrated +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.8 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_all_r269_r289_no275_024_rg101_mincorr0p1_dmin28 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin28 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_matched_cctbx_asinh_dmin2.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_matched_cctbx_asinh_dmin2.phil new file mode 100644 index 00000000..4211705b --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_matched_cctbx_asinh_dmin2.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/matched_cctbx_asinh +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_matched_cctbx_asinh_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_matched_cctbx_asinh_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_matched_cctbx_sqrt_dmin2.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_matched_cctbx_sqrt_dmin2.phil new file mode 100644 index 00000000..f6d44429 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_matched_cctbx_sqrt_dmin2.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/matched_cctbx_sqrt +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_matched_cctbx_sqrt_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_matched_cctbx_sqrt_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_r0269_018_combinedmask_dmin2p6.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_r0269_018_combinedmask_dmin2p6.phil new file mode 100644 index 00000000..77e3e443 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_r0269_018_combinedmask_dmin2p6.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/outputs/r0269/022_rg073/out +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.6 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_r0269_022_rg073_dmin26_combined_mask +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_r0269_022_rg073_dmin26_combined_mask +output.save_experiments_and_reflections = True diff --git a/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_r0269_018_combinedmask_dmin2p8.phil b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_r0269_018_combinedmask_dmin2p8.phil new file mode 100644 index 00000000..968ce4c9 --- /dev/null +++ b/workflows/mfx101555026/combinedmask_scale_merge_phil/scale_r0269_018_combinedmask_dmin2p8.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/outputs/r0269/022_rg073/out +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.8 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_r0269_022_rg073_dmin28_combined_mask +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_r0269_022_rg073_dmin28_combined_mask +output.save_experiments_and_reflections = True diff --git a/workflows/mfx101555026/configs/mfx_5000_asinh_foldednormal_imagewilson_perImage_B_G.yaml b/workflows/mfx101555026/configs/mfx_5000_asinh_foldednormal_imagewilson_perImage_B_G.yaml new file mode 100644 index 00000000..653128c1 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_5000_asinh_foldednormal_imagewilson_perImage_B_G.yaml @@ -0,0 +1,136 @@ +# MFX 10-file smoke test +# Goal: test latest code path: +# make_shoeboxes -> data loading -> training -> prediction/write-back +# Uses image-level Wilson B/G parameters. + +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + - image_id + - image_num + - source_refl + - source_expt + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +data_loader: + name: rotation_data + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_5000_269_289_no275_024_rg101 + batch_size: 2048 + validation_split: 0.2 + num_workers: 4 + + transform: asinh + resolution_cutoff: null + + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_5000_269_289_no275_024_rg101 + counts: counts.npy + masks: masks.npy + reference: metadata.npy + standardized_counts: null + +loss: + name: monochromatic_wilson + args: + observation_likelihood: normal + init_obs_scale: 10.0 + n_bins: 1 + image_level_wilson: true + lp_correction: false + +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 + +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +surrogates: + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_allruns_asinh_foldednormal_imagewilson_global_B_G.yaml b/workflows/mfx101555026/configs/mfx_allruns_asinh_foldednormal_imagewilson_global_B_G.yaml new file mode 100644 index 00000000..f9a082cf --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_allruns_asinh_foldednormal_imagewilson_global_B_G.yaml @@ -0,0 +1,135 @@ +# MFX 10-file smoke test +# Goal: test latest code path: +# make_shoeboxes -> data loading -> training -> prediction/write-back +# Uses image-level Wilson B/G parameters. + +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + - image_id + - image_num + - source_refl + - source_expt + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +data_loader: + name: rotation_data + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + batch_size: 2048 + validation_split: 0.2 + num_workers: 4 + + transform: asinh + resolution_cutoff: null + + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + counts: counts.npy + masks: masks.npy + reference: metadata.npy + standardized_counts: null + +loss: + name: monochromatic_wilson + args: + observation_likelihood: normal + init_obs_scale: 10.0 + n_bins: 1 + lp_correction: false + +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 + +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +surrogates: + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_allruns_asinh_foldednormal_imagewilson_perImage_B_G.yaml b/workflows/mfx101555026/configs/mfx_allruns_asinh_foldednormal_imagewilson_perImage_B_G.yaml new file mode 100644 index 00000000..78da0839 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_allruns_asinh_foldednormal_imagewilson_perImage_B_G.yaml @@ -0,0 +1,136 @@ +# MFX 10-file smoke test +# Goal: test latest code path: +# make_shoeboxes -> data loading -> training -> prediction/write-back +# Uses image-level Wilson B/G parameters. + +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + - image_id + - image_num + - source_refl + - source_expt + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +data_loader: + name: rotation_data + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + batch_size: 2048 + validation_split: 0.2 + num_workers: 4 + + transform: asinh + resolution_cutoff: null + + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + counts: counts.npy + masks: masks.npy + reference: metadata.npy + standardized_counts: null + +loss: + name: monochromatic_wilson + args: + observation_likelihood: normal + init_obs_scale: 10.0 + n_bins: 1 + image_level_wilson: true + lp_correction: false + +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 + +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +surrogates: + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_allruns_sqrt_squareplus_foldednormal_imagewilson_global_B_G.yaml b/workflows/mfx101555026/configs/mfx_allruns_sqrt_squareplus_foldednormal_imagewilson_global_B_G.yaml new file mode 100644 index 00000000..f0095ed2 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_allruns_sqrt_squareplus_foldednormal_imagewilson_global_B_G.yaml @@ -0,0 +1,136 @@ +# MFX all-runs smoke test +# Goal: test latest code path: +# make_shoeboxes -> data loading -> training -> prediction/write-back +# Uses image-level Wilson B/G parameters. + +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + - image_id + - image_num + - source_refl + - source_expt + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +data_loader: + name: rotation_data + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + batch_size: 1024 + validation_split: 0.2 + num_workers: 4 + + transform: sqrt_squareplus + resolution_cutoff: null + + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + counts: counts.npy + masks: masks.npy + reference: metadata.npy + standardized_counts: null + +loss: + name: monochromatic_wilson + args: + observation_likelihood: normal + init_obs_scale: 10.0 + n_bins: 1 + image_level_wilson: false + lp_correction: false + +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 + +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +surrogates: + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 diff --git a/workflows/mfx101555026/configs/mfx_allruns_sqrt_squareplus_foldednormal_imagewilson_perImage_B_G.yaml b/workflows/mfx101555026/configs/mfx_allruns_sqrt_squareplus_foldednormal_imagewilson_perImage_B_G.yaml new file mode 100644 index 00000000..1de27d33 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_allruns_sqrt_squareplus_foldednormal_imagewilson_perImage_B_G.yaml @@ -0,0 +1,136 @@ +# MFX all-runs smoke test +# Goal: test latest code path: +# make_shoeboxes -> data loading -> training -> prediction/write-back +# Uses image-level Wilson B/G parameters. + +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + - image_id + - image_num + - source_refl + - source_expt + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +data_loader: + name: rotation_data + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + batch_size: 1024 + validation_split: 0.2 + num_workers: 4 + + transform: sqrt_squareplus + resolution_cutoff: null + + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + counts: counts.npy + masks: masks.npy + reference: metadata.npy + standardized_counts: null + +loss: + name: monochromatic_wilson + args: + observation_likelihood: normal + init_obs_scale: 10.0 + n_bins: 1 + image_level_wilson: true + lp_correction: false + +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 + +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +surrogates: + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 diff --git a/workflows/mfx101555026/configs/mfx_asinh_normal_gamma.yaml b/workflows/mfx101555026/configs/mfx_asinh_normal_gamma.yaml new file mode 100644 index 00000000..5dbcfa0b --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_asinh_normal_gamma.yaml @@ -0,0 +1,153 @@ +# ADAPTED FOR MFX: +# Uses monochromatic mode because Kevin suggested avoiding the polychromatic Wilson prior here. +mode: monochromatic + + +# Main integrator model settings. +integrator: + name: hierarchical + + args: + encoder_out: 32 + mc_samples: 20 + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + +# Optimizer settings. +optimizer: + lr: 0.001 + weight_decay: 0.0 + + +# Data loader settings. +data_loader: + name: rotation_data + + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + batch_size: 1024 + validation_split: 0.2 + num_workers: 4 + + # For MFX/Jungfrau float pixels that can be negative. + transform: asinh + + resolution_cutoff: null + + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + counts: counts.npy + masks: masks.npy + reference: metadata_numeric.npy + standardized_counts: null + + +# Loss function settings. +loss: + name: monochromatic_wilson + + args: + observation_likelihood: normal + init_obs_scale: 10.0 + n_bins: 1 + + +# PyTorch Lightning trainer settings. +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + + +# Checkpoint settings. +checkpoint: + save_top_k: -1 + + +# Encoder networks. +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + +# Surrogate/posterior distributions. +surrogates: + # qp = profile posterior. + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + # qbg = background posterior. + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + # qi = intensity posterior. + qi: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_chunked_5000.yaml b/workflows/mfx101555026/configs/mfx_chunked_5000.yaml new file mode 100644 index 00000000..2dd318e9 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_chunked_5000.yaml @@ -0,0 +1,140 @@ +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + - image_id + - image_num + - reflection_id + - source_refl + - source_expt + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +# Chunked loader: +# - train/validation split was created during preprocessing +# - transform was applied during preprocessing +# - filtering was applied during preprocessing +# - file locations and metadata columns are read from manifest.yaml +data_loader: + name: chunked_rotation_data + args: + chunk_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101/chunks + batch_size: 2048 + num_workers: 20 + +# Global Wilson B/G: +# - image_level_wilson is explicitly false +# - init_log_B and init_log_G are passed through softplus +# - therefore the initial values are approximately B=20 and G=1000 +# - init_obs_scale controls Normal pixel-noise scale, not B or G +# - n_bins must match integrator.preprocess --n-bins +loss: + name: monochromatic_wilson + args: + image_level_wilson: false + + init_log_B: 20.0 + init_log_G: 1000.0 + + observation_likelihood: normal + init_obs_scale: 10.0 + + n_bins: 1 + lp_correction: false + + +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 + every_n_epochs: 1 + +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +surrogates: + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_chunked_sqrt_squareplus.yaml b/workflows/mfx101555026/configs/mfx_chunked_sqrt_squareplus.yaml new file mode 100644 index 00000000..2f0162ff --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_chunked_sqrt_squareplus.yaml @@ -0,0 +1,140 @@ +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + - image_id + - image_num + - reflection_id + - source_refl + - source_expt + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +# Chunked loader: +# - train/validation split was created during preprocessing +# - transform was applied during preprocessing +# - filtering was applied during preprocessing +# - file locations and metadata columns are read from manifest.yaml +data_loader: + name: chunked_rotation_data + args: + chunk_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101/chunks_sqrt_squareplus + batch_size: 2048 + num_workers: 18 + +# Global Wilson B/G: +# - image_level_wilson is explicitly false +# - init_log_B and init_log_G are passed through softplus +# - therefore the initial values are approximately B=20 and G=1000 +# - init_obs_scale controls Normal pixel-noise scale, not B or G +# - n_bins must match integrator.preprocess --n-bins +loss: + name: monochromatic_wilson + args: + image_level_wilson: false + + init_log_B: 20.0 + init_log_G: 1000.0 + + observation_likelihood: normal + init_obs_scale: 10.0 + + n_bins: 1 + lp_correction: false + + +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 + every_n_epochs: 1 + +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +surrogates: + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_chunked_test10.yaml b/workflows/mfx101555026/configs/mfx_chunked_test10.yaml new file mode 100644 index 00000000..9d6398cb --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_chunked_test10.yaml @@ -0,0 +1,169 @@ +# MFX chunked pipeline example config (v8 chunk layout) +# +# Template: +# mfx_allruns_asinh_foldednormal_imagewilson_global_B_G.yaml +# +# Purpose: +# First end-to-end smoke test of the chunked pipeline using: +# - global Wilson B/G +# - asinh-preprocessed shoeboxes +# - folded-normal qi +# - gamma qbg +# - one GPU +# - one training epoch +# +# Prerequisite: +# Run integrator.preprocess before training. +# +# Preprocess command: +# integrator.preprocess \ +# --data-dir /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 \ +# --validation-split 0.2 \ +# --split-seed 42 \ +# --transform asinh \ +# --chunk-size 50000 \ +# --n-bins 1 \ +# --out-dir /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101/chunks \ +# --min-valid-pixels 10 \ +# -v + +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + - image_id + - image_num + - reflection_id + - source_refl + - source_expt + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +# Chunked loader: +# - train/validation split was created during preprocessing +# - transform was applied during preprocessing +# - filtering was applied during preprocessing +# - file locations and metadata columns are read from manifest.yaml +data_loader: + name: chunked_rotation_data + args: + chunk_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_5000_269_289_no275_024_rg101/chunks + batch_size: 2048 + num_workers: 20 + +# Global Wilson B/G: +# - image_level_wilson is explicitly false +# - init_log_B and init_log_G are passed through softplus +# - therefore the initial values are approximately B=20 and G=1000 +# - init_obs_scale controls Normal pixel-noise scale, not B or G +# - n_bins must match integrator.preprocess --n-bins +loss: + name: monochromatic_wilson + args: + image_level_wilson: false + + init_log_B: 20.0 + init_log_G: 1000.0 + + observation_likelihood: normal + init_obs_scale: 10.0 + + n_bins: 1 + lp_correction: false + +# First smoke test: one epoch only. +# Increase after preprocess, train, validation, checkpointing, and prediction work. +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 + +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +surrogates: + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_normal_qpmean_config_log.yaml b/workflows/mfx101555026/configs/mfx_normal_qpmean_config_log.yaml new file mode 100644 index 00000000..608cfb50 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_normal_qpmean_config_log.yaml @@ -0,0 +1,289 @@ +# ADAPTED FOR MFX: +# Uses monochromatic mode because Kevin suggested avoiding the polychromatic Wilson prior here. +mode: monochromatic + + +# Main integrator model settings. +integrator: + # SAME AS LUIS CONFIG: + # Uses the hierarchical integrator model. + name: hierarchical + + args: + # ADJUSTED FOR MFX: + # Luis's HEWL config used encoder_out: 64. + # This uses 32 to make early MFX debugging runs faster and lighter on GPU memory. + encoder_out: 32 + + # ADJUSTED FOR MFX: + # Luis's HEWL config used mc_samples: 100. + # This uses 20 to reduce training/prediction time during MFX pipeline validation. + mc_samples: 20 + + # ADAPTED FOR MFX: + # Luis's HEWL config used 3D shoeboxes. + # MFX still-image data uses 2D detector shoeboxes. + data_dim: 2d + + # ADAPTED FOR MFX: + # For still images, each reflection comes from one detector image/event. + # Therefore the shoebox depth is 1. + d: 1 + + # ADAPTED FOR MFX: + # Uses 25x25 shoeboxes for MFX instead of Luis's 21x21 HEWL shoeboxes. + h: 25 + w: 25 + + # Prediction output fields. + # These values are saved during prediction, usually in pred.parquet. + predict_keys: + - refl_ids # copied metadata: reflection ID for tracking each reflection + - is_test # copied metadata: whether this reflection is in the test split + + - qi_mean # predicted mean reflection intensity + - qi_var # predicted intensity uncertainty/variance + + # ADDED FROM LUIS'S SUGGESTION: + # Saves the predicted 25x25 spot profile for each reflection. + # Useful for checking whether the model learns reasonable spot shapes. + - qp_mean # predicted 25x25 spot profile from the posterior mean + + - qbg_mean # predicted mean background + - qbg_var # predicted background uncertainty/variance + + #- intensity.prf.value # copied reference: traditional profile-fitted intensity, if present + #- intensity.prf.variance # copied reference: variance of profile-fitted intensity, if present + + - intensity.sum.value # copied reference: traditional cctbx/DIALS summed intensity + - intensity.sum.variance # copied reference: variance of summed intensity + + #- background.mean # copied reference: traditional background estimate, if present + + # ADDED/KEPT FOR VALIDATION: + # Useful for checking prediction quality by resolution and Miller index. + - d # copied metadata: d-spacing / resolution + - H # copied metadata: Miller index H + - K # copied metadata: Miller index K + - L # copied metadata: Miller index L + + +# Optimizer settings. +optimizer: + # Standard learning rate for model training. + lr: 0.001 + + # No extra L2 regularization in this first MFX setup. + weight_decay: 0.0 + + +# Data loader settings. +data_loader: + args: + # ADAPTED FOR MFX: + # Points to the generated MFX shoebox dataset. + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_stream_test_25 + + # Number of reflections per training batch. + batch_size: 1024 + + # Fraction of data used for validation. + validation_split: 0.2 + + # ADJUSTED FOR MFX: + # Moderate worker count for safer loading during MFX debugging. + num_workers: 4 + + # ADAPTED FOR MFX: + # Uses standardization because MFX/Jungfrau pixel values can be float/negative. + # This avoids Anscombe/Poisson-style assumptions. + transform: standardization + + # SIMPLE FIRST MFX SETUP: + # No resolution cutoff yet; keeps all reflections. + resolution_cutoff: null + + # ADAPTED FOR MFX: + # Luis's HEWL config used .pt files. + # This MFX extraction path saves NumPy .npy files. + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_stream_test_25 + + counts: counts.npy # MFX pixel shoebox values + masks: masks.npy # valid-pixel masks + reference: metadata.npy # reflection metadata + + # No precomputed standardized counts file. + # The loader computes standardization during loading. + standardized_counts: null + + # SAME AS LUIS STRUCTURE: + # Uses the rotation_data loader path with the current dataset format. + name: rotation_data + + +# Loss function settings. +loss: + # SAME LOSS FAMILY AS LUIS CONFIG: + # Uses monochromatic Wilson loss, not polychromatic_wilson. + name: monochromatic_wilson + + args: + # ADDED FOR MFX: + # Poisson assumes nonnegative count-like data. + # MFX/Jungfrau pixel values can be floating-point and sometimes negative. + # Normal likelihood is a better first-pass observation model. + observation_likelihood: normal + + # Initial noise scale for the Normal observation model. + init_obs_scale: 10.0 + + # ADJUSTED FOR MFX FIRST PASS: + # Luis's HEWL config used more resolution bins, such as 25. + # This uses one global bin first to simplify validation before testing resolution-bin priors. + n_bins: 1 + + +# PyTorch Lightning trainer settings. +trainer: + # Trains for 25 epochs in this MFX run. + max_epochs: 25 + + # Uses GPU training. + accelerator: gpu + + # Uses one GPU. + devices: 1 + + # Prevents very large gradients from destabilizing training. + gradient_clip_val: 1.0 + + # Runs validation every epoch for closer monitoring. + check_val_every_n_epoch: 1 + + +# Checkpoint settings. +checkpoint: + # Saves all checkpoints for later inspection. + # -1 means keep all checkpoints, not only the top k. + save_top_k: -1 + + +# Encoder networks. +# These turn input shoeboxes into learned feature vectors. +encoders: + - name: profile_encoder + args: + # ADAPTED FOR MFX: + # Profile encoder sees 2D shoebox images instead of 3D HEWL stacks. + data_dim: 2d + + # ADJUSTED FOR MFX: + # Matches integrator.args.encoder_out. + encoder_out: 32 + + # ADAPTED FOR MFX: + # Input shoebox shape is 25x25 pixels. + input_shape: + - 25 + - 25 + + # SAME STRUCTURE AS LUIS CONFIG: + # Multiple intensity encoders are used by the surrogate distributions. + # ADAPTED FOR MFX: + # Uses data_dim: 2d and encoder_out: 32. + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + +# Surrogate/posterior distributions. +# These are the three main hidden quantities predicted by the model: +# qp = profile, qbg = background, qi = intensity. +surrogates: + # SAME AS LUIS STRUCTURE: + # qp = profile posterior. + # Models the reflection spot shape across the shoebox pixels. + qp: + name: learned_basis_profile + args: + # ADJUSTED FOR MFX: + # Luis's config used latent_dim: 8. + # This run uses 12, but this is a tunable hyperparameter. + # It can be changed back to 8 for a simpler first-pass model. + latent_dim: 12 + + # Initial uncertainty for the hidden profile variable. + init_std: 0.5 + + # Prior scale used in profile KL regularization. + prior_scale: 3.0 + + # ADJUSTED FOR MFX: + # Must match encoder_out: 32. + input_dim: 32 + + # ADAPTED FOR MFX: + # Output profile size is 25x25 = 625 pixels. + # Luis's 3D HEWL config used 3x21x21 = 1323. + output_dim: 625 + + # SAME GENERAL ROLE AS LUIS CONFIG: + # qbg = background posterior. + # Models the background level. + qbg: + # ADJUSTED FROM LUIS EXAMPLE: + # Luis's example used gammaA. + # This config uses gamma, matching the current run that completed successfully. + name: gamma + args: + # Gamma parameterization used by this setup. + reparameterization: mean_fano + + # Small value for numerical stability. + eps: 1.0e-06 + + # Minimum Gamma parameter to avoid unstable tiny values. + k_min: 0.01 + + # ADJUSTED FOR MFX: + # Must match encoder_out: 32. + in_features: 32 + + # SAME GENERAL ROLE AS LUIS CONFIG: + # qi = intensity posterior. + # Models the reflection intensity. + qi: + # ADJUSTED FROM LUIS EXAMPLE: + # Luis's example used gammaA. + # This config uses gamma, matching the current run that completed successfully. + name: gamma + args: + # Gamma parameterization used by this setup. + reparameterization: mean_fano + + # Small value for numerical stability. + eps: 1.0e-06 + + # Minimum Gamma parameter to avoid unstable tiny values. + k_min: 0.01 + + # ADJUSTED FOR MFX: + # Must match encoder_out: 32. + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi.yaml b/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi.yaml new file mode 100644 index 00000000..105dab63 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi.yaml @@ -0,0 +1,146 @@ +# MFX still-image setup: use monochromatic Wilson loss, not polychromatic. +mode: monochromatic + +# Main model. +integrator: + name: hierarchical + args: + # Smaller encoder for faster MFX debugging. + encoder_out: 32 + + # Fewer Monte Carlo samples for faster train/predict. + mc_samples: 20 + + # MFX shoeboxes are 2D still images: 1 x 25 x 25. + data_dim: 2d + d: 1 + h: 25 + w: 25 + + # Fields saved during prediction. + predict_keys: + - refl_ids + - is_test + - qi_mean + - qi_var + - qp_mean + - qbg_mean + - qbg_var + - intensity.sum.value + - intensity.sum.variance + - d + - H + - K + - L + +# Optimizer. +optimizer: + lr: 0.001 + weight_decay: 0.0 + +# Dataset and loading. +data_loader: + name: rotation_data + args: + # Use the file MFX shoebox dataset. + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_stream_test_25 + + batch_size: 1024 + validation_split: 0.2 + num_workers: 4 + + # MFX/Jungfrau pixels can be float/negative, so use standardization. + transform: standardization + resolution_cutoff: null + + # Files inside data_dir. + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_stream_test_25 + counts: counts.npy + masks: masks.npy + reference: metadata.npy + standardized_counts: null + +# Loss. +loss: + name: monochromatic_wilson + args: + # Normal likelihood handles float/negative detector values better than Poisson. + observation_likelihood: normal + init_obs_scale: 10.0 + + # One global resolution bin for first-pass validation. + n_bins: 1 + +# Trainer. +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +# Save all checkpoints. +checkpoint: + save_top_k: -1 + +# Encoders create learned feature vectors from shoeboxes. +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + # Four intensity encoders are used by the hierarchical integrator. + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +# Surrogate/posterior distributions: +# qp = profile, qbg = background, qi = intensity. +surrogates: + # Profile posterior: predicts 25x25 spot shape. + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + # Keep background posterior as Gamma baseline. + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + # Test FoldedNormal intensity posterior. + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi_asinh.yaml b/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi_asinh.yaml new file mode 100644 index 00000000..4ed68512 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi_asinh.yaml @@ -0,0 +1,273 @@ +# ADAPTED FOR MFX: +# Uses monochromatic mode because Kevin suggested avoiding the polychromatic Wilson prior here. +mode: monochromatic + + +# Main integrator model settings. +integrator: + # SAME AS LUIS CONFIG: + # Uses the hierarchical integrator model. + name: hierarchical + + args: + # ADJUSTED FOR MFX: + # Luis's HEWL config used encoder_out: 64. + # This uses 32 to make early MFX debugging runs faster and lighter on GPU memory. + encoder_out: 32 + + # ADJUSTED FOR MFX: + # Luis's HEWL config used mc_samples: 100. + # This uses 20 to reduce training/prediction time during MFX pipeline validation. + mc_samples: 20 + + # ADAPTED FOR MFX: + # Luis's HEWL config used 3D shoeboxes. + # MFX still-image data uses 2D detector shoeboxes. + data_dim: 2d + + # ADAPTED FOR MFX: + # For still images, each reflection comes from one detector image/event. + # Therefore the shoebox depth is 1. + d: 1 + + # ADAPTED FOR MFX: + # Uses 25x25 shoeboxes for MFX instead of Luis's 21x21 HEWL shoeboxes. + h: 25 + w: 25 + + # Prediction output fields. + # These values are saved during prediction, usually in pred.parquet. + predict_keys: + - refl_ids # copied metadata: reflection ID for tracking each reflection + - is_test # copied metadata: whether this reflection is in the test split + + - qi_mean # predicted mean reflection intensity + - qi_var # predicted intensity uncertainty/variance + + # ADDED FROM LUIS'S SUGGESTION: + # Saves the predicted 25x25 spot profile for each reflection. + # Useful for checking whether the model learns reasonable spot shapes. + - qp_mean # predicted 25x25 spot profile from the posterior mean + + - qbg_mean # predicted mean background + - qbg_var # predicted background uncertainty/variance + + #- intensity.prf.value # copied reference: traditional profile-fitted intensity, if present + #- intensity.prf.variance # copied reference: variance of profile-fitted intensity, if present + + - intensity.sum.value # copied reference: traditional cctbx/DIALS summed intensity + - intensity.sum.variance # copied reference: variance of summed intensity + + #- background.mean # copied reference: traditional background estimate, if present + + # ADDED/KEPT FOR VALIDATION: + # Useful for checking prediction quality by resolution and Miller index. + - d # copied metadata: d-spacing / resolution + - H # copied metadata: Miller index H + - K # copied metadata: Miller index K + - L # copied metadata: Miller index L + + +# Optimizer settings. +optimizer: + # Standard learning rate for model training. + lr: 0.001 + + # No extra L2 regularization in this first MFX setup. + weight_decay: 0.0 + + +# Data loader settings. +data_loader: + args: + # ADAPTED FOR MFX: + # Points to the generated MFX shoebox dataset. + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + # Number of reflections per training batch. + batch_size: 1024 + + # Fraction of data used for validation. + validation_split: 0.2 + + # ADJUSTED FOR MFX: + # Moderate worker count for safer loading during MFX debugging. + num_workers: 4 + + # Uses asinh because MFX/Jungfrau pixel values can be float/negative. + # asinh preserves sign and compresses large values. + transform: asinh + + # SIMPLE FIRST MFX SETUP: + # No resolution cutoff yet; keeps all reflections. + resolution_cutoff: null + + # ADAPTED FOR MFX: + # Luis's HEWL config used .pt files. + # This MFX extraction path saves NumPy .npy files. + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + counts: counts.npy # MFX pixel shoebox values + masks: masks.npy # valid-pixel masks + reference: metadata_numeric.npy # reflection metadata + + # No precomputed standardized counts file. + # The loader computes standardization during loading. + standardized_counts: null + + # SAME AS LUIS STRUCTURE: + # Uses the rotation_data loader path with the current dataset format. + name: rotation_data + + +# Loss function settings. +loss: + # SAME LOSS FAMILY AS LUIS CONFIG: + # Uses monochromatic Wilson loss, not polychromatic_wilson. + name: monochromatic_wilson + + args: + # ADDED FOR MFX: + # Poisson assumes nonnegative count-like data. + # MFX/Jungfrau pixel values can be floating-point and sometimes negative. + # Normal likelihood is a better first-pass observation model. + observation_likelihood: normal + + # Initial noise scale for the Normal observation model. + init_obs_scale: 10.0 + + # ADJUSTED FOR MFX FIRST PASS: + # Luis's HEWL config used more resolution bins, such as 25. + # This uses one global bin first to simplify validation before testing resolution-bin priors. + n_bins: 1 + + +# PyTorch Lightning trainer settings. +trainer: + # Trains for 25 epochs in this MFX run. + max_epochs: 25 + + # Uses GPU training. + accelerator: gpu + + # Uses one GPU. + devices: 1 + + # Prevents very large gradients from destabilizing training. + gradient_clip_val: 1.0 + + # Runs validation every epoch for closer monitoring. + check_val_every_n_epoch: 1 + + +# Checkpoint settings. +checkpoint: + # Saves all checkpoints for later inspection. + # -1 means keep all checkpoints, not only the top k. + save_top_k: -1 + + +# Encoder networks. +# These turn input shoeboxes into learned feature vectors. +encoders: + - name: profile_encoder + args: + # ADAPTED FOR MFX: + # Profile encoder sees 2D shoebox images instead of 3D HEWL stacks. + data_dim: 2d + + # ADJUSTED FOR MFX: + # Matches integrator.args.encoder_out. + encoder_out: 32 + + # ADAPTED FOR MFX: + # Input shoebox shape is 25x25 pixels. + input_shape: + - 25 + - 25 + + # SAME STRUCTURE AS LUIS CONFIG: + # Multiple intensity encoders are used by the surrogate distributions. + # ADAPTED FOR MFX: + # Uses data_dim: 2d and encoder_out: 32. + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + +# Surrogate/posterior distributions. +# These are the three main hidden quantities predicted by the model: +# qp = profile, qbg = background, qi = intensity. +surrogates: + # SAME AS LUIS STRUCTURE: + # qp = profile posterior. + # Models the reflection spot shape across the shoebox pixels. + qp: + name: learned_basis_profile + args: + # ADJUSTED FOR MFX: + # Luis's config used latent_dim: 8. + # This run uses 12, but this is a tunable hyperparameter. + # It can be changed back to 8 for a simpler first-pass model. + latent_dim: 12 + + # Initial uncertainty for the hidden profile variable. + init_std: 0.5 + + # Prior scale used in profile KL regularization. + prior_scale: 3.0 + + # ADJUSTED FOR MFX: + # Must match encoder_out: 32. + input_dim: 32 + + # ADAPTED FOR MFX: + # Output profile size is 25x25 = 625 pixels. + # Luis's 3D HEWL config used 3x21x21 = 1323. + output_dim: 625 + + # SAME GENERAL ROLE AS LUIS CONFIG: + # qbg = background posterior. + # Models the background level. + qbg: + # ADJUSTED FROM LUIS EXAMPLE: + # Luis's example used gammaA. + # This config uses gamma, matching the current run that completed successfully. + name: gamma + args: + # Gamma parameterization used by this setup. + reparameterization: mean_fano + + # Small value for numerical stability. + eps: 1.0e-06 + + # Minimum Gamma parameter to avoid unstable tiny values. + k_min: 0.01 + + # ADJUSTED FOR MFX: + # Must match encoder_out: 32. + in_features: 32 + # Test FoldedNormal intensity posterior. + qi: + name: gamma + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 diff --git a/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi_log_softplus.yaml b/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi_log_softplus.yaml new file mode 100644 index 00000000..6cb7807c --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi_log_softplus.yaml @@ -0,0 +1,272 @@ +# ADAPTED FOR MFX: +# Uses monochromatic mode because Kevin suggested avoiding the polychromatic Wilson prior here. +mode: monochromatic + + +# Main integrator model settings. +integrator: + # SAME AS LUIS CONFIG: + # Uses the hierarchical integrator model. + name: hierarchical + + args: + # ADJUSTED FOR MFX: + # Luis's HEWL config used encoder_out: 64. + # This uses 32 to make early MFX debugging runs faster and lighter on GPU memory. + encoder_out: 32 + + # ADJUSTED FOR MFX: + # Luis's HEWL config used mc_samples: 100. + # This uses 20 to reduce training/prediction time during MFX pipeline validation. + mc_samples: 20 + + # ADAPTED FOR MFX: + # Luis's HEWL config used 3D shoeboxes. + # MFX still-image data uses 2D detector shoeboxes. + data_dim: 2d + + # ADAPTED FOR MFX: + # For still images, each reflection comes from one detector image/event. + # Therefore the shoebox depth is 1. + d: 1 + + # ADAPTED FOR MFX: + # Uses 25x25 shoeboxes for MFX instead of Luis's 21x21 HEWL shoeboxes. + h: 25 + w: 25 + + # Prediction output fields. + # These values are saved during prediction, usually in pred.parquet. + predict_keys: + - refl_ids # copied metadata: reflection ID for tracking each reflection + - is_test # copied metadata: whether this reflection is in the test split + + - qi_mean # predicted mean reflection intensity + - qi_var # predicted intensity uncertainty/variance + + # ADDED FROM LUIS'S SUGGESTION: + # Saves the predicted 25x25 spot profile for each reflection. + # Useful for checking whether the model learns reasonable spot shapes. + - qp_mean # predicted 25x25 spot profile from the posterior mean + + - qbg_mean # predicted mean background + - qbg_var # predicted background uncertainty/variance + + #- intensity.prf.value # copied reference: traditional profile-fitted intensity, if present + #- intensity.prf.variance # copied reference: variance of profile-fitted intensity, if present + + - intensity.sum.value # copied reference: traditional cctbx/DIALS summed intensity + - intensity.sum.variance # copied reference: variance of summed intensity + + #- background.mean # copied reference: traditional background estimate, if present + + # ADDED/KEPT FOR VALIDATION: + # Useful for checking prediction quality by resolution and Miller index. + - d # copied metadata: d-spacing / resolution + - H # copied metadata: Miller index H + - K # copied metadata: Miller index K + - L # copied metadata: Miller index L + + +# Optimizer settings. +optimizer: + # Standard learning rate for model training. + lr: 0.001 + + # No extra L2 regularization in this first MFX setup. + weight_decay: 0.0 + + +# Data loader settings. +data_loader: + args: + # ADAPTED FOR MFX: + # Points to the generated MFX shoebox dataset. + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + # Number of reflections per training batch. + batch_size: 1024 + + # Fraction of data used for validation. + validation_split: 0.2 + + # ADJUSTED FOR MFX: + # Moderate worker count for safer loading during MFX debugging. + num_workers: 4 + + + transform: log_softplus + + # SIMPLE FIRST MFX SETUP: + # No resolution cutoff yet; keeps all reflections. + resolution_cutoff: null + + # ADAPTED FOR MFX: + # Luis's HEWL config used .pt files. + # This MFX extraction path saves NumPy .npy files. + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + counts: counts.npy # MFX pixel shoebox values + masks: masks.npy # valid-pixel masks + reference: metadata_numeric.npy # reflection metadata + + # No precomputed standardized counts file. + # The loader computes standardization during loading. + standardized_counts: null + + # SAME AS LUIS STRUCTURE: + # Uses the rotation_data loader path with the current dataset format. + name: rotation_data + + +# Loss function settings. +loss: + # SAME LOSS FAMILY AS LUIS CONFIG: + # Uses monochromatic Wilson loss, not polychromatic_wilson. + name: monochromatic_wilson + + args: + # ADDED FOR MFX: + # Poisson assumes nonnegative count-like data. + # MFX/Jungfrau pixel values can be floating-point and sometimes negative. + # Normal likelihood is a better first-pass observation model. + observation_likelihood: normal + + # Initial noise scale for the Normal observation model. + init_obs_scale: 10.0 + + # ADJUSTED FOR MFX FIRST PASS: + # Luis's HEWL config used more resolution bins, such as 25. + # This uses one global bin first to simplify validation before testing resolution-bin priors. + n_bins: 1 + + +# PyTorch Lightning trainer settings. +trainer: + # Trains for 25 epochs in this MFX run. + max_epochs: 25 + + # Uses GPU training. + accelerator: gpu + + # Uses one GPU. + devices: 1 + + # Prevents very large gradients from destabilizing training. + gradient_clip_val: 1.0 + + # Runs validation every epoch for closer monitoring. + check_val_every_n_epoch: 1 + + +# Checkpoint settings. +checkpoint: + # Saves all checkpoints for later inspection. + # -1 means keep all checkpoints, not only the top k. + save_top_k: -1 + + +# Encoder networks. +# These turn input shoeboxes into learned feature vectors. +encoders: + - name: profile_encoder + args: + # ADAPTED FOR MFX: + # Profile encoder sees 2D shoebox images instead of 3D HEWL stacks. + data_dim: 2d + + # ADJUSTED FOR MFX: + # Matches integrator.args.encoder_out. + encoder_out: 32 + + # ADAPTED FOR MFX: + # Input shoebox shape is 25x25 pixels. + input_shape: + - 25 + - 25 + + # SAME STRUCTURE AS LUIS CONFIG: + # Multiple intensity encoders are used by the surrogate distributions. + # ADAPTED FOR MFX: + # Uses data_dim: 2d and encoder_out: 32. + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + +# Surrogate/posterior distributions. +# These are the three main hidden quantities predicted by the model: +# qp = profile, qbg = background, qi = intensity. +surrogates: + # SAME AS LUIS STRUCTURE: + # qp = profile posterior. + # Models the reflection spot shape across the shoebox pixels. + qp: + name: learned_basis_profile + args: + # ADJUSTED FOR MFX: + # Luis's config used latent_dim: 8. + # This run uses 12, but this is a tunable hyperparameter. + # It can be changed back to 8 for a simpler first-pass model. + latent_dim: 12 + + # Initial uncertainty for the hidden profile variable. + init_std: 0.5 + + # Prior scale used in profile KL regularization. + prior_scale: 3.0 + + # ADJUSTED FOR MFX: + # Must match encoder_out: 32. + input_dim: 32 + + # ADAPTED FOR MFX: + # Output profile size is 25x25 = 625 pixels. + # Luis's 3D HEWL config used 3x21x21 = 1323. + output_dim: 625 + + # SAME GENERAL ROLE AS LUIS CONFIG: + # qbg = background posterior. + # Models the background level. + qbg: + # ADJUSTED FROM LUIS EXAMPLE: + # Luis's example used gammaA. + # This config uses gamma, matching the current run that completed successfully. + name: gamma + args: + # Gamma parameterization used by this setup. + reparameterization: mean_fano + + # Small value for numerical stability. + eps: 1.0e-06 + + # Minimum Gamma parameter to avoid unstable tiny values. + k_min: 0.01 + + # ADJUSTED FOR MFX: + # Must match encoder_out: 32. + in_features: 32 + # Test FoldedNormal intensity posterior. + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi_sqrt_squareplus.yaml b/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi_sqrt_squareplus.yaml new file mode 100644 index 00000000..d4927c12 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_normal_qpmean_foldednormal_qi_sqrt_squareplus.yaml @@ -0,0 +1,275 @@ +# ADAPTED FOR MFX: +# Uses monochromatic mode because Kevin suggested avoiding the polychromatic Wilson prior here. +mode: monochromatic + + +# Main integrator model settings. +integrator: + # SAME AS LUIS CONFIG: + # Uses the hierarchical integrator model. + name: hierarchical + + args: + # ADJUSTED FOR MFX: + # Luis's HEWL config used encoder_out: 64. + # This uses 32 to make early MFX debugging runs faster and lighter on GPU memory. + encoder_out: 32 + + # ADJUSTED FOR MFX: + # Luis's HEWL config used mc_samples: 100. + # This uses 20 to reduce training/prediction time during MFX pipeline validation. + mc_samples: 20 + + # ADAPTED FOR MFX: + # Luis's HEWL config used 3D shoeboxes. + # MFX still-image data uses 2D detector shoeboxes. + data_dim: 2d + + # ADAPTED FOR MFX: + # For still images, each reflection comes from one detector image/event. + # Therefore the shoebox depth is 1. + d: 1 + + # ADAPTED FOR MFX: + # Uses 25x25 shoeboxes for MFX instead of Luis's 21x21 HEWL shoeboxes. + h: 25 + w: 25 + + # Prediction output fields. + # These values are saved during prediction, usually in pred.parquet. + predict_keys: + - refl_ids # copied metadata: reflection ID for tracking each reflection + - is_test # copied metadata: whether this reflection is in the test split + + - qi_mean # predicted mean reflection intensity + - qi_var # predicted intensity uncertainty/variance + + # ADDED FROM LUIS'S SUGGESTION: + # Saves the predicted 25x25 spot profile for each reflection. + # Useful for checking whether the model learns reasonable spot shapes. + - qp_mean # predicted 25x25 spot profile from the posterior mean + + - qbg_mean # predicted mean background + - qbg_var # predicted background uncertainty/variance + + #- intensity.prf.value # copied reference: traditional profile-fitted intensity, if present + #- intensity.prf.variance # copied reference: variance of profile-fitted intensity, if present + + - intensity.sum.value # copied reference: traditional cctbx/DIALS summed intensity + - intensity.sum.variance # copied reference: variance of summed intensity + + #- background.mean # copied reference: traditional background estimate, if present + + # ADDED/KEPT FOR VALIDATION: + # Useful for checking prediction quality by resolution and Miller index. + - d # copied metadata: d-spacing / resolution + - H # copied metadata: Miller index H + - K # copied metadata: Miller index K + - L # copied metadata: Miller index L + + +# Optimizer settings. +optimizer: + # Standard learning rate for model training. + lr: 0.001 + + # No extra L2 regularization in this first MFX setup. + weight_decay: 0.0 + + +# Data loader settings. +data_loader: + args: + # ADAPTED FOR MFX: + # Points to the generated MFX shoebox dataset. + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + # Number of reflections per training batch. + batch_size: 1024 + + # Fraction of data used for validation. + validation_split: 0.2 + + # ADJUSTED FOR MFX: + # Moderate worker count for safer loading during MFX debugging. + num_workers: 4 + + # ADAPTED FOR MFX: + # Uses standardization because MFX/Jungfrau pixel values can be float/negative. + # This avoids Anscombe/Poisson-style assumptions. + transform: sqrt_squareplus + + # SIMPLE FIRST MFX SETUP: + # No resolution cutoff yet; keeps all reflections. + resolution_cutoff: null + + # ADAPTED FOR MFX: + # Luis's HEWL config used .pt files. + # This MFX extraction path saves NumPy .npy files. + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + counts: counts.npy # MFX pixel shoebox values + masks: masks.npy # valid-pixel masks + reference: metadata_numeric.npy # reflection metadata + + # No precomputed standardized counts file. + # The loader computes standardization during loading. + standardized_counts: null + + # SAME AS LUIS STRUCTURE: + # Uses the rotation_data loader path with the current dataset format. + name: rotation_data + + +# Loss function settings. +loss: + # SAME LOSS FAMILY AS LUIS CONFIG: + # Uses monochromatic Wilson loss, not polychromatic_wilson. + name: monochromatic_wilson + + args: + # ADDED FOR MFX: + # Poisson assumes nonnegative count-like data. + # MFX/Jungfrau pixel values can be floating-point and sometimes negative. + # Normal likelihood is a better first-pass observation model. + observation_likelihood: normal + + # Initial noise scale for the Normal observation model. + init_obs_scale: 10.0 + + # ADJUSTED FOR MFX FIRST PASS: + # Luis's HEWL config used more resolution bins, such as 25. + # This uses one global bin first to simplify validation before testing resolution-bin priors. + n_bins: 1 + + +# PyTorch Lightning trainer settings. +trainer: + # Trains for 25 epochs in this MFX run. + max_epochs: 25 + + # Uses GPU training. + accelerator: gpu + + # Uses one GPU. + devices: 1 + + # Prevents very large gradients from destabilizing training. + gradient_clip_val: 1.0 + + # Runs validation every epoch for closer monitoring. + check_val_every_n_epoch: 1 + + +# Checkpoint settings. +checkpoint: + # Saves all checkpoints for later inspection. + # -1 means keep all checkpoints, not only the top k. + save_top_k: -1 + + +# Encoder networks. +# These turn input shoeboxes into learned feature vectors. +encoders: + - name: profile_encoder + args: + # ADAPTED FOR MFX: + # Profile encoder sees 2D shoebox images instead of 3D HEWL stacks. + data_dim: 2d + + # ADJUSTED FOR MFX: + # Matches integrator.args.encoder_out. + encoder_out: 32 + + # ADAPTED FOR MFX: + # Input shoebox shape is 25x25 pixels. + input_shape: + - 25 + - 25 + + # SAME STRUCTURE AS LUIS CONFIG: + # Multiple intensity encoders are used by the surrogate distributions. + # ADAPTED FOR MFX: + # Uses data_dim: 2d and encoder_out: 32. + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + +# Surrogate/posterior distributions. +# These are the three main hidden quantities predicted by the model: +# qp = profile, qbg = background, qi = intensity. +surrogates: + # SAME AS LUIS STRUCTURE: + # qp = profile posterior. + # Models the reflection spot shape across the shoebox pixels. + qp: + name: learned_basis_profile + args: + # ADJUSTED FOR MFX: + # Luis's config used latent_dim: 8. + # This run uses 12, but this is a tunable hyperparameter. + # It can be changed back to 8 for a simpler first-pass model. + latent_dim: 12 + + # Initial uncertainty for the hidden profile variable. + init_std: 0.5 + + # Prior scale used in profile KL regularization. + prior_scale: 3.0 + + # ADJUSTED FOR MFX: + # Must match encoder_out: 32. + input_dim: 32 + + # ADAPTED FOR MFX: + # Output profile size is 25x25 = 625 pixels. + # Luis's 3D HEWL config used 3x21x21 = 1323. + output_dim: 625 + + # SAME GENERAL ROLE AS LUIS CONFIG: + # qbg = background posterior. + # Models the background level. + qbg: + # ADJUSTED FROM LUIS EXAMPLE: + # Luis's example used gammaA. + # This config uses gamma, matching the current run that completed successfully. + name: gamma + args: + # Gamma parameterization used by this setup. + reparameterization: mean_fano + + # Small value for numerical stability. + eps: 1.0e-06 + + # Minimum Gamma parameter to avoid unstable tiny values. + k_min: 0.01 + + # ADJUSTED FOR MFX: + # Must match encoder_out: 32. + in_features: 32 + + # Test FoldedNormal intensity posterior. + qi: + name: lognormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_normal_qpmean_lognormal_qi.yaml b/workflows/mfx101555026/configs/mfx_normal_qpmean_lognormal_qi.yaml new file mode 100644 index 00000000..f0c1f75c --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_normal_qpmean_lognormal_qi.yaml @@ -0,0 +1,146 @@ +# MFX still-image setup: use monochromatic Wilson loss, not polychromatic. +mode: monochromatic + +# Main model. +integrator: + name: hierarchical + args: + # Smaller encoder for faster MFX debugging. + encoder_out: 32 + + # Fewer Monte Carlo samples for faster train/predict. + mc_samples: 20 + + # MFX shoeboxes are 2D still images: 1 x 25 x 25. + data_dim: 2d + d: 1 + h: 25 + w: 25 + + # Fields saved during prediction. + predict_keys: + - refl_ids + - is_test + - qi_mean + - qi_var + - qp_mean + - qbg_mean + - qbg_var + - intensity.sum.value + - intensity.sum.variance + - d + - H + - K + - L + +# Optimizer. +optimizer: + lr: 0.001 + weight_decay: 0.0 + +# Dataset and loading. +data_loader: + name: rotation_data + args: + # Use the MFX shoebox dataset. + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_combined_all + + batch_size: 1024 + validation_split: 0.2 + num_workers: 4 + + # MFX/Jungfrau pixels can be float/negative, so use standardization. + transform: standardization + resolution_cutoff: null + + # Files inside data_dir. + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_combined_all + counts: counts.npy + masks: masks.npy + reference: metadata.npy + standardized_counts: null + +# Loss. +loss: + name: monochromatic_wilson + args: + # Normal likelihood handles float/negative detector values better than Poisson. + observation_likelihood: normal + init_obs_scale: 10.0 + + # One global resolution bin for first-pass validation. + n_bins: 1 + +# Trainer. +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +# Save all checkpoints. +checkpoint: + save_top_k: -1 + +# Encoders create learned feature vectors from shoeboxes. +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + # Four intensity encoders are used by the hierarchical integrator. + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +# Surrogate/posterior distributions: +# qp = profile, qbg = background, qi = intensity. +surrogates: + # Profile posterior: predicts 25x25 spot shape. + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + # Keep background posterior as Gamma baseline. + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + # Test LogNormal intensity posterior. + qi: + name: lognormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_normal_qpmean_run_paths.yaml b/workflows/mfx101555026/configs/mfx_normal_qpmean_run_paths.yaml new file mode 100644 index 00000000..ede5c0e8 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_normal_qpmean_run_paths.yaml @@ -0,0 +1,23 @@ +config: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/runs/run_20260702-160655_3424/config_log.yaml +log_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/runs/run_20260702-160655_3424/files +output_root: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/runs/run_20260702-160655_3424 +plots_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/runs/run_20260702-160655_3424/plots +predictions_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/runs/run_20260702-160655_3424/predictions +resolved_paths: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_012_rg058_with_d_1500 + data_loader: + counts: + exists: true + path: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_012_rg058_with_d_1500/counts.npy + size_bytes: 2303997628 + masks: + exists: true + path: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_012_rg058_with_d_1500/masks.npy + size_bytes: 575999503 + reference: + exists: true + path: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_012_rg058_with_d_1500/metadata.npy + size_bytes: 95848117 + n_bins: 1 +slurm: + job_id: '30480765' diff --git a/workflows/mfx101555026/configs/mfx_smoke_train_with_d_125_normal_gpu.yaml b/workflows/mfx101555026/configs/mfx_smoke_train_with_d_125_normal_gpu.yaml new file mode 100644 index 00000000..a55f9cde --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_smoke_train_with_d_125_normal_gpu.yaml @@ -0,0 +1,36 @@ +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +data_loader: + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_012_rg058_with_d_125 + batch_size: 1024 + validation_split: 0.2 + num_workers: 4 + transform: standardization + resolution_cutoff: null + +loss: + name: monochromatic_wilson + args: + observation_likelihood: normal + init_obs_scale: 10.0 + +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 diff --git a/workflows/mfx101555026/configs/mfx_smoke_train_with_d_125_studentt_gpu.yaml b/workflows/mfx101555026/configs/mfx_smoke_train_with_d_125_studentt_gpu.yaml new file mode 100644 index 00000000..b3774f80 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_smoke_train_with_d_125_studentt_gpu.yaml @@ -0,0 +1,37 @@ +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +data_loader: + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_012_rg058_with_d_125 + batch_size: 1024 + validation_split: 0.2 + num_workers: 4 + transform: standardization + resolution_cutoff: null + +loss: + name: monochromatic_wilson + args: + observation_likelihood: student_t + init_obs_scale: 10.0 + student_t_df: 4.0 + +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 diff --git a/workflows/mfx101555026/configs/mfx_softplus_normal_gamma.yaml b/workflows/mfx101555026/configs/mfx_softplus_normal_gamma.yaml new file mode 100644 index 00000000..899ad676 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_softplus_normal_gamma.yaml @@ -0,0 +1,153 @@ +# ADAPTED FOR MFX: +# Uses monochromatic mode because Kevin suggested avoiding the polychromatic Wilson prior here. +mode: monochromatic + + +# Main integrator model settings. +integrator: + name: hierarchical + + args: + encoder_out: 32 + mc_samples: 20 + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + +# Optimizer settings. +optimizer: + lr: 0.001 + weight_decay: 0.0 + + +# Data loader settings. +data_loader: + name: rotation_data + + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + batch_size: 1024 + validation_split: 0.2 + num_workers: 4 + + # For MFX/Jungfrau float pixels that can be negative. + transform: log_softplus + + resolution_cutoff: null + + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + counts: counts.npy + masks: masks.npy + reference: metadata_numeric.npy + standardized_counts: null + + +# Loss function settings. +loss: + name: monochromatic_wilson + + args: + observation_likelihood: normal + init_obs_scale: 10.0 + n_bins: 1 + + +# PyTorch Lightning trainer settings. +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + + +# Checkpoint settings. +checkpoint: + save_top_k: -1 + + +# Encoder networks. +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + +# Surrogate/posterior distributions. +surrogates: + # qp = profile posterior. + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + # qbg = background posterior. + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + # qi = intensity posterior. + qi: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_sqrtsqureplus_normal_gamma.yaml b/workflows/mfx101555026/configs/mfx_sqrtsqureplus_normal_gamma.yaml new file mode 100644 index 00000000..2126b8f3 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_sqrtsqureplus_normal_gamma.yaml @@ -0,0 +1,153 @@ +# ADAPTED FOR MFX: +# Uses monochromatic mode because Kevin suggested avoiding the polychromatic Wilson prior here. +mode: monochromatic + + +# Main integrator model settings. +integrator: + name: hierarchical + + args: + encoder_out: 32 + mc_samples: 20 + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + + +# Optimizer settings. +optimizer: + lr: 0.001 + weight_decay: 0.0 + + +# Data loader settings. +data_loader: + name: rotation_data + + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + batch_size: 1024 + validation_split: 0.2 + num_workers: 4 + + # For MFX/Jungfrau float pixels that can be negative. + transform: sqrt_squareplus + + resolution_cutoff: null + + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + + counts: counts.npy + masks: masks.npy + reference: metadata_numeric.npy + standardized_counts: null + + +# Loss function settings. +loss: + name: monochromatic_wilson + + args: + observation_likelihood: normal + init_obs_scale: 10.0 + n_bins: 1 + + +# PyTorch Lightning trainer settings. +trainer: + max_epochs: 25 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + + +# Checkpoint settings. +checkpoint: + save_top_k: -1 + + +# Encoder networks. +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + +# Surrogate/posterior distributions. +surrogates: + # qp = profile posterior. + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + # qbg = background posterior. + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + # qi = intensity posterior. + qi: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/configs/mfx_test_10_files.yaml b/workflows/mfx101555026/configs/mfx_test_10_files.yaml new file mode 100644 index 00000000..a83ee645 --- /dev/null +++ b/workflows/mfx101555026/configs/mfx_test_10_files.yaml @@ -0,0 +1,128 @@ +# MFX 10-file smoke test +# Goal: test latest code path: +# make_shoeboxes -> data loading -> training -> prediction/write-back + +mode: monochromatic + +integrator: + name: hierarchical + args: + encoder_out: 32 + mc_samples: 20 + data_dim: 2d + d: 1 + h: 25 + w: 25 + + predict_keys: + - refl_ids + - is_test + + - qi_mean + - qi_var + + - qp_mean + + - qbg_mean + - qbg_var + + - intensity.sum.value + - intensity.sum.variance + + - d + - H + - K + - L + +optimizer: + lr: 0.001 + weight_decay: 0.0 + +data_loader: + name: rotation_data + args: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + batch_size: 1024 + validation_split: 0.2 + num_workers: 0 + + transform: asinh + resolution_cutoff: null + + shoebox_file_names: + data_dir: /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 + counts: counts.npy + masks: masks.npy + reference: metadata.npy + standardized_counts: null + +loss: + name: monochromatic_wilson + args: + observation_likelihood: normal + init_obs_scale: 10.0 + n_bins: 1 + +trainer: + max_epochs: 1 + accelerator: gpu + devices: 1 + gradient_clip_val: 1.0 + check_val_every_n_epoch: 1 + +checkpoint: + save_top_k: -1 + +encoders: + - name: profile_encoder + args: + data_dim: 2d + encoder_out: 32 + input_shape: + - 25 + - 25 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + + - name: intensity_encoder + args: + data_dim: 2d + encoder_out: 32 + +surrogates: + qp: + name: learned_basis_profile + args: + latent_dim: 12 + init_std: 0.5 + prior_scale: 3.0 + input_dim: 32 + output_dim: 625 + + qbg: + name: gamma + args: + reparameterization: mean_fano + eps: 1.0e-06 + k_min: 0.01 + in_features: 32 + + qi: + name: foldednormal + args: + eps: 1.0e-06 + scale_min: 1.0e-04 + in_features: 32 \ No newline at end of file diff --git a/workflows/mfx101555026/phil/PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil b/workflows/mfx101555026/phil/PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil new file mode 100644 index 00000000..41f969e7 --- /dev/null +++ b/workflows/mfx101555026/phil/PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_scale_all_r269_r289_asinh_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = PREDICTED_merge_all_r269_r289_asinh_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_merge_all_r269_r289_asinh_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/phil/PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil b/workflows/mfx101555026/phil/PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil new file mode 100644 index 00000000..74571626 --- /dev/null +++ b/workflows/mfx101555026/phil/PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_scale_all_r269_r289_sqrt_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = PREDICTED_merge_all_r269_r289_sqrt_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_merge_all_r269_r289_sqrt_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/phil/PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil b/workflows/mfx101555026/phil/PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil new file mode 100644 index 00000000..ef409864 --- /dev/null +++ b/workflows/mfx101555026/phil/PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/runs/run_20260805-011728_9704/predictions/mfx_refl_writeback +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = PREDICTED_scale_all_r269_r289_asinh_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_scale_all_r269_r289_asinh_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/phil/PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil b/workflows/mfx101555026/phil/PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil new file mode 100644 index 00000000..5ffca98a --- /dev/null +++ b/workflows/mfx101555026/phil/PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path =/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/runs/run_20260805-183129_a8b9/predictions/mfx_refl_writeback +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = PREDICTED_scale_all_r269_r289_sqrt_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/PREDICTED_scale_all_r269_r289_sqrt_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/phil/merge_all_r269_r289__dmin2.phil b/workflows/mfx101555026/phil/merge_all_r269_r289__dmin2.phil new file mode 100644 index 00000000..e62a118d --- /dev/null +++ b/workflows/mfx101555026/phil/merge_all_r269_r289__dmin2.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_all_r269_r289_no275_024_rg101_mm24_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_all_r269_r289_no275_024_rg101_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/phil/merge_matched_cctbx_asinh_dmin2.phil b/workflows/mfx101555026/phil/merge_matched_cctbx_asinh_dmin2.phil new file mode 100644 index 00000000..16e2688f --- /dev/null +++ b/workflows/mfx101555026/phil/merge_matched_cctbx_asinh_dmin2.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_matched_cctbx_asinh_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_matched_cctbx_asinh_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_matched_cctbx_asinh_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/phil/merge_matched_cctbx_sqrt_dmin2.phil b/workflows/mfx101555026/phil/merge_matched_cctbx_sqrt_dmin2.phil new file mode 100644 index 00000000..7551a916 --- /dev/null +++ b/workflows/mfx101555026/phil/merge_matched_cctbx_sqrt_dmin2.phil @@ -0,0 +1,19 @@ +dispatch.step_list = input model_scaling statistics_unitcell statistics_beam model_statistics group errors_merge statistics_intensity merge statistics_intensity_cxi + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_matched_cctbx_sqrt_dmin2 +input.reflections_suffix = .refl +input.experiments_suffix = .expt + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True +merging.error.model = mm24 + +statistics.n_bins = 20 + +output.prefix = mfx_merge_matched_cctbx_sqrt_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/merge_matched_cctbx_sqrt_dmin2 \ No newline at end of file diff --git a/workflows/mfx101555026/phil/scale_all_r269_r289__dmin2.phil b/workflows/mfx101555026/phil/scale_all_r269_r289__dmin2.phil new file mode 100644 index 00000000..f9a5ae6a --- /dev/null +++ b/workflows/mfx101555026/phil/scale_all_r269_r289__dmin2.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/all_runs_269_289_no275_024_rg101_integrated +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_all_r269_r289_no275_024_rg101_mincorr0p1_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_all_r269_r289_no275_024_rg101_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/phil/scale_matched_cctbx_asinh_dmin2.phil b/workflows/mfx101555026/phil/scale_matched_cctbx_asinh_dmin2.phil new file mode 100644 index 00000000..4211705b --- /dev/null +++ b/workflows/mfx101555026/phil/scale_matched_cctbx_asinh_dmin2.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/matched_cctbx_asinh +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_matched_cctbx_asinh_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_matched_cctbx_asinh_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/phil/scale_matched_cctbx_sqrt_dmin2.phil b/workflows/mfx101555026/phil/scale_matched_cctbx_sqrt_dmin2.phil new file mode 100644 index 00000000..f6d44429 --- /dev/null +++ b/workflows/mfx101555026/phil/scale_matched_cctbx_sqrt_dmin2.phil @@ -0,0 +1,28 @@ +dispatch.step_list = input balance model_scaling modify filter errors_premerge scale postrefine statistics_unitcell statistics_beam model_statistics statistics_resolution + +input.path = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/matched_cctbx_sqrt +input.reflections_suffix = _integrated.refl +input.experiments_suffix = _integrated.expt +input.parallel_file_load.method = uniform + +filter.algorithm = unit_cell +filter.unit_cell.value.target_unit_cell = 53.814 86.750 140.465 90 90 90 +filter.unit_cell.value.target_space_group = P 21 21 21 +filter.unit_cell.value.relative_length_tolerance = 0.1 +filter.unit_cell.value.absolute_angle_tolerance = 2.0 +filter.outlier.min_corr = 0.1 + +scaling.model = /sdf/data/lcls/ds/mfx/mfx101555026/results/offline/datasets/ClCRY4_wt_1us_74mJcm2_ellipse/processing/v01_new_defaults/diffmaps/dimple_dark/final.pdb +scaling.unit_cell = 53.814 86.750 140.465 90 90 90 +scaling.space_group = P 21 21 21 +scaling.resolution_scalar = 0.96 + +merging.d_min = 2.0 +merging.merge_anomalous = True + +postrefinement.enable = True +statistics.n_bins = 20 + +output.prefix = mfx_scale_matched_cctbx_sqrt_dmin2 +output.output_dir = /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/scale_matched_cctbx_sqrt_dmin2 +output.save_experiments_and_reflections = True \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/matching/match_cctbx_asinh.sbatch b/workflows/mfx101555026/sbatch/matching/match_cctbx_asinh.sbatch new file mode 100644 index 00000000..1261f655 --- /dev/null +++ b/workflows/mfx101555026/sbatch/matching/match_cctbx_asinh.sbatch @@ -0,0 +1,104 @@ +#!/bin/bash +#SBATCH --job-name=match_asinh +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --time=03:00:00 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=32G +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/match_asinh_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/match_asinh_%j.err + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +dials.python - <<'PY' +from pathlib import Path +import json +import os +import shutil + +BASE = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx") + +ORIG = BASE / "all_runs_269_289_no275_024_rg101_integrated" +WRITEBACK = BASE / "runs/run_20260805-011728_9704/predictions/mfx_refl_writeback" +SCALED = BASE / "PREDICTED_scale_all_r269_r289_asinh_dmin2" +MATCHED = BASE / "matched_cctbx_asinh" + +def read_ids(path): + with path.open() as fh: + data = json.load(fh) + + return [ + exp["identifier"] + for exp in data.get("experiment", []) + if exp.get("identifier") + ] + +# 1. UUIDs that survived ASINH scaling/filtering +survivors = set() + +scaled_files = list(SCALED.glob("*.expt")) + +for f in scaled_files: + survivors.update(read_ids(f)) + +print("Scaled chunk files:", len(scaled_files)) +print("Surviving ASINH experiments:", len(survivors)) + +# 2. Map survivor UUID -> write-back filename +mapping = {} +writeback_files = list(WRITEBACK.glob("*.expt")) + +for i, f in enumerate(writeback_files, 1): + + for uid in read_ids(f): + if uid in survivors: + mapping[uid] = f.name + + if i % 10000 == 0: + print( + f"Scanned {i}/{len(writeback_files)} " + f"write-back files; mapped {len(mapping)}" + ) + +unmapped = survivors - set(mapping) + +print("Mapped UUIDs:", len(mapping)) +print("Unmapped UUIDs:", len(unmapped)) + +if unmapped: + raise RuntimeError("Some survivor UUIDs could not be mapped") + +# 3. Rebuild matched cctbx directory +if MATCHED.exists(): + shutil.rmtree(MATCHED) + +MATCHED.mkdir(parents=True) + +linked = 0 +missing = 0 + +for uid, expt_name in mapping.items(): + + refl_name = expt_name[:-5] + ".refl" + + src_expt = ORIG / expt_name + src_refl = ORIG / refl_name + + if src_expt.is_file() and src_refl.is_file(): + + os.symlink(src_expt, MATCHED / expt_name) + os.symlink(src_refl, MATCHED / refl_name) + + linked += 1 + + else: + missing += 1 + +print() +print("Matched folder:", MATCHED) +print("Linked cctbx pairs:", linked) +print("Missing original pairs:", missing) +print("Total symlinks:", linked * 2) + +print("DONE ASINH") +PY \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/matching/match_cctbx_sqrt.sbatch b/workflows/mfx101555026/sbatch/matching/match_cctbx_sqrt.sbatch new file mode 100644 index 00000000..952efd55 --- /dev/null +++ b/workflows/mfx101555026/sbatch/matching/match_cctbx_sqrt.sbatch @@ -0,0 +1,104 @@ +#!/bin/bash +#SBATCH --job-name=match_sqrt +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --time=03:00:00 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=32G +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/match_sqrt_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/match_sqrt_%j.err + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +dials.python - <<'PY' +from pathlib import Path +import json +import os +import shutil + +BASE = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx") + +ORIG = BASE / "all_runs_269_289_no275_024_rg101_integrated" +WRITEBACK = BASE / "runs/run_20260805-183129_a8b9/predictions/mfx_refl_writeback" +SCALED = BASE / "PREDICTED_scale_all_r269_r289_sqrt_dmin2" +MATCHED = BASE / "matched_cctbx_sqrt" + +def read_ids(path): + with path.open() as fh: + data = json.load(fh) + + return [ + exp["identifier"] + for exp in data.get("experiment", []) + if exp.get("identifier") + ] + +# 1. UUIDs that survived sqrt scaling/filtering +survivors = set() + +scaled_files = list(SCALED.glob("*.expt")) + +for f in scaled_files: + survivors.update(read_ids(f)) + +print("Scaled chunk files:", len(scaled_files)) +print("Surviving sqrt experiments:", len(survivors)) + +# 2. Map survivor UUID -> write-back filename +mapping = {} +writeback_files = list(WRITEBACK.glob("*.expt")) + +for i, f in enumerate(writeback_files, 1): + + for uid in read_ids(f): + if uid in survivors: + mapping[uid] = f.name + + if i % 10000 == 0: + print( + f"Scanned {i}/{len(writeback_files)} " + f"write-back files; mapped {len(mapping)}" + ) + +unmapped = survivors - set(mapping) + +print("Mapped UUIDs:", len(mapping)) +print("Unmapped UUIDs:", len(unmapped)) + +if unmapped: + raise RuntimeError("Some survivor UUIDs could not be mapped") + +# 3. Rebuild matched cctbx directory +if MATCHED.exists(): + shutil.rmtree(MATCHED) + +MATCHED.mkdir(parents=True) + +linked = 0 +missing = 0 + +for uid, expt_name in mapping.items(): + + refl_name = expt_name[:-5] + ".refl" + + src_expt = ORIG / expt_name + src_refl = ORIG / refl_name + + if src_expt.is_file() and src_refl.is_file(): + + os.symlink(src_expt, MATCHED / expt_name) + os.symlink(src_refl, MATCHED / refl_name) + + linked += 1 + + else: + missing += 1 + +print() +print("Matched folder:", MATCHED) +print("Linked cctbx pairs:", linked) +print("Missing original pairs:", missing) +print("Total symlinks:", linked * 2) + +print("DONE SQRT") +PY \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/prediction/prediction_only_ampere_sqrt.sbatch b/workflows/mfx101555026/sbatch/prediction/prediction_only_ampere_sqrt.sbatch new file mode 100644 index 00000000..3a5b517f --- /dev/null +++ b/workflows/mfx101555026/sbatch/prediction/prediction_only_ampere_sqrt.sbatch @@ -0,0 +1,177 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=ampere +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gpus=1 +#SBATCH --mem=256G +#SBATCH --time=0-08:00 +#SBATCH --job-name=mfx_sqrt_ep24_prediction +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +############################################################################### +# Paths +############################################################################### + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +RUN_DIR="$BASE/runs/run_20260805-183129_a8b9" +FINAL_CKPT="$RUN_DIR/files/checkpoints/epoch=0024.ckpt" + +LOGS_DIR="$BASE/logs" +RUN_DIR_FILE="$LOGS_DIR/prediction_run_dir_${SLURM_JOB_ID}.txt" +TIMING_FILE="$LOGS_DIR/prediction_timing_${SLURM_JOB_ID}.txt" + +mkdir -p "$LOGS_DIR" + +############################################################################### +# Validate paths +############################################################################### + +if [[ ! -d "$RUN_DIR" ]]; then + echo "ERROR: run directory does not exist:" + echo " $RUN_DIR" + exit 1 +fi + +if [[ ! -f "$FINAL_CKPT" ]]; then + echo "ERROR: final checkpoint does not exist:" + echo " $FINAL_CKPT" + echo + echo "Available checkpoints:" + ls -lh "$RUN_DIR/files/checkpoints" + exit 1 +fi + +############################################################################### +# Helper +############################################################################### + +format_seconds() { + local total_seconds="$1" + + printf "%02d:%02d:%02d" \ + $((total_seconds / 3600)) \ + $(((total_seconds % 3600) / 60)) \ + $((total_seconds % 60)) +} + +############################################################################### +# Activate integrator-cuda-dev +############################################################################### + +cd "$REPO" + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + +eval "$( + /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba \ + shell hook --shell bash +)" + +micromamba activate integrator-cuda-dev + +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH="$REPO/src" +export PYTHONUNBUFFERED=1 + +############################################################################### +# Environment checks +############################################################################### + +JOB_START_EPOCH=$(date +%s) +JOB_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $JOB_START_TEXT" +echo "Run directory: $RUN_DIR" +echo "Prediction checkpoint: $FINAL_CKPT" +echo "Allocated CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Python: $(which python)" +echo "integrator.predict: $(which integrator.predict)" +echo "============================================================" + +python - <<'PY' +import torch + +print("CUDA available:", torch.cuda.is_available()) +print("GPU count:", torch.cuda.device_count()) + +if not torch.cuda.is_available(): + raise RuntimeError("CUDA is not available") + +print("GPU:", torch.cuda.get_device_name(0)) +print("PyTorch CUDA version:", torch.version.cuda) +PY + +nvidia-smi + +echo "$RUN_DIR" > "$RUN_DIR_FILE" + +############################################################################### +# Predict epoch 24 only +############################################################################### + +PREDICT_START_EPOCH=$(date +%s) +PREDICT_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Prediction started: $PREDICT_START_TEXT" +echo "Run directory: $RUN_DIR" +echo "Checkpoint: $FINAL_CKPT" +echo "============================================================" + +integrator.predict \ + --run-dir "$RUN_DIR" \ + --ckpt "$FINAL_CKPT" \ + --batch-size 2048 + +PREDICT_END_EPOCH=$(date +%s) +PREDICT_END_TEXT=$(date --iso-8601=seconds) +PREDICT_SECONDS=$((PREDICT_END_EPOCH - PREDICT_START_EPOCH)) + +echo "============================================================" +echo "Prediction finished: $PREDICT_END_TEXT" +echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" +echo "============================================================" + +############################################################################### +# Timing summary +############################################################################### + +JOB_END_EPOCH=$(date +%s) +JOB_END_TEXT=$(date --iso-8601=seconds) +TOTAL_SECONDS=$((JOB_END_EPOCH - JOB_START_EPOCH)) + +{ + echo "Job ID: $SLURM_JOB_ID" + echo "Run directory: $RUN_DIR" + echo "Prediction checkpoint: $FINAL_CKPT" + + echo "Job started: $JOB_START_TEXT" + echo "Prediction started: $PREDICT_START_TEXT" + echo "Prediction finished: $PREDICT_END_TEXT" + echo "Job finished: $JOB_END_TEXT" + + echo "Prediction seconds: $PREDICT_SECONDS" + echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" + + echo "Total seconds: $TOTAL_SECONDS" + echo "Total elapsed: $(format_seconds "$TOTAL_SECONDS")" +} | tee "$TIMING_FILE" + +echo +echo "Prediction finished successfully." +echo "Timing summary: $TIMING_FILE" +echo "Run directory file: $RUN_DIR_FILE" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/prediction/prediction_only_hopper_asinh.sbatch b/workflows/mfx101555026/sbatch/prediction/prediction_only_hopper_asinh.sbatch new file mode 100644 index 00000000..6c965304 --- /dev/null +++ b/workflows/mfx101555026/sbatch/prediction/prediction_only_hopper_asinh.sbatch @@ -0,0 +1,175 @@ +#!/bin/bash +#SBATCH --account=lcls:default +#SBATCH --partition=hopper +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gpus=1 +#SBATCH --mem=256G +#SBATCH --time=0-05:00 +#SBATCH --job-name=mfx_asinh_ep24_prediction +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +############################################################################### +# Paths +############################################################################### + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +RUN_DIR="$BASE/runs/run_20260805-011728_9704" +FINAL_CKPT="$RUN_DIR/files/checkpoints/epoch=0024.ckpt" + +LOGS_DIR="$BASE/logs" +RUN_DIR_FILE="$LOGS_DIR/prediction_run_dir_${SLURM_JOB_ID}.txt" +TIMING_FILE="$LOGS_DIR/prediction_timing_${SLURM_JOB_ID}.txt" + +mkdir -p "$LOGS_DIR" + +############################################################################### +# Validate paths +############################################################################### + +if [[ ! -d "$RUN_DIR" ]]; then + echo "ERROR: run directory does not exist:" + echo " $RUN_DIR" + exit 1 +fi + +if [[ ! -f "$FINAL_CKPT" ]]; then + echo "ERROR: final checkpoint does not exist:" + echo " $FINAL_CKPT" + echo + echo "Available checkpoints:" + ls -lh "$RUN_DIR/files/checkpoints" + exit 1 +fi + +############################################################################### +# Helper +############################################################################### + +format_seconds() { + local total_seconds="$1" + + printf "%02d:%02d:%02d" \ + $((total_seconds / 3600)) \ + $(((total_seconds % 3600) / 60)) \ + $((total_seconds % 60)) +} + +############################################################################### +# Activate integrator-cuda-dev +############################################################################### + +cd "$REPO" + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + +eval "$( + /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba \ + shell hook --shell bash +)" + +micromamba activate integrator-cuda-dev + +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH="$REPO/src" +export PYTHONUNBUFFERED=1 + +############################################################################### +# Environment checks +############################################################################### + +JOB_START_EPOCH=$(date +%s) +JOB_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $JOB_START_TEXT" +echo "Run directory: $RUN_DIR" +echo "Prediction checkpoint: $FINAL_CKPT" +echo "Allocated CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Python: $(which python)" +echo "integrator.predict: $(which integrator.predict)" +echo "============================================================" + +python - <<'PY' +import torch + +print("CUDA available:", torch.cuda.is_available()) +print("GPU count:", torch.cuda.device_count()) + +if not torch.cuda.is_available(): + raise RuntimeError("CUDA is not available") + +print("GPU:", torch.cuda.get_device_name(0)) +print("PyTorch CUDA version:", torch.version.cuda) +PY + +nvidia-smi + +echo "$RUN_DIR" > "$RUN_DIR_FILE" + +############################################################################### +# Predict epoch 24 only +############################################################################### + +PREDICT_START_EPOCH=$(date +%s) +PREDICT_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Prediction started: $PREDICT_START_TEXT" +echo "Run directory: $RUN_DIR" +echo "Checkpoint: $FINAL_CKPT" +echo "============================================================" + +integrator.predict \ + --run-dir "$RUN_DIR" \ + --ckpt "$FINAL_CKPT" \ + --batch-size 2048 + +PREDICT_END_EPOCH=$(date +%s) +PREDICT_END_TEXT=$(date --iso-8601=seconds) +PREDICT_SECONDS=$((PREDICT_END_EPOCH - PREDICT_START_EPOCH)) + +echo "============================================================" +echo "Prediction finished: $PREDICT_END_TEXT" +echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" +echo "============================================================" + +############################################################################### +# Timing summary +############################################################################### + +JOB_END_EPOCH=$(date +%s) +JOB_END_TEXT=$(date --iso-8601=seconds) +TOTAL_SECONDS=$((JOB_END_EPOCH - JOB_START_EPOCH)) + +{ + echo "Job ID: $SLURM_JOB_ID" + echo "Run directory: $RUN_DIR" + echo "Prediction checkpoint: $FINAL_CKPT" + + echo "Job started: $JOB_START_TEXT" + echo "Prediction started: $PREDICT_START_TEXT" + echo "Prediction finished: $PREDICT_END_TEXT" + echo "Job finished: $JOB_END_TEXT" + + echo "Prediction seconds: $PREDICT_SECONDS" + echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" + + echo "Total seconds: $TOTAL_SECONDS" + echo "Total elapsed: $(format_seconds "$TOTAL_SECONDS")" +} | tee "$TIMING_FILE" + +echo +echo "Prediction finished successfully." +echo "Timing summary: $TIMING_FILE" +echo "Run directory file: $RUN_DIR_FILE" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_all_dmin2_ASINH.sbatch b/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_all_dmin2_ASINH.sbatch new file mode 100644 index 00000000..f492239c --- /dev/null +++ b/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_all_dmin2_ASINH.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=08:00:00 +#SBATCH --job-name=predicted_asinh_dmin2 +#SBATCH --output=pred_asinh_dmin2_%j.out +#SBATCH --error=pred_asinh_dmin2_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting predicted ASINH scale dmin 2.0" +mpirun cctbx.xfel.merge PREDICTED_scale_all_r269_r289__dmin2_ASINH.phil + +echo "Starting predicted ASINH merge dmin 2.0" +mpirun cctbx.xfel.merge PREDICTED_merge_all_r269_r289__dmin2_ASINH.phil + +echo "All done for predicted ASINH dmin 2.0" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_all_dmin2_SQRT.sbatch b/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_all_dmin2_SQRT.sbatch new file mode 100644 index 00000000..0b45e346 --- /dev/null +++ b/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_all_dmin2_SQRT.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=08:00:00 +#SBATCH --job-name=predicted_sqrt_dmin2 +#SBATCH --output=pred_asinh_dmin2_%j.out +#SBATCH --error=pred_asinh_dmin2_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting predicted SQRT scale dmin 2.0" +mpirun cctbx.xfel.merge PREDICTED_scale_all_r269_r289__dmin2_SQRT.phil + +echo "Starting predicted SQRT merge dmin 2.0" +mpirun cctbx.xfel.merge PREDICTED_merge_all_r269_r289__dmin2_SQRT.phil + +echo "All done for predicted SQRT dmin 2.0" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_matched_cctbx_asinh_dmin2.sbatch b/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_matched_cctbx_asinh_dmin2.sbatch new file mode 100644 index 00000000..57932a34 --- /dev/null +++ b/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_matched_cctbx_asinh_dmin2.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=03:00:00 +#SBATCH --job-name=cctbx_asinh_dmin2 +#SBATCH --output=cctbx_asinh_dmin2_%j.out +#SBATCH --error=cctbx_asinh_dmin2_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting matched cctbx ASINH scale dmin 2.0" +mpirun cctbx.xfel.merge scale_matched_cctbx_asinh_dmin2.phil + +echo "Starting matched cctbx ASINH merge dmin 2.0" +mpirun cctbx.xfel.merge merge_matched_cctbx_asinh_dmin2.phil + +echo "All done for matched cctbx ASINH dmin 2.0" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_matched_cctbx_sqrt_dmin2.sbatch b/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_matched_cctbx_sqrt_dmin2.sbatch new file mode 100644 index 00000000..150c5060 --- /dev/null +++ b/workflows/mfx101555026/sbatch/scale_merge/run_scale_merge_matched_cctbx_sqrt_dmin2.sbatch @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +#SBATCH --partition=milano +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH --ntasks-per-node=120 +#SBATCH --time=08:00:00 +#SBATCH --job-name=cctbx_sqrt_dmin2 +#SBATCH --output=cctbx_sqrt_dmin2_%j.out +#SBATCH --error=cctbx_sqrt_dmin2_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +cd /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/combinedmask_scale_merge_phil + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +echo "Starting predicted SQRT scale dmin 2.0" +mpirun cctbx.xfel.merge scale_matched_cctbx_sqrt_dmin2.phil + +echo "Starting predicted SQRT merge dmin 2.0" +mpirun cctbx.xfel.merge merge_matched_cctbx_sqrt_dmin2.phil + +echo "All done for matched cctbx SQRT dmin 2.0" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/training/auto_preprocess.sbatch b/workflows/mfx101555026/sbatch/training/auto_preprocess.sbatch new file mode 100644 index 00000000..a835565b --- /dev/null +++ b/workflows/mfx101555026/sbatch/training/auto_preprocess.sbatch @@ -0,0 +1,63 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --qos=normal +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=64G +#SBATCH --time=0-04:00 +#SBATCH --job-name=preprocess_sqrt +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/preprocess_sqrt_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/preprocess_sqrt_%j.err + +set -eo pipefail + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +DATA_DIR=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_allruns_269_289_no275_024_rg101 +SQRT_CHUNK_DIR="$DATA_DIR/chunks_sqrt_squareplus" + +mkdir -p /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs + +cd "$REPO" + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + +eval "$( + /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba \ + shell hook --shell bash +)" + +micromamba activate integrator-cuda-dev + +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH="$REPO/src" +export PYTHONUNBUFFERED=1 + +echo "Host: $(hostname)" +echo "Started: $(date)" +echo "Data directory: $DATA_DIR" +echo "Output directory: $SQRT_CHUNK_DIR" +echo "Python: $(which python)" +echo "integrator.preprocess: $(which integrator.preprocess)" + +if [[ -e "$SQRT_CHUNK_DIR" ]]; then + echo "ERROR: output directory already exists:" + echo " $SQRT_CHUNK_DIR" + echo "Remove it or choose a new directory before rerunning." + exit 1 +fi + +integrator.preprocess \ + --data-dir "$DATA_DIR" \ + --validation-split 0.2 \ + --split-seed 42 \ + --transform sqrt_squareplus \ + --chunk-size 50000 \ + --n-bins 1 \ + --out-dir "$SQRT_CHUNK_DIR" \ + --min-valid-pixels 10 \ + -v + +echo "Finished: $(date)" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/training/auto_train_predict_chunked.sbatch b/workflows/mfx101555026/sbatch/training/auto_train_predict_chunked.sbatch new file mode 100644 index 00000000..a4c214cd --- /dev/null +++ b/workflows/mfx101555026/sbatch/training/auto_train_predict_chunked.sbatch @@ -0,0 +1,249 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=ampere +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gpus=1 +#SBATCH --mem=256G +#SBATCH --time=1-00:00 +#SBATCH --job-name=mfx_all_run_chunks +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +############################################################################### +# Paths +############################################################################### + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +CONFIG="$REPO/workflows/mfx101555026/configs/mfx_chunked_5000.yaml" +RUNS_DIR="$BASE/runs" +LOGS_DIR="$BASE/logs" + +mkdir -p "$RUNS_DIR" "$LOGS_DIR" + +if [[ ! -f "$CONFIG" ]]; then + echo "ERROR: config file does not exist:" + echo " $CONFIG" + exit 1 +fi + +BEFORE_FILE="$LOGS_DIR/before_runs_${SLURM_JOB_ID}.txt" +AFTER_FILE="$LOGS_DIR/after_runs_${SLURM_JOB_ID}.txt" +RUN_DIR_FILE="$LOGS_DIR/run_dir_${SLURM_JOB_ID}.txt" +TIMING_FILE="$LOGS_DIR/timing_${SLURM_JOB_ID}.txt" + +############################################################################### +# Helper: format seconds as HH:MM:SS +############################################################################### + +format_seconds() { + local total_seconds="$1" + + printf "%02d:%02d:%02d" \ + $((total_seconds / 3600)) \ + $(((total_seconds % 3600) / 60)) \ + $((total_seconds % 60)) +} + +############################################################################### +# Activate integrator-cuda-dev +############################################################################### + +cd "$REPO" + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + +eval "$( + /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba \ + shell hook --shell bash +)" + +micromamba activate integrator-cuda-dev + +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH="$REPO/src" +export PYTHONUNBUFFERED=1 + +############################################################################### +# Environment checks +############################################################################### + +JOB_START_EPOCH=$(date +%s) +JOB_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $JOB_START_TEXT" +echo "Config: $CONFIG" +echo "Repository: $REPO" +echo "Runs directory: $RUNS_DIR" +echo "Allocated CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Python: $(which python)" +echo "integrator.train: $(which integrator.train)" +echo "integrator.predict: $(which integrator.predict)" +echo "============================================================" + +python - <<'PY' +import reciprocalspaceship +import torch + +print("reciprocalspaceship import: OK") +print("CUDA available:", torch.cuda.is_available()) +print("GPU count:", torch.cuda.device_count()) + +if not torch.cuda.is_available(): + raise RuntimeError("CUDA is not available") + +print("GPU:", torch.cuda.get_device_name(0)) +print("PyTorch CUDA version:", torch.version.cuda) +PY + +nvidia-smi + +echo +echo "Relevant YAML settings:" +grep -nE \ + "chunk_dir:|batch_size:|num_workers:|max_epochs:|image_level_wilson:|init_log_B:|init_log_G:|n_bins:" \ + "$CONFIG" || true +echo + +############################################################################### +# Record existing run directories +############################################################################### + +find "$RUNS_DIR" \ + -maxdepth 1 \ + -type d \ + -name "run_*" \ + | sort > "$BEFORE_FILE" + +############################################################################### +# Training +############################################################################### + +TRAIN_START_EPOCH=$(date +%s) +TRAIN_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Training started: $TRAIN_START_TEXT" +echo "============================================================" + +integrator.train \ + --config "$CONFIG" \ + --log-dir "$RUNS_DIR" + +TRAIN_END_EPOCH=$(date +%s) +TRAIN_END_TEXT=$(date --iso-8601=seconds) +TRAIN_SECONDS=$((TRAIN_END_EPOCH - TRAIN_START_EPOCH)) + +echo "============================================================" +echo "Training finished: $TRAIN_END_TEXT" +echo "Training elapsed: $(format_seconds "$TRAIN_SECONDS")" +echo "============================================================" + +############################################################################### +# Find newly created run directory +############################################################################### + +find "$RUNS_DIR" \ + -maxdepth 1 \ + -type d \ + -name "run_*" \ + | sort > "$AFTER_FILE" + +RUN_DIR=$(comm -13 "$BEFORE_FILE" "$AFTER_FILE" | head -n 1) + +if [[ -z "${RUN_DIR:-}" ]]; then + echo "No new run found by directory comparison." + echo "Using the newest run_* directory." + + RUN_DIR=$( + find "$RUNS_DIR" \ + -maxdepth 1 \ + -type d \ + -name "run_*" \ + -printf "%T@ %p\n" \ + | sort -nr \ + | head -n 1 \ + | cut -d' ' -f2- + ) +fi + +if [[ -z "${RUN_DIR:-}" || ! -d "$RUN_DIR" ]]; then + echo "ERROR: could not find the training run directory" + exit 1 +fi + +echo "$RUN_DIR" > "$RUN_DIR_FILE" + +echo "Run directory: $RUN_DIR" +echo "Run directory saved in: $RUN_DIR_FILE" + +############################################################################### +# Prediction +############################################################################### + +PREDICT_START_EPOCH=$(date +%s) +PREDICT_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Prediction started: $PREDICT_START_TEXT" +echo "============================================================" + +integrator.predict \ + --run-dir "$RUN_DIR" \ + --batch-size 2048 + +PREDICT_END_EPOCH=$(date +%s) +PREDICT_END_TEXT=$(date --iso-8601=seconds) +PREDICT_SECONDS=$((PREDICT_END_EPOCH - PREDICT_START_EPOCH)) + +echo "============================================================" +echo "Prediction finished: $PREDICT_END_TEXT" +echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" +echo "============================================================" + +############################################################################### +# Final timing summary +############################################################################### + +JOB_END_EPOCH=$(date +%s) +JOB_END_TEXT=$(date --iso-8601=seconds) +TOTAL_SECONDS=$((JOB_END_EPOCH - JOB_START_EPOCH)) + +{ + echo "Job ID: $SLURM_JOB_ID" + echo "Config: $CONFIG" + echo "Run directory: $RUN_DIR" + + echo "Job started: $JOB_START_TEXT" + echo "Training started: $TRAIN_START_TEXT" + echo "Training finished: $TRAIN_END_TEXT" + echo "Prediction started: $PREDICT_START_TEXT" + echo "Prediction finished: $PREDICT_END_TEXT" + echo "Job finished: $JOB_END_TEXT" + + echo "Training seconds: $TRAIN_SECONDS" + echo "Training elapsed: $(format_seconds "$TRAIN_SECONDS")" + + echo "Prediction seconds: $PREDICT_SECONDS" + echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" + + echo "Total seconds: $TOTAL_SECONDS" + echo "Total elapsed: $(format_seconds "$TOTAL_SECONDS")" +} | tee "$TIMING_FILE" + +echo +echo "Finished successfully." +echo "Timing summary: $TIMING_FILE" +echo "Run directory file: $RUN_DIR_FILE" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/training/auto_train_predict_chunked_HOPPER.sbatch b/workflows/mfx101555026/sbatch/training/auto_train_predict_chunked_HOPPER.sbatch new file mode 100644 index 00000000..7bc6f837 --- /dev/null +++ b/workflows/mfx101555026/sbatch/training/auto_train_predict_chunked_HOPPER.sbatch @@ -0,0 +1,247 @@ +#!/bin/bash +#SBATCH --account=lcls:default +#SBATCH --partition=hopper +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gpus=1 +#SBATCH --mem=256G +#SBATCH --time=0-08:00 +#SBATCH --job-name=mfx_all_run_chunks_sqrt_squareplus +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +############################################################################### +# Paths +############################################################################### + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +CONFIG="$REPO/workflows/mfx101555026/configs/mfx_chunked_sqrt_squareplus.yaml" +RUNS_DIR="$BASE/runs" +LOGS_DIR="$BASE/logs" + +mkdir -p "$RUNS_DIR" "$LOGS_DIR" + +if [[ ! -f "$CONFIG" ]]; then + echo "ERROR: config file does not exist:" + echo " $CONFIG" + exit 1 +fi + +BEFORE_FILE="$LOGS_DIR/before_runs_${SLURM_JOB_ID}.txt" +AFTER_FILE="$LOGS_DIR/after_runs_${SLURM_JOB_ID}.txt" +RUN_DIR_FILE="$LOGS_DIR/run_dir_${SLURM_JOB_ID}.txt" +TIMING_FILE="$LOGS_DIR/timing_${SLURM_JOB_ID}.txt" + +############################################################################### +# Helper: format seconds as HH:MM:SS +############################################################################### + +format_seconds() { + local total_seconds="$1" + + printf "%02d:%02d:%02d" \ + $((total_seconds / 3600)) \ + $(((total_seconds % 3600) / 60)) \ + $((total_seconds % 60)) +} + +############################################################################### +# Activate integrator-cuda-dev +############################################################################### + +cd "$REPO" + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + +eval "$( + /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba \ + shell hook --shell bash +)" + +micromamba activate integrator-cuda-dev + +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH="$REPO/src" +export PYTHONUNBUFFERED=1 + +############################################################################### +# Environment checks +############################################################################### + +JOB_START_EPOCH=$(date +%s) +JOB_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $JOB_START_TEXT" +echo "Config: $CONFIG" +echo "Repository: $REPO" +echo "Runs directory: $RUNS_DIR" +echo "Allocated CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Python: $(which python)" +echo "integrator.train: $(which integrator.train)" +echo "integrator.predict: $(which integrator.predict)" +echo "============================================================" + +python - <<'PY' +import reciprocalspaceship +import torch + +print("reciprocalspaceship import: OK") +print("CUDA available:", torch.cuda.is_available()) +print("GPU count:", torch.cuda.device_count()) + +if not torch.cuda.is_available(): + raise RuntimeError("CUDA is not available") + +print("GPU:", torch.cuda.get_device_name(0)) +print("PyTorch CUDA version:", torch.version.cuda) +PY + +nvidia-smi + +echo +echo "Relevant YAML settings:" +grep -nE \ + "chunk_dir:|batch_size:|num_workers:|max_epochs:|image_level_wilson:|init_log_B:|init_log_G:|n_bins:" \ + "$CONFIG" || true +echo + +############################################################################### +# Record existing run directories +############################################################################### + +find "$RUNS_DIR" \ + -maxdepth 1 \ + -type d \ + -name "run_*" \ + | sort > "$BEFORE_FILE" + +############################################################################### +# Training +############################################################################### + +TRAIN_START_EPOCH=$(date +%s) +TRAIN_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Training started: $TRAIN_START_TEXT" +echo "============================================================" + +integrator.train \ + --config "$CONFIG" \ + --log-dir "$RUNS_DIR" + +TRAIN_END_EPOCH=$(date +%s) +TRAIN_END_TEXT=$(date --iso-8601=seconds) +TRAIN_SECONDS=$((TRAIN_END_EPOCH - TRAIN_START_EPOCH)) + +echo "============================================================" +echo "Training finished: $TRAIN_END_TEXT" +echo "Training elapsed: $(format_seconds "$TRAIN_SECONDS")" +echo "============================================================" + +############################################################################### +# Find newly created run directory +############################################################################### + +find "$RUNS_DIR" \ + -maxdepth 1 \ + -type d \ + -name "run_*" \ + | sort > "$AFTER_FILE" + +RUN_DIR=$(comm -13 "$BEFORE_FILE" "$AFTER_FILE" | head -n 1) + +if [[ -z "${RUN_DIR:-}" ]]; then + echo "No new run found by directory comparison." + echo "Using the newest run_* directory." + + RUN_DIR=$( + find "$RUNS_DIR" \ + -maxdepth 1 \ + -type d \ + -name "run_*" \ + -printf "%T@ %p\n" \ + | sort -nr \ + | head -n 1 \ + | cut -d' ' -f2- + ) +fi + +if [[ -z "${RUN_DIR:-}" || ! -d "$RUN_DIR" ]]; then + echo "ERROR: could not find the training run directory" + exit 1 +fi + +echo "$RUN_DIR" > "$RUN_DIR_FILE" + +echo "Run directory: $RUN_DIR" +echo "Run directory saved in: $RUN_DIR_FILE" + +############################################################################### +# Prediction +############################################################################### + +PREDICT_START_EPOCH=$(date +%s) +PREDICT_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Prediction started: $PREDICT_START_TEXT" +echo "============================================================" + +integrator.predict \ + --run-dir "$RUN_DIR" \ + --batch-size 2048 + +PREDICT_END_EPOCH=$(date +%s) +PREDICT_END_TEXT=$(date --iso-8601=seconds) +PREDICT_SECONDS=$((PREDICT_END_EPOCH - PREDICT_START_EPOCH)) + +echo "============================================================" +echo "Prediction finished: $PREDICT_END_TEXT" +echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" +echo "============================================================" + +############################################################################### +# Final timing summary +############################################################################### + +JOB_END_EPOCH=$(date +%s) +JOB_END_TEXT=$(date --iso-8601=seconds) +TOTAL_SECONDS=$((JOB_END_EPOCH - JOB_START_EPOCH)) + +{ + echo "Job ID: $SLURM_JOB_ID" + echo "Config: $CONFIG" + echo "Run directory: $RUN_DIR" + + echo "Job started: $JOB_START_TEXT" + echo "Training started: $TRAIN_START_TEXT" + echo "Training finished: $TRAIN_END_TEXT" + echo "Prediction started: $PREDICT_START_TEXT" + echo "Prediction finished: $PREDICT_END_TEXT" + echo "Job finished: $JOB_END_TEXT" + + echo "Training seconds: $TRAIN_SECONDS" + echo "Training elapsed: $(format_seconds "$TRAIN_SECONDS")" + + echo "Prediction seconds: $PREDICT_SECONDS" + echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" + + echo "Total seconds: $TOTAL_SECONDS" + echo "Total elapsed: $(format_seconds "$TOTAL_SECONDS")" +} | tee "$TIMING_FILE" + +echo +echo "Finished successfully." +echo "Timing summary: $TIMING_FILE" +echo "Run directory file: $RUN_DIR_FILE" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/training/resume_training.batch b/workflows/mfx101555026/sbatch/training/resume_training.batch new file mode 100644 index 00000000..46984974 --- /dev/null +++ b/workflows/mfx101555026/sbatch/training/resume_training.batch @@ -0,0 +1,239 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=ampere +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gpus=1 +#SBATCH --mem=256G +#SBATCH --time=2-00:00 +#SBATCH --job-name=mfx_allruns_resume_predict +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +############################################################################### +# Paths +############################################################################### + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +CONFIG="$REPO/workflows/mfx101555026/configs/mfx_chunked_5000.yaml" + +RUN_DIR="$BASE/runs/run_20260805-011728_9704" +RESUME_CKPT="$RUN_DIR/files/checkpoints/epoch=0023.ckpt" +FINAL_CKPT="$RUN_DIR/files/checkpoints/epoch=0024.ckpt" + +LOGS_DIR="$BASE/logs" +RUN_DIR_FILE="$LOGS_DIR/run_dir_${SLURM_JOB_ID}.txt" +TIMING_FILE="$LOGS_DIR/resume_predict_timing_${SLURM_JOB_ID}.txt" + +mkdir -p "$LOGS_DIR" + +############################################################################### +# Validate paths +############################################################################### + +if [[ ! -f "$CONFIG" ]]; then + echo "ERROR: config does not exist:" + echo " $CONFIG" + exit 1 +fi + +if [[ ! -d "$RUN_DIR" ]]; then + echo "ERROR: run directory does not exist:" + echo " $RUN_DIR" + exit 1 +fi + +if [[ ! -f "$RESUME_CKPT" ]]; then + echo "ERROR: resume checkpoint does not exist:" + echo " $RESUME_CKPT" + exit 1 +fi + +############################################################################### +# Helper +############################################################################### + +format_seconds() { + local total_seconds="$1" + + printf "%02d:%02d:%02d" \ + $((total_seconds / 3600)) \ + $(((total_seconds % 3600) / 60)) \ + $((total_seconds % 60)) +} + +############################################################################### +# Activate integrator-cuda-dev +############################################################################### + +cd "$REPO" + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + +eval "$( + /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba \ + shell hook --shell bash +)" + +micromamba activate integrator-cuda-dev + +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH="$REPO/src" +export PYTHONUNBUFFERED=1 + +############################################################################### +# Environment checks +############################################################################### + +JOB_START_EPOCH=$(date +%s) +JOB_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $JOB_START_TEXT" +echo "Config: $CONFIG" +echo "Run directory: $RUN_DIR" +echo "Resume checkpoint: $RESUME_CKPT" +echo "Final checkpoint: $FINAL_CKPT" +echo "Allocated CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Python: $(which python)" +echo "integrator.train: $(which integrator.train)" +echo "integrator.predict: $(which integrator.predict)" +echo "============================================================" + +python - <<'PY' +import torch + +print("CUDA available:", torch.cuda.is_available()) +print("GPU count:", torch.cuda.device_count()) + +if not torch.cuda.is_available(): + raise RuntimeError("CUDA is not available") + +print("GPU:", torch.cuda.get_device_name(0)) +print("PyTorch CUDA version:", torch.version.cuda) +PY + +nvidia-smi + +echo +echo "Relevant YAML settings:" +grep -nE \ + "chunk_dir:|batch_size:|num_workers:|max_epochs:|check_val_every_n_epoch:|image_level_wilson:|init_log_B:|init_log_G:|n_bins:" \ + "$CONFIG" || true +echo + +echo "$RUN_DIR" > "$RUN_DIR_FILE" + +############################################################################### +# Resume training +############################################################################### + +TRAIN_START_EPOCH=$(date +%s) +TRAIN_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Resume training started: $TRAIN_START_TEXT" +echo "Checkpoint: $RESUME_CKPT" +echo "============================================================" + +integrator.train \ + --config "$CONFIG" \ + --run-dir "$RUN_DIR" \ + --ckpt-path "$RESUME_CKPT" + +TRAIN_END_EPOCH=$(date +%s) +TRAIN_END_TEXT=$(date --iso-8601=seconds) +TRAIN_SECONDS=$((TRAIN_END_EPOCH - TRAIN_START_EPOCH)) + +echo "============================================================" +echo "Training finished: $TRAIN_END_TEXT" +echo "Training elapsed: $(format_seconds "$TRAIN_SECONDS")" +echo "============================================================" + +############################################################################### +# Verify final checkpoint +############################################################################### + +if [[ ! -f "$FINAL_CKPT" ]]; then + echo "ERROR: final checkpoint was not created:" + echo " $FINAL_CKPT" + echo + echo "Available checkpoints:" + ls -lh "$RUN_DIR/files/checkpoints" + exit 1 +fi + +############################################################################### +# Predict only the final checkpoint +############################################################################### + +PREDICT_START_EPOCH=$(date +%s) +PREDICT_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Prediction started: $PREDICT_START_TEXT" +echo "Run directory: $RUN_DIR" +echo "Checkpoint: $FINAL_CKPT" +echo "============================================================" + +integrator.predict \ + --run-dir "$RUN_DIR" \ + --ckpt "$FINAL_CKPT" \ + --batch-size 2048 + +PREDICT_END_EPOCH=$(date +%s) +PREDICT_END_TEXT=$(date --iso-8601=seconds) +PREDICT_SECONDS=$((PREDICT_END_EPOCH - PREDICT_START_EPOCH)) + +echo "============================================================" +echo "Prediction finished: $PREDICT_END_TEXT" +echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" +echo "============================================================" + +############################################################################### +# Timing summary +############################################################################### + +JOB_END_EPOCH=$(date +%s) +JOB_END_TEXT=$(date --iso-8601=seconds) +TOTAL_SECONDS=$((JOB_END_EPOCH - JOB_START_EPOCH)) + +{ + echo "Job ID: $SLURM_JOB_ID" + echo "Config: $CONFIG" + echo "Run directory: $RUN_DIR" + echo "Resume checkpoint: $RESUME_CKPT" + echo "Prediction checkpoint: $FINAL_CKPT" + + echo "Job started: $JOB_START_TEXT" + echo "Training started: $TRAIN_START_TEXT" + echo "Training finished: $TRAIN_END_TEXT" + echo "Prediction started: $PREDICT_START_TEXT" + echo "Prediction finished: $PREDICT_END_TEXT" + echo "Job finished: $JOB_END_TEXT" + + echo "Resume training seconds: $TRAIN_SECONDS" + echo "Resume training elapsed: $(format_seconds "$TRAIN_SECONDS")" + + echo "Prediction seconds: $PREDICT_SECONDS" + echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" + + echo "Total seconds: $TOTAL_SECONDS" + echo "Total elapsed: $(format_seconds "$TOTAL_SECONDS")" +} | tee "$TIMING_FILE" + +echo +echo "Finished successfully." +echo "Timing summary: $TIMING_FILE" +echo "Run directory file: $RUN_DIR_FILE" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/training/resume_training_HOPPER.batch b/workflows/mfx101555026/sbatch/training/resume_training_HOPPER.batch new file mode 100644 index 00000000..f39774b5 --- /dev/null +++ b/workflows/mfx101555026/sbatch/training/resume_training_HOPPER.batch @@ -0,0 +1,237 @@ +#!/bin/bash +#SBATCH --account=lcls:default +#SBATCH --partition=hopper +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --gpus=1 +#SBATCH --mem=256G +#SBATCH --time=0-05:00 +#SBATCH --job-name=mfx_all_run_chunks_sqrt_squareplus +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +############################################################################### +# Paths +############################################################################### + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +CONFIG="$REPO/workflows/mfx101555026/configs/mfx_chunked_sqrt_squareplus.yaml" + +RUN_DIR="$BASE/runs/run_20260805-183129_a8b9" +RESUME_CKPT="$RUN_DIR/files/checkpoints/last.ckpt" +FINAL_CKPT="$RUN_DIR/files/checkpoints/epoch=0024.ckpt" + +LOGS_DIR="$BASE/logs" +RUN_DIR_FILE="$LOGS_DIR/run_dir_${SLURM_JOB_ID}.txt" +TIMING_FILE="$LOGS_DIR/resume_predict_timing_${SLURM_JOB_ID}.txt" + +mkdir -p "$LOGS_DIR" + +############################################################################### +# Validate paths +############################################################################### + +if [[ ! -f "$CONFIG" ]]; then + echo "ERROR: config does not exist:" + echo " $CONFIG" + exit 1 +fi + +if [[ ! -d "$RUN_DIR" ]]; then + echo "ERROR: run directory does not exist:" + echo " $RUN_DIR" + exit 1 +fi + +if [[ ! -f "$RESUME_CKPT" ]]; then + echo "ERROR: resume checkpoint does not exist:" + echo " $RESUME_CKPT" + exit 1 +fi + +############################################################################### +# Helper +############################################################################### + +format_seconds() { + local total_seconds="$1" + + printf "%02d:%02d:%02d" \ + $((total_seconds / 3600)) \ + $(((total_seconds % 3600) / 60)) \ + $((total_seconds % 60)) +} + +############################################################################### +# Activate integrator-cuda-dev +############################################################################### + +cd "$REPO" + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + +eval "$( + /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba \ + shell hook --shell bash +)" + +micromamba activate integrator-cuda-dev + +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH="$REPO/src" +export PYTHONUNBUFFERED=1 + +############################################################################### +# Environment checks +############################################################################### + +JOB_START_EPOCH=$(date +%s) +JOB_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $JOB_START_TEXT" +echo "Config: $CONFIG" +echo "Run directory: $RUN_DIR" +echo "Resume checkpoint: $RESUME_CKPT" +echo "Final checkpoint: $FINAL_CKPT" +echo "Allocated CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Python: $(which python)" +echo "integrator.train: $(which integrator.train)" +echo "integrator.predict: $(which integrator.predict)" +echo "============================================================" + +python - <<'PY' +import torch + +print("CUDA available:", torch.cuda.is_available()) +print("GPU count:", torch.cuda.device_count()) + +if not torch.cuda.is_available(): + raise RuntimeError("CUDA is not available") + +print("GPU:", torch.cuda.get_device_name(0)) +print("PyTorch CUDA version:", torch.version.cuda) +PY + +nvidia-smi + +echo +echo "Relevant YAML settings:" +grep -nE \ + "chunk_dir:|batch_size:|num_workers:|max_epochs:|check_val_every_n_epoch:|image_level_wilson:|init_log_B:|init_log_G:|n_bins:" \ + "$CONFIG" || true +echo + +echo "$RUN_DIR" > "$RUN_DIR_FILE" + +############################################################################### +# Resume training +############################################################################### + +TRAIN_START_EPOCH=$(date +%s) +TRAIN_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Resume training started: $TRAIN_START_TEXT" +echo "Checkpoint: $RESUME_CKPT" +echo "============================================================" + +integrator.train \ + --config "$CONFIG" \ + --run-dir "$RUN_DIR" \ + --ckpt-path "$RESUME_CKPT" + +TRAIN_END_EPOCH=$(date +%s) +TRAIN_END_TEXT=$(date --iso-8601=seconds) +TRAIN_SECONDS=$((TRAIN_END_EPOCH - TRAIN_START_EPOCH)) + +echo "============================================================" +echo "Training finished: $TRAIN_END_TEXT" +echo "Training elapsed: $(format_seconds "$TRAIN_SECONDS")" +echo "============================================================" + +############################################################################### +# Verify final checkpoint +############################################################################### + +if [[ ! -f "$FINAL_CKPT" ]]; then + echo "ERROR: final checkpoint was not created:" + echo " $FINAL_CKPT" + echo + echo "Available checkpoints:" + ls -lh "$RUN_DIR/files/checkpoints" + exit 1 +fi + +############################################################################### +# Predict only the final checkpoint +############################################################################### + +PREDICT_START_EPOCH=$(date +%s) +PREDICT_START_TEXT=$(date --iso-8601=seconds) + +echo "============================================================" +echo "Prediction started: $PREDICT_START_TEXT" +echo "Run directory: $RUN_DIR" +echo "Checkpoint: $FINAL_CKPT" +echo "============================================================" + +integrator.predict \ + --run-dir "$RUN_DIR" \ + --ckpt "$FINAL_CKPT" \ + --batch-size 2048 + +PREDICT_END_EPOCH=$(date +%s) +PREDICT_END_TEXT=$(date --iso-8601=seconds) +PREDICT_SECONDS=$((PREDICT_END_EPOCH - PREDICT_START_EPOCH)) + +echo "============================================================" +echo "Prediction finished: $PREDICT_END_TEXT" +echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" +echo "============================================================" + +############################################################################### +# Timing summary +############################################################################### + +JOB_END_EPOCH=$(date +%s) +JOB_END_TEXT=$(date --iso-8601=seconds) +TOTAL_SECONDS=$((JOB_END_EPOCH - JOB_START_EPOCH)) + +{ + echo "Job ID: $SLURM_JOB_ID" + echo "Config: $CONFIG" + echo "Run directory: $RUN_DIR" + echo "Resume checkpoint: $RESUME_CKPT" + echo "Prediction checkpoint: $FINAL_CKPT" + + echo "Job started: $JOB_START_TEXT" + echo "Training started: $TRAIN_START_TEXT" + echo "Training finished: $TRAIN_END_TEXT" + echo "Prediction started: $PREDICT_START_TEXT" + echo "Prediction finished: $PREDICT_END_TEXT" + echo "Job finished: $JOB_END_TEXT" + + echo "Resume training seconds: $TRAIN_SECONDS" + echo "Resume training elapsed: $(format_seconds "$TRAIN_SECONDS")" + + echo "Prediction seconds: $PREDICT_SECONDS" + echo "Prediction elapsed: $(format_seconds "$PREDICT_SECONDS")" + + echo "Total seconds: $TOTAL_SECONDS" + echo "Total elapsed: $(format_seconds "$TOTAL_SECONDS")" +} | tee "$TIMING_FILE" + +echo +echo "Finished successfully." +echo "Timing summary: $TIMING_FILE" +echo "Run directory file: $RUN_DIR_FILE" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/writeback/asinh/export_asinh_ep24_npz.sbatch b/workflows/mfx101555026/sbatch/writeback/asinh/export_asinh_ep24_npz.sbatch new file mode 100644 index 00000000..40f2d145 --- /dev/null +++ b/workflows/mfx101555026/sbatch/writeback/asinh/export_asinh_ep24_npz.sbatch @@ -0,0 +1,130 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=ampere +#SBATCH --qos=normal +#SBATCH --gpus=1 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=128G +#SBATCH --time=0-05:00 +#SBATCH --job-name=mfx_asinh_ep24_export_npz +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +############################################################################### +# Paths +############################################################################### + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +RUN_DIR="$BASE/runs/run_20260805-011728_9704" + +PRED_DIR="$RUN_DIR/predictions/epoch_0024" +PRED_NPZ="$RUN_DIR/predictions/preds_epoch_0024.npz" + +LOGS_DIR="$BASE/logs" +mkdir -p "$LOGS_DIR" + +############################################################################### +# Validate prediction shards +############################################################################### + +if [[ ! -d "$PRED_DIR" ]]; then + echo "ERROR: prediction directory does not exist:" + echo " $PRED_DIR" + exit 1 +fi + +N_PARQUET=$(find "$PRED_DIR" -maxdepth 1 -type f -name '*.parquet' | wc -l) + +if [[ "$N_PARQUET" -ne 5758 ]]; then + echo "ERROR: expected 5758 prediction parquet files, found $N_PARQUET" + echo "Prediction directory:" + echo " $PRED_DIR" + exit 1 +fi + +echo "Prediction set complete: $N_PARQUET parquet files." + +############################################################################### +# Activate integrator-cuda-dev +############################################################################### + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + +eval "$( + /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba \ + shell hook --shell bash +)" + +micromamba activate integrator-cuda-dev + +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH="$REPO/src" +export PYTHONUNBUFFERED=1 + +cd "$REPO" + +############################################################################### +# Environment information +############################################################################### + +echo "============================================================" +echo "MFX ASINH epoch-24 NPZ export" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $(date --iso-8601=seconds)" +echo "Run: $RUN_DIR" +echo "Prediction dir:$PRED_DIR" +echo "Parquet files: $N_PARQUET" +echo "Output NPZ: $PRED_NPZ" +echo "CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Memory: 128G" +echo "Python: $(which python)" +echo "============================================================" + +############################################################################### +# Export all epoch-24 prediction shards -> NPZ +############################################################################### + +START=$(date +%s) + +/usr/bin/time -v \ +python "$REPO/workflows/mfx101555026/scripts/writeback/export_preds_for_dials.py" \ + --ckpt-dir "$PRED_DIR" \ + --out "$PRED_NPZ" + +END=$(date +%s) +ELAPSED=$((END - START)) + +############################################################################### +# Verify output +############################################################################### + +if [[ ! -f "$PRED_NPZ" ]]; then + echo "ERROR: expected NPZ was not created:" + echo " $PRED_NPZ" + exit 1 +fi + +echo +echo "============================================================" +echo "Export completed successfully." +echo "Finished: $(date --iso-8601=seconds)" + +printf "Elapsed: %02d:%02d:%02d\n" \ + $((ELAPSED / 3600)) \ + $(((ELAPSED % 3600) / 60)) \ + $((ELAPSED % 60)) + +echo +echo "Output:" +ls -lh "$PRED_NPZ" + +echo "============================================================" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/writeback/asinh/writeback_asinh_ep24.sbatch b/workflows/mfx101555026/sbatch/writeback/asinh/writeback_asinh_ep24.sbatch new file mode 100644 index 00000000..51f99437 --- /dev/null +++ b/workflows/mfx101555026/sbatch/writeback/asinh/writeback_asinh_ep24.sbatch @@ -0,0 +1,144 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=ampere +#SBATCH --qos=normal +#SBATCH --gpus=1 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=128G +#SBATCH --time=0-15:00 +#SBATCH --job-name=mfx_asinh_ep24_writeback +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +############################################################################### +# Paths +############################################################################### + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +DATA_DIR="$BASE/mfx_shoebox_allruns_269_289_no275_024_rg101" +RUN_DIR="$BASE/runs/run_20260805-011728_9704" +REFL_DIR="$BASE/all_runs_269_289_no275_024_rg101_integrated" + +PRED_NPZ="$RUN_DIR/predictions/preds_epoch_0024.npz" +WB_OUT="$RUN_DIR/predictions/mfx_refl_writeback" + +LOGS_DIR="$BASE/logs" +mkdir -p "$LOGS_DIR" + +############################################################################### +# Validate inputs +############################################################################### + +if [[ ! -f "$PRED_NPZ" ]]; then + echo "ERROR: NPZ prediction file does not exist:" + echo " $PRED_NPZ" + exit 1 +fi + +if [[ ! -f "$DATA_DIR/metadata.npy" ]]; then + echo "ERROR: metadata.npy does not exist:" + echo " $DATA_DIR/metadata.npy" + exit 1 +fi + +if [[ ! -d "$REFL_DIR" ]]; then + echo "ERROR: reflection directory does not exist:" + echo " $REFL_DIR" + exit 1 +fi + +############################################################################### +# Clean environment before cctbx / psana2 +############################################################################### + +unset PYTHONHOME +unset PYTHONNOUSERSITE +unset LD_LIBRARY_PATH +unset PYTHONPATH +hash -r + +############################################################################### +# Activate cctbx / psana2 +############################################################################### + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +export PYTHONPATH="$REPO/src:${PYTHONPATH:-}" +export PYTHONUNBUFFERED=1 + +cd "$REPO" + +############################################################################### +# Environment information +############################################################################### + +echo "============================================================" +echo "MFX ASINH epoch-24 reflection write-back" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $(date --iso-8601=seconds)" +echo "Run: $RUN_DIR" +echo "Prediction: $PRED_NPZ" +echo "Metadata: $DATA_DIR/metadata.npy" +echo "REFL dir: $REFL_DIR" +echo "Output: $WB_OUT" +echo "CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Memory: 128G" +echo "dials.python: $(which dials.python)" +echo "============================================================" + +############################################################################### +# Write back predictions +############################################################################### + +START=$(date +%s) + +/usr/bin/time -v \ +dials.python "$REPO/workflows/mfx101555026/scripts/writeback/run_writeback.py" \ + --npz "$PRED_NPZ" \ + --metadata "$DATA_DIR/metadata.npy" \ + --refl-dir "$REFL_DIR" \ + --out-dir "$WB_OUT" + +END=$(date +%s) +ELAPSED=$((END - START)) + +############################################################################### +# Verify output +############################################################################### + +if [[ ! -d "$WB_OUT" ]]; then + echo "ERROR: expected write-back directory was not created:" + echo " $WB_OUT" + exit 1 +fi + +############################################################################### +# Summary +############################################################################### + +echo +echo "============================================================" +echo "Write-back completed successfully." +echo "Finished: $(date --iso-8601=seconds)" + +printf "Elapsed: %02d:%02d:%02d\n" \ + $((ELAPSED / 3600)) \ + $(((ELAPSED % 3600) / 60)) \ + $((ELAPSED % 60)) + +echo +echo "Output directory:" +echo " $WB_OUT" + +echo +echo "Top-level output files:" +find "$WB_OUT" -maxdepth 1 -type f | wc -l + +echo "============================================================" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/writeback/sqrt/export_sqrt_ep24_npz.sbatch b/workflows/mfx101555026/sbatch/writeback/sqrt/export_sqrt_ep24_npz.sbatch new file mode 100644 index 00000000..516391bd --- /dev/null +++ b/workflows/mfx101555026/sbatch/writeback/sqrt/export_sqrt_ep24_npz.sbatch @@ -0,0 +1,121 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=ampere +#SBATCH --qos=normal +#SBATCH --gpus=1 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=128G +#SBATCH --time=0-05:00 +#SBATCH --job-name=mfx_sqrt_ep24_export_npz +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +RUN_DIR="$BASE/runs/run_20260805-183129_a8b9" +PRED_DIR="$RUN_DIR/predictions/epoch_0024" +PRED_NPZ="$RUN_DIR/predictions/preds_epoch_0024.npz" + +LOGS_DIR="$BASE/logs" +mkdir -p "$LOGS_DIR" + +############################################################################### +# Validate existing prediction +############################################################################### + +if [[ ! -d "$PRED_DIR" ]]; then + echo "ERROR: prediction directory does not exist:" + echo " $PRED_DIR" + exit 1 +fi + +N_PARQUET=$(find "$PRED_DIR" -maxdepth 1 -type f -name '*.parquet' | wc -l) + +if [[ "$N_PARQUET" -eq 0 ]]; then + echo "ERROR: no parquet prediction files found in:" + echo " $PRED_DIR" + exit 1 +fi + +echo "Found $N_PARQUET parquet prediction files." + +############################################################################### +# Activate integrator-cuda-dev +############################################################################### + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + +eval "$( + /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba \ + shell hook --shell bash +)" + +micromamba activate integrator-cuda-dev + +unset PYTHONHOME +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH="$CONDA_PREFIX/lib:${LD_LIBRARY_PATH:-}" +export PYTHONPATH="$REPO/src" +export PYTHONUNBUFFERED=1 + +cd "$REPO" + +############################################################################### +# Environment information +############################################################################### + +echo "============================================================" +echo "MFX SQRT epoch-24 prediction export" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $(date --iso-8601=seconds)" +echo "Run: $RUN_DIR" +echo "Pred dir: $PRED_DIR" +echo "Output NPZ: $PRED_NPZ" +echo "CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Memory: 128G" +echo "Python: $(which python)" +echo "============================================================" + +############################################################################### +# Export parquet -> NPZ +############################################################################### + +START=$(date +%s) + +/usr/bin/time -v \ +python "$REPO/workflows/mfx101555026/scripts/writeback/export_preds_for_dials.py" \ + --ckpt-dir "$PRED_DIR" \ + --out "$PRED_NPZ" + +END=$(date +%s) +ELAPSED=$((END - START)) + +############################################################################### +# Verify output +############################################################################### + +if [[ ! -f "$PRED_NPZ" ]]; then + echo "ERROR: expected NPZ was not created:" + echo " $PRED_NPZ" + exit 1 +fi + +echo +echo "============================================================" +echo "Export completed successfully." +echo "Finished: $(date --iso-8601=seconds)" + +printf "Elapsed: %02d:%02d:%02d\n" \ + $((ELAPSED / 3600)) \ + $(((ELAPSED % 3600) / 60)) \ + $((ELAPSED % 60)) + +echo +ls -lh "$PRED_NPZ" +echo "============================================================" \ No newline at end of file diff --git a/workflows/mfx101555026/sbatch/writeback/sqrt/writeback_sqrt_ep24.sbatch b/workflows/mfx101555026/sbatch/writeback/sqrt/writeback_sqrt_ep24.sbatch new file mode 100644 index 00000000..87c8d799 --- /dev/null +++ b/workflows/mfx101555026/sbatch/writeback/sqrt/writeback_sqrt_ep24.sbatch @@ -0,0 +1,144 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=ampere +#SBATCH --qos=normal +#SBATCH --gpus=1 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=128G +#SBATCH --time=0-02:00 +#SBATCH --job-name=mfx_sqrt_ep24_writeback +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/%x_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +############################################################################### +# Paths +############################################################################### + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +DATA_DIR="$BASE/mfx_shoebox_allruns_269_289_no275_024_rg101" +RUN_DIR="$BASE/runs/run_20260805-183129_a8b9" +REFL_DIR="$BASE/all_runs_269_289_no275_024_rg101_integrated" + +PRED_NPZ="$RUN_DIR/predictions/preds_epoch_0024.npz" +WB_OUT="$RUN_DIR/predictions/mfx_refl_writeback" + +LOGS_DIR="$BASE/logs" +mkdir -p "$LOGS_DIR" + +############################################################################### +# Validate inputs +############################################################################### + +if [[ ! -f "$PRED_NPZ" ]]; then + echo "ERROR: NPZ prediction file does not exist:" + echo " $PRED_NPZ" + exit 1 +fi + +if [[ ! -f "$DATA_DIR/metadata.npy" ]]; then + echo "ERROR: metadata.npy does not exist:" + echo " $DATA_DIR/metadata.npy" + exit 1 +fi + +if [[ ! -d "$REFL_DIR" ]]; then + echo "ERROR: reflection directory does not exist:" + echo " $REFL_DIR" + exit 1 +fi + +############################################################################### +# Clean environment before cctbx / psana2 +############################################################################### + +unset PYTHONHOME +unset PYTHONNOUSERSITE +unset LD_LIBRARY_PATH +unset PYTHONPATH +hash -r + +############################################################################### +# Activate cctbx / psana2 +############################################################################### + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +export PYTHONPATH="$REPO/src:${PYTHONPATH:-}" +export PYTHONUNBUFFERED=1 + +cd "$REPO" + +############################################################################### +# Environment information +############################################################################### + +echo "============================================================" +echo "MFX SQRT epoch-24 reflection write-back" +echo "Job ID: $SLURM_JOB_ID" +echo "Host: $(hostname)" +echo "Started: $(date --iso-8601=seconds)" +echo "Run: $RUN_DIR" +echo "Prediction: $PRED_NPZ" +echo "Metadata: $DATA_DIR/metadata.npy" +echo "REFL dir: $REFL_DIR" +echo "Output: $WB_OUT" +echo "CPUs: ${SLURM_CPUS_PER_TASK:-unknown}" +echo "Memory: 128G" +echo "dials.python: $(which dials.python)" +echo "============================================================" + +############################################################################### +# Write back predictions +############################################################################### + +START=$(date +%s) + +/usr/bin/time -v \ +dials.python "$REPO/workflows/mfx101555026/scripts/writeback/run_writeback.py" \ + --npz "$PRED_NPZ" \ + --metadata "$DATA_DIR/metadata.npy" \ + --refl-dir "$REFL_DIR" \ + --out-dir "$WB_OUT" + +END=$(date +%s) +ELAPSED=$((END - START)) + +############################################################################### +# Verify output +############################################################################### + +if [[ ! -d "$WB_OUT" ]]; then + echo "ERROR: expected write-back directory was not created:" + echo " $WB_OUT" + exit 1 +fi + +############################################################################### +# Summary +############################################################################### + +echo +echo "============================================================" +echo "Write-back completed successfully." +echo "Finished: $(date --iso-8601=seconds)" + +printf "Elapsed: %02d:%02d:%02d\n" \ + $((ELAPSED / 3600)) \ + $(((ELAPSED % 3600) / 60)) \ + $((ELAPSED % 60)) + +echo +echo "Output directory:" +echo " $WB_OUT" + +echo +echo "Top-level output files and symlinks:" +find "$WB_OUT" -maxdepth 1 \( -type f -o -type l \) | wc -l + +echo "============================================================" \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/compare_qi_25files.py b/workflows/mfx101555026/scripts/archive/diagnostics/compare_qi_25files.py new file mode 100644 index 00000000..ffecafc0 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/compare_qi_25files.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +""" +Compare Gamma, LogNormal, and FoldedNormal qi prediction outputs. + +This script searches each run folder for prediction parquet files, +loads them, summarizes key columns, and writes a CSV comparison table. + +Expected run folders: + + mfx_gamma_qi_25files + mfx_lognormal_qi_25files_mc_kl + mfx_foldednormal_qi_25files_mc_kl +""" + +from __future__ import annotations + +from pathlib import Path + +import pandas as pd + + +RUNS_ROOT = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/" + "mfx101555026_cctbx/runs" +) + +RUNS = { + "gamma": RUNS_ROOT / "mfx_gamma_qi_25files", + "lognormal": RUNS_ROOT / "mfx_lognormal_qi_25files_mc_kl", + "foldednormal": RUNS_ROOT / "mfx_foldednormal_qi_25files_mc_kl", +} + +OUT_CSV = RUNS_ROOT / "compare_qi_25files_summary.csv" + + +def find_prediction_file(run_dir: Path) -> Path: + """ + Find the main prediction parquet file under a run folder. + + integrator.predict may create files such as: + pred.parquet + test_preds_all.parquet + + We prefer test_preds_all.parquet if it exists, otherwise pred.parquet. + """ + if not run_dir.exists(): + raise FileNotFoundError(f"Run folder does not exist: {run_dir}") + + candidates = list(run_dir.rglob("test_preds_all.parquet")) + if candidates: + return sorted(candidates)[-1] + + candidates = list(run_dir.rglob("pred.parquet")) + if candidates: + return sorted(candidates)[-1] + + raise FileNotFoundError(f"No prediction parquet found under: {run_dir}") + + +def summarize_prediction(label: str, parquet_path: Path) -> dict: + """ + Load one prediction parquet and summarize useful columns. + """ + df = pd.read_parquet(parquet_path) + + summary = { + "model": label, + "parquet_path": str(parquet_path), + "n_rows": len(df), + } + + # Summarize common prediction columns if they exist. + cols = [ + "qi_mean", + "qi_var", + "qbg_mean", + "qbg_var", + "intensity.sum.value", + "intensity.sum.variance", + "d", + ] + + for col in cols: + if col in df.columns: + x = pd.to_numeric(df[col], errors="coerce") + summary[f"{col}_mean"] = x.mean() + summary[f"{col}_std"] = x.std() + summary[f"{col}_min"] = x.min() + summary[f"{col}_max"] = x.max() + summary[f"{col}_nan_count"] = x.isna().sum() + else: + summary[f"{col}_missing"] = True + + # Test-set count if available. + if "is_test" in df.columns: + summary["n_test"] = int(df["is_test"].sum()) + summary["n_train_flagged"] = int((~df["is_test"].astype(bool)).sum()) + + return summary + + +def main() -> None: + rows = [] + + for label, run_dir in RUNS.items(): + print(f"\n=== {label} ===") + print(f"run_dir: {run_dir}") + + parquet_path = find_prediction_file(run_dir) + print(f"prediction file: {parquet_path}") + + row = summarize_prediction(label, parquet_path) + rows.append(row) + + summary_df = pd.DataFrame(rows) + summary_df.to_csv(OUT_CSV, index=False) + + print("\n=== Summary ===") + print(summary_df.to_string(index=False)) + + print(f"\nWrote summary CSV:\n{OUT_CSV}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/dump_one_raw_panel.py b/workflows/mfx101555026/scripts/archive/diagnostics/dump_one_raw_panel.py new file mode 100644 index 00000000..8facd396 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/dump_one_raw_panel.py @@ -0,0 +1,56 @@ +from pathlib import Path +import time +import numpy as np + +from dxtbx.model.experiment_list import ExperimentListFactory + + +BASE = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx") +IN_DIR = BASE / "outputs/r0269/018_rg070/out" +OUT_DIR = BASE / "raw_panel_cache_r0269_018_rg070" +OUT_DIR.mkdir(parents=True, exist_ok=True) + +IMAGE_NUM = 15833 + +expt_path = IN_DIR / f"idx-data_{IMAGE_NUM:05d}_integrated.expt" +out_path = OUT_DIR / f"raw_panels_image_{IMAGE_NUM:05d}.npz" + +print("expt:", expt_path) +print("out: ", out_path) + +if not expt_path.exists(): + raise FileNotFoundError(expt_path) + +t0 = time.perf_counter() +experiments = ExperimentListFactory.from_json_file( + str(expt_path), + check_format=True, +) +t_load = time.perf_counter() - t0 + +t0 = time.perf_counter() +raw = experiments[0].imageset.get_raw_data(0) +t_raw = time.perf_counter() - t0 + +panels = { + f"panel_{i:02d}": raw[i].as_numpy_array().astype(np.float32, copy=False) + for i in range(len(raw)) +} + +t0 = time.perf_counter() +np.savez_compressed(out_path, **panels) +t_save = time.perf_counter() - t0 + +try: + experiments[0].imageset.clear_cache() +except Exception: + pass + +size_mb = out_path.stat().st_size / 1024 / 1024 + +print(f"saved: {out_path}") +print(f"size: {size_mb:.1f} MB") +print(f"load .expt: {t_load:.2f} sec") +print(f"get_raw_data: {t_raw:.2f} sec") +print(f"save .npz: {t_save:.2f} sec") +print(f"total: {t_load + t_raw + t_save:.2f} sec") diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/dump_raw_panels_test.py b/workflows/mfx101555026/scripts/archive/diagnostics/dump_raw_panels_test.py new file mode 100644 index 00000000..0e1358a1 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/dump_raw_panels_test.py @@ -0,0 +1,154 @@ +from pathlib import Path +import argparse +import re +import time +import numpy as np + +from dxtbx.model.experiment_list import ExperimentListFactory + + +BASE = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx") +DEFAULT_IN_DIR = BASE / "outputs/r0269/018_rg070/out" +DEFAULT_OUT_DIR = BASE / "raw_panel_cache_r0269_018_rg070" + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Dump MFX raw detector panels from .expt files into cached .npz files." + ) + + parser.add_argument( + "--in-dir", + type=str, + default=str(DEFAULT_IN_DIR), + help="Input folder containing idx-data_XXXXX_integrated.expt files.", + ) + + parser.add_argument( + "--out-dir", + type=str, + default=str(DEFAULT_OUT_DIR), + help="Output folder for raw_panels_image_XXXXX.npz files.", + ) + + parser.add_argument( + "--start-file", + type=int, + default=0, + help="Starting index in the sorted .expt file list.", + ) + + parser.add_argument( + "--max-files", + type=int, + default=None, + help="Maximum number of .expt files to process.", + ) + + parser.add_argument( + "--overwrite", + action="store_true", + help="Overwrite existing cached .npz files.", + ) + + return parser.parse_args() + + +def image_num_from_path(path: Path) -> int: + m = re.search(r"idx-data_(\d+)_integrated\.expt$", path.name) + if not m: + raise ValueError(f"Cannot parse image number from {path}") + return int(m.group(1)) + + +def main(): + args = parse_args() + + in_dir = Path(args.in_dir) + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + expt_paths = sorted(in_dir.glob("idx-data_*_integrated.expt")) + + start = args.start_file + stop = None if args.max_files is None else start + args.max_files + expt_paths = expt_paths[start:stop] + + print(f"Input dir: {in_dir}") + print(f"Output dir: {out_dir}") + print(f"Start file: {args.start_file}") + print(f"Max files: {args.max_files}") + print(f"Selected {len(expt_paths)} expt files") + + total_load_expt = 0.0 + total_get_raw = 0.0 + total_save = 0.0 + total_wall_start = time.perf_counter() + + n_saved = 0 + n_skipped_existing = 0 + n_failed = 0 + + for local_i, expt_path in enumerate(expt_paths): + image_num = image_num_from_path(expt_path) + out_path = out_dir / f"raw_panels_image_{image_num:05d}.npz" + + if out_path.exists() and not args.overwrite: + n_skipped_existing += 1 + print(f"[{local_i+1}/{len(expt_paths)}] exists, skipping {out_path.name}") + continue + + try: + t0 = time.perf_counter() + experiments = ExperimentListFactory.from_json_file( + str(expt_path), + check_format=True, + ) + total_load_expt += time.perf_counter() - t0 + + t0 = time.perf_counter() + raw = experiments[0].imageset.get_raw_data(0) + total_get_raw += time.perf_counter() - t0 + + panels = { + f"panel_{i:02d}": raw[i].as_numpy_array().astype(np.float32, copy=False) + for i in range(len(raw)) + } + + t0 = time.perf_counter() + + # Uncompressed .npz: larger files, but faster save/load than savez_compressed. + np.savez(out_path, **panels) + + total_save += time.perf_counter() - t0 + + try: + experiments[0].imageset.clear_cache() + except Exception: + pass + + size_mb = out_path.stat().st_size / 1024 / 1024 + n_saved += 1 + print(f"[{local_i+1}/{len(expt_paths)}] saved {out_path.name}: {size_mb:.1f} MB") + + except Exception as e: + n_failed += 1 + print(f"[{local_i+1}/{len(expt_paths)}] FAILED {expt_path.name}: {type(e).__name__}: {e}") + + total_wall = time.perf_counter() - total_wall_start + + print("") + print("Timing summary") + print(f"load .expt: {total_load_expt:.2f} sec") + print(f"get_raw_data: {total_get_raw:.2f} sec") + print(f"save .npz: {total_save:.2f} sec") + print(f"accounted total: {total_load_expt + total_get_raw + total_save:.2f} sec") + print(f"wall time: {total_wall:.2f} sec") + print("") + print(f"saved: {n_saved}") + print(f"skipped existing: {n_skipped_existing}") + print(f"failed: {n_failed}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/inspect_check_format.py b/workflows/mfx101555026/scripts/archive/diagnostics/inspect_check_format.py new file mode 100644 index 00000000..23866934 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/inspect_check_format.py @@ -0,0 +1,66 @@ +from pathlib import Path +from dxtbx.model.experiment_list import ExperimentListFactory + +EXPT = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/outputs/r0269/018_rg070/out/idx-data_01418_integrated.expt") + +def show_obj(name, obj): + print(f"\n{name}") + print(" type:", type(obj)) + print(" repr:", repr(obj)) + + for attr in [ + "_reader", + "_format_class", + "format_class", + "_format_kwargs", + "kwargs", + "_filename", + "_paths", + "_indices", + "_image_range", + ]: + if hasattr(obj, attr): + try: + print(f" {attr}:", getattr(obj, attr)) + except Exception as e: + print(f" {attr}: ") + +def inspect(check_format): + print("\n" + "=" * 80) + print("check_format =", check_format) + + experiments = ExperimentListFactory.from_json_file( + str(EXPT), + check_format=check_format, + ) + + print("num experiments:", len(experiments)) + print("num imagesets:", len(experiments.imagesets())) + + exp = experiments[0] + imageset = experiments.imagesets()[0] + + show_obj("experiment", exp) + show_obj("imageset", imageset) + + try: + reader = imageset.reader() + show_obj("imageset.reader()", reader) + except Exception as e: + print("imageset.reader() failed:", type(e).__name__, e) + + try: + raw = imageset.get_raw_data(0) + print("\nget_raw_data(0): OK") + print(" raw type:", type(raw)) + print(" num panels:", len(raw)) + for i in range(min(3, len(raw))): + panel = raw[i] + arr = panel.as_numpy_array() + print(f" panel {i}: flex shape={panel.all()}, numpy shape={arr.shape}, dtype={arr.dtype}") + except Exception as e: + print("\nget_raw_data(0): FAILED") + print(" error:", type(e).__name__, e) + +for check_format in [True, False]: + inspect(check_format) diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/inspect_missing_mfx_files.py b/workflows/mfx101555026/scripts/archive/diagnostics/inspect_missing_mfx_files.py new file mode 100644 index 00000000..dcfa5def --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/inspect_missing_mfx_files.py @@ -0,0 +1,150 @@ +from pathlib import Path +import re +import csv +import traceback +import numpy as np + +from dials.array_family import flex +from dxtbx.model.experiment_list import ExperimentListFactory + + +ORIGINAL_DIR = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/outputs/r0269/018_rg070/out" +) + +METADATA = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_combined_all/metadata.npy" +) + +OUT_CSV = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/diagnostics_missing_0269_files.csv" +) + + +def image_num_from_name(path: Path): + match = re.search(r"idx-data_(\d+)_integrated\.refl$", path.name) + if not match: + return None + return int(match.group(1)) + + +def matching_expt(refl_path: Path): + return refl_path.with_suffix(".expt") + + +def main(): + meta = np.load(METADATA, allow_pickle=True).item() + represented = set(int(x) for x in np.unique(meta["image_num"])) + + refl_paths = sorted(ORIGINAL_DIR.glob("idx-data_*_integrated.refl")) + + rows = [] + + for i, refl_path in enumerate(refl_paths, start=1): + image_num = image_num_from_name(refl_path) + expt_path = matching_expt(refl_path) + + status = "UNKNOWN" + n_refl = None + n_experiments = None + raw_ok = False + error_type = "" + error_message = "" + + try: + if image_num is None: + status = "BAD_FILENAME" + raise RuntimeError("Could not parse image_num from filename") + + if image_num in represented: + status = "OK_IN_METADATA" + else: + status = "MISSING_FROM_METADATA" + + if not expt_path.exists(): + status = "MISSING_EXPT" + raise FileNotFoundError(f"Missing matching expt: {expt_path}") + + refl = flex.reflection_table.from_file(str(refl_path)) + n_refl = len(refl) + + if n_refl == 0: + status = "EMPTY_REFL" + raise RuntimeError("Reflection table has zero rows") + + experiments = ExperimentListFactory.from_json_file( + str(expt_path), + check_format=True, + ) + n_experiments = len(experiments) + + if n_experiments != 1: + status = "BAD_EXPERIMENT_COUNT" + raise RuntimeError(f"Expected 1 experiment, got {n_experiments}") + + imageset = experiments[0].imageset + + # This is the important test: can dxtbx/psana2 actually open raw data? + raw = imageset.get_raw_data(0) + raw_ok = True + + # If it was missing from metadata but all basic open tests pass, + # then the reason is probably later filtering/cropping/mask logic. + if image_num not in represented: + status = "MISSING_FROM_METADATA_BUT_OPENS" + + except Exception as e: + error_type = type(e).__name__ + error_message = str(e).replace("\n", " ")[:500] + + rows.append( + { + "image_num": image_num, + "refl_path": str(refl_path), + "expt_path": str(expt_path), + "status": status, + "in_metadata": image_num in represented if image_num is not None else False, + "n_refl": n_refl, + "n_experiments": n_experiments, + "raw_ok": raw_ok, + "error_type": error_type, + "error_message": error_message, + } + ) + + if i % 100 == 0: + print(f"Checked {i}/{len(refl_paths)} files...") + + with OUT_CSV.open("w", newline="") as f: + writer = csv.DictWriter( + f, + fieldnames=[ + "image_num", + "status", + "in_metadata", + "n_refl", + "n_experiments", + "raw_ok", + "error_type", + "error_message", + "refl_path", + "expt_path", + ], + ) + writer.writeheader() + writer.writerows(rows) + + print(f"Wrote: {OUT_CSV}") + + # Summary + counts = {} + for row in rows: + counts[row["status"]] = counts.get(row["status"], 0) + 1 + + print("\nSummary:") + for k, v in sorted(counts.items()): + print(f"{k}: {v}") + + +if __name__ == "__main__": + main() diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/inspect_rawcache_metadata.py b/workflows/mfx101555026/scripts/archive/diagnostics/inspect_rawcache_metadata.py new file mode 100644 index 00000000..3b7d2bfc --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/inspect_rawcache_metadata.py @@ -0,0 +1,92 @@ +from pathlib import Path +import numpy as np + +OUT = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_r0269_018_rg070_rawcache_all_timing") + +metadata_path = OUT / "metadata.npy" +counts_path = OUT / "counts.npy" +masks_path = OUT / "masks.npy" + +print("Dataset:", OUT) +print("metadata exists:", metadata_path.exists()) +print("counts exists: ", counts_path.exists()) +print("masks exists: ", masks_path.exists()) +print() + +raw = np.load(metadata_path, allow_pickle=True) +counts = np.load(counts_path, mmap_mode="r") +masks = np.load(masks_path, mmap_mode="r") + +print("counts shape:", counts.shape, counts.dtype) +print("masks shape: ", masks.shape, masks.dtype) +print("raw metadata type:", type(raw)) +print("raw metadata shape:", raw.shape) +print("raw metadata dtype:", raw.dtype) +print() + +if raw.shape == () and isinstance(raw.item(), dict): + metadata = raw.item() +else: + raise TypeError("Expected metadata.npy to be a saved dictionary object.") + +print("metadata format: dictionary") +print("metadata keys:") +for key in metadata.keys(): + print(" -", key) + +print() +first_key = next(iter(metadata)) +n_rows = len(metadata[first_key]) +print("rows:", n_rows) + +print() +print("field summaries:") +for key, value in metadata.items(): + arr = np.asarray(value) + + print() + print(f"{key}:") + print(" shape:", arr.shape) + print(" dtype:", arr.dtype) + + if arr.ndim > 0: + print(" first 5:", arr[:5]) + else: + print(" value:", arr) + + if arr.ndim > 0 and len(arr) == n_rows: + print(" length matches rows: yes") + else: + print(" length matches rows: no") + + if np.issubdtype(arr.dtype, np.number): + finite = np.isfinite(arr) + print(" finite:", finite.sum(), "/", arr.size) + print(" min:", np.nanmin(arr)) + print(" max:", np.nanmax(arr)) + print(" mean:", np.nanmean(arr)) + + if key in ["image_num", "source_refl", "source_expt", "raw_cache_file", "panel"]: + unique = np.unique(arr) + print(" unique count:", len(unique)) + print(" first unique:", unique[:5]) + print(" last unique: ", unique[-5:]) + +print() +print("important checks:") +for key in ["image_num", "source_refl", "source_expt", "raw_cache_file", "refl_ids", "reflection_id", "d", "wavelength"]: + if key in metadata: + arr = np.asarray(metadata[key]) + print(f" {key}: present, shape={arr.shape}, dtype={arr.dtype}") + else: + print(f" {key}: MISSING") + +print() +print("first 3 rows:") +keys = list(metadata.keys()) +for i in range(min(3, n_rows)): + print(f"row {i}:") + for key in keys: + arr = np.asarray(metadata[key]) + if arr.ndim > 0 and len(arr) == n_rows: + print(f" {key}: {arr[i]}") diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/plot_qi_meeting_25files.py b/workflows/mfx101555026/scripts/archive/diagnostics/plot_qi_meeting_25files.py new file mode 100644 index 00000000..708d97a3 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/plot_qi_meeting_25files.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +""" +Minimal meeting plots for Gamma, LogNormal, and FoldedNormal qi comparison. +""" + +from pathlib import Path + +import matplotlib.pyplot as plt +import pandas as pd + + +RUNS_ROOT = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/" + "mfx101555026_cctbx/runs" +) + +RUNS = { + "Gamma": RUNS_ROOT / "mfx_gamma_qi_25files", + "LogNormal": RUNS_ROOT / "mfx_lognormal_qi_25files_mc_kl", + "FoldedNormal": RUNS_ROOT / "mfx_foldednormal_qi_25files_mc_kl", +} + +OUT_DIR = RUNS_ROOT / "diagnostics_qi_meeting_25files" + + +def find_prediction_file(run_dir: Path) -> Path: + files = list(run_dir.rglob("test_preds_all.parquet")) + if files: + return sorted(files)[-1] + + files = list(run_dir.rglob("pred.parquet")) + if files: + return sorted(files)[-1] + + raise FileNotFoundError(f"No prediction parquet found under {run_dir}") + + +def load_all() -> dict[str, pd.DataFrame]: + dfs = {} + + for name, run_dir in RUNS.items(): + path = find_prediction_file(run_dir) + print(f"{name}: {path}") + dfs[name] = pd.read_parquet(path) + + return dfs + + +def plot_qi_mean_hist(dfs: dict[str, pd.DataFrame]) -> None: + plt.figure(figsize=(8, 5)) + + for name, df in dfs.items(): + x = pd.to_numeric(df["qi_mean"], errors="coerce").dropna() + x = x[x > 0] + plt.hist(x, bins=80, alpha=0.5, label=name) + + plt.xscale("log") + plt.xlabel("qi_mean") + plt.ylabel("count") + plt.title("Predicted intensity mean: Gamma vs LogNormal vs FoldedNormal") + plt.legend() + plt.tight_layout() + plt.savefig(OUT_DIR / "meeting_01_qi_mean_hist_logx.png", dpi=200) + plt.close() + + +def plot_qi_var_hist(dfs: dict[str, pd.DataFrame]) -> None: + plt.figure(figsize=(8, 5)) + + for name, df in dfs.items(): + x = pd.to_numeric(df["qi_var"], errors="coerce").dropna() + x = x[x > 0] + plt.hist(x, bins=80, alpha=0.5, label=name) + + plt.xscale("log") + plt.xlabel("qi_var") + plt.ylabel("count") + plt.title("Predicted intensity variance") + plt.legend() + plt.tight_layout() + plt.savefig(OUT_DIR / "meeting_02_qi_var_hist_logx.png", dpi=200) + plt.close() + + +def plot_qi_vs_integrated_intensity(dfs: dict[str, pd.DataFrame]) -> None: + plt.figure(figsize=(7, 6)) + + for name, df in dfs.items(): + x = pd.to_numeric(df["intensity.sum.value"], errors="coerce") + y = pd.to_numeric(df["qi_mean"], errors="coerce") + + mask = x.notna() & y.notna() & (y > 0) + plt.scatter(x[mask], y[mask], s=4, alpha=0.25, label=name) + + plt.yscale("log") + plt.xlabel("integrated intensity.sum.value") + plt.ylabel("qi_mean") + plt.title("qi_mean vs integrated intensity") + plt.legend() + plt.tight_layout() + plt.savefig(OUT_DIR / "meeting_03_qi_mean_vs_integrated_intensity.png", dpi=200) + plt.close() + + +def main() -> None: + OUT_DIR.mkdir(parents=True, exist_ok=True) + + dfs = load_all() + + plot_qi_mean_hist(dfs) + plot_qi_var_hist(dfs) + plot_qi_vs_integrated_intensity(dfs) + + print(f"\nWrote meeting plots to:\n{OUT_DIR}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/ray_mfx_combined_smoke.py b/workflows/mfx101555026/scripts/archive/diagnostics/ray_mfx_combined_smoke.py new file mode 100644 index 00000000..dc452e7c --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/ray_mfx_combined_smoke.py @@ -0,0 +1,128 @@ +from pathlib import Path +import time +import numpy as np + +import ray +from dials.array_family import flex +from dxtbx.model.experiment_list import ExperimentListFactory + + +BASE = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx") +COMBINED = BASE / "outputs/r0269/018_rg070/out/combined_all_018_rg070" + +EXPT = COMBINED / "combined_018_rg070.expt" +REFL = COMBINED / "combined_018_rg070.refl" + + +@ray.remote +class CombinedWorker: + def __init__(self, worker_id): + self.worker_id = worker_id + + t0 = time.perf_counter() + self.experiments = ExperimentListFactory.from_json_file( + str(EXPT), + check_format=True, + ) + self.reflections = flex.reflection_table.from_file(str(REFL)) + self.ids_np = self.reflections["id"].as_numpy_array() + self.load_time = time.perf_counter() - t0 + + def process_ids(self, experiment_ids): + rows = [] + + for experiment_id in experiment_ids: + t0 = time.perf_counter() + + try: + select_mask_np = self.ids_np == experiment_id + select_mask = flex.bool(select_mask_np.tolist()) + refl_subset = self.reflections.select(select_mask) + + experiment = self.experiments[experiment_id] + + t_raw0 = time.perf_counter() + raw = experiment.imageset.get_raw_data(0) + raw_time = time.perf_counter() - t_raw0 + + n_panels = len(raw) + + try: + experiment.imageset.clear_cache() + except Exception: + pass + + rows.append( + { + "worker_id": self.worker_id, + "experiment_id": experiment_id, + "status": "OK", + "n_refl": len(refl_subset), + "n_panels": n_panels, + "load_time": self.load_time, + "raw_time": raw_time, + "total_time": time.perf_counter() - t0, + "error": "", + } + ) + + except Exception as e: + rows.append( + { + "worker_id": self.worker_id, + "experiment_id": experiment_id, + "status": "ERROR", + "n_refl": -1, + "n_panels": -1, + "load_time": self.load_time, + "raw_time": -1, + "total_time": time.perf_counter() - t0, + "error": f"{type(e).__name__}: {e}", + } + ) + + return rows + + +def chunk_list(xs, n_chunks): + return [xs[i::n_chunks] for i in range(n_chunks)] + + +def main(): + num_workers = 4 + test_ids = list(range(20)) + + ray.init(local_mode=True, include_dashboard=False) + + print(f"Ray cluster resources: {ray.cluster_resources()}") + print(f"Testing experiment ids: {test_ids}") + + workers = [CombinedWorker.remote(i) for i in range(num_workers)] + chunks = chunk_list(test_ids, num_workers) + + t0 = time.perf_counter() + futures = [ + worker.process_ids.remote(chunk) + for worker, chunk in zip(workers, chunks) + ] + + results_nested = ray.get(futures) + total_time = time.perf_counter() - t0 + + results = [row for group in results_nested for row in group] + + print("") + print("Results:") + for row in sorted(results, key=lambda r: r["experiment_id"]): + print(row) + + print("") + print(f"Total Ray processing time: {total_time:.2f} sec") + print(f"OK: {sum(r['status'] == 'OK' for r in results)}") + print(f"ERROR: {sum(r['status'] == 'ERROR' for r in results)}") + + ray.shutdown() + + +if __name__ == "__main__": + main() diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/test_dxtbx_panel_access.py b/workflows/mfx101555026/scripts/archive/diagnostics/test_dxtbx_panel_access.py new file mode 100644 index 00000000..11654bac --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/test_dxtbx_panel_access.py @@ -0,0 +1,41 @@ +from pathlib import Path +import time + +from dxtbx.model.experiment_list import ExperimentListFactory + +EXPT = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/outputs/r0269/018_rg070/out/idx-data_01418_integrated.expt") + +def inspect(check_format): + print() + print("=" * 80) + print(f"check_format={check_format}") + + t0 = time.perf_counter() + experiments = ExperimentListFactory.from_json_file(str(EXPT), check_format=check_format) + print(f"load expt: {time.perf_counter() - t0:.2f} sec") + + exp = experiments[0] + imageset = exp.imageset + + print("imageset type:", type(imageset)) + print("detector panels:", len(exp.detector)) + + t0 = time.perf_counter() + raw = imageset.get_raw_data(0) + print(f"get_raw_data(0): {time.perf_counter() - t0:.2f} sec") + + print("raw type:", type(raw)) + print("number of raw panels:", len(raw)) + + for i in range(min(5, len(raw))): + panel = raw[i] + arr = panel.as_numpy_array() + print(f"panel {i}: flex shape={panel.all()}, numpy shape={arr.shape}, dtype={arr.dtype}") + + try: + imageset.clear_cache() + except Exception: + pass + +inspect(check_format=True) +inspect(check_format=False) diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/test_force_xtcjungfrau.py b/workflows/mfx101555026/scripts/archive/diagnostics/test_force_xtcjungfrau.py new file mode 100644 index 00000000..af8b416d --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/test_force_xtcjungfrau.py @@ -0,0 +1,24 @@ +from pathlib import Path + +from dxtbx.model.experiment_list import ExperimentListFactory +from dxtbx.format.FormatXTCJungfrau import FormatXTCJungfrau + +EXPT = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/outputs/r0269/018_rg070/out/idx-data_01418_integrated.expt") + +experiments = ExperimentListFactory.from_json_file( + str(EXPT), + check_format=False, + format_class=FormatXTCJungfrau, +) + +imageset = experiments.imagesets()[0] +reader = imageset.reader() + +print("reader type:", type(reader)) +print("format_class:", getattr(reader, "format_class", None)) +print("filename:", getattr(reader, "_filename", None)) + +raw = imageset.get_raw_data(0) +print("raw OK") +print("num panels:", len(raw)) +print("panel 0 shape:", raw[0].all()) diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/test_override_format_class.py b/workflows/mfx101555026/scripts/archive/diagnostics/test_override_format_class.py new file mode 100644 index 00000000..0289d5fa --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/test_override_format_class.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from dxtbx.model.experiment_list import ExperimentListFactory +from dxtbx.format.FormatXTCJungfrau import FormatXTCJungfrau + +EXPT = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/outputs/r0269/018_rg070/out/idx-data_01418_integrated.expt") + +experiments = ExperimentListFactory.from_json_file( + str(EXPT), + check_format=False, +) + +imageset = experiments.imagesets()[0] + +print("before override reader:", type(imageset.reader())) +print("before format_class:", getattr(imageset.reader(), "format_class", None)) + +imageset.format_class = FormatXTCJungfrau + +print("after override reader:", type(imageset.reader())) +print("after format_class:", getattr(imageset.reader(), "format_class", None)) + +raw = imageset.get_raw_data(0) + +print("raw OK") +print("num panels:", len(raw)) +print("panel 0 shape:", raw[0].all()) diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/test_override_reader_format_class.py b/workflows/mfx101555026/scripts/archive/diagnostics/test_override_reader_format_class.py new file mode 100644 index 00000000..f77a67c6 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/test_override_reader_format_class.py @@ -0,0 +1,28 @@ +from pathlib import Path + +from dxtbx.model.experiment_list import ExperimentListFactory +from dxtbx.format.FormatXTCJungfrau import FormatXTCJungfrau + +EXPT = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/outputs/r0269/018_rg070/out/idx-data_01418_integrated.expt") + +experiments = ExperimentListFactory.from_json_file( + str(EXPT), + check_format=False, +) + +imageset = experiments.imagesets()[0] +reader = imageset.reader() + +print("before reader:", type(reader)) +print("before format_class:", getattr(reader, "format_class", None)) + +reader.format_class = FormatXTCJungfrau + +print("after reader:", type(reader)) +print("after format_class:", getattr(reader, "format_class", None)) + +raw = imageset.get_raw_data(0) + +print("raw OK") +print("num panels:", len(raw)) +print("panel 0 shape:", raw[0].all()) diff --git a/workflows/mfx101555026/scripts/archive/diagnostics/time_fast_reader_loading_10.py b/workflows/mfx101555026/scripts/archive/diagnostics/time_fast_reader_loading_10.py new file mode 100644 index 00000000..e791efd2 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/diagnostics/time_fast_reader_loading_10.py @@ -0,0 +1,55 @@ +from pathlib import Path +import time + +from dxtbx.model.experiment_list import ExperimentListFactory +from dxtbx.format.FormatXTCJungfrau import FormatXTCJungfrau + +BASE = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx") +IN_DIR = BASE / "outputs/r0269/018_rg070/out" + +expt_paths = sorted(IN_DIR.glob("idx-data_*_integrated.expt"))[:10] + +print("num files:", len(expt_paths)) + +total_load = 0.0 +total_override = 0.0 +total_raw = 0.0 + +for i, expt_path in enumerate(expt_paths, start=1): + print() + print(f"[{i}/{len(expt_paths)}] {expt_path.name}") + + t0 = time.perf_counter() + experiments = ExperimentListFactory.from_json_file( + str(expt_path), + check_format=False, + ) + total_load += time.perf_counter() - t0 + + t0 = time.perf_counter() + imageset = experiments.imagesets()[0] + reader = imageset.reader() + reader.format_class = FormatXTCJungfrau + total_override += time.perf_counter() - t0 + + print(" reader:", type(reader)) + print(" format_class:", getattr(reader, "format_class", None)) + + t0 = time.perf_counter() + raw = imageset.get_raw_data(0) + total_raw += time.perf_counter() - t0 + + print(" raw panels:", len(raw)) + print(" panel 0 shape:", raw[0].all()) + + try: + imageset.clear_cache() + except Exception: + pass + +print() +print("Timing summary") +print(f"load expt check_format=False: {total_load:.2f} sec") +print(f"override reader format_class: {total_override:.2f} sec") +print(f"get_raw_data total: {total_raw:.2f} sec") +print(f"accounted total: {total_load + total_override + total_raw:.2f} sec") diff --git a/workflows/mfx101555026/scripts/archive/extraction/dump_raw_panels_array.sh b/workflows/mfx101555026/scripts/archive/extraction/dump_raw_panels_array.sh new file mode 100644 index 00000000..1c690edf --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/extraction/dump_raw_panels_array.sh @@ -0,0 +1,50 @@ +#!/bin/bash +#SBATCH --job-name=dump_raw_panels +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=64G +#SBATCH --time=0-6:00 +#SBATCH --array=0-15 +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/dump_raw_panels_%A_%a.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/dump_raw_panels_%A_%a.err + +set -eo pipefail + +export PS1="${PS1:-}" + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +ulimit -n 65535 + +export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH + +cd /sdf/home/t/thaoh/s3df_practice/integrator + +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx +IN=$BASE/outputs/r0269/018_rg070/out +CACHE=$BASE/raw_panel_cache_r0269_018_rg070 +LOGDIR=$BASE/logs + +mkdir -p "$CACHE" +mkdir -p "$LOGDIR" + +CHUNK_SIZE=283 +START_FILE=$((SLURM_ARRAY_TASK_ID * CHUNK_SIZE)) + +echo "SLURM_ARRAY_TASK_ID=$SLURM_ARRAY_TASK_ID" +echo "START_FILE=$START_FILE" +echo "CHUNK_SIZE=$CHUNK_SIZE" +echo "CACHE=$CACHE" +echo "ulimit -n=$(ulimit -n)" +date + +dials.python workflows/mfx101555026/scripts/diagnostics/dump_raw_panels_test.py \ + --in-dir "$IN" \ + --out-dir "$CACHE" \ + --start-file "$START_FILE" \ + --max-files "$CHUNK_SIZE" + +date +echo "done" diff --git a/workflows/mfx101555026/scripts/archive/extraction/make_mfx_018_rg070_1500_to_end_array.sh b/workflows/mfx101555026/scripts/archive/extraction/make_mfx_018_rg070_1500_to_end_array.sh new file mode 100644 index 00000000..046f2bb2 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/extraction/make_mfx_018_rg070_1500_to_end_array.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH --time=0-4:00 +#SBATCH --job-name=mfx018box2 +#SBATCH --array=0-7 +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx018_box2_%A_%a.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx018_box2_%A_%a.err + +set -eo pipefail + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +# Important: avoid "Too many open files" error from psana2/dxtbx +ulimit -n 8192 +echo "open file limit: $(ulimit -n)" + +export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH + +cd /sdf/home/t/thaoh/s3df_practice/integrator + +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx +DATA_DIR=$BASE/outputs/r0269/018_rg070/out +OUT_BASE=$BASE/mfx_shoebox_r0269_018_rg070_chunks_1500_to_end +MASK=/sdf/data/lcls/ds/mfx/mfx101555026/results/pam/hot_lines_combined5.mask + +mkdir -p "$OUT_BASE" +mkdir -p "$BASE/logs" + +# Files 0-1499 were already processed. +# These 8 jobs process files 1500-4068. +STARTS=(1500 1821 2142 2463 2784 3105 3426 3747) +COUNTS=(321 321 321 321 321 321 321 322) + +START=${STARTS[$SLURM_ARRAY_TASK_ID]} +COUNT=${COUNTS[$SLURM_ARRAY_TASK_ID]} + +OUT_DIR=$OUT_BASE/chunk_${SLURM_ARRAY_TASK_ID}_start_${START}_n_${COUNT} + +echo "SLURM_ARRAY_TASK_ID=$SLURM_ARRAY_TASK_ID" +echo "START=$START" +echo "COUNT=$COUNT" +echo "OUT_DIR=$OUT_DIR" +echo "DATA_DIR=$DATA_DIR" + +dials.python src/integrator/cli/make_shoeboxes.py \ + --mfx \ + --data-dir "$DATA_DIR" \ + --out-dir "$OUT_DIR" \ + --mfx-pattern "idx-data_*_integrated.refl" \ + --start-file "$START" \ + --max-files "$COUNT" \ + --detector-mask "$MASK" \ + --w 25 \ + --h 25 \ + --d 1 \ + --counts-dtype float32 \ + --write-chunk-size 10000 \ + --no-mask-overlap + +echo "Done: $OUT_DIR" diff --git a/workflows/mfx101555026/scripts/archive/extraction/make_mfx_018_rg070_fastreader_array.sh b/workflows/mfx101555026/scripts/archive/extraction/make_mfx_018_rg070_fastreader_array.sh new file mode 100644 index 00000000..20757315 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/extraction/make_mfx_018_rg070_fastreader_array.sh @@ -0,0 +1,56 @@ +#!/bin/bash +#SBATCH --job-name=mfx_fastreader_270 +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=4 +#SBATCH --mem=64G +#SBATCH --time=0-2:00 +#SBATCH --array=0-15 +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx_fastreader_r0270_019_%A_%a.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx_fastreader_r0270_019_%A_%a.err + +set -eo pipefail +export PS1="${PS1:-}" + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh +ulimit -n 65535 + +export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH +cd /sdf/home/t/thaoh/s3df_practice/integrator + +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx +IN=$BASE/outputs/r0270/019_rg070/out +OUT_BASE=$BASE/mfx_shoebox_r0270_019_rg070_fastreader_chunks +LOGDIR=$BASE/logs + +mkdir -p "$OUT_BASE" "$LOGDIR" + +CHUNK_SIZE=126 +START_FILE=$((SLURM_ARRAY_TASK_ID * CHUNK_SIZE)) +OUT=$OUT_BASE/chunk_${SLURM_ARRAY_TASK_ID} + +echo "SLURM_ARRAY_TASK_ID=$SLURM_ARRAY_TASK_ID" +echo "START_FILE=$START_FILE" +echo "CHUNK_SIZE=$CHUNK_SIZE" +echo "OUT=$OUT" +echo "ulimit -n=$(ulimit -n)" +date + +time dials.python src/integrator/cli/make_shoeboxes.py \ + --mfx \ + --data-dir "$IN" \ + --out-dir "$OUT" \ + --mfx-pattern "idx-data_*_integrated.refl" \ + --start-file "$START_FILE" \ + --max-files "$CHUNK_SIZE" \ + --detector-mask /sdf/data/lcls/ds/mfx/mfx101555026/results/pam/hot_lines_combined5.mask \ + --w 25 \ + --h 25 \ + --d 1 \ + --counts-dtype float32 \ + --write-chunk-size 50000 \ + --no-stats + +date +echo "done" \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/archive/extraction/make_mfx_125_chunks_ulimit.sh b/workflows/mfx101555026/scripts/archive/extraction/make_mfx_125_chunks_ulimit.sh new file mode 100644 index 00000000..5cbcd2ac --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/extraction/make_mfx_125_chunks_ulimit.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --job-name=mfx125u +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=64G +#SBATCH --time=0-6:00 +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx125u_%A_%a.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx125u_%A_%a.err + +ulimit -n 4096 +echo "ulimit -n: $(ulimit -n)" + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh +export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH +cd /sdf/home/t/thaoh/s3df_practice/integrator + +CHUNK=$(printf "%03d" $SLURM_ARRAY_TASK_ID) +IN_DIR=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/chunk_inputs/chunk_${CHUNK} +OUT_DIR=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_chunks/chunk_${CHUNK} + +dials.python src/integrator/cli/make_shoeboxes.py \ + --mfx \ + --data-dir "$IN_DIR" \ + --out-dir "$OUT_DIR" \ + --mfx-pattern "idx-data_*_integrated.refl" \ + --max-files 125 \ + --detector-mask /sdf/data/lcls/ds/mfx/mfx101555026/results/pam/hot_lines_combined5.mask \ + --w 25 \ + --h 25 \ + --d 1 \ + --counts-dtype float32 \ + --no-mask-overlap diff --git a/workflows/mfx101555026/scripts/archive/extraction/make_mfx_next1000_chunks_ulimit.sh b/workflows/mfx101555026/scripts/archive/extraction/make_mfx_next1000_chunks_ulimit.sh new file mode 100644 index 00000000..6e7a85c8 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/extraction/make_mfx_next1000_chunks_ulimit.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --job-name=mfxnext1k +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=1 +#SBATCH --mem=64G +#SBATCH --time=0-6:00 +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfxnext1k_%A_%a.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfxnext1k_%A_%a.err + +ulimit -n 4096 +echo "ulimit -n: $(ulimit -n)" + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh +export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH +cd /sdf/home/t/thaoh/s3df_practice/integrator + +CHUNK=$(printf "%03d" $SLURM_ARRAY_TASK_ID) +IN_DIR=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/chunk_inputs/chunk_${CHUNK} +OUT_DIR=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/mfx_shoebox_chunks_next1000/chunk_${CHUNK} + +dials.python src/integrator/cli/make_shoeboxes.py \ + --mfx \ + --data-dir "$IN_DIR" \ + --out-dir "$OUT_DIR" \ + --mfx-pattern "idx-data_*_integrated.refl" \ + --max-files 125 \ + --detector-mask /sdf/data/lcls/ds/mfx/mfx101555026/results/pam/hot_lines_combined5.mask \ + --w 25 \ + --h 25 \ + --d 1 \ + --counts-dtype float32 \ + --no-mask-overlap diff --git a/workflows/mfx101555026/scripts/archive/extraction/make_shoeboxes_5000_024_rg101_16array.sh b/workflows/mfx101555026/scripts/archive/extraction/make_shoeboxes_5000_024_rg101_16array.sh new file mode 100644 index 00000000..52e038dc --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/extraction/make_shoeboxes_5000_024_rg101_16array.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH --time=0-8:00 +#SBATCH --job-name=mfx_5000_box +#SBATCH --array=0-15 +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx_5000_box_%A_%a.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx_5000_box_%A_%a.err + +set -eo pipefail + +############################################################################### +# Environment +############################################################################### + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +# Avoid "Too many open files" errors from psana2/dxtbx. +ulimit -n 65535 +echo "Open file limit: $(ulimit -n)" + +export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:${PYTHONPATH:-} + +cd /sdf/home/t/thaoh/s3df_practice/integrator + +############################################################################### +# Paths +############################################################################### + +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx + +DATA_DIR="$BASE/all_runs_269_289_no275_024_rg101_integrated" +OUT_BASE="$BASE/mfx_shoebox_5000_269_289_no275_024_rg101_chunks" +MASK="$BASE/masks/r0269_pam_stddev_border2_combined.mask" + +mkdir -p "$OUT_BASE" +mkdir -p "$BASE/logs" + +############################################################################### +# Split the first 5,000 source files across 16 Slurm array tasks +############################################################################### + +TOTAL_FILES=5000 +N_TASKS=16 +TASK_ID=${SLURM_ARRAY_TASK_ID} + +# Ceiling division: +# 5000 files / 16 tasks gives about 313 files per task. +FILES_PER_TASK=$(( (TOTAL_FILES + N_TASKS - 1) / N_TASKS )) + +START=$(( TASK_ID * FILES_PER_TASK )) +END=$(( START + FILES_PER_TASK )) + +if [ "$END" -gt "$TOTAL_FILES" ]; then + END=$TOTAL_FILES +fi + +COUNT=$(( END - START )) + +OUT_DIR="$OUT_BASE/chunk_${TASK_ID}_start_${START}_n_${COUNT}" + +############################################################################### +# Print task information +############################################################################### + +echo "============================================================" +echo "Job ID: $SLURM_JOB_ID" +echo "Array job ID: $SLURM_ARRAY_JOB_ID" +echo "Array task ID: $TASK_ID" +echo "Host: $(hostname)" +echo "Time: $(date)" +echo "TOTAL_FILES=$TOTAL_FILES" +echo "N_TASKS=$N_TASKS" +echo "FILES_PER_TASK=$FILES_PER_TASK" +echo "START=$START" +echo "END=$END" +echo "COUNT=$COUNT" +echo "DATA_DIR=$DATA_DIR" +echo "OUT_DIR=$OUT_DIR" +echo "MASK=$MASK" +echo "============================================================" + +if [ "$COUNT" -le 0 ]; then + echo "No files assigned to this task. Exiting." + exit 0 +fi + +if [ ! -d "$DATA_DIR" ]; then + echo "ERROR: data directory does not exist:" + echo "$DATA_DIR" + exit 1 +fi + +if [ ! -f "$MASK" ]; then + echo "ERROR: detector mask does not exist:" + echo "$MASK" + exit 1 +fi + +############################################################################### +# Create shoeboxes for this task's file range +############################################################################### + +dials.python src/integrator/cli/make_shoeboxes.py \ + --mfx \ + --data-dir "$DATA_DIR" \ + --out-dir "$OUT_DIR" \ + --mfx-pattern "*_integrated.refl" \ + --start-file "$START" \ + --max-files "$COUNT" \ + --detector-mask "$MASK" \ + --w 25 \ + --h 25 \ + --d 1 \ + --counts-dtype float32 \ + --write-chunk-size 10000 \ + --no-mask-overlap + +echo "Finished task $TASK_ID at $(date)" +echo "Output: $OUT_DIR" \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/archive/extraction/merge_fastreader_chunks.py b/workflows/mfx101555026/scripts/archive/extraction/merge_fastreader_chunks.py new file mode 100644 index 00000000..77b8164f --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/extraction/merge_fastreader_chunks.py @@ -0,0 +1,150 @@ +from pathlib import Path +import numpy as np +import yaml +from numpy.lib.format import open_memmap + +BASE = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx") + +IN_BASE = BASE / "mfx_shoebox_r0269_018_rg070_fastreader_chunks" +OUT = BASE / "mfx_shoebox_r0269_018_rg070_fastreader_merged_all" +OUT.mkdir(parents=True, exist_ok=True) + +DATASET_DIRS = sorted(IN_BASE.glob("chunk_*")) + +good_dirs = [] +for d in DATASET_DIRS: + needed = [d / "counts.npy", d / "masks.npy", d / "metadata.npy"] + if all(p.exists() for p in needed): + good_dirs.append(d) + else: + print(f"SKIP missing final files: {d}") + +if not good_dirs: + raise SystemExit(f"No completed dataset folders found under {IN_BASE}") + +print("Datasets to merge:") +for d in good_dirs: + print(" ", d) + +total_rows = 0 +n_pixels = None +counts_dtype = None +all_meta = [] + +for d in good_dirs: + counts = np.load(d / "counts.npy", mmap_mode="r") + masks = np.load(d / "masks.npy", mmap_mode="r") + meta = np.load(d / "metadata.npy", allow_pickle=True).item() + + print(d.name, counts.shape) + + if masks.shape != counts.shape: + raise ValueError(f"mask/count shape mismatch in {d}") + + if n_pixels is None: + n_pixels = counts.shape[1] + counts_dtype = counts.dtype + elif counts.shape[1] != n_pixels: + raise ValueError(f"pixel count mismatch in {d}") + + total_rows += counts.shape[0] + all_meta.append(meta) + +print("total rows:", total_rows) +print("pixels per shoebox:", n_pixels) + +counts_out = open_memmap( + OUT / "counts.npy", + mode="w+", + dtype=counts_dtype, + shape=(total_rows, n_pixels), +) +masks_out = open_memmap( + OUT / "masks.npy", + mode="w+", + dtype=np.bool_, + shape=(total_rows, n_pixels), +) + +row0 = 0 +for d in good_dirs: + counts = np.load(d / "counts.npy", mmap_mode="r") + masks = np.load(d / "masks.npy", mmap_mode="r") + row1 = row0 + counts.shape[0] + + counts_out[row0:row1] = counts + masks_out[row0:row1] = masks + + row0 = row1 + +counts_out.flush() +masks_out.flush() +del counts_out, masks_out + +common_keys = set(all_meta[0].keys()) +for meta in all_meta[1:]: + common_keys &= set(meta.keys()) + +common_keys = sorted(common_keys) +print("common metadata keys:", common_keys) + +merged_meta = {} +for key in common_keys: + merged_meta[key] = np.concatenate([meta[key] for meta in all_meta], axis=0) + +merged_meta["refl_ids"] = np.arange(total_rows, dtype=np.int64) + +np.save(OUT / "metadata.npy", merged_meta) + +counts = np.load(OUT / "counts.npy", mmap_mode="r") +masks = np.load(OUT / "masks.npy", mmap_mode="r") + +sum_c = 0.0 +sumsq_c = 0.0 +nel = 0 +chunk = 10000 + +for i in range(0, total_rows, chunk): + c = counts[i:i+chunk].astype(np.float64) + m = masks[i:i+chunk] + valid = c[m] + sum_c += valid.sum() + sumsq_c += (valid * valid).sum() + nel += valid.size + +mean_c = sum_c / nel +var_c = sumsq_c / nel - mean_c * mean_c +stats = {"raw": [float(mean_c), float(var_c)]} + +concentration = np.zeros(total_rows, dtype=np.float32) +for i in range(0, total_rows, chunk): + c = counts[i:i+chunk].astype(np.float32) + m = masks[i:i+chunk] + concentration[i:i+chunk] = (c * m).sum(axis=1) + +np.save(OUT / "concentration.npy", concentration) + +dataset_yaml = { + "geometry": {"d": 1, "h": 25, "w": 25}, + "n_reflections": int(total_rows), + "polychromatic": False, + "anscombe": False, + "files": { + "counts": "counts.npy", + "masks": "masks.npy", + "reference": "metadata.npy", + }, + "crystal": None, + "stats": stats, + "refl_file": None, +} + +with open(OUT / "dataset.yaml", "w") as f: + yaml.safe_dump(dataset_yaml, f, sort_keys=False) + +print("WROTE:", OUT) +print(" counts.npy") +print(" masks.npy") +print(" metadata.npy") +print(" concentration.npy") +print(" dataset.yaml") diff --git a/workflows/mfx101555026/scripts/archive/extraction/merge_mfx_shoebox_datasets.py b/workflows/mfx101555026/scripts/archive/extraction/merge_mfx_shoebox_datasets.py new file mode 100644 index 00000000..1ccae66f --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/extraction/merge_mfx_shoebox_datasets.py @@ -0,0 +1,201 @@ +from pathlib import Path + +import numpy as np +import yaml +from numpy.lib.format import open_memmap + + +BASE = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx") + +#EDIT THIS: +CHUNKS_DIR = BASE / "mfx_shoebox_allruns_269_289_no275_024_rg101_chunks" +OUT = BASE / "mfx_shoebox_allruns_269_289_no275_024_rg101" + +OUT.mkdir(parents=True, exist_ok=True) + +DATASET_DIRS = sorted(CHUNKS_DIR.glob("chunk_*")) + +# Keep only completed dataset folders. +good_dirs = [] +for d in DATASET_DIRS: + needed = [d / "counts.npy", d / "masks.npy", d / "metadata.npy"] + if all(p.exists() for p in needed): + good_dirs.append(d) + else: + print(f"SKIP missing final files: {d}") + +if not good_dirs: + raise SystemExit("No completed dataset folders found.") + +print("Datasets to merge:") +for d in good_dirs: + print(" ", d) + +# Inspect shapes. +total_rows = 0 +n_pixels = None +counts_dtype = None +all_meta = [] + +for d in good_dirs: + counts = np.load(d / "counts.npy", mmap_mode="r") + masks = np.load(d / "masks.npy", mmap_mode="r") + meta = np.load(d / "metadata.npy", allow_pickle=True).item() + + print(d.name, counts.shape) + + if masks.shape != counts.shape: + raise ValueError(f"mask/count shape mismatch in {d}") + + if n_pixels is None: + n_pixels = counts.shape[1] + counts_dtype = counts.dtype + elif counts.shape[1] != n_pixels: + raise ValueError(f"pixel count mismatch in {d}") + + total_rows += counts.shape[0] + all_meta.append(meta) + +print("total rows:", total_rows) +print("pixels per shoebox:", n_pixels) +print("counts dtype:", counts_dtype) + +# Write combined counts/masks. +counts_out = open_memmap( + OUT / "counts.npy", + mode="w+", + dtype=counts_dtype, + shape=(total_rows, n_pixels), +) + +masks_out = open_memmap( + OUT / "masks.npy", + mode="w+", + dtype=np.bool_, + shape=(total_rows, n_pixels), +) + +row0 = 0 +for d in good_dirs: + counts = np.load(d / "counts.npy", mmap_mode="r") + masks = np.load(d / "masks.npy", mmap_mode="r") + + row1 = row0 + counts.shape[0] + + print(f"copying {d.name}: rows {row0}:{row1}") + + counts_out[row0:row1] = counts + masks_out[row0:row1] = masks + + row0 = row1 + +counts_out.flush() +masks_out.flush() +del counts_out, masks_out + +# Merge metadata using keys common to all datasets. +common_keys = set(all_meta[0].keys()) +for meta in all_meta[1:]: + common_keys &= set(meta.keys()) + +common_keys = sorted(common_keys) +print("common metadata keys:", common_keys) + +merged_meta = {} +for key in common_keys: + merged_meta[key] = np.concatenate([meta[key] for meta in all_meta], axis=0) + +# Reassign refl_ids so they are unique and sequential in the combined dataset. +merged_meta["refl_ids"] = np.arange(total_rows, dtype=np.int64) + +# IMPORTANT: recompute global image_id/n_images after merging all 16 chunks. +# Each Slurm task may have local image_id values, so do not keep those. +if "image_index" in merged_meta: + image_key = "image_index" +elif "image_num" in merged_meta: + image_key = "image_num" +else: + raise KeyError("metadata has neither image_index nor image_num") + +image_num = np.asarray(merged_meta[image_key]).astype(np.int64) + +unique_images = np.array(sorted(set(image_num.tolist())), dtype=np.int64) +image_to_id = {img: i for i, img in enumerate(unique_images)} + +merged_meta["image_num"] = image_num +merged_meta["image_id"] = np.array( + [image_to_id[x] for x in image_num], + dtype=np.int64, +) +merged_meta["n_images"] = np.full( + total_rows, + len(unique_images), + dtype=np.int64, +) + +print("global n_images:", len(unique_images)) +print("image_id min/max:", merged_meta["image_id"].min(), merged_meta["image_id"].max()) + +np.save(OUT / "metadata.npy", merged_meta) + +# Compute raw mean/variance from valid pixels only. +counts = np.load(OUT / "counts.npy", mmap_mode="r") +masks = np.load(OUT / "masks.npy", mmap_mode="r") + +sum_c = 0.0 +sumsq_c = 0.0 +nel = 0 +chunk = 10000 + +for i in range(0, total_rows, chunk): + c = counts[i:i + chunk].astype(np.float64) + m = masks[i:i + chunk].astype(bool) + + valid = c[m] + sum_c += valid.sum() + sumsq_c += (valid * valid).sum() + nel += valid.size + +if nel == 0: + raise RuntimeError("No valid pixels found while computing stats.") + +mean_c = sum_c / nel +var_c = sumsq_c / nel - mean_c * mean_c +stats = {"raw": [float(mean_c), float(var_c)]} + +print("raw stats:", stats) + +# concentration.npy: simple per-shoebox total valid counts. +concentration = np.zeros(total_rows, dtype=np.float32) + +for i in range(0, total_rows, chunk): + c = counts[i:i + chunk].astype(np.float32) + m = masks[i:i + chunk] + concentration[i:i + chunk] = (c * m).sum(axis=1) + +np.save(OUT / "concentration.npy", concentration) + +dataset_yaml = { + "geometry": {"d": 1, "h": 25, "w": 25}, + "n_reflections": int(total_rows), + "polychromatic": False, + "anscombe": False, + "files": { + "counts": "counts.npy", + "masks": "masks.npy", + "reference": "metadata.npy", + }, + "crystal": None, + "stats": stats, + "refl_file": None, +} + +with open(OUT / "dataset.yaml", "w") as f: + yaml.safe_dump(dataset_yaml, f, sort_keys=False) + +print("WROTE:", OUT) +print(" counts.npy") +print(" masks.npy") +print(" metadata.npy") +print(" concentration.npy") +print(" dataset.yaml") \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/archive/training/train_predict_gamma_full_sequential.sh b/workflows/mfx101555026/scripts/archive/training/train_predict_gamma_full_sequential.sh new file mode 100755 index 00000000..4bb1d902 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/training/train_predict_gamma_full_sequential.sh @@ -0,0 +1,167 @@ +#!/bin/bash +#SBATCH --partition=ampere +#SBATCH --account=lcls:prjlumine22 +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH -N 1 +#SBATCH -c 10 +#SBATCH -G 1 +#SBATCH --mem=96G +#SBATCH -t 0-12:00 +#SBATCH -o /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/gamma_full_sequential_%j.out +#SBATCH -e /sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/gamma_full_sequential_%j.err + +set -eo pipefail +export PS1="${PS1:-}" + +################################################################################ +# Paths +################################################################################ + +REPO=/sdf/home/t/thaoh/s3df_practice/integrator +CONFIG_DIR=$REPO/workflows/mfx101555026/configs + +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx +RUNS=$BASE/runs +LOGS=$BASE/logs +DATASET=$BASE/mfx_shoebox_r0269_018_rg070_fastreader_merged_all + +mkdir -p "$RUNS" +mkdir -p "$LOGS" + +################################################################################ +# Environment setup +################################################################################ + +cd "$REPO" + +export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root +eval "$(/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/hewl_1118_tutorial/micromamba/bin/micromamba shell hook --shell bash)" +micromamba activate integrator-cuda-dev + +export PYTHONNOUSERSITE=1 +export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH +export PYTHONPATH=$REPO/src + +echo "========================================" +echo "Gamma full sequential training/prediction" +echo "Time: $(date)" +echo "Node: $HOSTNAME" +echo "Repo: $REPO" +echo "Runs folder: $RUNS" +echo "Dataset: $DATASET" +echo "========================================" + +echo "Python: $(which python)" +echo "integrator.train: $(which integrator.train)" +echo "integrator.predict: $(which integrator.predict)" + +echo "CUDA check:" +python - <<'PY' +import torch +print("torch.cuda.is_available():", torch.cuda.is_available()) +if torch.cuda.is_available(): + print("GPU:", torch.cuda.get_device_name(0)) +PY + +echo "nvidia-smi:" +nvidia-smi + +################################################################################ +# Configs to run sequentially +################################################################################ + +CONFIGS=( + #"$CONFIG_DIR/mfx_asinh_normal_gamma.yaml" + #"$CONFIG_DIR/mfx_softplus_normal_gamma.yaml" + "$CONFIG_DIR/mfx_sqrtsqureplus_normal_gamma.yaml" +) + +NAMES=( + #"asinh_normal_gamma" + #"softplus_normal_gamma" + "sqrt_squareplus_normal_gamma" +) + +################################################################################ +# Train + predict each config +################################################################################ + +for i in "${!CONFIGS[@]}"; do + CONFIG="${CONFIGS[$i]}" + NAME="${NAMES[$i]}" + + echo "" + echo "################################################################################" + echo "# Starting $NAME" + echo "# Config: $CONFIG" + echo "# Time: $(date)" + echo "################################################################################" + + if [[ ! -f "$CONFIG" ]]; then + echo "ERROR: Config file does not exist:" + echo "$CONFIG" + exit 1 + fi + + echo "Checking dataset path mentions in config:" + grep -n "mfx_shoebox" "$CONFIG" || true + grep -n "metadata" "$CONFIG" || true + + BEFORE_FILE="$LOGS/${NAME}_before_runs_${SLURM_JOB_ID}.txt" + AFTER_FILE="$LOGS/${NAME}_after_runs_${SLURM_JOB_ID}.txt" + RUN_DIR_FILE="$LOGS/${NAME}_run_dir_${SLURM_JOB_ID}.txt" + + echo "Recording existing run folders before training..." + find "$RUNS" -maxdepth 1 -type d -name "run_*" | sort > "$BEFORE_FILE" + + echo "Training started for $NAME: $(date)" + integrator.train \ + --config "$CONFIG" \ + --log-dir "$RUNS" + + echo "Training finished for $NAME: $(date)" + + echo "Recording run folders after training..." + find "$RUNS" -maxdepth 1 -type d -name "run_*" | sort > "$AFTER_FILE" + + echo "Finding new run directory..." + RUN_DIR=$(comm -13 "$BEFORE_FILE" "$AFTER_FILE" | head -n 1) + + if [[ -z "${RUN_DIR:-}" ]]; then + echo "WARNING: No new run found by before/after comparison." + echo "Falling back to newest run_* under $RUNS" + RUN_DIR=$(find "$RUNS" -maxdepth 1 -type d -name "run_*" -printf "%T@ %p\n" | sort -nr | head -n 1 | awk '{print $2}') + fi + + if [[ -z "${RUN_DIR:-}" ]]; then + echo "ERROR: could not find run directory under $RUNS" + exit 1 + fi + + echo "Using run directory for $NAME:" + echo "$RUN_DIR" + + echo "$RUN_DIR" > "$RUN_DIR_FILE" + + echo "Prediction started for $NAME: $(date)" + integrator.predict \ + --run-dir "$RUN_DIR" + + echo "Prediction finished for $NAME: $(date)" + + echo "Prediction folders for $NAME:" + find "$RUN_DIR/predictions" -maxdepth 2 -type d || true + + echo "Finished $NAME" + echo "Run directory saved to:" + echo "$RUN_DIR_FILE" +done + +echo "" +echo "========================================" +echo "All gamma full jobs finished." +echo "Finished time: $(date)" +echo "Run directory pointer files:" +ls -lh "$LOGS"/*_run_dir_${SLURM_JOB_ID}.txt || true +echo "========================================" diff --git a/workflows/mfx101555026/scripts/archive/writeback/code_changes_review.txt b/workflows/mfx101555026/scripts/archive/writeback/code_changes_review.txt new file mode 100644 index 00000000..fa96700e --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/writeback/code_changes_review.txt @@ -0,0 +1,267 @@ +================================================================================ +CODE CHANGES REVIEW +MFX Write-Back Performance Fixes +================================================================================ +Date : 2026-08-06 +Author : opencode +Branch : mfx-shoebox-qi-posteriors + +Two files changed. No new files. No interface changes. +Both changes are drop-in replacements — all callers unchanged. + +================================================================================ +FILE 1 — src/integrator/io/pred_io.py +================================================================================ + +LOCATION : get_pred_files(), lines 37-48 (parquet branch) + +WHAT CHANGED: + 4 separate polars lazy-scan collect() calls replaced with 1. + +BEFORE: + lf = pl.scan_parquet(pred_files) + refl_ids = lf.select("refl_ids").collect().to_numpy().ravel() # scan 1 + qi_mean = lf.select("qi_mean").collect().to_numpy().ravel() # scan 2 + qi_var = lf.select("qi_var").collect().to_numpy().ravel() # scan 3 + qbg_mean = lf.select("qbg_mean").collect().to_numpy().ravel() # scan 4 + + data = { + "refl_ids": refl_ids, + "qi_mean": qi_mean, + "qi_var": qi_var, + "qbg_mean": qbg_mean, + } + +AFTER: + lf = pl.scan_parquet(pred_files) + df = lf.select(["refl_ids", "qi_mean", "qi_var", "qbg_mean"]).collect() + refl_ids = df["refl_ids"].to_numpy() + qi_mean = df["qi_mean"].to_numpy() + qi_var = df["qi_var"].to_numpy() + qbg_mean = df["qbg_mean"].to_numpy() + + data = { + "refl_ids": refl_ids, + "qi_mean": qi_mean, + "qi_var": qi_var, + "qbg_mean": qbg_mean, + } + +WHY: + Each .collect() re-opens and re-reads all 5758 parquet files for one column. + 4 calls = 4 full scans = 23,032 file operations. + 1 call = 1 scan = 5,758 file operations. + +CORRECTNESS: + Identical output. Column dtypes and values unchanged. + ravel() removed — polars .to_numpy() already returns a 1-D array for + single-column selections; no change in shape. + +SPEEDUP: ~4x on parquet loading stage (~90s -> ~25s). + +================================================================================ +FILE 2 — src/integrator/io/refl_io.py +================================================================================ + +LOCATION : write_mfx_refl_with_predictions(), lines 183-252 + +WHAT CHANGED: + Replaced two 117M-iteration Python loops (building source_refl / + source_expt string object arrays) and one O(N x n_files) scan loop + with an integer-sort groupby using numpy argsort and contiguous + group boundaries. + + The change splits into two code paths inside the same function: + + FAST PATH — image_num or image_index is in metadata (your dataset) + SLOW PATH — only source_refl / source_expt in metadata (fallback) + +-------------------------------------------------------------------- +BEFORE (all cases, lines 183-252): +-------------------------------------------------------------------- + + # Build source_refl — Python loop over 117 M rows + source_refl = np.array( + [_mfx_source_name(metadata, "source_refl", row, image_numbers[row], ".refl") + for row in metadata_rows], + dtype=object, + ) + + # Build source_expt — Python loop over 117 M rows + source_expt = np.array( + [_mfx_source_name(metadata, "source_expt", row, image_numbers[row], ".expt") + for row in metadata_rows], + dtype=object, + ) + + row_in_source = reflection_id[metadata_rows] + + # Write loop — O(N x n_files): scans all 117 M rows for each of 124,975 files + for refl_name in sorted(set(source_refl.tolist())): + group = np.nonzero(source_refl == refl_name)[0] # 117 M comparison each time + ... + expt_names = sorted(set(source_expt[group].tolist())) + if len(expt_names) != 1: + raise ValueError(...) + _copy_or_link_expt(src_expt, out_dir / expt_names[0], copy_expt=copy_expt) + +-------------------------------------------------------------------- +AFTER — FAST PATH (image_num available): +-------------------------------------------------------------------- + + row_in_source = reflection_id[metadata_rows] + + has_image_num = "image_num" in metadata or "image_index" in metadata + + if has_image_num: + # Subset of image numbers for predictions only (int64) + image_nums_subset = image_numbers[metadata_rows] + + # Single integer sort — O(N log N), SIMD-optimized + order = np.argsort(image_nums_subset, kind="stable") + sorted_images = image_nums_subset[order] + + # Contiguous group boundaries — no np.split, no repeated full-array scan + boundaries = np.flatnonzero(sorted_images[1:] != sorted_images[:-1]) + 1 + starts = np.concatenate(([0], boundaries)) + ends = np.concatenate((boundaries, [len(order)])) + + for start, end in zip(starts, ends): + group = order[start:end] # O(1) slice + image_num = int(sorted_images[start]) + refl_name = f"idx-data_{image_num:05d}_integrated.refl" + expt_name = f"idx-data_{image_num:05d}_integrated.expt" + ... + _copy_or_link_expt(src_expt, out_dir / expt_name, copy_expt=copy_expt) + + KEY DIFFERENCES vs old code: + - No source_refl / source_expt string arrays built at all + - Filename reconstructed deterministically from integer image_num + - group = O(1) slice instead of O(N) np.nonzero scan + - expt_name derived directly — no set() dedup needed + +-------------------------------------------------------------------- +AFTER — SLOW PATH (source_refl only, no image_num): +-------------------------------------------------------------------- + + else: + # Pull full string arrays once; fancy-index in C (no Python loop) + source_refl_all = _as_numpy(metadata["source_refl"]) + source_expt_all = _as_numpy(metadata["source_expt"]) + source_refl_subset = source_refl_all[metadata_rows] # vectorized + source_expt_subset = source_expt_all[metadata_rows] # vectorized + + # np.unique: encode unique strings to sorted int IDs — one C pass + unique_refl, encoded = np.unique(source_refl_subset, return_inverse=True) + + order = np.argsort(encoded, kind="stable") + sorted_enc = encoded[order] + + boundaries = np.flatnonzero(sorted_enc[1:] != sorted_enc[:-1]) + 1 + starts = np.concatenate(([0], boundaries)) + ends = np.concatenate((boundaries, [len(order)])) + + # One expt per group — first element of each sorted group, vectorized + unique_expt = source_expt_subset[order[starts]] + + for i, (start, end) in enumerate(zip(starts, ends)): + group = order[start:end] + refl_name = _as_text(unique_refl[i]) + expt_name = _as_text(unique_expt[i]) + ... + + KEY DIFFERENCES vs old code: + - source_refl/expt_subset built with vectorized numpy fancy index (not 117M Python loop) + - np.unique encodes strings to int IDs in one C pass (not 117M Python loop) + - group = O(1) slice; no O(N) np.nonzero scan per file + - unique_expt from vectorized indexing; no per-group set() dedup + +================================================================================ +CORRECTNESS NOTES +================================================================================ + + 1. FAST PATH naming assumption: + Filenames are reconstructed as idx-data_{image_num:05d}_integrated.refl/expt. + This matches the MFX/cctbx naming convention used by make_shoeboxes and + confirmed by the old write_mfx_predictions_to_refl.py script. + If source_refl in metadata differs from this pattern for any image, + the fast path will produce the wrong filename. Verify with: + python - <<'PY' + import numpy as np + meta = np.load("metadata.npy", allow_pickle=True).item() + sr = meta["source_refl"] + nums = meta["image_num"] + expected = [f"idx-data_{int(n):05d}_integrated.refl" for n in nums[:5]] + print(list(sr[:5])) + print(expected) + PY + + 2. SLOW PATH expt lookup: + unique_expt = source_expt_subset[order[starts]] takes the first element + of each sorted group as the expt name. This is correct when each source + .refl has exactly one matching .expt (which the old code enforced via + len(expt_names) != 1 check). That check is implicitly preserved: if two + different expt names mapped to the same refl name, they would appear as + different source_refl values after np.unique (since refl and expt names + are generated together), so they would form separate groups. + + 3. row_in_source: + Moved above the if/else block (line 185). Previously it was between the + two Python loops (line 209). Semantics identical — reflection_id[metadata_rows] + produces the same array regardless of position. + + 4. _set_column closure: + Still defined inside the per-file loop. Captures rt, n_rows, rows from + the enclosing scope at call time. Semantics unchanged from original. + + 5. Unchanged behavior: + - variance_floor clipping (qi_var = np.clip(...)) — unchanged, happens before + - refl_ids matching via searchsorted — unchanged + - reflection_id range check — unchanged + - _copy_or_link_expt skip-if-exists — unchanged + - return dict (out_dir, n_prediction_rows, n_refl_files, written_refl_files) — unchanged + +================================================================================ +WHAT WAS NOT CHANGED +================================================================================ + + - All function signatures unchanged + - All callers unchanged (predict.py, pred_io.py call sites) + - write_refl_with_predictions() — not touched + - _mfx_source_name() — kept in file, no longer called in main path + (still available as utility function) + - _copy_or_link_expt() — unchanged + - _as_numpy(), _as_text() — unchanged + +================================================================================ +ESTIMATED SPEEDUP (your dataset: ~117M rows, 124,975 files) +================================================================================ + + Stage Before After + ------------------------------------------------------- + Parquet load (pred_io.py) ~90 s ~25 s + source_refl/expt build ~46 s ~0 s (fast path: skipped entirely) + Write loop groupby ~80 hours ~seconds (argsort + slice) + DIALS file I/O ~3-7 hours ~3-7 hours (unchanged, I/O bound) + ------------------------------------------------------- + Total ~82-84 hours ~3-7 hours + +================================================================================ +RECOMMENDED VALIDATION BEFORE FULL RUN +================================================================================ + + 1. Quick sanity check — verify source_refl naming matches image_num pattern: + (see correctness note 1 above) + + 2. Dry-run with first 10 images using the old two-script approach and compare + output .refl files against new integrated code output for the same 10 images. + + 3. Full run with: + integrator.predict \ + --run-dir "$RUN_DIR" \ + --ckpt "epoch=0024.ckpt" \ + --write-refl --mfx-writeback \ + --original-refl-dir "$REFL_DIR" \ + --batch-size 2048 + +================================================================================ diff --git a/workflows/mfx101555026/scripts/archive/writeback/refl_io_final_fix.txt b/workflows/mfx101555026/scripts/archive/writeback/refl_io_final_fix.txt new file mode 100644 index 00000000..2815f26c --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/writeback/refl_io_final_fix.txt @@ -0,0 +1,183 @@ +================================================================================ +FINAL FIX — src/integrator/io/refl_io.py +write_mfx_refl_with_predictions() +================================================================================ +Date : 2026-08-06 +Reason : Validation test showed source_refl has run-prefixed filenames + (e.g. r0269_idx-data_01418_integrated.refl) that cannot be + reconstructed from image_num alone. + Previous fast-path was removed; code is now a single unified path. + +Validation result that triggered this fix: + Total rows : 115,146,952 + Checked rows : 100,000 + Mismatches : 100,000 (100%) + Actual : r0269_idx-data_01418_integrated.refl + Expected : idx-data_01418_integrated.refl (missing run prefix) + +================================================================================ +WHAT CHANGES +================================================================================ + +Only the groupby/write-loop section changes (~lines 185-272 in current file). +Everything before (refl_ids matching, reflection_id lookup, image_numbers +extraction) and everything after (return dict) is untouched. + +================================================================================ +BEFORE (current code in file) +================================================================================ + + row_in_source = reflection_id[metadata_rows] + + # Pull source_refl / source_expt via vectorized fancy-index ... + source_refl_all = _as_numpy(metadata["source_refl"]) + source_expt_all = _as_numpy(metadata["source_expt"]) + source_refl_subset = source_refl_all[metadata_rows] + source_expt_subset = source_expt_all[metadata_rows] + + has_image_num = "image_num" in metadata or "image_index" in metadata + + if has_image_num: + # Fast sort key: integer image_num + sort_key = image_numbers[metadata_rows] + order = np.argsort(sort_key, kind="stable") + sorted_key = sort_key[order] + boundaries = np.flatnonzero(sorted_key[1:] != sorted_key[:-1]) + 1 + else: + # Encode unique source_refl strings to sorted int IDs + _, encoded = np.unique(source_refl_subset, return_inverse=True) + order = np.argsort(encoded, kind="stable") + sorted_enc = encoded[order] + boundaries = np.flatnonzero(sorted_enc[1:] != sorted_enc[:-1]) + 1 + + starts = np.concatenate(([0], boundaries)) + ends = np.concatenate((boundaries, [len(order)])) + + # One refl/expt filename per group: first element silently (no check). + unique_refl = source_refl_subset[order[starts]] + unique_expt = source_expt_subset[order[starts]] + + written = [] + for i, (start, end) in enumerate(zip(starts, ends)): + group = order[start:end] + refl_name = _as_text(unique_refl[i]) + expt_name = _as_text(unique_expt[i]) # <- silent, no validation + + src_refl = original_refl_dir / refl_name + ... + _copy_or_link_expt(src_expt, out_dir / expt_name, copy_expt=copy_expt) + +================================================================================ +AFTER (proposed replacement) +================================================================================ + + row_in_source = reflection_id[metadata_rows] + + # Pull source_refl / source_expt via vectorized fancy-index — one C call, + # no per-row Python loop. metadata["source_refl"] is the source of truth + # for filenames; run-prefixed names (e.g. r0269_idx-data_01418_integrated.refl) + # are NOT reconstructible from image_num alone. + source_refl_all = _as_numpy(metadata["source_refl"]) + source_expt_all = _as_numpy(metadata["source_expt"]) + source_refl_subset = source_refl_all[metadata_rows] + source_expt_subset = source_expt_all[metadata_rows] + + # Encode unique source_refl strings to sorted int IDs — one C pass. + # Groupby via argsort on encoded IDs: O(N log N), no O(N x n_files) scan. + unique_refl, encoded = np.unique( + source_refl_subset, return_inverse=True + ) + order = np.argsort(encoded, kind="stable") + sorted_encoded = encoded[order] + boundaries = ( + np.flatnonzero(sorted_encoded[1:] != sorted_encoded[:-1]) + 1 + ) + starts = np.concatenate(([0], boundaries)) + ends = np.concatenate((boundaries, [len(order)])) + + written = [] + for i, (start, end) in enumerate(zip(starts, ends)): + group = order[start:end] + refl_name = _as_text(unique_refl[i]) + + # Validate exactly one .expt per .refl group (restored correctness check). + expt_names = np.unique(source_expt_subset[group]) + if len(expt_names) != 1: + raise ValueError( + f"Expected one source_expt for {refl_name}, " + f"got {expt_names.tolist()}" + ) + expt_name = _as_text(expt_names[0]) + + src_refl = original_refl_dir / refl_name + ... + _copy_or_link_expt(src_expt, out_dir / expt_name, copy_expt=copy_expt) + +================================================================================ +DIFF SUMMARY +================================================================================ + +REMOVED: + - has_image_num = "image_num" in metadata or "image_index" in metadata + - if has_image_num / else two-branch block (sort by image_num vs encoded) + - unique_refl = source_refl_subset[order[starts]] (silent pre-computation) + - unique_expt = source_expt_subset[order[starts]] (silent pre-computation) + - expt_name = _as_text(unique_expt[i]) (silent first-element pick) + +ADDED: + + unique_refl, encoded = np.unique(source_refl_subset, return_inverse=True) + (always groups by source_refl; image_num branch removed entirely) + + expt_names = np.unique(source_expt_subset[group]) + + if len(expt_names) != 1: raise ValueError(...) (restored correctness check) + + expt_name = _as_text(expt_names[0]) + +UNCHANGED: + - source_refl_all / source_expt_all vectorized fancy-index extraction + - order = np.argsort(encoded, kind="stable") + - boundaries / starts / ends computation + - entire per-file write body (src_refl check, rt load, _set_column x5, + rt.as_file, src_expt check, _copy_or_link_expt) + - all code before the groupby block (refl_ids match, reflection_id, image_numbers) + - return dict + +================================================================================ +WHY +================================================================================ + +1. source_refl is the only correct source for filenames. + Validation confirmed 100% mismatch between: + metadata source_refl : r0269_idx-data_01418_integrated.refl + image_num reconstruct: idx-data_01418_integrated.refl (missing run prefix) + +2. Grouping by source_refl encoded IDs is correct by definition: + each group = all prediction rows belonging to one source .refl file. + Grouping by image_num is wrong here because image_num is not unique + across runs (run r0269 and run r0270 could both have image_num=1418). + +3. np.unique expt check per group restores the original correctness check + that was present in the very first write_mfx_refl_with_predictions() + and was accidentally dropped in the previous fast-path version. + +4. Single unified code path: simpler, no hidden naming assumptions, + works for any dataset regardless of filename convention. + +================================================================================ +PERFORMANCE (unchanged from previous fix) +================================================================================ + + Stage Before all fixes After this fix + ----------------------------------------------------------------- + source_refl/expt extraction ~46 s ~1 s + Groupby scan ~80 hours ~30-60 s + DIALS file I/O ~3-7 hours ~3-7 hours + ----------------------------------------------------------------- + Total ~84 hours ~3-7 hours + +================================================================================ +FILES CHANGED +================================================================================ + + src/integrator/io/refl_io.py — groupby/write-loop section only + src/integrator/io/pred_io.py — already fixed (single polars collect) + +================================================================================ diff --git a/workflows/mfx101555026/scripts/archive/writeback/write_mfx_predictions_to_refl.py b/workflows/mfx101555026/scripts/archive/writeback/write_mfx_predictions_to_refl.py new file mode 100644 index 00000000..db5b9c63 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/writeback/write_mfx_predictions_to_refl.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python +""" +Write Integrator predictions back into separate MFX DIALS .refl files. + +This script is designed for the current MFX workflow where: + + 1. Predictions were exported from parquet to NPZ first. + 2. Shoebox metadata contains: + refl_ids + image_num + reflection_id + 3. Original cctbx/DIALS output contains many separate files: + idx-data_01418_integrated.refl + idx-data_01418_integrated.expt + idx-data_02660_integrated.refl + idx-data_02660_integrated.expt + ... + +Mapping used: + + prediction refl_ids + -> metadata.npy row + -> image_num + reflection_id + -> idx-data__integrated.refl + -> row reflection_id inside that .refl file + +This follows Luis's write_refl_with_predictions() idea, but supports many +separate MFX .refl/.expt files instead of one combined .refl with refl_ids. + +Run with dials.python, not regular python: + + dials.python workflows/mfx101555026/scripts/writeback/write_mfx_predictions_to_refl.py ... +""" + +from __future__ import annotations + +import argparse +import shutil +from pathlib import Path + +import numpy as np +import pandas as pd + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Write MFX Integrator qi_mean/qi_var predictions back to " + "separate DIALS .refl files." + ) + ) + + parser.add_argument( + "--pred-npz", + required=True, + type=Path, + help=( + "NPZ prediction file made from parquet. Expected arrays: " + "refl_ids, qi_mean, qi_var, qbg_mean." + ), + ) + parser.add_argument( + "--metadata", + required=True, + type=Path, + help="metadata.npy from the shoebox dataset", + ) + parser.add_argument( + "--original-refl-dir", + required=True, + type=Path, + help=( + "Original cctbx output folder containing " + "idx-data_*_integrated.refl/.expt" + ), + ) + parser.add_argument( + "--out-dir", + required=True, + type=Path, + help="New output folder for predicted .refl/.expt files", + ) + parser.add_argument( + "--max-images", + type=int, + default=None, + help="Optional small dry-run limit, e.g. --max-images 5", + ) + parser.add_argument( + "--copy-expt", + action="store_true", + help="Copy .expt files instead of symlinking them", + ) + parser.add_argument( + "--variance-floor", + type=float, + default=1e-6, + help="Minimum allowed variance written to intensity variance columns", + ) + + return parser.parse_args() + + +def load_prediction_table(pred_npz: Path) -> pd.DataFrame: + """ + Load prediction arrays from NPZ. + + Expected arrays: + refl_ids, qi_mean, qi_var, qbg_mean + """ + if not pred_npz.exists(): + raise FileNotFoundError(f"Missing prediction NPZ: {pred_npz}") + + data = np.load(pred_npz) + + required = ["refl_ids", "qi_mean", "qi_var", "qbg_mean"] + missing = [k for k in required if k not in data] + if missing: + raise KeyError(f"{pred_npz} is missing required array(s): {missing}") + + pred_df = pd.DataFrame( + { + "refl_ids": data["refl_ids"].astype(np.int64), + "qi_mean": data["qi_mean"].astype(np.float64), + "qi_var": data["qi_var"].astype(np.float64), + "qbg_mean": data["qbg_mean"].astype(np.float64), + } + ) + + return pred_df + + +def add_metadata_columns( + pred_df: pd.DataFrame, + metadata_path: Path, +) -> pd.DataFrame: + """ + Join predictions to metadata using refl_ids. + + In the current MFX dataset, refl_ids are row ids into metadata.npy. + """ + if not metadata_path.exists(): + raise FileNotFoundError(f"Missing metadata file: {metadata_path}") + + meta = np.load(metadata_path, allow_pickle=True).item() + + required = ["image_num", "reflection_id", "refl_ids"] + missing = [k for k in required if k not in meta] + if missing: + raise KeyError(f"metadata.npy is missing required key(s): {missing}") + + meta_df = pd.DataFrame( + { + "refl_ids": np.asarray(meta["refl_ids"], dtype=np.int64), + "image_num": np.asarray(meta["image_num"], dtype=np.int64), + "reflection_id": np.asarray(meta["reflection_id"], dtype=np.int64), + } + ) + + joined = pred_df.merge(meta_df, on="refl_ids", how="inner") + + if len(joined) != len(pred_df): + print( + "WARNING: joined " + f"{len(joined)} rows, but predictions had {len(pred_df)} rows" + ) + + duplicate_count = joined.duplicated( + subset=["image_num", "reflection_id"] + ).sum() + if duplicate_count: + raise ValueError( + "Duplicate target rows found after join: " + f"{duplicate_count} duplicate image_num/reflection_id pairs. " + "This would overwrite the same reflection row more than once." + ) + + return joined + + +def link_or_copy_expt(src: Path, dst: Path, copy_expt: bool) -> None: + dst.parent.mkdir(parents=True, exist_ok=True) + + if dst.exists() or dst.is_symlink(): + dst.unlink() + + if copy_expt: + shutil.copy2(src, dst) + else: + dst.symlink_to(src) + + +def get_or_zeros(rt, column_name: str, n_refl: int) -> np.ndarray: + """ + Return an existing DIALS reflection column as a NumPy float64 array, + or a zero array if the column is missing. + """ + from dxtbx import flumpy + + if column_name in rt: + return flumpy.to_numpy(rt[column_name]).astype(np.float64) + + return np.zeros(n_refl, dtype=np.float64) + + +def write_one_image( + image_num: int, + rows: pd.DataFrame, + original_refl_dir: Path, + out_dir: Path, + copy_expt: bool, + variance_floor: float, +) -> tuple[int, int]: + """ + Open one original .refl, write predictions into selected rows, save new .refl. + + Returns: + (image_num, number_of_written_rows) + """ + from dials.array_family import flex + from dxtbx import flumpy + + refl_name = f"idx-data_{image_num:05d}_integrated.refl" + expt_name = f"idx-data_{image_num:05d}_integrated.expt" + + src_refl = original_refl_dir / refl_name + src_expt = original_refl_dir / expt_name + + if not src_refl.exists(): + raise FileNotFoundError(f"Missing source .refl: {src_refl}") + if not src_expt.exists(): + raise FileNotFoundError(f"Missing source .expt: {src_expt}") + + dst_refl = out_dir / refl_name + dst_expt = out_dir / expt_name + + rt = flex.reflection_table.from_file(str(src_refl)) + n_refl = len(rt) + + reflection_ids = rows["reflection_id"].to_numpy(dtype=np.int64) + qi_mean = rows["qi_mean"].to_numpy(dtype=np.float64) + qi_var = rows["qi_var"].to_numpy(dtype=np.float64) + qbg_mean = rows["qbg_mean"].to_numpy(dtype=np.float64) + + if np.any(reflection_ids < 0) or np.any(reflection_ids >= n_refl): + bad = reflection_ids[(reflection_ids < 0) | (reflection_ids >= n_refl)] + raise IndexError( + f"{refl_name}: reflection_id out of range. " + f"n_refl={n_refl}, first bad={bad[:10]}" + ) + + qi_var = np.maximum(qi_var, variance_floor) + + intensity_sum_value = get_or_zeros(rt, "intensity.sum.value", n_refl) + intensity_sum_variance = get_or_zeros( + rt, "intensity.sum.variance", n_refl + ) + intensity_prf_value = get_or_zeros(rt, "intensity.prf.value", n_refl) + intensity_prf_variance = get_or_zeros( + rt, "intensity.prf.variance", n_refl + ) + background_mean = get_or_zeros(rt, "background.mean", n_refl) + + intensity_sum_value[reflection_ids] = qi_mean + intensity_sum_variance[reflection_ids] = qi_var + intensity_prf_value[reflection_ids] = qi_mean + intensity_prf_variance[reflection_ids] = qi_var + background_mean[reflection_ids] = qbg_mean + + rt["intensity.sum.value"] = flumpy.from_numpy(intensity_sum_value) + rt["intensity.sum.variance"] = flumpy.from_numpy(intensity_sum_variance) + rt["intensity.prf.value"] = flumpy.from_numpy(intensity_prf_value) + rt["intensity.prf.variance"] = flumpy.from_numpy(intensity_prf_variance) + rt["background.mean"] = flumpy.from_numpy(background_mean) + + out_dir.mkdir(parents=True, exist_ok=True) + rt.as_file(str(dst_refl)) + link_or_copy_expt(src_expt, dst_expt, copy_expt=copy_expt) + + return image_num, len(reflection_ids) + + +def main() -> None: + args = parse_args() + args.out_dir.mkdir(parents=True, exist_ok=True) + + print("Loading predictions from NPZ...") + pred_df = load_prediction_table(args.pred_npz) + print(f"Prediction rows: {len(pred_df)}") + + print("Joining predictions to metadata...") + df = add_metadata_columns(pred_df, args.metadata) + print(f"Joined rows: {len(df)}") + + image_nums = sorted(df["image_num"].unique().tolist()) + if args.max_images is not None: + image_nums = image_nums[: args.max_images] + + print(f"Images to write: {len(image_nums)}") + print(f"Output folder: {args.out_dir}") + + total_rows = 0 + + for i, image_num in enumerate(image_nums, start=1): + rows = df[df["image_num"] == image_num] + + written_image_num, n_rows = write_one_image( + image_num=int(image_num), + rows=rows, + original_refl_dir=args.original_refl_dir, + out_dir=args.out_dir, + copy_expt=args.copy_expt, + variance_floor=args.variance_floor, + ) + + total_rows += n_rows + + if i <= 10 or i % 100 == 0: + print( + f"[{i}/{len(image_nums)}] wrote image_num={written_image_num} " + f"rows={n_rows}" + ) + + print("Done.") + print(f"Images written: {len(image_nums)}") + print(f"Prediction rows written: {total_rows}") + print(f"Output folder: {args.out_dir}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/archive/writeback/writeback_performance_diagnosis.txt b/workflows/mfx101555026/scripts/archive/writeback/writeback_performance_diagnosis.txt new file mode 100644 index 00000000..9209b885 --- /dev/null +++ b/workflows/mfx101555026/scripts/archive/writeback/writeback_performance_diagnosis.txt @@ -0,0 +1,218 @@ +================================================================================ +MFX WRITE-BACK PERFORMANCE DIAGNOSIS +write_mfx_refl_with_predictions() — src/integrator/io/refl_io.py +pred_io.py — src/integrator/io/pred_io.py +================================================================================ +Date: 2026-08-06 +Author: code review (opencode) + +DATASET SCALE + Prediction rows (N) : ~117 M (5758 parquet files x 10 batches x 2048 rows) + Source .refl files : 124,975 + Metadata rows : ~117 M (metadata.npy — one row per original reflection) + +================================================================================ +ISSUE 1 — 4 SEPARATE POLARS SCANS +File: src/integrator/io/pred_io.py lines 38-41 +================================================================================ + +CURRENT CODE: + lf = pl.scan_parquet(pred_files) + refl_ids = lf.select("refl_ids").collect().to_numpy().ravel() # scan 1 + qi_mean = lf.select("qi_mean").collect().to_numpy().ravel() # scan 2 + qi_var = lf.select("qi_var").collect().to_numpy().ravel() # scan 3 + qbg_mean = lf.select("qbg_mean").collect().to_numpy().ravel() # scan 4 + +PROBLEM: + Each .collect() re-opens and re-reads all 5758 parquet files to extract + one column. Parquet is column-oriented so each pass reads only the target + column, but there is still file-open + footer-parse + column-read overhead + per file, repeated four times. + + 5758 files x 4 passes = 23,032 file operations + vs optimized: 5758 file operations (1 pass) + +ESTIMATE: + Current : ~60-120 seconds + Optimized: ~15-30 seconds + Speedup : ~4x + +FIX: + df = lf.select(["refl_ids", "qi_mean", "qi_var", "qbg_mean"]).collect() + refl_ids = df["refl_ids"].to_numpy() + qi_mean = df["qi_mean"].to_numpy() + qi_var = df["qi_var"].to_numpy() + qbg_mean = df["qbg_mean"].to_numpy() + +SEVERITY: Minor — parquet loading is already fast per file with column pruning. + +================================================================================ +ISSUE 2 — TWO PYTHON LOOPS OVER ~117 M ROWS +File: src/integrator/io/refl_io.py lines 183-208 +================================================================================ + +CURRENT CODE: + source_refl = np.array( + [_mfx_source_name(metadata, "source_refl", row, image_numbers[row], ".refl") + for row in metadata_rows], # <- Python loop, 117 M iterations + dtype=object, + ) + source_expt = np.array( + [_mfx_source_name(metadata, "source_expt", row, image_numbers[row], ".expt") + for row in metadata_rows], # <- Python loop, 117 M iterations + dtype=object, + ) + +PROBLEM: + Pure Python list comprehension over 117 M elements. Each iteration does: + - NumPy array index: metadata["source_refl"][row] + - isinstance() check inside _as_numpy() + - _as_text() conversion (bytes decode or str()) + - image_numbers[row] fallback logic + Python loops run at ~3-7 M iterations/second for this access pattern. + +ESTIMATE: + 117 M / 5 M iter/s = ~23 seconds per loop + Two loops = ~46 seconds total + Optimized (vectorized numpy fancy indexing): ~0.5 seconds + Speedup: ~90x + +FIX: + # Extract entire arrays once, then index with metadata_rows in one shot. + raw_refl = _as_numpy(metadata["source_refl"]) # full array, one call + raw_expt = _as_numpy(metadata["source_expt"]) # full array, one call + source_refl = raw_refl[metadata_rows] # vectorized fancy index + source_expt = raw_expt[metadata_rows] # vectorized fancy index + + # Convert bytes -> str in one vectorized pass if needed: + if source_refl.dtype.kind in ('S', 'V'): + source_refl = source_refl.astype('U') + source_expt = source_expt.astype('U') + else: + # dtype=object: apply _as_text once per element (still faster because + # metadata lookup happens only once outside the loop) + source_refl = np.array([_as_text(x) for x in source_refl], dtype=object) + source_expt = np.array([_as_text(x) for x in source_expt], dtype=object) + + NOTE: The key win is pulling the metadata dict lookup and _as_numpy() call + outside the loop. The full metadata["source_refl"] array is loaded once + and indexed with a numpy fancy-index in C rather than 117 M Python steps. + +SEVERITY: Moderate — slow but completes in under a minute. + +================================================================================ +ISSUE 3 — O(N x n_files) SCAN IN THE WRITE LOOP [CRITICAL] +File: src/integrator/io/refl_io.py lines 212-213 +================================================================================ + +CURRENT CODE: + for refl_name in sorted(set(source_refl.tolist())): # 124,975 unique files + group = np.nonzero(source_refl == refl_name)[0] # scans all 117 M each time + ... + +PROBLEM: + For each of the 124,975 source .refl files, the code scans the entire + source_refl array (dtype=object, 117 M elements) to find matching rows. + numpy object-array comparison calls PyObject_RichCompareBool() for every + element from a C loop — faster than pure Python but still expensive because + it cannot use SIMD and must dereference 117 M Python object pointers. + +TIMING BREAKDOWN: + numpy dtype=object string comparison speed : ~10-30 ns per element + Per-file scan : 117 M x 20 ns = ~2.3 seconds + 124,975 files x 2.3 seconds = ~288,000 seconds + ~ 80 HOURS + + This is the dominant bottleneck. The previous successful run (124,975 pairs) + almost certainly spent most of its 2-day wall-time budget here. + +FIX OPTION A — dict-based groupby, O(N): + from collections import defaultdict + + groups = defaultdict(list) + for i, name in enumerate(source_refl): # one Python pass, builds index + groups[name].append(i) + + for refl_name, idx_list in groups.items(): + group = np.array(idx_list, dtype=np.int64) + src_refl = original_refl_dir / refl_name + ... + + Estimated time: ~60 seconds (one Python loop + dict insertions) + +FIX OPTION B — numpy argsort, O(N log N), fully vectorized: + sort_idx = np.argsort(source_refl) # sorts once in C + sorted_refl = source_refl[sort_idx] + boundaries = np.flatnonzero(sorted_refl[:-1] != sorted_refl[1:]) + 1 + splits = np.split(sort_idx, boundaries) + names = sorted_refl[np.concatenate([[0], boundaries])] + + for refl_name, group in zip(names, splits): + src_refl = original_refl_dir / refl_name + ... + + Estimated time: ~5-30 seconds (numpy argsort on object array + split) + NOTE: np.argsort on dtype=object uses Python comparisons but only O(N log N) + total comparisons instead of O(N x n_files). + +RECOMMENDATION: Option B (argsort) for minimum Python overhead. + +SEVERITY: CRITICAL — this is the ~80-hour bottleneck. + Estimated current : ~80 hours + Estimated optimized: ~30 seconds (argsort) + Speedup : ~10,000x + +================================================================================ +OVERALL TIMING SUMMARY +================================================================================ + + Stage Current Optimized Speedup + ----------------------------------------------------------------------- + Parquet loading (4 scans) ~90 s ~25 s ~4x + Python loops (source arrays x2) ~46 s ~1 s ~90x + Write loop (O(N x n_files)) ~80 hours ~30 s ~10,000x + DIALS file I/O (124,975 files) ~2-4 hours ~2-4 hours 1x (I/O bound) + ----------------------------------------------------------------------- + TOTAL (estimated) ~82-84 hours ~2-4 hours ~20-40x + + The DIALS file I/O — reading 124,975 source .refl files, writing 124,975 + output .refl files, linking .expt files — is unavoidably sequential and + disk-bound. That stage cannot be improved without multiprocessing. + +================================================================================ +NO CORRECTNESS BUGS FOUND +================================================================================ + + The code is functionally correct. The previous run (124,975 pairs written) + confirms this. All changes described above are performance optimizations only. + + Additional design notes: + - out_dir is hardcoded as pred_dir/"mfx_refl_writeback" (predict.py:328). + Running write-back twice for different epochs overwrites .refl files but + skips existing .expt symlinks (_copy_or_link_expt returns early if dst + already exists). This is intentional and correct since .expt files do + not change between epochs. + - .expt symlinks use absolute paths (os.symlink). Works fine on S3DF but + would break if storage is remounted under a different path. + - qi_var is clipped to variance_floor=1e-6 before writing. Correct. + +================================================================================ +PROPOSED IMPLEMENTATION ORDER +================================================================================ + + 1. Fix Issue 3 (write loop) first — largest impact by far. + Change: np.nonzero(source_refl == refl_name) inside loop + To: argsort-based groupby computed once before loop + File: src/integrator/io/refl_io.py lines 211-213 + + 2. Fix Issue 2 (Python loops) — easy vectorization. + Change: list comprehensions over metadata_rows + To: raw_arr = _as_numpy(metadata[key]); result = raw_arr[metadata_rows] + File: src/integrator/io/refl_io.py lines 183-208 + + 3. Fix Issue 1 (polars scans) — minor, one-liner. + Change: 4 separate .collect() calls + To: single lf.select([...]).collect() + File: src/integrator/io/pred_io.py lines 37-48 + +================================================================================ diff --git a/workflows/mfx101555026/scripts/diagnostics/inspect_cctbx_refl_expt.py b/workflows/mfx101555026/scripts/diagnostics/inspect_cctbx_refl_expt.py new file mode 100644 index 00000000..1d630200 --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/inspect_cctbx_refl_expt.py @@ -0,0 +1,47 @@ +from pathlib import Path +from dials.array_family import flex +from dxtbx.model.experiment_list import ExperimentListFactory + +folder = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/" + "mfx101555026_cctbx/scale_r0269_018_p212121" +) + +refl_path = folder / "mfx_r0269_018_scale.refl" +expt_path = folder / "mfx_r0269_018_scale.expt" + +refl = flex.reflection_table.from_file(str(refl_path)) +expts = ExperimentListFactory.from_json_file(str(expt_path), check_format=False) + +print("reflection rows:", len(refl)) +print("experiment count:", len(expts)) + +print("\nColumns:") +for key in refl.keys(): + print(" ", key) + +print("\nFirst 10 scaled reflection rows:") +for i in range(10): + print("row", i) + print(" id:", refl["id"][i]) + print(" hkl_asym:", refl["miller_index_asymmetric"][i]) + print(" I:", refl["intensity.sum.value"][i]) + print(" var:", refl["intensity.sum.variance"][i]) + +print("\nFirst 10 experiments:") +for i in range(min(10, len(expts))): + exp = expts[i] + print("\nexperiment", i) + print(" identifier:", exp.identifier) + + try: + paths = exp.imageset.paths() + print(" paths:", paths[:3]) + except Exception as e: + print(" paths: ERROR", e) + + try: + indices = exp.imageset.indices() + print(" indices:", indices[:10]) + except Exception as e: + print(" indices: ERROR", e) \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/diagnostics/inspect_integrator_parquet.py b/workflows/mfx101555026/scripts/diagnostics/inspect_integrator_parquet.py new file mode 100644 index 00000000..ec27d535 --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/inspect_integrator_parquet.py @@ -0,0 +1,34 @@ +from pathlib import Path +import pandas as pd + +path = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/" + "mfx101555026_cctbx/runs/mfx_gamma_qi_25files/" + "run_20260707-175519_bda0/predictions/test_preds_all.parquet" +) + +df = pd.read_parquet(path) + +print("FILE:", path) +print("shape:", df.shape) +print("\ncolumns:") +for c in df.columns: + print(" ", c, df[c].dtype) + +print("\nfirst rows:") +print(df.head()) + +print("\npossible matching columns:") +for c in df.columns: + if any(k in c.lower() for k in ["h", "k", "l", "miller", "id", "image", "file", "panel", "bbox", "xyz", "d", "intensity", "qi", "epoch"]): + print(" ", c) + +if "epoch" in df.columns: + print("\nepoch values:") + print(df["epoch"].value_counts().sort_index()) + + final_epoch = df["epoch"].max() + final_df = df[df["epoch"] == final_epoch] + print("\nfinal epoch:", final_epoch) + print("final epoch shape:", final_df.shape) + diff --git a/workflows/mfx101555026/scripts/diagnostics/inspect_mfx_shoeboxes.py b/workflows/mfx101555026/scripts/diagnostics/inspect_mfx_shoeboxes.py new file mode 100644 index 00000000..1b63d3d4 --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/inspect_mfx_shoeboxes.py @@ -0,0 +1,92 @@ +import numpy as np +import matplotlib.pyplot as plt +from pathlib import Path + +project_dir = Path("/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx") +data_dir = project_dir / "mfx_shoebox_r0269_012_rg058_with_d_1500" +out_dir = project_dir/"scripts/diagnostics/shoebox_inspection_1500_masked" + + +counts = np.load(data_dir / "counts.npy", mmap_mode="r") +masks = np.load(data_dir / "masks.npy", mmap_mode="r") +metadata = np.load(data_dir / "metadata.npy", allow_pickle=True).item() + +intensity = metadata["intensity.sum.value"] + +indices = [ + # 10% + int(np.nanargmin(np.abs(intensity - np.nanpercentile(intensity, 10)))), + + # 50% + int(np.nanargmin(np.abs(intensity - np.nanpercentile(intensity, 50)))), + + # 90% + int(np.nanargmin(np.abs(intensity - np.nanpercentile(intensity, 90)))), +] + + +# Remove duplicate indices while keeping the same order +indices = list(dict.fromkeys(indices)) + +for idx in indices: + img = counts[idx].reshape(25, 25) + mask_img = masks[idx].reshape(25, 25) + + masked_img = np.ma.masked_where(~mask_img, img) + + plt.figure(figsize=(5, 5)) + plt.imshow(masked_img, vmin=0, vmax=500) + plt.colorbar() + plt.title( + f"idx={idx}, intensity={intensity[idx]:.2f}, d={metadata['d'][idx]:.2f}", + fontsize=8 + ) + plt.tight_layout() + plt.savefig(out_dir / f"shoebox_{idx}_masked.png", dpi=150, bbox_inches="tight") + plt.close() + +print("Wrote masked images to:", out_dir) + + +# Top 20 strongest reflections +# Make one grid image for the top 20 strongest reflections +top20_indices = np.argsort(intensity)[-20:][::-1] + +#fig: the whole image, axes: the 5 x 4 collection + +""" +fig = whole page + +axes = +[ +[plot1, plot2, plot3, plot4 ], +[plot5, plot6, plot7, plot8 ], +[plot9, plot10, plot11, plot12], +[plot13, plot14, plot15, plot16], +[plot17, plot18, plot19, plot20] +] +""" + +fig, axes= plt.subplots(5, 4, figsize=(12,15)) + +#axes.flat: return plot1, plot2, plot3, ..., plot20 +#top20_indices: return index1, index2, index3, ..., index20 +#zip(axes.flat, top20_indices): pair (plot1, index1) + +for ax, idx in zip(axes.flat, top20_indices): + img = counts[idx].reshape(25, 25) + mask_img = masks[idx].reshape(25, 25) + + masked_img = np.ma.masked_where(~mask_img, img) + + + ax.imshow(masked_img, vmin=0, vmax=500) + ax.set_title( + f"idx={idx}\nI={intensity[idx]:.1f}, d={metadata['d'][idx]:.2f}", + fontsize=7) + ax.axis("off") + +plt.tight_layout() +plt.savefig(out_dir / "top20_strongest_grid.png", dpi=150, bbox_inches="tight") +plt.close() +print("Wrote top 20 grid to:", out_dir / "top20_strongest_grid.png") diff --git a/workflows/mfx101555026/scripts/diagnostics/inspect_qp_mean_profiles.py b/workflows/mfx101555026/scripts/diagnostics/inspect_qp_mean_profiles.py new file mode 100644 index 00000000..00922bfd --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/inspect_qp_mean_profiles.py @@ -0,0 +1,205 @@ +from pathlib import Path + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + + +""" +Compare real MFX input shoeboxes with the model-predicted profile qp_mean. + +For each selected reflection, this script saves one side-by-side plot: + + left = real input shoebox from counts.npy, with invalid pixels masked + right = predicted profile from qp_mean + +This helps check whether the model's predicted profile is spatially reasonable: + - Is the predicted spot in the right location? + - Does qp_mean look like a reasonable spot shape? + - Does the profile behavior change from weak to strong reflections? +""" + + +DATA_DIR = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/" + "mfx_shoebox_r0269_012_rg058_with_d_1500" +) + +RUN_DIR = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/" + "runs/run_20260702-160655_3424" +) + +PRED_PATH = RUN_DIR / "predictions" / "test_preds_all.parquet" + +OUT_DIR = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/" + "diagnostics_qp_mean_profiles" +) + +OUT_DIR.mkdir(parents=True, exist_ok=True) + + +# Load dataset arrays. +# counts.npy: flattened 25 x 25 shoebox pixel values, one row per reflection. +# masks.npy: matching valid-pixel masks. True = valid, False = invalid/masked/padded. +# metadata.npy: reflection metadata saved as a Python dictionary. +counts = np.load(DATA_DIR / "counts.npy", mmap_mode="r") +masks = np.load(DATA_DIR / "masks.npy", mmap_mode="r") +metadata = np.load(DATA_DIR / "metadata.npy", allow_pickle=True).item() + +# Load prediction table from integrator.predict. +# This should include qp_mean if predict_keys was configured correctly. +pred = pd.read_parquet(PRED_PATH) + + +print("counts shape:", counts.shape) +print("masks shape:", masks.shape) +print("metadata keys:", list(metadata.keys())[:30]) +print("prediction columns:", list(pred.columns)[:30]) +print("number of predictions:", len(pred)) + + +# Required prediction columns. +if "qp_mean" not in pred.columns: + raise ValueError("qp_mean is missing from prediction output.") + +if "refl_ids" not in pred.columns: + raise ValueError("refl_ids is missing from prediction output.") + + +def get_profile_array(row): + """ + Return qp_mean for one prediction row as a 25 x 25 image. + """ + + qp = np.asarray(row["qp_mean"], dtype=float) + + if qp.size != 625: + raise ValueError(f"Expected qp_mean size 625, got {qp.size}") + + return qp.reshape(25, 25) + + +def get_counts_array(refl_id): + """ + Return the real input shoebox for one reflection as a 25 x 25 image. + + This assumes refl_ids match row indices in counts.npy. + """ + + img = np.asarray(counts[int(refl_id)], dtype=float) + + if img.size != 625: + raise ValueError(f"Expected counts size 625, got {img.size}") + + return img.reshape(25, 25) + + +def get_mask_array(refl_id): + """ + Return the saved valid-pixel mask for one reflection as a 25 x 25 image. + """ + + mask_img = np.asarray(masks[int(refl_id)], dtype=bool) + + if mask_img.size != 625: + raise ValueError(f"Expected mask size 625, got {mask_img.size}") + + return mask_img.reshape(25, 25) + + +def save_comparison(row, label): + """ + Save one side-by-side comparison: + + left = masked real input shoebox + right = model-predicted profile qp_mean + """ + + refl_id = int(row["refl_ids"]) + + # Real input shoebox, with invalid pixels hidden for visualization. + input_img = get_counts_array(refl_id) + mask_img = get_mask_array(refl_id) + input_img = np.ma.masked_where(~mask_img, input_img) + + # Model-predicted profile. + qp_img = get_profile_array(row) + + fig, axes = plt.subplots(1, 2, figsize=(8, 4)) + + im0 = axes[0].imshow(input_img) + axes[0].set_title(f"Input shoebox\nrefl_id={refl_id}", fontsize=9) + axes[0].axis("off") + plt.colorbar(im0, ax=axes[0], fraction=0.046) + + im1 = axes[1].imshow(qp_img) + axes[1].set_title("Predicted profile\nqp_mean", fontsize=9) + axes[1].axis("off") + plt.colorbar(im1, ax=axes[1], fraction=0.046) + + title_parts = [label] + + if "qi_mean" in row.index: + title_parts.append(f"qi_mean={row['qi_mean']:.2f}") + + if "intensity.sum.value" in row.index: + title_parts.append(f"I_sum={row['intensity.sum.value']:.2f}") + + if "d" in row.index: + title_parts.append(f"d={row['d']:.2f}") + + fig.suptitle(" | ".join(title_parts), fontsize=10) + fig.tight_layout() + + out_path = OUT_DIR / f"{label}_refl_{refl_id}.png" + fig.savefig(out_path, dpi=150, bbox_inches="tight") + plt.close(fig) + + print("saved:", out_path) + + +# Choose examples across weak-to-strong reflections. +# Prefer traditional cctbx/DIALS intensity if available. +if "intensity.sum.value" in pred.columns: + score_col = "intensity.sum.value" +elif "qi_mean" in pred.columns: + score_col = "qi_mean" +else: + raise ValueError("Need either intensity.sum.value or qi_mean to choose examples.") + + +score = pred[score_col].replace([np.inf, -np.inf], np.nan) + +percentiles = [10, 50, 90, 99] +chosen_rows = [] + +for p in percentiles: + target = np.nanpercentile(score, p) + idx = (score - target).abs().idxmin() + chosen_rows.append((f"p{p}_{score_col}", pred.loc[idx])) + +# Also inspect the strongest reflection. +max_idx = score.idxmax() +chosen_rows.append((f"max_{score_col}", pred.loc[max_idx])) + + +for label, row in chosen_rows: + save_comparison(row, label) + + +print() +print("Done.") +print("Diagnostics folder:", OUT_DIR) + + +""" +Important assumption: + + refl_ids in the prediction table must match row indices in counts.npy. + +If the input shoebox and qp_mean look mismatched, check whether refl_ids are +original cctbx/DIALS reflection IDs instead of counts.npy row numbers. If so, +this script needs a row-mapping fix before the comparison is valid. +""" \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/diagnostics/plot_cc12_vs_resolution.py b/workflows/mfx101555026/scripts/diagnostics/plot_cc12_vs_resolution.py new file mode 100644 index 00000000..b7c5e515 --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/plot_cc12_vs_resolution.py @@ -0,0 +1,173 @@ +""" +Plot CC1/2 vs resolution for cctbx, sqrt_squareplus, and asinh. + +Data source: merging_refinement_results.txt + CC1/2 values are in the scaling table (NOT the Intensity Statistics table). + The scaling table column labeled "Completeness" shows [N/M] X% where + X% = CC1/2 * 100. This was confirmed by cross-checking against the + explicit "CC 1/2" list printed after each Intensity Statistics block. + + All three methods have ~100% crystallographic completeness in all bins + (verified from Intensity Statistics tables, lines 261-284, 678-701, + 1100-1123). Do NOT make a "completeness vs resolution" plot. + +Key observations: + - cctbx CC1/2 decreases smoothly from 0.998 to 0.041 at 2.0 A + - Integrator CC1/2 drops faster at mid-resolution then plateaus + erratically at ~0.35-0.55 in the high-resolution tail + - The non-monotonic Integrator behavior at high resolution requires + further investigation — do not interpret as better signal +""" + +import matplotlib.pyplot as plt + +# ── Resolution shell upper bounds (d_min, Angstroms) ───────────────────────── +resolution = [ + 5.4288, + 4.3089, + 3.7641, + 3.4200, + 3.1748, + 2.9876, + 2.8380, + 2.7144, + 2.6099, + 2.5198, + 2.4410, + 2.3713, + 2.3088, + 2.2525, + 2.2013, + 2.1544, + 2.1113, + 2.0715, + 2.0345, + 2.0000, +] + +# ── CC1/2 values (0-1) verified from merging_refinement_results.txt ────────── +# sqrt_squareplus — scaling table lines 385-404, X% / 100 +sqrt_squareplus = [ + 0.993, + 0.986, + 0.976, + 0.949, + 0.842, + 0.776, + 0.657, + 0.587, + 0.565, + 0.558, + 0.538, + 0.580, + 0.512, + 0.581, + 0.513, + 0.458, + 0.574, + 0.335, + 0.477, + 0.551, +] + +# asinh — scaling table lines 802-821, X% / 100 +asinh = [ + 0.993, + 0.985, + 0.976, + 0.946, + 0.856, + 0.763, + 0.675, + 0.554, + 0.525, + 0.556, + 0.512, + 0.500, + 0.565, + 0.542, + 0.525, + 0.522, + 0.545, + 0.496, + 0.572, + 0.473, +] + +# cctbx baseline — scaling table lines 1224-1243, X% / 100 +cctbx = [ + 0.998, + 0.997, + 0.996, + 0.994, + 0.991, + 0.987, + 0.980, + 0.953, + 0.935, + 0.906, + 0.819, + 0.710, + 0.521, + 0.449, + 0.329, + 0.234, + 0.156, + 0.058, + 0.092, + 0.041, +] + +# ── Plot ────────────────────────────────────────────────────────────────────── +fig, ax = plt.subplots(figsize=(8, 5.5)) + +ax.plot( + resolution, + cctbx, + marker="o", + linewidth=2.5, + color="#2166ac", + label="cctbx.xfel", +) + +ax.plot( + resolution, + sqrt_squareplus, + marker="s", + linewidth=2.5, + color="#d6604d", + label="Integrator — sqrt-squareplus", +) + +ax.plot( + resolution, + asinh, + marker="^", + linestyle="--", + linewidth=2.5, + color="#f4a582", + label="Integrator — asinh", +) + +ax.axhline( + 0.5, + color="gray", + linestyle=":", + linewidth=1.0, + label="CC½ = 0.5 reference", +) + +ax.set_xlabel("Resolution (Å)", fontsize=13) +ax.set_ylabel("CC½", fontsize=13) +ax.set_title("CC½ vs. Resolution", fontsize=15) +ax.set_ylim(0, 1.05) +ax.invert_xaxis() # crystallography convention: high resolution on the right +ax.grid(alpha=0.25) +ax.legend(frameon=False, fontsize=10) + +fig.tight_layout() +fig.savefig("cc12_vs_resolution.png", dpi=300, bbox_inches="tight") +fig.savefig("cc12_vs_resolution.pdf", bbox_inches="tight") +plt.show() + +print("Saved: cc12_vs_resolution.png / .pdf") diff --git a/workflows/mfx101555026/scripts/diagnostics/plot_cctbx_selected_profiles.py b/workflows/mfx101555026/scripts/diagnostics/plot_cctbx_selected_profiles.py new file mode 100644 index 00000000..8b750e20 --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/plot_cctbx_selected_profiles.py @@ -0,0 +1,919 @@ +from pathlib import Path + +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + + +# ============================================================ +# Paths +# ============================================================ + +BASE = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/" + "mfx101555026_cctbx" +) + +DATA_DIR = ( + BASE + / "mfx_shoebox_allruns_269_289_no275_024_rg101" +) + +RUNS = { + "ASINH": ( + BASE + / "runs" + / "run_20260805-011728_9704" + ), + "sqrt-squareplus": ( + BASE + / "runs" + / "run_20260805-183129_a8b9" + ), +} + +OUT_DIR = ( + BASE + / "diagnostics_qp_mean_profiles_allruns_cctbx_thresholds" +) + +OUT_DIR.mkdir( + parents=True, + exist_ok=True, +) + + +# ============================================================ +# cctbx intensity categories +# +# Thresholds from Luis: +# +# Weak : I < 20 +# Medium : 20 <= I <= 200 +# Strong : I > 200 +# +# The "target" values below are ONLY used to choose one +# representative example inside each category. +# +# They do NOT define the categories. +# ============================================================ + +CATEGORIES = { + "Weak": { + "target": 10.0, + }, + "Medium": { + "target": 100.0, + }, + "Strong": { + "target": 500.0, + }, +} + + +# ============================================================ +# Load observed shoebox arrays +# +# mmap_mode="r" keeps the huge arrays on disk instead of +# loading the entire 115M-reflection dataset into RAM. +# ============================================================ + +counts = np.load( + DATA_DIR / "counts.npy", + mmap_mode="r", +) + +masks = np.load( + DATA_DIR / "masks.npy", + mmap_mode="r", +) + +print("counts shape:", counts.shape) +print("masks shape :", masks.shape) + + +# ============================================================ +# Helpers +# ============================================================ + +def prediction_files(run_dir): + """ + Return all prediction parquet shards for epoch 24. + """ + + pred_dir = ( + run_dir + / "predictions" + / "epoch_0024" + ) + + files = sorted( + pred_dir.glob("*.parquet") + ) + + if not files: + raise FileNotFoundError( + f"No parquet files found in {pred_dir}" + ) + + return files + + +def category_mask(values, label): + """ + Return a Boolean mask for Luis's cctbx intensity ranges. + + Weak: + I < 20 + + Medium: + 20 <= I <= 200 + + Strong: + I > 200 + """ + + finite = np.isfinite(values) + + if label == "Weak": + return ( + finite + & (values < 20.0) + ) + + if label == "Medium": + return ( + finite + & (values >= 20.0) + & (values <= 200.0) + ) + + if label == "Strong": + return ( + finite + & (values > 200.0) + ) + + raise ValueError( + f"Unknown category: {label}" + ) + + +def validate_category(label, intensity): + """ + Make sure the selected reflection actually satisfies + Luis's threshold definition. + """ + + if label == "Weak": + valid = intensity < 20.0 + + elif label == "Medium": + valid = ( + 20.0 <= intensity <= 200.0 + ) + + elif label == "Strong": + valid = intensity > 200.0 + + else: + raise ValueError(label) + + if not valid: + raise RuntimeError( + f"{label} reflection has invalid cctbx " + f"intensity: {intensity}" + ) + + +# ============================================================ +# PASS 1 +# +# Select Weak / Medium / Strong using ONLY cctbx intensity. +# +# We search the ASINH prediction files because they contain: +# +# refl_ids +# cctbx intensity +# qi_mean +# qp_mean +# d +# +# ASINH qi_mean does NOT determine the category. +# ============================================================ + +def find_cctbx_selected_reflections(run_dir): + + files = prediction_files(run_dir) + + print( + "\n=== PASS 1: selecting reflections " + "using cctbx intensity ===" + ) + + print( + "Thresholds:" + ) + + print( + " Weak : I < 20" + ) + + print( + " Medium : 20 <= I <= 200" + ) + + print( + " Strong : I > 200" + ) + + best = { + label: { + "distance": np.inf, + "refl_id": None, + "cctbx_I": None, + "qi_mean": None, + "qp_mean": None, + "d": None, + } + for label in CATEGORIES + } + + columns = [ + "refl_ids", + "qi_mean", + "qp_mean", + "d", + "intensity.sum.value", + ] + + for i, path in enumerate( + files, + start=1, + ): + + df = pd.read_parquet( + path, + columns=columns, + ) + + cctbx_I = ( + df["intensity.sum.value"] + .to_numpy( + dtype=np.float64, + copy=False, + ) + ) + + for label, config in CATEGORIES.items(): + + mask = category_mask( + cctbx_I, + label, + ) + + if not np.any(mask): + continue + + valid_indices = np.flatnonzero( + mask + ) + + valid_I = cctbx_I[mask] + + target = config["target"] + + # Pick the reflection inside the category + # whose cctbx intensity is nearest to the + # representative target. + local_pos = int( + np.argmin( + np.abs( + valid_I - target + ) + ) + ) + + dataframe_pos = int( + valid_indices[local_pos] + ) + + selected_I = float( + valid_I[local_pos] + ) + + distance = abs( + selected_I - target + ) + + if ( + distance + < best[label]["distance"] + ): + + row = df.iloc[ + dataframe_pos + ] + + best[label] = { + "distance": float( + distance + ), + "refl_id": int( + row["refl_ids"] + ), + "cctbx_I": float( + row[ + "intensity.sum.value" + ] + ), + "qi_mean": float( + row["qi_mean"] + ), + "qp_mean": np.asarray( + row["qp_mean"], + dtype=np.float32, + ), + "d": float( + row["d"] + ), + } + + if ( + i % 500 == 0 + or i == len(files) + ): + print( + f" searched " + f"{i:,} / {len(files):,}" + ) + + del df + + # -------------------------------------------------------- + # Validate results + # -------------------------------------------------------- + + for label, info in best.items(): + + if info["refl_id"] is None: + raise RuntimeError( + f"No reflection found for " + f"{label}" + ) + + validate_category( + label, + info["cctbx_I"], + ) + + return best + + +# ============================================================ +# PASS 2 +# +# Find those exact SAME reflection IDs in another Integrator +# model. +# +# IMPORTANT: +# +# We intentionally do NOT read cctbx intensity here. +# +# The original cctbx intensity from PASS 1 is preserved later. +# This fixes the previous mismatch bug. +# ============================================================ + +def find_same_reflections_in_model( + run_dir, + selected_ids, +): + + files = prediction_files(run_dir) + + wanted = set( + selected_ids + ) + + found = {} + + print( + "\n=== PASS 2: finding same reflections " + "in second model ===" + ) + + columns = [ + "refl_ids", + "qi_mean", + "qp_mean", + "d", + ] + + for i, path in enumerate( + files, + start=1, + ): + + df = pd.read_parquet( + path, + columns=columns, + ) + + ids = ( + df["refl_ids"] + .to_numpy( + dtype=np.int64, + copy=False, + ) + ) + + for target_id in list( + wanted + ): + + positions = np.flatnonzero( + ids == target_id + ) + + if len(positions) == 0: + continue + + row = df.iloc[ + int(positions[0]) + ] + + found[target_id] = { + "refl_id": target_id, + "qi_mean": float( + row["qi_mean"] + ), + "qp_mean": np.asarray( + row["qp_mean"], + dtype=np.float32, + ), + "d": float( + row["d"] + ), + } + + wanted.remove( + target_id + ) + + if ( + i % 500 == 0 + or not wanted + ): + print( + f" searched " + f"{i:,} / {len(files):,} " + f"| remaining IDs: " + f"{len(wanted)}" + ) + + del df + + if not wanted: + break + + if wanted: + raise RuntimeError( + "Could not find these reflection IDs " + "in the second model: " + f"{sorted(wanted)}" + ) + + return found + + +# ============================================================ +# Run selection +# ============================================================ + +asinh_selected = ( + find_cctbx_selected_reflections( + RUNS["ASINH"] + ) +) + + +# ------------------------------------------------------------ +# Store the selected IDs by category. +# ------------------------------------------------------------ + +selected_ids = { + label: info["refl_id"] + for label, info + in asinh_selected.items() +} + + +# ------------------------------------------------------------ +# Find those SAME IDs in sqrt-squareplus. +# ------------------------------------------------------------ + +sqrt_by_id = ( + find_same_reflections_in_model( + RUNS["sqrt-squareplus"], + selected_ids.values(), + ) +) + + +# ============================================================ +# CRITICAL BUG FIX +# +# Build sqrt_selected while preserving: +# +# 1. same refl_id +# 2. same original cctbx intensity +# 3. same Weak / Medium / Strong classification +# +# Only the Integrator prediction changes between models. +# ============================================================ + +sqrt_selected = {} + +for label, refl_id in selected_ids.items(): + + sqrt_info = ( + sqrt_by_id[ + refl_id + ].copy() + ) + + # Preserve EXACT cctbx intensity + # used for classification. + sqrt_info["cctbx_I"] = ( + asinh_selected[ + label + ]["cctbx_I"] + ) + + sqrt_selected[label] = ( + sqrt_info + ) + + +# ============================================================ +# Sanity checks +# ============================================================ + +print( + "\n=== SELECTED REFLECTIONS ===" +) + +for label in [ + "Weak", + "Medium", + "Strong", +]: + + a = asinh_selected[label] + s = sqrt_selected[label] + + validate_category( + label, + a["cctbx_I"], + ) + + # Make sure both models use the + # exact same reflection. + if ( + a["refl_id"] + != s["refl_id"] + ): + raise RuntimeError( + f"{label}: ASINH and sqrt " + "reflection IDs do not match." + ) + + # Make sure displayed cctbx intensity + # is identical in both models. + if not np.isclose( + a["cctbx_I"], + s["cctbx_I"], + ): + raise RuntimeError( + f"{label}: cctbx intensity " + "changed between models." + ) + + print( + f"{label:6s} | " + f"refl_id={a['refl_id']} | " + f"cctbx I={a['cctbx_I']:.2f} | " + f"ASINH I={a['qi_mean']:.2f} | " + f"sqrt I={s['qi_mean']:.2f} | " + f"d={a['d']:.2f} A" + ) + + +# ============================================================ +# Extra threshold check +# ============================================================ + +weak_I = ( + asinh_selected[ + "Weak" + ]["cctbx_I"] +) + +medium_I = ( + asinh_selected[ + "Medium" + ]["cctbx_I"] +) + +strong_I = ( + asinh_selected[ + "Strong" + ]["cctbx_I"] +) + +print( + "\n=== THRESHOLD CHECK ===" +) + +print( + f"Weak : {weak_I:.2f} " + "(must be < 20)" +) + +print( + f"Medium : {medium_I:.2f} " + "(must be 20-200)" +) + +print( + f"Strong : {strong_I:.2f} " + "(must be > 200)" +) + + +# ============================================================ +# Image helpers +# ============================================================ + +def get_observed(refl_id): + """ + Get the real experimental 25x25 detector shoebox. + """ + + img = np.asarray( + counts[refl_id], + dtype=float, + ).reshape( + 25, + 25, + ) + + mask = np.asarray( + masks[refl_id], + dtype=bool, + ).reshape( + 25, + 25, + ) + + return np.ma.masked_where( + ~mask, + img, + ) + + +def get_profile(info): + """ + Convert Integrator qp_mean into a 25x25 learned profile. + """ + + qp = np.asarray( + info["qp_mean"], + dtype=float, + ) + + if qp.size != 625: + raise ValueError( + "Expected qp_mean size 625, " + f"got {qp.size}" + ) + + return qp.reshape( + 25, + 25, + ) + + +# ============================================================ +# Make figure +# ============================================================ + +def make_figure( + model_name, + model_selected, +): + + fig, axes = plt.subplots( + 2, + 3, + figsize=(11, 7), + ) + + labels = [ + "Weak", + "Medium", + "Strong", + ] + + for col, label in enumerate( + labels + ): + + info = ( + model_selected[ + label + ] + ) + + refl_id = ( + info["refl_id"] + ) + + observed = get_observed( + refl_id + ) + + profile = get_profile( + info + ) + + cctbx_I = ( + info["cctbx_I"] + ) + + integrator_I = ( + info["qi_mean"] + ) + + d = info["d"] + + # ---------------------------------------------------- + # Top row: + # real experimental shoebox + # ---------------------------------------------------- + + ax = axes[ + 0, + col, + ] + + ax.imshow( + observed, + interpolation="nearest", + ) + + ax.set_title( + f"{label}\n" + f"cctbx I = {cctbx_I:.1f}", + fontsize=15, + fontweight="bold", + ) + + ax.set_xticks([]) + ax.set_yticks([]) + + # ---------------------------------------------------- + # Bottom row: + # Integrator learned profile + # ---------------------------------------------------- + + ax = axes[ + 1, + col, + ] + + ax.imshow( + profile, + interpolation="nearest", + ) + + ax.set_title( + f"Integrator I = " + f"{integrator_I:.1f}\n" + f"d = {d:.2f} A", + fontsize=14, + ) + + ax.set_xticks([]) + ax.set_yticks([]) + + + # -------------------------------------------------------- + # Row labels + # -------------------------------------------------------- + + axes[ + 0, + 0, + ].set_ylabel( + "Observed shoebox", + fontsize=16, + fontweight="bold", + labelpad=15, + ) + + axes[ + 1, + 0, + ].set_ylabel( + "Learned profile", + fontsize=16, + fontweight="bold", + labelpad=15, + ) + + + # -------------------------------------------------------- + # Main title + # -------------------------------------------------------- + + fig.suptitle( + f"{model_name}: " + "Observed Shoeboxes and Learned Profiles", + fontsize=18, + fontweight="bold", + ) + + + fig.tight_layout( + rect=[ + 0, + 0, + 1, + 0.94, + ] + ) + + + safe_name = ( + model_name.lower() + .replace( + "-", + "_", + ) + .replace( + " ", + "_", + ) + ) + + + out = ( + OUT_DIR + / ( + f"cctbx_threshold_" + f"{safe_name}_" + f"weak_medium_strong.png" + ) + ) + + + fig.savefig( + out, + dpi=300, + bbox_inches="tight", + ) + + plt.close( + fig + ) + + print( + "saved:", + out, + ) + + +# ============================================================ +# Generate ASINH and sqrt-squareplus figures +# ============================================================ + +make_figure( + "ASINH", + asinh_selected, +) + +make_figure( + "sqrt-squareplus", + sqrt_selected, +) + + +print( + "\nDone." +) + +print( + "Output folder:", + OUT_DIR, +) \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/diagnostics/plot_completeness_vs_resolution.py b/workflows/mfx101555026/scripts/diagnostics/plot_completeness_vs_resolution.py new file mode 100644 index 00000000..9313c08f --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/plot_completeness_vs_resolution.py @@ -0,0 +1,186 @@ +""" +Plot completeness vs. resolution for cctbx, sqrt_squareplus, and asinh. + +Data source: merging_refinement_results.txt + - sqrt_squareplus: lines 383-407 + - asinh: lines 800-824 + - cctbx: lines 1222-1245 + +Completeness = [observed unique / total possible] per resolution shell. + +NOTE: The Integrator is NOT 100% complete in all bins. + - Low-to-mid resolution (bins 1-10, > 2.52 A): cctbx is near-complete + (~90-100%); Integrator drops to ~55-85%. + - High resolution (bins 12-20, < 2.44 A): cctbx drops sharply to 4%; + Integrator maintains 33-58%. + This is the key comparison — not a simple "Integrator = 100%" story. +""" + +import matplotlib.pyplot as plt +import numpy as np + +# ── Resolution shell upper bounds (d_min of each shell, Angstroms) ────────── +# Bins run from low resolution (bin 1, ~5.4 A) to high resolution (bin 20, 2.0 A). +# X-axis is inverted so high resolution appears on the right. +resolution = [ + 5.4288, # bin 1 + 4.3089, # bin 2 + 3.7641, # bin 3 + 3.4200, # bin 4 + 3.1748, # bin 5 + 2.9876, # bin 6 + 2.8380, # bin 7 + 2.7144, # bin 8 + 2.6099, # bin 9 + 2.5198, # bin 10 + 2.4410, # bin 11 + 2.3713, # bin 12 + 2.3088, # bin 13 + 2.2525, # bin 14 + 2.2013, # bin 15 + 2.1544, # bin 16 + 2.1113, # bin 17 + 2.0715, # bin 18 + 2.0345, # bin 19 + 2.0000, # bin 20 +] + +# ── Completeness (%) — verified from merging_refinement_results.txt ───────── + +# sqrt_squareplus (lines 383-407) +sqrt_squareplus = [ + 99.3, + 98.6, + 97.6, + 94.9, + 84.2, + 77.6, + 65.7, + 58.7, + 56.5, + 55.8, + 53.8, + 58.0, + 51.2, + 58.1, + 51.3, + 45.8, + 57.4, + 33.5, + 47.7, + 55.1, +] + +# asinh (lines 800-824) +asinh = [ + 99.3, + 98.5, + 97.6, + 94.6, + 85.5, + 76.4, + 67.5, + 55.4, + 52.5, + 55.6, + 51.2, + 50.0, + 56.5, + 54.2, + 52.5, + 52.2, + 54.5, + 49.6, + 57.2, + 47.3, +] + +# cctbx baseline (lines 1222-1245) +cctbx = [ + 99.8, + 99.7, + 99.6, + 99.4, + 99.1, + 98.7, + 98.0, + 95.3, + 93.5, + 90.6, + 81.9, + 71.0, + 52.1, + 44.9, + 32.9, + 23.4, + 15.6, + 5.8, + 9.2, + 4.1, +] + +# ── Plot ───────────────────────────────────────────────────────────────────── +fig, ax = plt.subplots(figsize=(8, 5.5)) + +ax.plot( + resolution, + cctbx, + marker="o", + linewidth=2.5, + color="#2166ac", + label="cctbx.xfel", +) + +ax.plot( + resolution, + sqrt_squareplus, + marker="s", + linewidth=2.5, + color="#d6604d", + label="Integrator — sqrt-squareplus", +) + +ax.plot( + resolution, + asinh, + marker="^", + linestyle="--", + linewidth=2.5, + color="#f4a582", + label="Integrator — asinh", +) + +# Shade the high-resolution region where the key difference is visible +ax.axvspan( + 2.0, 2.44, alpha=0.07, color="gray", label="High-res region (< 2.44 Å)" +) + +ax.set_xlabel("Resolution (Å)", fontsize=13) +ax.set_ylabel("Completeness (%)", fontsize=13) +ax.set_title("Completeness vs. Resolution", fontsize=15) + +ax.set_ylim(0, 105) +ax.invert_xaxis() # crystallography convention: high resolution on the right + +ax.grid(alpha=0.25) +ax.legend(frameon=False, fontsize=10) + +fig.tight_layout() + +fig.savefig("completeness_vs_resolution.png", dpi=300, bbox_inches="tight") +fig.savefig("completeness_vs_resolution.pdf", bbox_inches="tight") + +plt.show() + +print("Saved: completeness_vs_resolution.png / .pdf") +print() +print("Key values at high resolution (< 2.44 A):") +print(f"{'Bin':>4} {'d_min':>6} {'cctbx':>8} {'sqrt_sq':>10} {'asinh':>8}") +print("-" * 50) +for i in range(11, 20): + print( + f"{i + 1:>4} {resolution[i]:>6.4f} " + f"{cctbx[i]:>7.1f}% " + f"{sqrt_squareplus[i]:>9.1f}% " + f"{asinh[i]:>7.1f}%" + ) diff --git a/workflows/mfx101555026/scripts/diagnostics/plot_loss_history.py b/workflows/mfx101555026/scripts/diagnostics/plot_loss_history.py new file mode 100644 index 00000000..2040b5d9 --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/plot_loss_history.py @@ -0,0 +1,38 @@ +from pathlib import Path + +import pandas as pd +import matplotlib.pyplot as plt + + +# Fill this in with the path to loss_history.csv +input_path = "" + +if not input_path: + raise ValueError("Please set input_path to your loss_history.csv file.") + +csv_path = Path(input_path) +df = pd.read_csv(csv_path) + +print("Loaded:", csv_path) +print("Columns:", df.columns.tolist()) + +required_cols = ["epoch", "train_loss", "val_loss"] +missing = [col for col in required_cols if col not in df.columns] + +if missing: + raise ValueError(f"Missing required columns: {missing}") + +plt.figure() +plt.plot(df["epoch"], df["train_loss"], marker="o", label="train_loss") +plt.plot(df["epoch"], df["val_loss"], marker="o", label="val_loss") + +plt.xlabel("Epoch") +plt.ylabel("Loss") +plt.title("Training and validation loss vs epoch") +plt.legend() +plt.tight_layout() + +out_path = csv_path.with_name("train_val_loss_vs_epoch.png") +plt.savefig(out_path, dpi=200) + +print("Saved:", out_path) diff --git a/workflows/mfx101555026/scripts/diagnostics/plot_mfx_negative_pixels.py b/workflows/mfx101555026/scripts/diagnostics/plot_mfx_negative_pixels.py new file mode 100644 index 00000000..431b2075 --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/plot_mfx_negative_pixels.py @@ -0,0 +1,109 @@ +from pathlib import Path +import numpy as np +import matplotlib.pyplot as plt + +# Change this to the dataset you want to inspect +DATASET_DIR = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/" + "mfx101555026_cctbx/mfx_shoebox_stream_test_25" +) + +OUT_DIR = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/" + "mfx101555026_cctbx/figures" +) +OUT_DIR.mkdir(parents=True, exist_ok=True) + +counts_path = DATASET_DIR / "counts.npy" +masks_path = DATASET_DIR / "masks.npy" + +counts = np.load(counts_path, mmap_mode="r") +masks = np.load(masks_path, mmap_mode="r") + +print("counts path:", counts_path) +print("masks path:", masks_path) +print("counts shape:", counts.shape) +print("masks shape:", masks.shape) +print("counts dtype:", counts.dtype) +print("masks dtype:", masks.dtype) + +# Collect valid pixels without loading everything at once +chunk_size = 1000 +sample_values = [] + +valid_count = 0 +negative_count = 0 +below_anscombe_count = 0 +valid_min = np.inf +valid_max = -np.inf + +# Limit histogram sample size so plotting stays fast +max_sample_pixels = 2_000_000 +rng = np.random.default_rng(42) + +for start in range(0, counts.shape[0], chunk_size): + end = min(start + chunk_size, counts.shape[0]) + + c = counts[start:end] + m = masks[start:end].astype(bool) + + valid = c[m] + + if valid.size == 0: + continue + + valid_count += valid.size + negative_count += np.count_nonzero(valid < 0) + below_anscombe_count += np.count_nonzero(valid < -0.375) + valid_min = min(valid_min, float(valid.min())) + valid_max = max(valid_max, float(valid.max())) + + # Randomly sample pixels for histogram + remaining = max_sample_pixels - sum(x.size for x in sample_values) + if remaining > 0: + if valid.size <= remaining: + sample_values.append(np.asarray(valid)) + else: + idx = rng.choice(valid.size, size=remaining, replace=False) + sample_values.append(np.asarray(valid[idx])) + +sample = np.concatenate(sample_values) + +frac_negative = negative_count / valid_count +frac_below_anscombe = below_anscombe_count / valid_count + +print("\nValid pixel statistics") +print("valid pixel count:", valid_count) +print("min valid pixel:", valid_min) +print("max valid pixel:", valid_max) +print("fraction valid pixels < 0:", frac_negative) +print("fraction valid pixels < -0.375:", frac_below_anscombe) + +# Save text summary for slide notes +summary_path = OUT_DIR / "mfx_negative_pixel_summary.txt" +with open(summary_path, "w") as f: + f.write(f"Dataset: {DATASET_DIR}\n") + f.write(f"counts dtype: {counts.dtype}\n") + f.write(f"counts shape: {counts.shape}\n") + f.write(f"valid pixel count: {valid_count}\n") + f.write(f"min valid pixel: {valid_min}\n") + f.write(f"max valid pixel: {valid_max}\n") + f.write(f"fraction valid pixels < 0: {frac_negative}\n") + f.write(f"fraction valid pixels < -0.375: {frac_below_anscombe}\n") + +# Histogram zoomed around the negative/low-value region +fig_path = OUT_DIR / "mfx_valid_pixel_hist_anscombe_threshold.png" + +plt.figure(figsize=(8, 5)) +plt.hist(sample, bins=300, range=(-10, 50), log=True) +plt.axvline(-0.375, linestyle="--", linewidth=2, label="Anscombe threshold: -0.375") +plt.xlabel("valid pixel value") +plt.ylabel("count (log scale)") +plt.title("MFX valid pixel values include negative floats") +plt.legend() +plt.tight_layout() +plt.savefig(fig_path, dpi=200) + +print("\nSaved:") +print("summary:", summary_path) +print("figure:", fig_path) \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/diagnostics/plot_qi_by_strength.py b/workflows/mfx101555026/scripts/diagnostics/plot_qi_by_strength.py new file mode 100644 index 00000000..4251b12b --- /dev/null +++ b/workflows/mfx101555026/scripts/diagnostics/plot_qi_by_strength.py @@ -0,0 +1,353 @@ +#!/usr/bin/env python3 +""" +plot_qi_by_strength.py — qi distribution stratified by reflection strength. + +Bins reflections into weak / medium / strong using cctbx intensity percentiles +computed from intensity.sum.value: + weak : I < p10 (bottom 10th percentile) + medium : p10 ≤ I < p90 + strong : I ≥ p90 (top 10th percentile) + +The p10/p50/p90 boundaries are derived from the cctbx column +intensity.sum.value written into the prediction parquet by integrator.predict. + +Usage +----- +python plot_qi_by_strength.py \\ + --run-dir $RUN_DIR \\ + --epoch 24 \\ + --out-dir /tmp/qi_by_strength \\ + [--p-weak 10] [--p-strong 90] [--n-profile 625] [--dpi 150] +""" + +from __future__ import annotations + +import argparse +import logging +from pathlib import Path + +import numpy as np +import polars as pl +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser( + description=( + "Plot qi distribution stratified by weak/medium/strong " + "using cctbx intensity percentiles (p10/p50/p90)." + ), + ) + p.add_argument( + "--run-dir", + required=True, + type=Path, + help="integrator run directory (contains files/predictions/)", + ) + p.add_argument( + "--epoch", + type=int, + default=None, + help="Epoch index (e.g. 24). Defaults to the latest epoch found.", + ) + p.add_argument( + "--out-dir", + type=Path, + default=None, + help="Output directory for PNG files. Defaults to /diagnostics/", + ) + p.add_argument( + "--p-weak", + type=int, + default=10, + help="Percentile upper bound for 'weak' category (default: 10 → p10)", + ) + p.add_argument( + "--p-strong", + type=int, + default=90, + help="Percentile lower bound for 'strong' category (default: 90 → p90)", + ) + p.add_argument( + "--n-profile", + type=int, + default=625, + help="Number of pixels in the flattened profile (default: 625 = 25×25)", + ) + p.add_argument( + "--max-rows", + type=int, + default=500_000, + help="Max rows to sample per category for speed (default: 500 000)", + ) + p.add_argument("--dpi", type=int, default=150) + p.add_argument("-v", "--verbose", action="store_true") + return p.parse_args() + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _find_pred_dir(run_dir: Path, epoch: int | None) -> Path: + pred_root = run_dir / "files" / "predictions" + if not pred_root.exists(): + raise FileNotFoundError( + f"No predictions directory found at {pred_root}" + ) + if epoch is not None: + d = pred_root / f"epoch_{epoch:04d}" + if not d.exists(): + raise FileNotFoundError(f"Epoch directory not found: {d}") + return d + # find latest epoch_XXXX directory + epoch_dirs = sorted(pred_root.glob("epoch_*")) + if not epoch_dirs: + raise FileNotFoundError(f"No epoch_* directories found in {pred_root}") + return epoch_dirs[-1] + + +def _load_predictions(pred_dir: Path, max_rows: int) -> pl.DataFrame: + parquet_files = sorted(pred_dir.glob("*.parquet")) + if not parquet_files: + raise FileNotFoundError(f"No parquet files found in {pred_dir}") + logger.info( + "Scanning %d parquet files from %s", len(parquet_files), pred_dir + ) + + needed = [ + "qi_mean", + "qi_var", + "intensity.sum.value", + "intensity.sum.variance", + "d", + ] + # also load qp_mean columns if present (flattened profile) + lf = pl.scan_parquet(parquet_files) + schema_cols = lf.collect_schema().names() + qp_cols = [c for c in schema_cols if c.startswith("qp_mean")] + select_cols = [c for c in needed if c in schema_cols] + qp_cols + + df = lf.select(select_cols).collect() + logger.info("Loaded %d rows, %d columns", len(df), len(df.columns)) + + if len(df) > max_rows: + logger.info("Sampling %d rows from %d total", max_rows, len(df)) + df = df.sample(max_rows, seed=42) + + return df + + +def _bin_by_strength( + df: pl.DataFrame, + p_weak: int, + p_strong: int, +) -> tuple[dict[str, np.ndarray], float, float]: + """Bin by cctbx intensity percentiles. + + Uses intensity.sum.value directly (cctbx column). + Returns (bins dict, p_weak boundary, p_strong boundary). + """ + I = df["intensity.sum.value"].to_numpy().astype(np.float64) + lo = float(np.percentile(I, p_weak)) + hi = float(np.percentile(I, p_strong)) + bins = { + f"weak (I < p{p_weak} = {lo:.1f})": np.where(I < lo)[0], + f"medium (p{p_weak} ≤ I < p{p_strong})": np.where( + (I >= lo) & (I < hi) + )[0], + f"strong (I ≥ p{p_strong} = {hi:.1f})": np.where(I >= hi)[0], + } + return bins, lo, hi + + +# --------------------------------------------------------------------------- +# Plots +# --------------------------------------------------------------------------- + + +def plot_qi_distributions( + df: pl.DataFrame, + bins: dict[str, np.ndarray], + out_dir: Path, + dpi: int, +) -> None: + """Histogram of qi_mean for each strength category on the same axes.""" + qi = df["qi_mean"].to_numpy().astype(np.float32) + + fig, ax = plt.subplots(figsize=(8, 5)) + colors = ["steelblue", "darkorange", "seagreen"] + + for (label, idx), color in zip(bins.items(), colors): + if len(idx) == 0: + logger.warning("No reflections in category: %s", label) + continue + vals = qi[idx] + ax.hist( + vals, + bins=100, + range=(0, np.percentile(qi, 99)), + density=True, + alpha=0.55, + label=f"{label} (n={len(idx):,})", + color=color, + ) + + ax.set_xlabel("qi_mean (predicted intensity)", fontsize=12) + ax.set_ylabel("Density", fontsize=12) + ax.set_title( + "qi distribution by reflection strength (cctbx intensity percentiles)", + fontsize=13, + ) + ax.legend(fontsize=9) + fig.tight_layout() + out = out_dir / "qi_by_strength.png" + fig.savefig(out, dpi=dpi) + plt.close(fig) + logger.info("Saved: %s", out) + + +def plot_mean_profiles( + df: pl.DataFrame, + bins: dict[str, np.ndarray], + n_profile: int, + out_dir: Path, + dpi: int, +) -> None: + """Mean learned spot profile (qp_mean) for each strength category.""" + qp_cols = [c for c in df.columns if c.startswith("qp_mean")] + if len(qp_cols) != n_profile: + logger.warning( + "Expected %d qp_mean columns but found %d — skipping profile plot.", + n_profile, + len(qp_cols), + ) + return + + qp = df.select(qp_cols).to_numpy().astype(np.float32) # (N, pixels) + h = w = int(np.sqrt(n_profile)) + colors = ["steelblue", "darkorange", "seagreen"] + + fig, axes = plt.subplots(1, 3, figsize=(12, 4)) + for ax, (label, idx), color in zip(axes, bins.items(), colors): + if len(idx) == 0: + ax.set_title(f"{label}\n(no data)") + ax.axis("off") + continue + mean_profile = qp[idx].mean(axis=0).reshape(h, w) + im = ax.imshow(mean_profile, cmap="inferno", origin="lower") + ax.set_title(f"{label}\nn={len(idx):,}", fontsize=9) + ax.axis("off") + plt.colorbar(im, ax=ax, fraction=0.046, pad=0.04) + + fig.suptitle( + "Mean learned spot profile (qp_mean) by reflection strength", + fontsize=12, + ) + fig.tight_layout() + out = out_dir / "mean_profile_by_strength.png" + fig.savefig(out, dpi=dpi) + plt.close(fig) + logger.info("Saved: %s", out) + + +def plot_qi_vs_d( + df: pl.DataFrame, + bins: dict[str, np.ndarray], + out_dir: Path, + dpi: int, +) -> None: + """Scatter / 2D hex of qi_mean vs resolution d for each category.""" + if "d" not in df.columns: + return + qi = df["qi_mean"].to_numpy().astype(np.float32) + d = df["d"].to_numpy().astype(np.float32) + colors = ["steelblue", "darkorange", "seagreen"] + + fig, axes = plt.subplots(1, 3, figsize=(14, 4), sharey=True) + q99 = np.percentile(qi, 99) + for ax, (label, idx), color in zip(axes, bins.items(), colors): + if len(idx) == 0: + ax.set_title(f"{label}\n(no data)") + continue + sample = np.random.default_rng(0).choice( + idx, min(len(idx), 50_000), replace=False + ) + ax.hexbin( + d[sample], + qi[sample], + gridsize=60, + extent=(d.min(), d.max(), 0, q99), + cmap="Blues", + mincnt=1, + ) + ax.set_xlabel("d (Å)", fontsize=10) + ax.set_title(f"{label}\nn={len(idx):,}", fontsize=9) + axes[0].set_ylabel("qi_mean", fontsize=10) + fig.suptitle("qi_mean vs resolution by reflection strength", fontsize=12) + fig.tight_layout() + out = out_dir / "qi_vs_d_by_strength.png" + fig.savefig(out, dpi=dpi) + plt.close(fig) + logger.info("Saved: %s", out) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def main() -> None: + args = parse_args() + logging.basicConfig( + level=logging.DEBUG if args.verbose else logging.INFO, + format="%(levelname)s %(message)s", + ) + + out_dir = args.out_dir or (args.run_dir / "diagnostics") + out_dir.mkdir(parents=True, exist_ok=True) + + pred_dir = _find_pred_dir(args.run_dir, args.epoch) + logger.info("Prediction dir: %s", pred_dir) + + df = _load_predictions(pred_dir, args.max_rows) + + for col in ("intensity.sum.value", "qi_mean"): + if col not in df.columns: + raise RuntimeError( + f"Required column '{col}' not found in predictions.\n" + "Make sure the training YAML includes it in predict_keys." + ) + + bins, lo, hi = _bin_by_strength(df, args.p_weak, args.p_strong) + logger.info( + "Intensity percentile boundaries: p%d=%.2f p%d=%.2f", + args.p_weak, + lo, + args.p_strong, + hi, + ) + + for label, idx in bins.items(): + logger.info(" %-50s %8d reflections", label, len(idx)) + + plot_qi_distributions(df, bins, out_dir, args.dpi) + plot_mean_profiles(df, bins, args.n_profile, out_dir, args.dpi) + plot_qi_vs_d(df, bins, out_dir, args.dpi) + + print(f"\nOutputs written to: {out_dir}") + + +if __name__ == "__main__": + main() diff --git a/workflows/mfx101555026/scripts/extraction/make_shoeboxes_allruns_024_rg101_16array.sh b/workflows/mfx101555026/scripts/extraction/make_shoeboxes_allruns_024_rg101_16array.sh new file mode 100644 index 00000000..07a4c79b --- /dev/null +++ b/workflows/mfx101555026/scripts/extraction/make_shoeboxes_allruns_024_rg101_16array.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=32 +#SBATCH --mem=256G +#SBATCH --time=0-8:00 +#SBATCH --job-name=mfx_all_box +#SBATCH --array=0-15 +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx_all_box_%A_%a.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/mfx_all_box_%A_%a.err + +set -eo pipefail + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + +# Avoid "Too many open files" errors from psana2/dxtbx. +ulimit -n 65535 +echo "open file limit: $(ulimit -n)" + +export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH + +cd /sdf/home/t/thaoh/s3df_practice/integrator + +BASE=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx +DATA_DIR=$BASE/all_runs_269_289_no275_024_rg101_integrated +OUT_BASE=$BASE/mfx_shoebox_allruns_269_289_no275_024_rg101_chunks +MASK=$BASE/masks/r0269_pam_stddev_border2_combined.mask + +mkdir -p "$OUT_BASE" +mkdir -p "$BASE/logs" + +TOTAL_FILES=124975 +N_TASKS=16 + +TASK_ID=${SLURM_ARRAY_TASK_ID} + +# Ceiling division: files per task. +FILES_PER_TASK=$(( (TOTAL_FILES + N_TASKS - 1) / N_TASKS )) + +START=$(( TASK_ID * FILES_PER_TASK )) +END=$(( START + FILES_PER_TASK )) + +if [ "$END" -gt "$TOTAL_FILES" ]; then + END=$TOTAL_FILES +fi + +COUNT=$(( END - START )) + +OUT_DIR=$OUT_BASE/chunk_${TASK_ID}_start_${START}_n_${COUNT} + +echo "SLURM_ARRAY_TASK_ID=$TASK_ID" +echo "TOTAL_FILES=$TOTAL_FILES" +echo "N_TASKS=$N_TASKS" +echo "FILES_PER_TASK=$FILES_PER_TASK" +echo "START=$START" +echo "END=$END" +echo "COUNT=$COUNT" +echo "DATA_DIR=$DATA_DIR" +echo "OUT_DIR=$OUT_DIR" +echo "MASK=$MASK" + +if [ "$COUNT" -le 0 ]; then + echo "No files assigned to this task. Exiting." + exit 0 +fi + +dials.python src/integrator/cli/make_shoeboxes.py \ + --mfx \ + --data-dir "$DATA_DIR" \ + --out-dir "$OUT_DIR" \ + --mfx-pattern "*_integrated.refl" \ + --start-file "$START" \ + --max-files "$COUNT" \ + --detector-mask "$MASK" \ + --w 25 \ + --h 25 \ + --d 1 \ + --counts-dtype float32 \ + --write-chunk-size 10000 \ + --no-mask-overlap + +echo "Done: $OUT_DIR" diff --git a/workflows/mfx101555026/scripts/extraction/merge_allruns_024_rg101_shoebox_chunks.py b/workflows/mfx101555026/scripts/extraction/merge_allruns_024_rg101_shoebox_chunks.py new file mode 100644 index 00000000..3f620370 --- /dev/null +++ b/workflows/mfx101555026/scripts/extraction/merge_allruns_024_rg101_shoebox_chunks.py @@ -0,0 +1,477 @@ +from pathlib import Path + +import numpy as np +import yaml +from numpy.lib.format import open_memmap + + +BASE = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/" + "mfx101555026_cctbx" +) + +CHUNKS_DIR = ( + BASE / "mfx_shoebox_5000_269_289_no275_024_rg101_chunks" +) + +OUT = ( + BASE / "mfx_shoebox_5000_269_289_no275_024_rg101" +) + +OUT.mkdir(parents=True, exist_ok=True) + + +def chunk_number(path: Path) -> int: + """Extract the Slurm task number from names such as chunk_3_start_939_n_313.""" + return int(path.name.split("_")[1]) + + +# Sort chunks numerically: chunk_0, chunk_1, ..., chunk_15. +DATASET_DIRS = sorted( + CHUNKS_DIR.glob("chunk_*"), + key=chunk_number, +) + +good_dirs = [] + +for dataset_dir in DATASET_DIRS: + needed = [ + dataset_dir / "counts.npy", + dataset_dir / "masks.npy", + dataset_dir / "metadata.npy", + ] + + if all(path.exists() for path in needed): + good_dirs.append(dataset_dir) + else: + print(f"SKIP missing final files: {dataset_dir}") + +if not good_dirs: + raise SystemExit( + f"No completed dataset folders found in {CHUNKS_DIR}" + ) + +print("Datasets to merge:") + +for dataset_dir in good_dirs: + print(" ", dataset_dir) + +if len(good_dirs) != 16: + print( + f"WARNING: expected 16 completed chunks, " + f"but found {len(good_dirs)}" + ) + +############################################################################### +# Inspect chunk shapes and metadata +############################################################################### + +total_rows = 0 +n_pixels = None +counts_dtype = None +all_meta = [] + +for dataset_dir in good_dirs: + counts = np.load( + dataset_dir / "counts.npy", + mmap_mode="r", + ) + + masks = np.load( + dataset_dir / "masks.npy", + mmap_mode="r", + ) + + metadata = np.load( + dataset_dir / "metadata.npy", + allow_pickle=True, + ).item() + + print( + dataset_dir.name, + "counts:", + counts.shape, + "masks:", + masks.shape, + ) + + if counts.ndim != 2: + raise ValueError( + f"Expected 2D counts array in {dataset_dir}, " + f"got shape {counts.shape}" + ) + + if masks.shape != counts.shape: + raise ValueError( + f"Mask/count shape mismatch in {dataset_dir}: " + f"{masks.shape} versus {counts.shape}" + ) + + if n_pixels is None: + n_pixels = counts.shape[1] + counts_dtype = counts.dtype + elif counts.shape[1] != n_pixels: + raise ValueError( + f"Pixel-count mismatch in {dataset_dir}: " + f"{counts.shape[1]} versus {n_pixels}" + ) + + n_rows = counts.shape[0] + + for key, value in metadata.items(): + try: + value_length = len(value) + except TypeError: + continue + + if value_length != n_rows: + raise ValueError( + f"Metadata key {key!r} in {dataset_dir} has " + f"{value_length} rows, but counts has {n_rows}" + ) + + total_rows += n_rows + all_meta.append(metadata) + +print("Total rows:", total_rows) +print("Pixels per shoebox:", n_pixels) +print("Counts dtype:", counts_dtype) + +############################################################################### +# Create merged counts and masks arrays on disk +############################################################################### + +counts_out = open_memmap( + OUT / "counts.npy", + mode="w+", + dtype=counts_dtype, + shape=(total_rows, n_pixels), +) + +masks_out = open_memmap( + OUT / "masks.npy", + mode="w+", + dtype=np.bool_, + shape=(total_rows, n_pixels), +) + +row0 = 0 + +for dataset_dir in good_dirs: + counts = np.load( + dataset_dir / "counts.npy", + mmap_mode="r", + ) + + masks = np.load( + dataset_dir / "masks.npy", + mmap_mode="r", + ) + + row1 = row0 + counts.shape[0] + + print( + f"Copying {dataset_dir.name}: " + f"rows {row0}:{row1}" + ) + + counts_out[row0:row1] = counts + masks_out[row0:row1] = masks + + row0 = row1 + +counts_out.flush() +masks_out.flush() + +del counts_out +del masks_out + +############################################################################### +# Merge metadata +############################################################################### + +common_keys = set(all_meta[0].keys()) + +for metadata in all_meta[1:]: + common_keys &= set(metadata.keys()) + +common_keys = sorted(common_keys) + +print("Common metadata keys:") +for key in common_keys: + print(" ", key) + +merged_meta = {} + +for key in common_keys: + arrays = [ + np.asarray(metadata[key]) + for metadata in all_meta + ] + + merged_meta[key] = np.concatenate( + arrays, + axis=0, + ) + +# Give every merged reflection a unique global row ID. +merged_meta["refl_ids"] = np.arange( + total_rows, + dtype=np.int64, +) + +############################################################################### +# Rebuild global image_id and n_images +############################################################################### + +if "image_index" in merged_meta: + image_key = "image_index" +elif "image_num" in merged_meta: + image_key = "image_num" +else: + raise KeyError( + "Metadata has neither image_index nor image_num" + ) + +image_num = np.asarray( + merged_meta[image_key], + dtype=np.int64, +) + +unique_images = np.unique(image_num) + +image_to_id = { + image_number: image_id + for image_id, image_number in enumerate(unique_images) +} + +merged_meta["image_num"] = image_num + +merged_meta["image_id"] = np.fromiter( + (image_to_id[value] for value in image_num), + dtype=np.int64, + count=total_rows, +) + +merged_meta["n_images"] = np.full( + total_rows, + len(unique_images), + dtype=np.int64, +) + +print("Global n_images:", len(unique_images)) +print( + "image_id min/max:", + merged_meta["image_id"].min(), + merged_meta["image_id"].max(), +) + +np.save( + OUT / "metadata.npy", + merged_meta, +) + +############################################################################### +# Compute raw pixel statistics in chunks +############################################################################### + +counts = np.load( + OUT / "counts.npy", + mmap_mode="r", +) + +masks = np.load( + OUT / "masks.npy", + mmap_mode="r", +) + +sum_counts = 0.0 +sum_squared_counts = 0.0 +number_of_valid_pixels = 0 + +ROW_CHUNK_SIZE = 10000 + +for start in range( + 0, + total_rows, + ROW_CHUNK_SIZE, +): + end = min( + start + ROW_CHUNK_SIZE, + total_rows, + ) + + chunk_counts = counts[start:end].astype( + np.float64, + copy=False, + ) + + chunk_masks = masks[start:end].astype( + bool, + copy=False, + ) + + valid_counts = chunk_counts[chunk_masks] + + sum_counts += valid_counts.sum( + dtype=np.float64, + ) + + sum_squared_counts += np.square( + valid_counts, + dtype=np.float64, + ).sum( + dtype=np.float64, + ) + + number_of_valid_pixels += valid_counts.size + +if number_of_valid_pixels == 0: + raise RuntimeError( + "No valid pixels found while computing statistics" + ) + +mean_counts = ( + sum_counts / number_of_valid_pixels +) + +variance_counts = ( + sum_squared_counts / number_of_valid_pixels + - mean_counts * mean_counts +) + +# Protect against a tiny negative number caused by floating-point rounding. +variance_counts = max( + variance_counts, + 0.0, +) + +stats = { + "raw": [ + float(mean_counts), + float(variance_counts), + ] +} + +print("Raw stats:", stats) + +############################################################################### +# Compute concentration in chunks +############################################################################### + +concentration = open_memmap( + OUT / "concentration.npy", + mode="w+", + dtype=np.float32, + shape=(total_rows,), +) + +for start in range( + 0, + total_rows, + ROW_CHUNK_SIZE, +): + end = min( + start + ROW_CHUNK_SIZE, + total_rows, + ) + + chunk_counts = counts[start:end].astype( + np.float32, + copy=False, + ) + + chunk_masks = masks[start:end] + + concentration[start:end] = ( + chunk_counts * chunk_masks + ).sum( + axis=1, + dtype=np.float32, + ) + +concentration.flush() +del concentration + +############################################################################### +# Write dataset.yaml +############################################################################### + +dataset_yaml = { + "geometry": { + "d": 1, + "h": 25, + "w": 25, + }, + "n_reflections": int(total_rows), + "polychromatic": False, + "anscombe": False, + "files": { + "counts": "counts.npy", + "masks": "masks.npy", + "reference": "metadata.npy", + }, + "crystal": None, + "stats": stats, + "refl_file": None, +} + +with open( + OUT / "dataset.yaml", + "w", + encoding="utf-8", +) as stream: + yaml.safe_dump( + dataset_yaml, + stream, + sort_keys=False, + ) + +############################################################################### +# Final validation +############################################################################### + +final_counts = np.load( + OUT / "counts.npy", + mmap_mode="r", +) + +final_masks = np.load( + OUT / "masks.npy", + mmap_mode="r", +) + +final_metadata = np.load( + OUT / "metadata.npy", + allow_pickle=True, +).item() + +if final_counts.shape != final_masks.shape: + raise RuntimeError( + "Final counts and masks shapes do not match" + ) + +if final_counts.shape[0] != total_rows: + raise RuntimeError( + "Final counts row count is incorrect" + ) + +if len(final_metadata["refl_ids"]) != total_rows: + raise RuntimeError( + "Final metadata row count is incorrect" + ) + +print() +print("Merge completed successfully") +print("Output directory:", OUT) +print("Counts shape:", final_counts.shape) +print("Masks shape:", final_masks.shape) +print("Metadata rows:", len(final_metadata["refl_ids"])) +print("Unique images:", len(unique_images)) +print() +print("Wrote:") +print(" counts.npy") +print(" masks.npy") +print(" metadata.npy") +print(" concentration.npy") +print(" dataset.yaml") \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/extraction/merge_allruns_024_rg101_shoebox_chunks.sbatch b/workflows/mfx101555026/scripts/extraction/merge_allruns_024_rg101_shoebox_chunks.sbatch new file mode 100644 index 00000000..558f37a9 --- /dev/null +++ b/workflows/mfx101555026/scripts/extraction/merge_allruns_024_rg101_shoebox_chunks.sbatch @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +#SBATCH --account=lcls:prjlumine22 +#SBATCH --partition=milano +#SBATCH --qos=normal +#SBATCH --constraint=OS_VER:8.6 +#SBATCH --nodes=1 +#SBATCH --cpus-per-task=8 +#SBATCH --mem=480G +#SBATCH --time=0-8:00 +#SBATCH --job-name=merge_mfx_boxes +#SBATCH --output=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/merge_allruns_024_rg101_%j.out +#SBATCH --error=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx/logs/merge_allruns_024_rg101_%j.err + +set -eo pipefail + +source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh +export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH + +cd /sdf/home/t/thaoh/s3df_practice/integrator + +time dials.python workflows/mfx101555026/scripts/merge_allruns_024_rg101_shoebox_chunks.py diff --git a/workflows/mfx101555026/scripts/writeback/diagnose_refl_ids.py b/workflows/mfx101555026/scripts/writeback/diagnose_refl_ids.py new file mode 100644 index 00000000..c1a5be6c --- /dev/null +++ b/workflows/mfx101555026/scripts/writeback/diagnose_refl_ids.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python +""" +Diagnostic: check why prediction refl_ids do not match metadata refl_ids. + +Run in integrator-cuda-dev (no DIALS needed): + + python diagnose_refl_ids.py + +Paths are hard-coded for run_20260805-011728_9704 (asinh, epoch 24). +""" + +from __future__ import annotations + +from pathlib import Path + +import numpy as np + +BASE = Path( + "/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/mfx101555026_cctbx" +) +RUN_DIR = BASE / "runs/run_20260805-011728_9704" +DATA_DIR = BASE / "mfx_shoebox_allruns_269_289_no275_024_rg101" + +NPZ_PATH = RUN_DIR / "predictions/preds_epoch_0024.npz" +META_PATH = DATA_DIR / "metadata.npy" + + +def main(): + # ── Load predictions ────────────────────────────────────────────────────── + print(f"Loading NPZ: {NPZ_PATH}") + pred = np.load(NPZ_PATH, allow_pickle=False) + pred_ids_raw = pred["refl_ids"] + + # ── Load metadata ───────────────────────────────────────────────────────── + print(f"Loading metadata: {META_PATH}") + meta = np.load(META_PATH, allow_pickle=True).item() + meta_ids_raw = np.asarray(meta["refl_ids"]) + + # ── Basic info ──────────────────────────────────────────────────────────── + print() + print("=" * 60) + print("BASIC INFO") + print("=" * 60) + print(f"pred refl_ids dtype : {pred_ids_raw.dtype}") + print(f"meta refl_ids dtype : {meta_ids_raw.dtype}") + print(f"pred count : {len(pred_ids_raw):,}") + print(f"meta count : {len(meta_ids_raw):,}") + print( + f"pred range : {pred_ids_raw.min()} .. {pred_ids_raw.max()}" + ) + print( + f"meta range : {meta_ids_raw.min()} .. {meta_ids_raw.max()}" + ) + + # ── Float32 precision check ─────────────────────────────────────────────── + print() + print("=" * 60) + print("FLOAT32 PRECISION CHECK") + print("=" * 60) + FLOAT32_EXACT_MAX = 2**23 # 8,388,608 + pred_max = float(pred_ids_raw.max()) + if pred_max > FLOAT32_EXACT_MAX: + print( + f"WARNING: pred refl_ids max ({pred_max:.0f}) > float32 exact range " + f"({FLOAT32_EXACT_MAX:,})" + ) + print(" Large refl_ids stored as float32 will lose precision.") + else: + print( + f"pred refl_ids max ({pred_max:.0f}) is within float32 exact range — no precision loss." + ) + + # ── Duplicate check ─────────────────────────────────────────────────────── + print() + print("=" * 60) + print("DUPLICATE CHECK") + print("=" * 60) + pred_int = pred_ids_raw.astype(np.int64) + meta_int = meta_ids_raw.astype(np.int64) + + n_unique_pred = len(np.unique(pred_int)) + n_unique_meta = len(np.unique(meta_int)) + print( + f"pred unique count : {n_unique_pred:,} (total: {len(pred_int):,})" + ) + print( + f"meta unique count : {n_unique_meta:,} (total: {len(meta_int):,})" + ) + + if n_unique_pred < len(pred_int): + n_dups = len(pred_int) - n_unique_pred + print( + f"PROBLEM: {n_dups:,} duplicate pred refl_ids — caused by float32 precision loss" + ) + else: + print("No duplicates in pred refl_ids.") + + # ── Mismatch check ──────────────────────────────────────────────────────── + print() + print("=" * 60) + print("MISMATCH CHECK") + print("=" * 60) + meta_sorted = np.sort(meta_int) + pred_sorted = np.sort(pred_int) + + found = np.searchsorted(meta_sorted, pred_sorted) + found = np.clip(found, 0, len(meta_sorted) - 1) + bad_mask = meta_sorted[found] != pred_sorted + n_bad = int(bad_mask.sum()) + print(f"mismatching pred_ids : {n_bad:,} / {len(pred_sorted):,}") + + if n_bad > 0: + bad = pred_sorted[bad_mask][:5] + print(f"first bad pred_ids : {bad.tolist()}") + print() + print("Nearest metadata values for first bad pred_ids:") + for b in bad: + i = int(np.searchsorted(meta_sorted, b)) + lo = max(0, i - 2) + hi = min(len(meta_sorted), i + 3) + print( + f" pred {b:>15,} → nearest meta: {meta_sorted[lo:hi].tolist()}" + ) + else: + print("All pred refl_ids found in metadata — no mismatch.") + + # ── Set difference ──────────────────────────────────────────────────────── + print() + print("=" * 60) + print("SET DIFFERENCE (sample)") + print("=" * 60) + pred_set = set(pred_int[:10000].tolist()) + meta_set = set(meta_int[:10000].tolist()) + only_in_pred = sorted(pred_set - meta_set)[:5] + only_in_meta = sorted(meta_set - pred_set)[:5] + print(f"In pred but not meta (first 5): {only_in_pred}") + print(f"In meta but not pred (first 5): {only_in_meta}") + + # ── Summary ─────────────────────────────────────────────────────────────── + print() + print("=" * 60) + print("SUMMARY") + print("=" * 60) + if n_unique_pred < len(pred_int): + print("ROOT CAUSE: float32 precision loss in refl_ids") + print(" BatchPredWriter stored refl_ids as float32.") + print(" Large refl_ids (> 8,388,608) lost precision → duplicates.") + print() + print("FIX: re-export the NPZ with rounding to nearest integer,") + print(" OR modify write_mfx_refl_with_predictions to match") + print(" using source_refl + reflection_id directly from parquets") + print(" instead of refl_ids → metadata lookup.") + elif n_bad > 0: + print( + "ROOT CAUSE: pred refl_ids are different values from metadata refl_ids." + ) + print(" This may mean the wrong metadata.npy is being used,") + print( + " or refl_ids were assigned differently in predictions vs metadata." + ) + else: + print("No issue found — refl_ids match correctly.") + print("The error may be intermittent or already resolved.") + + +if __name__ == "__main__": + main() diff --git a/workflows/mfx101555026/scripts/writeback/export_preds_for_dials.py b/workflows/mfx101555026/scripts/writeback/export_preds_for_dials.py new file mode 100644 index 00000000..438b04a1 --- /dev/null +++ b/workflows/mfx101555026/scripts/writeback/export_preds_for_dials.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Export Integrator parquet predictions to NPZ for dials.python write-back." + ) + parser.add_argument("--ckpt-dir", required=True, type=Path) + parser.add_argument("--out", required=True, type=Path) + args = parser.parse_args() + + single = args.ckpt_dir / "pred.parquet" + if single.exists(): + pred_files = [single] + else: + pred_files = sorted(args.ckpt_dir.glob("preds_epoch_*.parquet")) + + if not pred_files: + raise RuntimeError(f"No parquet files found in {args.ckpt_dir}") + + print(f"Found {len(pred_files)} parquet file(s).") + + needed_cols = ["refl_ids", "qi_mean", "qi_var", "qbg_mean"] + + dfs = [] + for i, p in enumerate(pred_files, start=1): + if i <= 5 or i % 50 == 0: + print(f"[{i}/{len(pred_files)}] reading {p.name}") + dfs.append(pd.read_parquet(p, columns=needed_cols)) + + df = pd.concat(dfs, ignore_index=True) + + args.out.parent.mkdir(parents=True, exist_ok=True) + + np.savez( + args.out, + refl_ids=df["refl_ids"].to_numpy(dtype=np.int64), + qi_mean=df["qi_mean"].to_numpy(dtype=np.float64), + qi_var=df["qi_var"].to_numpy(dtype=np.float64), + qbg_mean=df["qbg_mean"].to_numpy(dtype=np.float64), + ) + + print("Wrote:", args.out) + print("Rows:", len(df)) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/workflows/mfx101555026/scripts/writeback/run_writeback.py b/workflows/mfx101555026/scripts/writeback/run_writeback.py new file mode 100644 index 00000000..d1117c76 --- /dev/null +++ b/workflows/mfx101555026/scripts/writeback/run_writeback.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python +""" +Production MFX write-back script (run with dials.python). + +Reads prediction .npz produced by export_preds_for_dials.py in +integrator-cuda-dev, then writes qi_mean / qi_var / qbg_mean back into +the original MFX .refl files using the optimised write_mfx_refl_with_predictions(). + +Dependencies (cctbx/psana2 environment only): + numpy, dials, dxtbx — NO polars, NO pandas, NO pyarrow. + +================================================================================ +Full two-environment workflow +================================================================================ + +Step 1 — integrator-cuda-dev (GPU node) +----------------------------------------- +Run prediction: + + integrator.predict \\ + --run-dir "$RUN_DIR" \\ + --ckpt "$RUN_DIR/files/checkpoints/epoch=0024.ckpt" \\ + --batch-size 2048 + +This writes parquet files to: + $RUN_DIR/files/predictions/epoch_0024/ + +Export parquets → .npz (no DIALS needed): + + python workflows/mfx101555026/scripts/writeback/export_preds_for_dials.py \\ + --ckpt-dir "$RUN_DIR/files/predictions/epoch_0024" \\ + --out "$RUN_DIR/files/predictions/preds_epoch_0024.npz" + +Step 2 — cctbx/psana2 + dials.python (CPU node) +-------------------------------------------------- +Activate cctbx environment and run write-back: + + source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH + cd /sdf/home/t/thaoh/s3df_practice/integrator + + dials.python workflows/mfx101555026/scripts/writeback/run_writeback.py \\ + --npz "$RUN_DIR/files/predictions/preds_epoch_0024.npz" \\ + --metadata "$DATA_DIR/metadata.npy" \\ + --refl-dir "$REFL_DIR" \\ + --out-dir "$RUN_DIR/files/predictions/mfx_refl_writeback" + +================================================================================ +""" + +from __future__ import annotations + +import argparse +import sys +import time +from pathlib import Path + +import numpy as np + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _as_text(v) -> str: + if hasattr(v, "item"): + v = v.item() + if isinstance(v, bytes): + return v.decode() + return str(v) + + +def _fmt(seconds: float) -> str: + h = int(seconds) // 3600 + m = (int(seconds) % 3600) // 60 + s = int(seconds) % 60 + return f"{h:02d}:{m:02d}:{s:02d}" + + +# --------------------------------------------------------------------------- +# Load .npz predictions (numpy only — no polars/pandas) +# --------------------------------------------------------------------------- + + +def load_npz(npz_path: Path) -> dict: + print(f"Loading predictions from {npz_path} ...") + t0 = time.time() + data = np.load(npz_path, allow_pickle=False) + + required = ["refl_ids", "qi_mean", "qi_var", "qbg_mean"] + missing = [k for k in required if k not in data] + if missing: + raise KeyError( + f"Missing arrays in {npz_path}: {missing}\n" + "Re-run export_preds_for_dials.py to regenerate the .npz." + ) + + pred_data = { + "refl_ids": data["refl_ids"].astype(np.int64), + "qi_mean": data["qi_mean"].astype(np.float64), + "qi_var": data["qi_var"].astype(np.float64), + "qbg_mean": data["qbg_mean"].astype(np.float64), + } + + n = len(pred_data["refl_ids"]) + print(f" {n:,} prediction rows loaded in {time.time() - t0:.1f}s") + print( + f" qi_mean : {pred_data['qi_mean'].min():.4f} .. {pred_data['qi_mean'].max():.4f}" + ) + print( + f" qi_var : {pred_data['qi_var'].min():.6f} .. {pred_data['qi_var'].max():.6f}" + ) + print( + f" qbg_mean : {pred_data['qbg_mean'].min():.4f} .. {pred_data['qbg_mean'].max():.4f}" + ) + return pred_data + + +# --------------------------------------------------------------------------- +# Run write-back +# --------------------------------------------------------------------------- + + +def run_writeback( + pred_data: dict, + metadata_path: Path, + refl_dir: Path, + out_dir: Path, + variance_floor: float, + copy_expt: bool, +) -> dict: + from integrator.io.refl_io import write_mfx_refl_with_predictions + + out_dir.mkdir(parents=True, exist_ok=True) + + print(f"\nStarting write-back ...") + print(f" metadata : {metadata_path}") + print(f" refl-dir : {refl_dir}") + print(f" out-dir : {out_dir}") + print(f" variance_floor: {variance_floor}") + + t0 = time.time() + result = write_mfx_refl_with_predictions( + pred_data=pred_data, + metadata_path=metadata_path, + original_refl_dir=refl_dir, + out_dir=out_dir, + variance_floor=variance_floor, + copy_expt=copy_expt, + ) + elapsed = time.time() - t0 + + print(f"\nWrite-back complete in {_fmt(elapsed)}") + print(f" refl files written : {result['n_refl_files']:,}") + print(f" prediction rows : {result['n_prediction_rows']:,}") + print(f" output dir : {result['out_dir']}") + return result + + +# --------------------------------------------------------------------------- +# Quick spot-check on a sample of written .refl files +# --------------------------------------------------------------------------- + + +def spot_check( + result: dict, + refl_dir: Path, + pred_data: dict, + meta: dict, + n_files: int = 3, + n_rows: int = 5, +) -> None: + from dials.array_family import flex + from dxtbx import flumpy + + print( + f"\nSpot-check: first {n_files} written files, {n_rows} rows each ..." + ) + + meta_refl_ids = np.asarray(meta["refl_ids"], dtype=np.int64) + meta_rid = np.asarray(meta["reflection_id"], dtype=np.int64) + meta_sr = np.asarray(meta["source_refl"]) + + pred_ids = pred_data["refl_ids"] + pred_mean = pred_data["qi_mean"].astype(np.float64) + pred_var = np.clip(pred_data["qi_var"].astype(np.float64), 1e-6, None) + + meta_order = np.argsort(meta_refl_ids) + meta_sorted = meta_refl_ids[meta_order] + + written = result["written_refl_files"][:n_files] + all_ok = True + + for out_path in written: + src_n = _as_text(out_path.name) + rt = flex.reflection_table.from_file(str(out_path)) + i_sum = flumpy.to_numpy(rt["intensity.sum.value"]) + v_sum = flumpy.to_numpy(rt["intensity.sum.variance"]) + + # Find predictions for this file + file_mask = np.array( + [_as_text(v) == src_n for v in meta_sr], dtype=bool + ) + file_rows = np.flatnonzero(file_mask) + + found_idx = np.searchsorted( + meta_sorted, meta_refl_ids[file_rows[:n_rows]] + ) + file_ok = True + for j, fi in enumerate(file_rows[:n_rows]): + rid = meta_refl_ids[fi] + row = int(meta_rid[fi]) + p = np.searchsorted(pred_ids, rid) + if p >= len(pred_ids) or pred_ids[p] != rid: + continue + exp_i = float(pred_mean[p]) + exp_v = float(pred_var[p]) + got_i = float(i_sum[row]) + got_v = float(v_sum[row]) + if not ( + np.isclose(got_i, exp_i, rtol=1e-4, atol=1e-6) + and np.isclose(got_v, exp_v, rtol=1e-4, atol=1e-6) + ): + print( + f" MISMATCH {src_n} row {row}: " + f"qi_mean exp={exp_i:.4f} got={got_i:.4f} " + f"qi_var exp={exp_v:.6f} got={got_v:.6f}" + ) + file_ok = False + + print( + f" {'OK' if file_ok else 'MISMATCH'} {src_n} n_rows={len(rt)}" + ) + if not file_ok: + all_ok = False + + if all_ok: + print(" Spot-check PASSED.") + else: + print(" Spot-check found MISMATCHES — review before using output.") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def parse_args(): + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "--npz", + required=True, + type=Path, + help="preds_epoch_XXXX.npz from export_preds_for_dials.py", + ) + p.add_argument( + "--metadata", + required=True, + type=Path, + help="metadata.npy from the shoebox dataset (DATA_DIR)", + ) + p.add_argument( + "--refl-dir", + required=True, + type=Path, + help="Original .refl/.expt directory (REFL_DIR)", + ) + p.add_argument( + "--out-dir", + required=True, + type=Path, + help="Output directory for written .refl files", + ) + p.add_argument( + "--variance-floor", + type=float, + default=1e-6, + help="Minimum allowed variance (default: 1e-6)", + ) + p.add_argument( + "--copy-expt", + action="store_true", + help="Copy .expt files instead of symlinking them", + ) + p.add_argument( + "--skip-spot-check", + action="store_true", + help="Skip the post-write spot-check", + ) + return p.parse_args() + + +def main(): + args = parse_args() + + print("=" * 65) + print("MFX write-back [dials.python]") + print(f" npz : {args.npz}") + print(f" metadata : {args.metadata}") + print(f" refl-dir : {args.refl_dir}") + print(f" out-dir : {args.out_dir}") + print("=" * 65) + + t_start = time.time() + + # 1. Load predictions (.npz, numpy only) + pred_data = load_npz(args.npz) + + # 2. Load metadata (numpy only) + print(f"\nLoading metadata ...") + t0 = time.time() + meta = np.load(args.metadata, allow_pickle=True).item() + print( + f" {len(np.asarray(meta['refl_ids'])):,} rows loaded in {time.time() - t0:.1f}s" + ) + + # 3. Write-back + result = run_writeback( + pred_data, + args.metadata, + args.refl_dir, + args.out_dir, + variance_floor=args.variance_floor, + copy_expt=args.copy_expt, + ) + + # 4. Optional spot-check + if not args.skip_spot_check: + spot_check(result, args.refl_dir, pred_data, meta) + + total = time.time() - t_start + print(f"\nTotal elapsed: {_fmt(total)}") + print("Done.") + + +if __name__ == "__main__": + main() diff --git a/workflows/mfx101555026/scripts/writeback/test_predict_small.py b/workflows/mfx101555026/scripts/writeback/test_predict_small.py new file mode 100644 index 00000000..0fcb2c9a --- /dev/null +++ b/workflows/mfx101555026/scripts/writeback/test_predict_small.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python +""" +Step 1 of 2 — Prediction smoke test (run in integrator-cuda-dev). + +Runs model inference on the first N source .refl files and saves predictions +to a .npz file that can be read by the cctbx/dials.python environment. + +The .npz format is used as the interchange file because: + - integrator-cuda-dev has polars/torch but NOT dials/dxtbx + - cctbx/psana2 has dials/dxtbx but NOT polars/pandas/pyarrow + - numpy (.npz) is available in BOTH environments + +Usage (integrator-cuda-dev): + + python test_predict_small.py \\ + --config "$CONFIG" \\ + --ckpt "$RUN_DIR/files/checkpoints/epoch=0024.ckpt" \\ + --metadata "$DATA_DIR/metadata.npy" \\ + --out-dir "/tmp/test_smoke" \\ + --n-images 5 + +Output: + /tmp/test_smoke/preds_small.npz <- read by test_writeback_small.py + /tmp/test_smoke/preds_small_info.txt <- human-readable summary + +The .npz contains: + refl_ids int64 (N,) + qi_mean float32 (N,) + qi_var float32 (N,) + qbg_mean float32 (N,) + +Activating integrator-cuda-dev: + + export MAMBA_ROOT_PREFIX=/sdf/scratch/lcls/ds/prj/prjlumine22/scratch/thaoh/micromamba_root + eval "$(/sdf/scratch/.../micromamba/bin/micromamba shell hook --shell bash)" + micromamba activate integrator-cuda-dev + export PYTHONNOUSERSITE=1 + export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH + export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src +""" + +from __future__ import annotations + +import argparse +import sys +import time +from pathlib import Path + +import numpy as np +import torch +import yaml + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _as_text(v) -> str: + if hasattr(v, "item"): + v = v.item() + if isinstance(v, bytes): + return v.decode() + return str(v) + + +# --------------------------------------------------------------------------- +# Load config and model +# --------------------------------------------------------------------------- + + +def load_model(config: dict, ckpt_path: Path, device: torch.device): + from integrator.utils.factory_utils import construct_integrator + + print(f"Building model ...") + model = construct_integrator(config) + print(f"Loading checkpoint: {ckpt_path.name}") + state = torch.load( + ckpt_path.as_posix(), map_location="cpu", weights_only=False + ) + model.load_state_dict(state["state_dict"]) + model.eval() + model.to(device) + print(f"Model loaded on {device}") + return model + + +# --------------------------------------------------------------------------- +# Select first N source .refl files from metadata +# --------------------------------------------------------------------------- + + +def select_target_files(meta: dict, n_images: int) -> dict: + source_refl_all = np.asarray(meta["source_refl"]) + refl_ids_all = np.asarray(meta["refl_ids"], dtype=np.int64) + + unique_sr, sr_encoded, sr_counts = np.unique( + source_refl_all, return_inverse=True, return_counts=True + ) + n_files = min(n_images, len(unique_sr)) + + target_names = [_as_text(unique_sr[i]) for i in range(n_files)] + target_counts = { + target_names[i]: int(sr_counts[i]) for i in range(n_files) + } + + sel_mask = np.isin(sr_encoded, np.arange(n_files)) + sel_rows = np.flatnonzero(sel_mask) + target_refl_ids = refl_ids_all[sel_rows] + + print(f"\nSelected {n_files} source .refl files:") + for name in target_names: + print(f" {name} ({target_counts[name]} rows)") + print(f" Total target rows: {len(target_refl_ids):,}") + + return { + "names": target_names, + "counts": target_counts, + "refl_ids": target_refl_ids, + } + + +# --------------------------------------------------------------------------- +# Run inference — stop when all target refl_ids are collected +# --------------------------------------------------------------------------- + + +def run_inference( + model, config: dict, target_info: dict, device: torch.device +) -> dict[int, dict]: + from integrator.utils.factory_utils import construct_data_loader + + target_set = set(target_info["refl_ids"].tolist()) + total_target = len(target_set) + + print(f"\nBuilding predict dataloader ...") + dm = construct_data_loader(config) + dm.setup() + dl = dm.predict_dataloader() + + predictions: dict[int, dict] = {} + batches = 0 + t0 = time.time() + + print( + f"Scanning batches (stops as soon as {total_target:,} rows found) ..." + ) + with torch.no_grad(): + for batch in dl: + counts_b, shoebox_b, mask_b, meta_b = batch + batch_rids = meta_b["refl_ids"].numpy().astype(np.int64) + hits = np.isin(batch_rids, list(target_set)) + batches += 1 + + if hits.any(): + counts_b = counts_b.to(device) + shoebox_b = shoebox_b.to(device) + mask_b = mask_b.to(device) + meta_dev = { + k: v.to(device) if isinstance(v, torch.Tensor) else v + for k, v in meta_b.items() + } + pred = model.predict_step( + (counts_b, shoebox_b, mask_b, meta_dev), 0 + ) + for j in np.flatnonzero(hits): + rid = int(batch_rids[j]) + if rid in target_set: + predictions[rid] = { + k: float(v[j].detach().cpu()) + for k, v in pred.items() + if isinstance(v, torch.Tensor) and v.dim() == 1 + } + target_set.discard(rid) + + if not target_set: + break + + if batches % 50 == 0: + found = total_target - len(target_set) + elapsed = time.time() - t0 + print( + f" {batches} batches | {found}/{total_target} found | {elapsed:.0f}s" + ) + + elapsed = time.time() - t0 + found = total_target - len(target_set) + print( + f"Done: {batches} batches scanned, {found}/{total_target} rows found in {elapsed:.1f}s" + ) + + if target_set: + print( + f"WARNING: {len(target_set)} target rows not found in dataloader." + ) + + return predictions + + +# --------------------------------------------------------------------------- +# Save predictions as .npz (numpy interchange format) +# --------------------------------------------------------------------------- + + +def save_npz( + predictions: dict, target_info: dict, out_dir: Path, epoch: int +) -> Path: + out_dir.mkdir(parents=True, exist_ok=True) + npz_path = out_dir / "preds_small.npz" + + rids = target_info["refl_ids"] + found_rids = [int(r) for r in rids if int(r) in predictions] + missing = len(rids) - len(found_rids) + + if missing: + print( + f"WARNING: {missing} rows have no prediction — not included in .npz" + ) + if not found_rids: + raise RuntimeError("No predictions found for any target refl_id.") + + def _col(key: str) -> np.ndarray: + vals = [predictions[r].get(key, 0.0) for r in found_rids] + return np.array(vals, dtype=np.float32) + + np.savez( + npz_path, + refl_ids=np.array(found_rids, dtype=np.int64), + qi_mean=_col("qi_mean"), + qi_var=_col("qi_var"), + qbg_mean=_col("qbg_mean"), + epoch=np.array([epoch], dtype=np.int32), + ) + + print(f"\nSaved {len(found_rids):,} prediction rows → {npz_path}") + + # Human-readable summary + info_path = out_dir / "preds_small_info.txt" + lines = [ + f"epoch: {epoch}", + f"n_rows: {len(found_rids)}", + f"source_refl_files: {', '.join(target_info['names'])}", + f"qi_mean range: {_col('qi_mean').min():.4f} .. {_col('qi_mean').max():.4f}", + f"qi_var range: {_col('qi_var').min():.6f} .. {_col('qi_var').max():.6f}", + ] + info_path.write_text("\n".join(lines) + "\n") + print(f"Summary → {info_path}") + for line in lines: + print(f" {line}") + + return npz_path + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def parse_args(): + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument("--config", required=True, type=Path) + p.add_argument("--ckpt", required=True, type=Path) + p.add_argument("--metadata", required=True, type=Path) + p.add_argument("--out-dir", required=True, type=Path) + p.add_argument("--n-images", type=int, default=5) + p.add_argument("--epoch", type=int, default=24) + return p.parse_args() + + +def main(): + args = parse_args() + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + print("=" * 60) + print("Prediction smoke test [integrator-cuda-dev]") + print(f" config : {args.config}") + print(f" ckpt : {args.ckpt}") + print(f" metadata : {args.metadata}") + print(f" out-dir : {args.out_dir}") + print(f" n-images : {args.n_images}") + print(f" device : {device}") + print("=" * 60) + + config = yaml.safe_load(args.config.read_text()) + + # 1. Load model + model = load_model(config, args.ckpt, device) + + # 2. Load metadata and pick N files + print(f"\nLoading metadata ...") + t0 = time.time() + meta = np.load(args.metadata, allow_pickle=True).item() + print( + f" {len(np.asarray(meta['refl_ids'])):,} rows in {time.time() - t0:.1f}s" + ) + + target_info = select_target_files(meta, args.n_images) + + # 3. Run inference + predictions = run_inference(model, config, target_info, device) + + # 4. Save .npz + npz_path = save_npz(predictions, target_info, args.out_dir, args.epoch) + + print("\n" + "=" * 60) + print("DONE — prediction complete.") + print(f"Next step: run test_writeback_small.py with dials.python") + print(f" dials.python test_writeback_small.py \\") + print(f" --npz {npz_path} \\") + print(f" --metadata {args.metadata} \\") + print(f" --refl-dir \\") + print(f" --out-dir {args.out_dir}/writeback") + print("=" * 60) + + +if __name__ == "__main__": + main() diff --git a/workflows/mfx101555026/scripts/writeback/test_predict_writeback_small.py b/workflows/mfx101555026/scripts/writeback/test_predict_writeback_small.py new file mode 100644 index 00000000..a5332025 --- /dev/null +++ b/workflows/mfx101555026/scripts/writeback/test_predict_writeback_small.py @@ -0,0 +1,571 @@ +#!/usr/bin/env python +""" +End-to-end predict + write-back smoke test on first N source .refl files. + +Steps: + 1. Load config and model from checkpoint. + 2. Load metadata — identify first N unique source .refl files. + 3. Iterate predict_dataloader; stop as soon as all target refl_ids + have been seen (first N files are in first chunks — exits quickly). + 4. Write predictions to out_dir/epoch_XXXX/ as parquet. + 5. Run write-back for those N files via write_mfx_refl_with_predictions. + 6. Validate: + Check 1 source refl row count == output refl row count + Check 2 predicted rows written == expected + Check 3 no duplicate (source_refl, reflection_id) in metadata subset + Check 4 sample qi_mean / qi_var values match parquet predictions + +Run with integrator-cuda-dev active (GPU optional — falls back to CPU): + + python test_predict_writeback_small.py \\ + --config "$CONFIG" \\ + --ckpt "$RUN_DIR/files/checkpoints/epoch=0024.ckpt" \\ + --metadata "$DATA_DIR/metadata.npy" \\ + --refl-dir "$REFL_DIR" \\ + --out-dir "/tmp/test_smoke" \\ + --n-images 5 + +Exit 0 = all checks passed. Exit 1 = at least one check failed. +""" + +from __future__ import annotations + +import argparse +import sys +import time +from pathlib import Path + +import numpy as np +import polars as pl +import torch +import yaml + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _as_text(v) -> str: + if hasattr(v, "item"): + v = v.item() + if isinstance(v, bytes): + return v.decode() + return str(v) + + +def _load_config(config_path: Path) -> dict: + return yaml.safe_load(config_path.read_text()) + + +def _load_metadata(metadata_path: Path) -> dict: + print(f" Loading metadata from {metadata_path} ...") + t0 = time.time() + meta = np.load(metadata_path, allow_pickle=True).item() + print( + f" Done in {time.time() - t0:.1f}s rows={len(np.asarray(meta['refl_ids'])):,}" + ) + return meta + + +# --------------------------------------------------------------------------- +# Step 1 — build model + load checkpoint +# --------------------------------------------------------------------------- + + +def load_model(config: dict, ckpt_path: Path, device: torch.device): + from integrator.utils.factory_utils import construct_integrator + + print(f"\nBuilding model from config...") + integrator = construct_integrator(config) + print(f" Loading checkpoint: {ckpt_path.name}") + state = torch.load( + ckpt_path.as_posix(), map_location="cpu", weights_only=False + ) + integrator.load_state_dict(state["state_dict"]) + integrator.eval() + integrator.to(device) + print(f" Model on {device}") + return integrator + + +# --------------------------------------------------------------------------- +# Step 2 — find first N source .refl files and their target refl_ids +# --------------------------------------------------------------------------- + + +def select_target_files(meta: dict, n_images: int): + source_refl_all = np.asarray(meta["source_refl"]) + refl_ids_all = np.asarray(meta["refl_ids"], dtype=np.int64) + reflection_id_all = np.asarray(meta["reflection_id"], dtype=np.int64) + source_expt_all = np.asarray(meta["source_expt"]) + + # First N unique source_refl names (alphabetically sorted by np.unique) + unique_sr, sr_encoded, sr_counts = np.unique( + source_refl_all, return_inverse=True, return_counts=True + ) + n_files = min(n_images, len(unique_sr)) + target_names = [_as_text(unique_sr[i]) for i in range(n_files)] + target_counts = { + target_names[i]: int(sr_counts[i]) for i in range(n_files) + } + + # All metadata rows belonging to those N files + sel_enc = np.arange(n_files) + sel_mask = np.isin(sr_encoded, sel_enc) + sel_rows = np.flatnonzero(sel_mask) + + target_refl_ids = refl_ids_all[sel_rows] + target_source_refl = source_refl_all[sel_rows] + target_source_expt = source_expt_all[sel_rows] + target_reflection_id = reflection_id_all[sel_rows] + + print(f"\nSelected {n_files} source .refl files:") + for name in target_names: + print(f" {name} ({target_counts[name]} rows)") + print(f" Total target rows: {len(target_refl_ids):,}") + + return { + "names": target_names, + "counts": target_counts, + "refl_ids": target_refl_ids, + "source_refl": target_source_refl, + "source_expt": target_source_expt, + "reflection_id": target_reflection_id, + } + + +# --------------------------------------------------------------------------- +# Step 3 — run inference, stop when all target refl_ids collected +# --------------------------------------------------------------------------- + + +def run_inference( + integrator, config: dict, target_info: dict, device: torch.device +) -> dict: + from integrator.utils.factory_utils import construct_data_loader + + target_ids_set = set(target_info["refl_ids"].tolist()) + total_target = len(target_ids_set) + + print(f"\nBuilding predict dataloader ...") + data_module = construct_data_loader(config) + data_module.setup() + predict_dl = data_module.predict_dataloader() + + predictions: dict[int, dict] = {} # refl_id -> {key: scalar_value} + batches_scanned = 0 + t0 = time.time() + + print(f" Scanning batches for {total_target:,} target rows ...") + with torch.no_grad(): + for batch in predict_dl: + counts_b, shoebox_b, mask_b, meta_b = batch + batch_rids = meta_b["refl_ids"].numpy().astype(np.int64) + + # Check if this batch contains any target rows + hits = np.isin(batch_rids, list(target_ids_set)) + batches_scanned += 1 + + if hits.any(): + # Move batch to device + counts_b = counts_b.to(device) + shoebox_b = shoebox_b.to(device) + mask_b = mask_b.to(device) + meta_dev = { + k: v.to(device) if isinstance(v, torch.Tensor) else v + for k, v in meta_b.items() + } + + pred = integrator.predict_step( + (counts_b, shoebox_b, mask_b, meta_dev), 0 + ) + + # Store predictions for target rows only + hit_idx = np.flatnonzero(hits) + for j in hit_idx: + rid = int(batch_rids[j]) + if rid in target_ids_set: + row_pred = {} + for k, v in pred.items(): + if isinstance(v, torch.Tensor): + row_pred[k] = v[j].detach().cpu().numpy() + predictions[rid] = row_pred + target_ids_set.discard(rid) + + if not target_ids_set: + break + + if batches_scanned % 50 == 0: + found = total_target - len(target_ids_set) + elapsed = time.time() - t0 + print( + f" ... {batches_scanned} batches, " + f"{found}/{total_target} target rows found " + f"({elapsed:.0f}s)" + ) + + elapsed = time.time() - t0 + found = total_target - len(target_ids_set) + print( + f" Done: {batches_scanned} batches scanned, " + f"{found}/{total_target} target rows found in {elapsed:.1f}s" + ) + + if target_ids_set: + print( + f" WARNING: {len(target_ids_set)} target refl_ids not found in dataloader." + ) + + return predictions + + +# --------------------------------------------------------------------------- +# Step 4 — write predictions to parquet +# --------------------------------------------------------------------------- + + +def write_predictions( + predictions: dict, target_info: dict, pred_dir: Path, epoch: int +) -> Path: + pred_dir.mkdir(parents=True, exist_ok=True) + + # Build aligned arrays in refl_id order + rids = target_info["refl_ids"] + rows = [] + missing = 0 + + for rid in rids: + if int(rid) in predictions: + rows.append(predictions[int(rid)]) + else: + missing += 1 + + if missing: + print( + f" WARNING: {missing} rows have no prediction (not in dataloader)" + ) + + # Get available rows + found_rids = [int(r) for r in rids if int(r) in predictions] + if not found_rids: + raise RuntimeError("No predictions found for target refl_ids.") + + # Collect scalar prediction columns (skip large arrays like qp_mean) + needed = ["refl_ids", "qi_mean", "qi_var", "qbg_mean"] + available = {k for k in predictions[found_rids[0]].keys()} + + col_data: dict = {"refl_ids": np.array(found_rids, dtype=np.int64)} + for key in ["qi_mean", "qi_var", "qbg_mean"]: + if key in available: + col_data[key] = np.array( + [float(predictions[r][key]) for r in found_rids], + dtype=np.float32, + ) + else: + print(f" WARNING: {key} not in predictions — filling zeros") + col_data[key] = np.zeros(len(found_rids), dtype=np.float32) + + col_data["epoch"] = np.full(len(found_rids), epoch, dtype=np.int32) + + df = pl.DataFrame(col_data) + out_path = ( + pred_dir / f"preds_epoch_{epoch:04d}_rank=0_flush=000000.parquet" + ) + df.write_parquet(out_path) + print(f" Wrote {len(df):,} prediction rows to {out_path.name}") + return pred_dir + + +# --------------------------------------------------------------------------- +# Step 5 — run write-back +# --------------------------------------------------------------------------- + + +def run_writeback( + pred_dir: Path, metadata_path: Path, refl_dir: Path, out_dir: Path +) -> dict: + from integrator.io.pred_io import write_mfx_refl_from_preds + + out_dir.mkdir(parents=True, exist_ok=True) + print(f"\nRunning write-back ...") + result = write_mfx_refl_from_preds( + ckpt_dir=pred_dir, + metadata_path=metadata_path, + original_refl_dir=refl_dir, + out_dir=out_dir, + filetype="parquet", + variance_floor=1e-6, + ) + print( + f" Written: {result['n_refl_files']} .refl files, " + f"{result['n_prediction_rows']:,} prediction rows" + ) + return result + + +# --------------------------------------------------------------------------- +# Step 6 — validation checks +# --------------------------------------------------------------------------- + + +def check_row_counts( + refl_dir: Path, out_dir: Path, written_files: list +) -> bool: + from dials.array_family import flex + + print("\n[Check 1] Source row count == output row count ...") + passed = True + for out_path in written_files: + src_path = refl_dir / out_path.name + n_src = len(flex.reflection_table.from_file(str(src_path))) + n_out = len(flex.reflection_table.from_file(str(out_path))) + ok = n_src == n_out + print( + f" {'PASS' if ok else 'FAIL'} {out_path.name} " + f"src={n_src} out={n_out}" + ) + if not ok: + passed = False + return passed + + +def check_predicted_rows(n_prediction_rows: int, pred_dir: Path) -> bool: + print("\n[Check 2] Predicted rows written == parquet rows ...") + df = pl.scan_parquet( + list(pred_dir.glob("preds_epoch_*.parquet")) + ).collect() + parquet_rows = len(df) + ok = parquet_rows == n_prediction_rows + print( + f" {'PASS' if ok else 'FAIL'} " + f"parquet={parquet_rows:,} written={n_prediction_rows:,}" + ) + return ok + + +def check_no_duplicates(meta: dict, target_info: dict) -> bool: + print( + "\n[Check 3] No duplicate (source_refl, reflection_id) in subset ..." + ) + sr = target_info["source_refl"] + rid = target_info["reflection_id"].astype(np.int64) + + _, encoded = np.unique(sr, return_inverse=True) + order = np.lexsort((rid, encoded)) + enc_s = encoded[order] + rid_s = rid[order] + dups = np.flatnonzero( + (enc_s[1:] == enc_s[:-1]) & (rid_s[1:] == rid_s[:-1]) + ) + + ok = len(dups) == 0 + if ok: + print(f" PASS no duplicates in {len(sr):,} rows") + else: + i = dups[0] + print( + f" FAIL {len(dups):,} duplicates — first: " + f"source_refl={_as_text(sr[order[i]])} " + f"reflection_id={rid_s[i]}" + ) + return ok + + +def check_values( + pred_dir: Path, + out_dir: Path, + target_info: dict, + predictions: dict, + n_sample: int = 10, +) -> bool: + from dials.array_family import flex + from dxtbx import flumpy + + print("\n[Check 4] Sample qi_mean / qi_var values match predictions ...") + rng = np.random.default_rng(42) + passed = True + + # Group target rows by source_refl + sr = target_info["source_refl"] + rids = target_info["refl_ids"].astype(np.int64) + rflid = target_info["reflection_id"].astype(np.int64) + + unique_sr, enc = np.unique(sr, return_inverse=True) + for i, sr_bytes in enumerate(unique_sr): + sr_name = _as_text(sr_bytes) + out_path = out_dir / sr_name + if not out_path.exists(): + print(f" SKIP {sr_name} (not written)") + continue + + group = np.flatnonzero(enc == i) + # Filter to rows that have predictions + group_with_pred = [j for j in group if int(rids[j]) in predictions] + if not group_with_pred: + print(f" SKIP {sr_name} (no predictions)") + continue + + rt = flex.reflection_table.from_file(str(out_path)) + i_sum = flumpy.to_numpy(rt["intensity.sum.value"]) + v_sum = flumpy.to_numpy(rt["intensity.sum.variance"]) + + sample = rng.choice( + group_with_pred, + size=min(n_sample, len(group_with_pred)), + replace=False, + ) + file_ok = True + for j in sample: + rid = int(rids[j]) + row = int(rflid[j]) + exp_i = float(predictions[rid].get("qi_mean", np.nan)) + exp_v = float(predictions[rid].get("qi_var", np.nan)) + # qi_var is clipped to 1e-6 + exp_v = max(exp_v, 1e-6) + got_i = float(i_sum[row]) + got_v = float(v_sum[row]) + if not ( + np.isclose(got_i, exp_i, rtol=1e-4, atol=1e-6) + and np.isclose(got_v, exp_v, rtol=1e-4, atol=1e-6) + ): + print( + f" MISMATCH {sr_name} row {row}: " + f"qi_mean exp={exp_i:.6f} got={got_i:.6f} " + f"qi_var exp={exp_v:.6f} got={got_v:.6f}" + ) + file_ok = False + + print(f" {'PASS' if file_ok else 'FAIL'} {sr_name}") + if not file_ok: + passed = False + + return passed + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def parse_args(): + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "--config", + required=True, + type=Path, + help="Training YAML config (same one used for training)", + ) + p.add_argument( + "--ckpt", + required=True, + type=Path, + help="Checkpoint file, e.g. epoch=0024.ckpt", + ) + p.add_argument( + "--metadata", + required=True, + type=Path, + help="metadata.npy from the shoebox dataset (DATA_DIR)", + ) + p.add_argument( + "--refl-dir", + required=True, + type=Path, + help="Original .refl/.expt directory (REFL_DIR)", + ) + p.add_argument( + "--out-dir", + required=True, + type=Path, + help="Output directory for test predictions and write-back", + ) + p.add_argument( + "--n-images", + type=int, + default=5, + help="Number of source .refl files to test (default: 5)", + ) + p.add_argument( + "--epoch", + type=int, + default=24, + help="Epoch number label for parquet filenames (default: 24)", + ) + return p.parse_args() + + +def main(): + args = parse_args() + + pred_dir = args.out_dir / f"epoch_{args.epoch:04d}" + wb_dir = args.out_dir / "mfx_refl_writeback" + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + print("=" * 65) + print("Predict + write-back smoke test") + print(f" config : {args.config}") + print(f" ckpt : {args.ckpt}") + print(f" metadata : {args.metadata}") + print(f" refl-dir : {args.refl_dir}") + print(f" out-dir : {args.out_dir}") + print(f" n-images : {args.n_images}") + print(f" device : {device}") + print("=" * 65) + + config = _load_config(args.config) + + # 1. Model + model = load_model(config, args.ckpt, device) + + # 2. Target files + meta = _load_metadata(args.metadata) + target_info = select_target_files(meta, args.n_images) + + # 3. Inference + predictions = run_inference(model, config, target_info, device) + + # 4. Write parquet + write_predictions(predictions, target_info, pred_dir, args.epoch) + + # 5. Write-back + result = run_writeback(pred_dir, args.metadata, args.refl_dir, wb_dir) + + # 6. Validate + written = result["written_refl_files"] + + p1 = check_row_counts(args.refl_dir, wb_dir, written) + p2 = check_predicted_rows(result["n_prediction_rows"], pred_dir) + p3 = check_no_duplicates(meta, target_info) + p4 = check_values(pred_dir, wb_dir, target_info, predictions) + + print("\n" + "=" * 65) + print("SUMMARY") + print("=" * 65) + checks = { + "Check 1 source row count == output row count": p1, + "Check 2 predicted rows == parquet rows": p2, + "Check 3 no duplicate (source_refl, refl_id)": p3, + "Check 4 qi_mean/qi_var spot-check": p4, + } + all_ok = True + for label, ok in checks.items(): + print(f" {'PASS' if ok else 'FAIL'} {label}") + if not ok: + all_ok = False + + print("=" * 65) + if all_ok: + print("ALL CHECKS PASSED — safe to run full predict + write-back job.") + sys.exit(0) + else: + print("ONE OR MORE CHECKS FAILED — investigate before full run.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/workflows/mfx101555026/scripts/writeback/test_writeback_small.py b/workflows/mfx101555026/scripts/writeback/test_writeback_small.py new file mode 100644 index 00000000..2b5571dc --- /dev/null +++ b/workflows/mfx101555026/scripts/writeback/test_writeback_small.py @@ -0,0 +1,366 @@ +#!/usr/bin/env python +""" +Step 2 of 2 — Write-back and validation smoke test (run with dials.python). + +Reads the .npz produced by test_predict_small.py, runs write-back for the +5 test .refl files using the optimised write_mfx_refl_with_predictions(), +and validates 4 correctness checks. + +Dependencies (all present in cctbx/psana2 environment): + numpy, dials, dxtbx — NO polars, NO pandas, NO pyarrow needed. + +Usage (cctbx/psana2 environment): + + source /sdf/group/lcls/ds/tools/cctbx/psana2_setup.sh + export PYTHONPATH=/sdf/home/t/thaoh/s3df_practice/integrator/src:$PYTHONPATH + cd /sdf/home/t/thaoh/s3df_practice/integrator + + dials.python workflows/mfx101555026/scripts/writeback/test_writeback_small.py \\ + --npz "/tmp/test_smoke/preds_small.npz" \\ + --metadata "$DATA_DIR/metadata.npy" \\ + --refl-dir "$REFL_DIR" \\ + --out-dir "/tmp/test_smoke/writeback" + +Validates: + Check 1 source refl row count == output refl row count + Check 2 predicted rows written == npz row count + Check 3 no duplicate (source_refl, reflection_id) in metadata subset + Check 4 sample qi_mean / qi_var values match npz predictions + +Exit 0 = all passed. Exit 1 = at least one check failed. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import numpy as np + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _as_text(v) -> str: + if hasattr(v, "item"): + v = v.item() + if isinstance(v, bytes): + return v.decode() + return str(v) + + +# --------------------------------------------------------------------------- +# Load .npz predictions (numpy only — no polars/pandas) +# --------------------------------------------------------------------------- + + +def load_npz(npz_path: Path) -> dict: + data = np.load(npz_path, allow_pickle=False) + required = ["refl_ids", "qi_mean", "qi_var", "qbg_mean"] + missing = [k for k in required if k not in data] + if missing: + raise KeyError(f"Missing arrays in {npz_path}: {missing}") + pred_data = { + "refl_ids": data["refl_ids"].astype(np.int64), + "qi_mean": data["qi_mean"].astype(np.float64), + "qi_var": data["qi_var"].astype(np.float64), + "qbg_mean": data["qbg_mean"].astype(np.float64), + } + epoch = int(data["epoch"][0]) if "epoch" in data else -1 + print( + f"Loaded {len(pred_data['refl_ids']):,} prediction rows from {npz_path.name}" + ) + print(f" epoch : {epoch}") + print( + f" qi_mean : {pred_data['qi_mean'].min():.4f} .. {pred_data['qi_mean'].max():.4f}" + ) + print( + f" qi_var : {pred_data['qi_var'].min():.6f} .. {pred_data['qi_var'].max():.6f}" + ) + return pred_data + + +# --------------------------------------------------------------------------- +# Run write-back +# --------------------------------------------------------------------------- + + +def run_writeback( + pred_data: dict, metadata_path: Path, refl_dir: Path, out_dir: Path +) -> dict: + # Import here so the module-level import of refl_io (now fixed) works fine + # without dials being on the path at import time. + from integrator.io.refl_io import write_mfx_refl_with_predictions + + out_dir.mkdir(parents=True, exist_ok=True) + print(f"\nRunning write-back ...") + result = write_mfx_refl_with_predictions( + pred_data=pred_data, + metadata_path=metadata_path, + original_refl_dir=refl_dir, + out_dir=out_dir, + variance_floor=1e-6, + copy_expt=False, + ) + print( + f" Written: {result['n_refl_files']} .refl files, " + f"{result['n_prediction_rows']:,} prediction rows" + ) + return result + + +# --------------------------------------------------------------------------- +# Validation checks +# --------------------------------------------------------------------------- + + +def check_row_counts( + refl_dir: Path, out_dir: Path, written_files: list +) -> bool: + from dials.array_family import flex + + print("\n[Check 1] Source row count == output row count ...") + passed = True + for out_path in written_files: + src_path = refl_dir / out_path.name + n_src = len(flex.reflection_table.from_file(str(src_path))) + n_out = len(flex.reflection_table.from_file(str(out_path))) + ok = n_src == n_out + print( + f" {'PASS' if ok else 'FAIL'} {out_path.name} " + f"src={n_src} out={n_out}" + ) + if not ok: + passed = False + return passed + + +def check_predicted_rows(n_prediction_rows: int, n_npz_rows: int) -> bool: + print("\n[Check 2] Predicted rows written == npz rows ...") + ok = n_prediction_rows == n_npz_rows + print( + f" {'PASS' if ok else 'FAIL'} " + f"written={n_prediction_rows:,} npz={n_npz_rows:,}" + ) + return ok + + +def check_no_duplicates(meta: dict, pred_data: dict) -> bool: + print( + "\n[Check 3] No duplicate (source_refl, reflection_id) in subset ..." + ) + + meta_refl_ids = np.asarray(meta["refl_ids"], dtype=np.int64) + meta_sr = np.asarray(meta["source_refl"]) + meta_rid = np.asarray(meta["reflection_id"], dtype=np.int64) + + # Find metadata rows for prediction refl_ids + pred_ids = pred_data["refl_ids"] + meta_order = np.argsort(meta_refl_ids) + meta_sorted = meta_refl_ids[meta_order] + found = np.searchsorted(meta_sorted, pred_ids) + valid = (found < len(meta_sorted)) & (meta_sorted[found] == pred_ids) + rows = meta_order[found[valid]] + + sr = meta_sr[rows] + rid = meta_rid[rows].astype(np.int64) + + _, encoded = np.unique(sr, return_inverse=True) + order = np.lexsort((rid, encoded)) + enc_s = encoded[order] + rid_s = rid[order] + dups = np.flatnonzero( + (enc_s[1:] == enc_s[:-1]) & (rid_s[1:] == rid_s[:-1]) + ) + + ok = len(dups) == 0 + if ok: + print(f" PASS no duplicates in {len(sr):,} rows") + else: + i = dups[0] + print( + f" FAIL {len(dups):,} duplicates — first: " + f"source_refl={_as_text(sr[order[i]])} " + f"reflection_id={rid_s[i]}" + ) + return ok + + +def check_values( + refl_dir: Path, + out_dir: Path, + pred_data: dict, + meta: dict, + n_sample: int = 10, +) -> bool: + from dials.array_family import flex + from dxtbx import flumpy + + print( + "\n[Check 4] Sample qi_mean / qi_var values match npz predictions ..." + ) + + meta_refl_ids = np.asarray(meta["refl_ids"], dtype=np.int64) + meta_sr = np.asarray(meta["source_refl"]) + meta_rid = np.asarray(meta["reflection_id"], dtype=np.int64) + + pred_ids = pred_data["refl_ids"] + pred_imean = pred_data["qi_mean"].astype(np.float64) + pred_ivar = np.clip(pred_data["qi_var"].astype(np.float64), 1e-6, None) + + # Match pred_ids to metadata rows + meta_order = np.argsort(meta_refl_ids) + meta_sorted = meta_refl_ids[meta_order] + found = np.searchsorted(meta_sorted, pred_ids) + valid = (found < len(meta_sorted)) & (meta_sorted[found] == pred_ids) + rows = meta_order[found[valid]] + + sr = meta_sr[rows] + rid = meta_rid[rows].astype(np.int64) + qi_mean_matched = pred_imean[valid] + qi_var_matched = pred_ivar[valid] + + rng = np.random.default_rng(42) + passed = True + + unique_sr, enc = np.unique(sr, return_inverse=True) + for i, sr_bytes in enumerate(unique_sr): + sr_name = _as_text(sr_bytes) + out_path = out_dir / sr_name + if not out_path.exists(): + print(f" SKIP {sr_name}") + continue + + group = np.flatnonzero(enc == i) + sample = rng.choice( + group, size=min(n_sample, len(group)), replace=False + ) + + rt = flex.reflection_table.from_file(str(out_path)) + i_sum = flumpy.to_numpy(rt["intensity.sum.value"]) + v_sum = flumpy.to_numpy(rt["intensity.sum.variance"]) + + file_ok = True + for j in sample: + row = int(rid[j]) + exp_i = float(qi_mean_matched[j]) + exp_v = float(qi_var_matched[j]) + got_i = float(i_sum[row]) + got_v = float(v_sum[row]) + if not ( + np.isclose(got_i, exp_i, rtol=1e-4, atol=1e-6) + and np.isclose(got_v, exp_v, rtol=1e-4, atol=1e-6) + ): + print( + f" MISMATCH {sr_name} row {row}: " + f"qi_mean exp={exp_i:.6f} got={got_i:.6f} " + f"qi_var exp={exp_v:.6f} got={got_v:.6f}" + ) + file_ok = False + + print(f" {'PASS' if file_ok else 'FAIL'} {sr_name}") + if not file_ok: + passed = False + + return passed + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +def parse_args(): + p = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + p.add_argument( + "--npz", + required=True, + type=Path, + help="preds_small.npz from test_predict_small.py", + ) + p.add_argument( + "--metadata", + required=True, + type=Path, + help="metadata.npy from the shoebox dataset (DATA_DIR)", + ) + p.add_argument( + "--refl-dir", + required=True, + type=Path, + help="Original .refl/.expt directory (REFL_DIR)", + ) + p.add_argument( + "--out-dir", + required=True, + type=Path, + help="Output directory for test write-back .refl files", + ) + return p.parse_args() + + +def main(): + args = parse_args() + + print("=" * 60) + print("Write-back validation smoke test [dials.python]") + print(f" npz : {args.npz}") + print(f" metadata : {args.metadata}") + print(f" refl-dir : {args.refl_dir}") + print(f" out-dir : {args.out_dir}") + print("=" * 60) + + # Load .npz predictions (numpy only) + pred_data = load_npz(args.npz) + n_npz = len(pred_data["refl_ids"]) + + # Load metadata (numpy only) + print(f"\nLoading metadata ...") + meta = np.load(args.metadata, allow_pickle=True).item() + print(f" {len(np.asarray(meta['refl_ids'])):,} metadata rows") + + # Run write-back + result = run_writeback( + pred_data, args.metadata, args.refl_dir, args.out_dir + ) + written = result["written_refl_files"] + + # Validate + p1 = check_row_counts(args.refl_dir, args.out_dir, written) + p2 = check_predicted_rows(result["n_prediction_rows"], n_npz) + p3 = check_no_duplicates(meta, pred_data) + p4 = check_values(args.refl_dir, args.out_dir, pred_data, meta) + + # Summary + print("\n" + "=" * 60) + print("SUMMARY") + print("=" * 60) + checks = { + "Check 1 source row count == output row count": p1, + "Check 2 written rows == npz rows": p2, + "Check 3 no duplicate (source_refl, refl_id)": p3, + "Check 4 qi_mean / qi_var spot-check": p4, + } + all_ok = True + for label, ok in checks.items(): + print(f" {'PASS' if ok else 'FAIL'} {label}") + if not ok: + all_ok = False + + print("=" * 60) + if all_ok: + print("ALL CHECKS PASSED — safe to run full write-back.") + sys.exit(0) + else: + print("ONE OR MORE CHECKS FAILED — investigate before full run.") + sys.exit(1) + + +if __name__ == "__main__": + main()