diff --git a/.gitignore b/.gitignore index db20f308..4c4b6d0b 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ code # .felt here is a machine-local symlink into it. Never track it in this repo. /.felt/ /.felt +.snakemake/ diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml new file mode 100644 index 00000000..e4ad8e2c --- /dev/null +++ b/profiles/nibi/config.yaml @@ -0,0 +1,87 @@ +# Snakemake profile for the Nibi cluster (Digital Research Alliance). +# +# SLURM-EXECUTOR mode (PRD #848 D-profile): one SLURM job per rule instance, +# each carrying its rule's own attempt-scaled resources (cpus_per_task, +# mem_mb, runtime). Snakemake feeds the queue as jobs finish, so DR6's ~170k +# total jobs never queue at once, and multi-node scaling is inherent — this +# supersedes the earlier one-allocation/local-scheduler mode. +# +# Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). +# software-deployment-method wraps every job's shell in `apptainer exec` — the +# user never types apptainer; the container is set in the Snakefile (local .sif). + +executor: slurm + +# Per-user submit cap, queried 2026-07-30 on nibi: +# sacctmgr show assoc user=cdaley format=Account,MaxSubmitPU -P +# -> def-mjhudson_cpu 1000, def-mjhudson_gpu 1000 (MaxSubmitPU; no site-wide +# MaxSubmitJobs in `scontrol show config`, so the association limit governs) +# Set to ~80% of that (1000) so this workflow never starves other submissions +# under the same account. Re-query if the association limits change. +jobs: 800 + +default-resources: + mem_mb: 2000 + runtime: 120 # minutes + slurm_account: def-mjhudson_cpu + +software-deployment-method: [apptainer] +# Explicit container environment (finding 2/3): the apptainer SDM otherwise drops +# the proven-recipe env and hardcodes --home , hiding ~/.ssl/cadcproxy.pem. +# --cleanenv strict host-env isolation (APPTAINERENV_*/SINGULARITYENV_* survive) +# OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) +# MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) +# --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp +# PYTHONPATH SETTLED CALL 3 — P0 pins THIS branch's src/ (identical to +# develop@97e16d50: orchestration commits never touch src/). +# NOT shapepipe-prod (drifted to a PR branch mid-run, and the +# live p3-batch1 job reads it) and not the sif default +# (frozen pre-#843). Production later rebuilds the sif at the +# validated commit and DROPS this --env line. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch" + +latency-wait: 60 # NFS: wait for outputs to appear after a job +keep-going: true # a failed job poisons only its cone; siblings run on +rerun-incomplete: true # re-do jobs left incomplete by an unclean death +show-failed-logs: true +printshellcmds: true + +# D3 DEPENDS ON THIS. Snakemake's default is to DELETE the declared outputs of a +# failed job — and this workflow's declared output IS the manifest, the only +# record of *why* a unit failed and the only thing `sp report` reads. Without +# keep-incomplete, every failed unit reads back as "not_run" and the post-mortem +# debris is gone. Safe here because each rule `rm -rf`s its own run dir at start, +# so a rerun never sees stale products. +keep-incomplete: true + +# rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still +# propagate — completeness.py writes the manifest only on change, so mtimes move +# only when reality moves. +# +# `input` is dropped because reclamation needs the tile->exposure edge to be +# CONDITIONAL: a tile whose final_cat is on disk declares no exposure inputs, so +# a neighbour rebuilding a shared exposure cannot drag it along (see tile.smk). +# With the `input` trigger on, that same conditional reads as "set of input files +# has changed" and reruns every finished tile — against an exposure store that +# reclamation has already deleted. Measured on fixture t4: 70 jobs with the +# trigger, 28 without, for one damaged tile in a four-tile chain. +# +# Nothing this workflow relied on is lost. The two genuinely data-derived input +# sets are covered another way: a changed exposure list arrives through the +# (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides +# in params. And clean scheduling is structurally gated to bin/sp (SP_PHASE=compute +# + this profile), so a bare snakemake invocation cannot quietly recombine +# reclamation with the `input` trigger; a runtime assertion is impossible +# (the trigger set is unreadable at parse time) — the gate is the launcher. +rerun-triggers: [mtime, params, code, software-env] + +# NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a +# rule's own values (verified snakemake 9.23), which would kill the +# attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +# ngmix thread count. The RULES own threads and resources; this profile only +# sets defaults for rules that state nothing (default-resources above). +# +# `group:` fusion of the short rules (uncompress, merges) into their chunky +# neighbours (PRD D-profile) is NOT set here — it requires labels in the rule +# files themselves, which is out of scope for this profile-only pass. Deferred +# to whichever slice next touches workflow/rules/*.smk. diff --git a/src/shapepipe/modules/merge_sep_cats_runner.py b/src/shapepipe/modules/merge_sep_cats_runner.py index 927c79b7..6e99cf3e 100644 --- a/src/shapepipe/modules/merge_sep_cats_runner.py +++ b/src/shapepipe/modules/merge_sep_cats_runner.py @@ -28,7 +28,7 @@ def merge_sep_cats_runner( ): """Define The Merge SEP Catalogues Runner.""" # Get config entries - n_split_max = config.getint(module_config_sec, "N_SPLIT_MAX") + n_split_max = int(config.getexpanded(module_config_sec, "N_SPLIT_MAX")) file_pattern = config.getlist(module_config_sec, "FILE_PATTERN") file_ext = config.getlist(module_config_sec, "FILE_EXT") diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index 230606bb..fca6ee0b 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -40,15 +40,27 @@ (no batch saving) ID_OBJ_MIN : int ID of first galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, so an orchestrator can + set the object range per chunk, for example + ``ID_OBJ_MIN = $SP_NGMIX_ID_OBJ_MIN``. ID_OBJ_MAX : int ID of last galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, as for ``ID_OBJ_MIN``. BKG_RMS_VIGNET_PATH : str, optional Path to a ``background_rms_vignet*.sqlite`` file produced by ``vignetmaker_runner``. The string may contain ``{file_number_string}``, which is replaced by the current tile ID. +Random number generation +======================== + +Each object gets its own random number stream, seeded from its sky position +and CCD (``position_seed``, ngmix#796). The output is therefore identical +whether a tile is processed in one go or split into object chunks with +``ID_OBJ_MIN``/``ID_OBJ_MAX``. The older tile-seeded mode is retired; the +config option ``SEED_FROM_POSITION`` is obsolete. Setting it to ``False`` +raises an error, so a stale config cannot silently change the RNG. + """ __all__ = ["ngmix"] diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index d4f64a74..fc9779c6 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -335,13 +335,12 @@ class Ngmix(object): (robust for galaxies); ``"wcs"`` uses the catalog sky position projected through the WCS (better for stars, whose HSM moments are noisy). See :func:`make_ngmix_observation`. - seed_from_position : bool, optional - If ``True``, replace the tile-level RNG with a per-object RNG seeded - from the object's sky position (:func:`position_seed`) inside the - object loop, so metacal's ``fixnoise`` counter-noise and the fit - guesses cancel across Pujol image-simulation branches (ngmix#796). The - default ``False`` leaves the production path byte-identical. See - :func:`position_seed` for the physics and the seed construction. + Notes + ----- + The RNG is always per object and seeded from that object's sky position + (:func:`position_seed`). Results therefore do not depend on how the tile is + split into object chunks, and metacal's ``fixnoise`` counter-noise and the + fit guesses cancel across Pujol image-simulation branches (ngmix#796). Raises ------ @@ -364,7 +363,6 @@ def __init__( id_obj_max=-1, bkg_sub=True, centroid_source="hsm", - seed_from_position=False, metacal_psf="fitgauss", ): @@ -418,20 +416,14 @@ def __init__( self._id_obj_max = id_obj_max self._bkg_sub = bkg_sub self._centroid_source = centroid_source - self._seed_from_position = seed_from_position self._metacal_psf = metacal_psf self._w_log = w_log - # Initiatlise random generator - seed = int(''.join(re.findall(r'\d+', self._file_number_string))) - self._rng = np.random.RandomState(seed) - self._w_log.info(f'Random generator initialisation seed = {seed}') - if self._seed_from_position: - self._w_log.info( - 'SEED_FROM_POSITION on: per-object RNG seeded from sky position' - ' for Pujol noise cancellation (image sims, ngmix#796)' - ) + self._w_log.info( + 'Per-object RNG seeded from sky position (ngmix#796): results are' + ' invariant to how the tile is split into object chunks' + ) @classmethod def MegaCamFlip(self, vign, ccd_nb): @@ -461,18 +453,6 @@ def MegaCamFlip(self, vign, ccd_nb): # swap y axis so origin is on bottom-left return vign - def get_prior(self, T_range=None, F_range=None): - """Get Prior. - - Returns - ------- - ngmix.joint_prior.PriorSimpleSep - """ - return get_prior( - self._pixel_scale, self._rng, - T_range=T_range, F_range=F_range, - ) - def compile_results(self, results): """Compile Results. @@ -809,7 +789,6 @@ def process(self): vignet_cat = self._vignet_cat final_res = [] - prior = self.get_prior() count = 0 n_empty_cat = 0 @@ -843,24 +822,20 @@ def process(self): n_no_epoch += 1 continue - # Position-seeded per-object RNG for Pujol noise cancellation in - # image sims (ngmix#796): the same object gets the same fixnoise - # counter-noise and fit guesses in every shear branch, so both - # cancel in the branch difference. The prior is rebuilt from the - # same per-object RNG because the guesser draws its initial guess - # via prior.sample() (ngmix guessers.py), which consumes the RNG the - # prior was CONSTRUCTED with — so a per-object rng alone would leave - # the guess drawing from the shared tile stream and break - # cancellation. Off in production, where the single tile-level - # self._rng and the tile-level prior carry the whole loop. - if self._seed_from_position: - obj_rng = np.random.RandomState( - position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) - ) - obj_prior = get_prior(self._pixel_scale, obj_rng) - else: - obj_rng = self._rng - obj_prior = prior + # Position-seeded per-object RNG (ngmix#796). Each object draws from + # a stream fixed by its own (ra, dec, ccd), so the result is + # independent of which chunk the object lands in and of detection + # order, and the same object gets the same fixnoise counter-noise + # and fit guesses in every Pujol shear branch, so both cancel in the + # branch difference. The prior is rebuilt from the same per-object + # RNG because the guesser draws its initial guess via prior.sample() + # (ngmix guessers.py), which consumes the RNG the prior was + # CONSTRUCTED with — a per-object rng alone would leave the guess + # drawing from a shared stream and break both properties. + obj_rng = np.random.RandomState( + position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) + ) + obj_prior = get_prior(self._pixel_scale, obj_rng) try: flux_guess = ( diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 3d98356c..fcb50be9 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -86,9 +86,12 @@ def ngmix_runner( # No batch saving save_batch = -1 - # First and last galaxy ID to process - id_obj_min = config.getint(module_config_sec, "ID_OBJ_MIN") - id_obj_max = config.getint(module_config_sec, "ID_OBJ_MAX") + # First and last galaxy ID to process. Read via ``getexpanded`` so an + # orchestrator can drive the chunk bounds from environment variables + # (``$SP_NGMIX_ID_OBJ_MIN`` and friends); ``getexpanded`` is the only + # accessor in ShapePipe's config that expands ``$VAR``. + id_obj_min = int(config.getexpanded(module_config_sec, "ID_OBJ_MIN")) + id_obj_max = int(config.getexpanded(module_config_sec, "ID_OBJ_MAX")) # Centroid source for the galaxy Jacobian origin: "wcs" (default -- the # catalog sky position projected through the WCS, trusting the astrometry) @@ -99,16 +102,20 @@ def ngmix_runner( else: centroid_source = "wcs" - # Seed the per-object RNG from sky position instead of per tile, so - # metacal's fixnoise counter-noise (and the fit guesses) cancel across - # Pujol image-simulation shear branches (ngmix#796). Default False leaves - # the production path byte-identical. + # Position-seeded RNG is the only mode: every object's RNG comes from its + # own (ra, dec, ccd), so results do not depend on how the tile is split + # into chunks, and metacal's fixnoise counter-noise cancels across Pujol + # image-simulation shear branches (ngmix#796). The retired tile-seed mode + # had neither property. Old configs that disable it must fail loudly. if config.has_option(module_config_sec, "SEED_FROM_POSITION"): - seed_from_position = config.getboolean( - module_config_sec, "SEED_FROM_POSITION" - ) - else: - seed_from_position = False + if not config.getboolean(module_config_sec, "SEED_FROM_POSITION"): + raise ValueError( + "SEED_FROM_POSITION = False is no longer supported: the" + " tile-seeded RNG mode has been retired because it makes" + " results depend on the object chunking. Remove the" + " SEED_FROM_POSITION entry from the ngmix config section" + " (position-seeded RNG is now the only mode)." + ) # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. @@ -161,7 +168,6 @@ def ngmix_runner( id_obj_max=id_obj_max, bkg_sub=bkg_sub, centroid_source=centroid_source, - seed_from_position=seed_from_position, metacal_psf=metacal_psf, ) diff --git a/src/shapepipe/run.py b/src/shapepipe/run.py index 962953ed..fee0dad3 100644 --- a/src/shapepipe/run.py +++ b/src/shapepipe/run.py @@ -84,7 +84,7 @@ def _set_run_name(self): Set the name of the current pipeline run. """ - self._run_name = self.config.get("DEFAULT", "RUN_NAME") + self._run_name = self.config.getexpanded("DEFAULT", "RUN_NAME") if self.config.getboolean("DEFAULT", "RUN_DATETIME"): self._run_name += datetime.now().strftime("_%Y-%m-%d_%H-%M-%S") diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 00000000..c64047e7 --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,172 @@ +# ShapePipe Snakemake orchestration + +Snakemake workflow that orchestrates real-data ShapePipe runs. It replaces the +`curl_canfar_local.sh → run_job_sp_canfar_v2.0.bash → job_sp_canfar_v2.0.bash` +bash layers and the per-site sbatch reimplementations. **Module code is +untouched**: rules call `shapepipe_run -c ` on the existing config +chains. Design and rationale: +[CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) +(the living PRD). + +Use `workflow/bin/sp` for everything. Bare `snakemake all` outside `sp` is +unsupported: `sp` sets the state directory, the SLURM profile, and +`SP_PHASE`, which the Snakefile needs to build the tile/exposure index at +parse time. Running snakemake directly skips all of that. + +## Quick start (nibi) + +```bash +# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which +# is node-local; the SLURM executor re-invokes this python inside every job). +uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 +source /project/def-mjhudson/cdaley/snakemake-env/bin/activate +uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' + +# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats. + +# The committed launcher loads apptainer/1.4.5 + the /project venv, so a +# fresh shell always has the right state. +workflow/bin/sp run # bring products on disk up to date with the tile list +workflow/bin/sp report # emit run_report.json now (mid-run is fine) +workflow/bin/sp cancel # scancel this workflow's jobs +``` + +Installed and pinned versions on nibi (`/project/def-mjhudson/cdaley/snakemake-env`, +queried 2026-07-30): `snakemake==9.23.1`, `snakemake-executor-plugin-slurm==2.7.1`. +Pin range: `snakemake>=9,<10`, `snakemake-executor-plugin-slurm>=2.7,<3`. The +v8→v9 breaks matter here: `--use-singularity` became `--sdm`, executors became +plugins, and full `rerun-triggers` became the default. + +Anything other than `run`, `report`, `cancel` passes straight through to +snakemake with the workflow's profile and state dir — the escape hatch for +`sp --unlock`, `sp --dag`, `sp exp_psf ...`. + +## Execution: two static invocations + +The exposure job set is data-derived from the tiles' `find_exposures` output, +so it cannot live in the same static DAG that produces it. `sp run` is +therefore two snakemake invocations over one Snakefile: + +1. **PREPARE** — `snakemake prepare_all_tiles`: per-tile static DAG + (`Git_vos → Uz → Fe`), `keep-going` so tile failures are independent. A + nonzero exit here is not fatal to the run — tiles that lost their + exposure list are dropped at the compute parse — but it is a warning: + `SP_MISSING_THRESHOLD` (default 0.0) is the real gate. +2. **COMPUTE** — `snakemake all`: this invocation's *parse* builds the + tile↔exposure index (`build_index.py`, imported at parse time, not a DAG + node) and runs the full tile/exposure compute chain. The index + accumulates across invocations, so appending tiles later changes which + jobs exist without invalidating completed work. + +`sp run` chains both so the UX is one command; both exit codes are checked +and the run fails if either phase failed. + +## Execution mode: one SLURM job per rule + +The profile (`profiles/nibi/config.yaml`) sets `executor: slurm`. Every rule +instance becomes its own SLURM job carrying that rule's own attempt-scaled +resources (`cpus_per_task = threads`, `mem_mb`, `runtime`) and runs inside +`apptainer exec` via the profile's software-deployment method — the workflow +never calls apptainer directly. Snakemake feeds the queue as jobs finish, so +the full campaign's job count never needs to queue at once, and multi-node +scaling is inherent. `jobs:` in the profile caps concurrent submissions at a +fraction of the cluster's per-user submit limit (queried, not invented — see +the comment in the profile file for the query and date). + +The profile intentionally sets no `set-resources` / `set-threads` overrides: +those replace a rule's own values wholesale, which would kill the +attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +ngmix thread count. Rules own their own resources; the profile only supplies +defaults for rules that state nothing. + +`group:` labels that fuse short rules (uncompress, merges) into their chunky +neighbours (queue-latency amortization, per the PRD) are **not yet wired**: +they require labels in `workflow/rules/*.smk`, out of scope for this +profile-only pass. + +## Layout + +``` +workflow/ + Snakefile parse-time index load; global container:; onsuccess/onerror report hooks + config.yaml the run: tile list, paths, container, chunk count + bin/sp committed launcher (module load + /project venv + run/report/cancel) + rules/ + prepare.smk tile get_images/uncompress/find_exposures + per-unit star cats + exposure.smk per-exposure: get_images, split, mask, psf (no temp()) + tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat + scripts/ + sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) + build_index.py prepare-phase run_index.sqlite builder (plain script) + build_forest.py per-tile exposure symlink forest (group-compatible shell) + completeness.py the ported count-floor table (shared by sp_rule + run_report) + run_report.py standalone report (NOT a DAG node; run_report hooks call it) + clean_exposure.py ONE exposure's store + manifests -> tombstone (the clean_exposure rule) +profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going +``` + +## How it works + +- **The atom is one rule == one `shapepipe_run` on one unit.** Its single + declared output is that unit's manifest + (`/manifests/.json`), not its product files — a missing + CCD is often legitimate, and at DR6 scale per-CCD declaration means + millions of paths. +- **Manifests are the DAG's currency.** `completeness.py check` writes the + manifest, and it is the only record of *why* a unit failed. That is why + the profile sets `keep-incomplete: true` — Snakemake's default deletes a + failed job's declared outputs, which would erase the manifest the report + needs. +- **Completeness is a count floor, not a taxonomy.** After a run, + `sp_rule.py` counts products per mandatory runner against + `completeness.py`'s floor and exits nonzero below it. Per-CCD attrition + between floor and `expect` is tolerated. No 3-class taxonomy, no + error-signature whitelist. `--keep-going` isolates a failure to its own + DAG cone. +- **Stores are sharded.** Every tile/exposure runs its own `shapepipe_run` + in `tiles/<2-char prefix>//` or `exp///`, with a `cfis` + config symlink, `star_cat_{exp,tiles}` symlinks, and a config copy setting + `RUN_DATETIME=False`. Configs are committed under `workflow/config/cfis/` + and version with the rules that set the env vars they interpolate — there + is no `config_src` knob, by construction. +- **The index is parse-time data, never a rule input.** Appending tiles + changes which jobs exist without invalidating completed work. Star cats + are re-keyed per unit for the same reason. `final_cat` is `protected()`. +- **Exposure products are not `temp()`.** Exposures overlap tiles, so + `temp()` would cascade destructive reruns when a tile is appended later. + Reclamation is the in-DAG `clean_exposure` rule instead: one job per + exposure, taking every consuming tile's `tile_vignets` manifest as input + (the campaign-wide consumer set comes from the accumulating index), which + deletes the store *and* the exposure's manifests and leaves a `cleaned.json` + tombstone. Deleting the manifests is what makes a late append correct: the + appended tile finds an unbuilt chain and regenerates it. The `clean:` flag + in `config.yaml` gates it; flipping it on later reclaims retroactively, + since the missing tombstones schedule exactly the outstanding clean jobs. + The tombstone is written *before* anything is deleted, so a crash can cost + disk but never the record. +- **A finished tile declares no reclaimed exposures.** Deleting an exposure's + manifests would otherwise rerun every other tile that reads it, and those + reruns spread across the exposure-overlap component. So a tile whose + `final_cat` exists drops the exposure manifests that are gone from its input + list, and holds the rest through `ancient()`. This is why the profile runs + with `rerun-triggers: [mtime, params, code, software-env]`: the `input` + trigger reads that cut as a reason to rerun the very tiles it protects. + Know the consequence — `--forcerun` on a tile whose `final_cat` exists will + not rebuild its reclaimed exposures. Delete the `final_cat` first. +- **A dead tile can be told to stop pinning exposures.** An exposure is + cleanable only once every consuming tile has its vignets, so one + permanently-failed tile holds its ~80 exposures for the life of the + campaign. List it under `clean_ignore_tiles:` in `config.yaml` and it leaves + the consumer sets. Retrying an ignored tile later is legal and expensive: + its exposure chains are gone and rebuild from scratch. +- **A reclaimed exposure reports as `cleaned`.** `run_report.py` reads the + absorbed manifests out of `cleaned.json`, so a reclaimed exposure keeps its + per-runner counts and blocks no tile. The `exp_psf` benchmark tsv lives + beside `manifests/`, not inside it, so reclamation does not eat the + memory-sizing data. +- **Failure is a report, not a gate.** `run_report.py` disk-scans the trees + against the count table and enumerates shortfalls (whole-unit absence vs + per-CCD attrition). It runs standalone — a DAG report node would itself be + poisoned by the failures it must enumerate — and fires automatically from + the COMPUTE invocation's `onsuccess`/`onerror` hooks, or on demand via + `sp report`. diff --git a/workflow/Snakefile b/workflow/Snakefile new file mode 100644 index 00000000..419e56cb --- /dev/null +++ b/workflow/Snakefile @@ -0,0 +1,392 @@ +"""ShapePipe real-data orchestration — Snakemake workflow. + +Design / rationale: CosmoStat/shapepipe#848 (the living PRD), D1-D5. + +`sp run` is TWO snakemake invocations over this one Snakefile: + + SP_PHASE=prepare snakemake prepare_all_tiles # Git -> Uz -> Fe, per tile + SP_PHASE=compute snakemake all # everything else + +They are two because the exposure job set is *data-derived*: it comes from the +tiles' find_exposures output, and a Snakemake DAG is fixed at parse time. The +join between them is the tile<->exposure index, built HERE at parse time of +invocation 2 (build_index.build(), imported — there is no `sp index` verb) and +loaded into plain dicts. The index is never a rule input, so appending tiles and +rebuilding it changes which jobs exist without invalidating completed work; it +ACCUMULATES across invocations, so a later clean_exposure (S5) sees every +consuming tile of the whole campaign, not just this tile list. + +The atom (D2): one rule == one `shapepipe_run` on one unit; its single declared +output is its MANIFEST (`/manifests/.json`), written by +`completeness.py check`. Product files are not declared — a missing CCD is often +legitimate, and at DR6 scale per-CCD declaration means millions of paths. +""" + +import hashlib +import os +import sqlite3 +import sys +from pathlib import Path + +from snakemake.exceptions import WorkflowError + +# Resolved relative to THIS file, not the working directory: snakemake runs with +# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project +# (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). +configfile: str(Path(workflow.snakefile).parent / "config.yaml") + +# Every job's shell runs inside this container (apptainer software-deployment in +# the profile); the user never types apptainer. +container: config["container"] + +# --- paths ----------------------------------------------------------------- +RUN_DIR = Path(config["run_dir"]) +STAR_CATS = Path(config["star_cats"]) +INDEX_DB = Path(config["index_db"]) +SCRIPTS = Path(workflow.basedir) / "scripts" +# The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the +# configs and the rules that set the env vars they interpolate are one artefact +# and must version together. Hence no `config_src` knob. +CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis" + +sys.path.insert(0, str(SCRIPTS)) +import build_index # noqa: E402 +from completeness import STAGE_DIR # noqa: E402 + +# SP_PHASE is set by bin/sp on the two invocations of `sp run` and NOWHERE else. +# It gates the two parse-time side effects — the index build and the report +# hooks — so that a passthrough invocation (`sp --unlock`, `sp --dag`, `sp +# exp_psf ...`) never mutates durable state or dies on a threshold it was not +# asked about. Unset means "read the index, build nothing". +PHASE = os.environ.get("SP_PHASE", "") + +with open(config["tile_list"]) as f: + TILES = [ln.strip() for ln in f if ln.strip()] + +# --- ngmix scatter (D4) ---------------------------------------------------- +# Native directive: `--set-scatter ngmix=N` overrides it, N=1 degenerates to one +# ngmix job per tile. We take the count and drive our own integer `chunk` +# wildcard rather than snakemake's `{scatteritem}` ("3-of-8") token, because the +# chunk number is not ours alone: it names the run dir +# (`run_sp_tile_ngmix_Ngu`, from the template's RUN_NAME), and +# merge_sep_cats derives chunks 2..N from chunk 1's path by substituting the +# first "1" — which only works for bare integers. +scattergather: + ngmix=int(config.get("ngmix_chunks", 8)) + +NGMIX_CHUNKS = workflow._scatter["ngmix"] + +# --- parse-time index build + load (D1) ------------------------------------ +# The COMPUTE invocation's parse IS the index build, and it runs UNCONDITIONALLY +# there — no "some Fe output exists" guard. That guard used to make a +# totally-failed prepare produce an empty index, an empty DAG and a green exit 0; +# with it gone, zero Fe outputs means a missing fraction of 1.0, which trips the +# SP_MISSING_THRESHOLD gate and fails loudly, as it should. +# +# In every other phase (prepare, or unset for a passthrough invocation) the parse +# builds NOTHING and only loads whatever index is already on disk. +if PHASE == "compute": + build_index.build( + TILES, RUN_DIR, INDEX_DB, + missing_threshold=float(os.environ.get("SP_MISSING_THRESHOLD", "0.0"))) + +# EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes +# verbatim into the fabricated per-unit exp_numbers list so get_images matches +# .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. +# EXP_TILES is the inverse edge — the CAMPAIGN-WIDE consumer set clean_exposure +# is keyed on (D5). +EXP, TILE_EXP, EXP_TILES = {}, {}, {} +if INDEX_DB.exists(): + _con = sqlite3.connect(INDEX_DB) + EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) + for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + TILE_EXP.setdefault(_tile, []).append(_exp) + EXP_TILES.setdefault(_exp, []).append(_tile) + _con.close() + +# Tiles this run can actually compute: declared AND indexed. The index spans the +# campaign, so it is intersected with the declared list, not used as it. +TILES_READY = [t for t in TILES if t in TILE_EXP] +READY_SET = set(TILES_READY) + +# A compute invocation with nothing to compute is never a success. Without this, +# an empty intersection yields `rule all` with no inputs, an empty DAG and exit +# 0 — the silent green run the threshold gate exists to prevent. +if PHASE == "compute" and not TILES_READY: + raise WorkflowError( + f"No declared tile has an indexed exposure list: 0 of {len(TILES)} tiles " + f"are ready to compute (index {INDEX_DB}). Run the prepare phase first " + f"(`sp run`), or check {INDEX_DB.parent / 'missing.json'}.") + +wildcard_constraints: + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + shard = r"\d{2}", + chunk = r"\d+", + +# --- the sharded stores (D2) ---------------------------------------------- +# tiles/<2-char prefix>// and exp/<2-char prefix>// — no directory +# exceeds ~1k entries at full-UNIONS scale. Rules carry the shard as its own +# wildcard because an output pattern cannot compute it; every path the DAG uses +# is built by these helpers, so a mismatched (shard, id) pair is never requested. +TILE_DIR = str(RUN_DIR / "tiles" / "{shard}" / "{tile}") +EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") + +def tile_dir(tile): + return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" + +def exp_dir(exp): + return f"{RUN_DIR}/exp/{exp[:2]}/{exp}" + +def tile_manifest(tile, stage): + return f"{tile_dir(tile)}/manifests/{stage}.json" + +def exp_manifest(exp, stage): + return f"{exp_dir(exp)}/manifests/{stage}.json" + +def forest_dir(tile): + return f"{tile_dir(tile)}/exp_forest" + +def final_cat(tile): + return f"{tile_dir(tile)}/final_cat-{tile}.fits" + +def unit_num(unit): + """$SP_UNIT_NUM: ShapePipe's image-number convention, dot -> dash, leading + dash (tile ``210.282`` -> ``-210-282``; exposure ``2605805`` -> ``-2605805``). + The RULES do this transform; the configs just interpolate $SP_UNIT_NUM into + NUMBER_LIST (the `set_config_number_list` mechanism that replaced the retired + -e/--exclusive flag, #746).""" + return "-" + unit.replace(".", "-") + +# Content hash of completeness.py, computed once at parse time and carried as a +# param on every rule: the default rerun-triggers' `code` trigger hashes only the +# rule's own shell string, NOT external scripts it calls — without this, a fix to +# the count table silently leaves stale manifests in place (bitten live). Scoped +# to completeness.py alone, the one script every shell line runs; build_forest.py +# gets its own hash, on the forest rule only. +SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] +FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] +CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] + +# --- exposure reclamation (D5, S5) ----------------------------------------- + + +def flag(value, default=False): + """Truthiness for a config value that may arrive as a STRING. + + `--config clean=false` delivers the string "false", and every non-empty + string is truthy in Python — a plain bool() read that as ON and scheduled + the deletions the user had just switched off. YAML booleans pass through + unchanged; only strings are parsed, and an unparseable one is an error, not + a guess. + """ + if value is None: + return default + if isinstance(value, str): + v = value.strip().lower() + if v in ("1", "true", "yes", "on"): + return True + if v in ("0", "false", "no", "off", ""): + return False + raise WorkflowError(f"Cannot read {value!r} as a boolean (use true/false).") + return bool(value) + + +# `clean:` in config.yaml gates the whole mechanism. Off => the rule generates no +# jobs at all (nothing requests a tombstone); flipping it on later reclaims +# RETROACTIVELY, because the exposures already cleaned are exactly the ones with +# a tombstone, so the missing tombstones schedule exactly the clean jobs. +# Only ever active under SP_PHASE=compute: the prepare and passthrough parses +# read no index of their own and must schedule no deletions. +CLEAN = flag(config.get("clean", False)) and PHASE == "compute" + +# Reclamation REQUIRES the `input` rerun-trigger to be off (profiles/nibi sets +# the list; tile.smk explains why). The two are already tied together: CLEAN is +# gated on SP_PHASE=compute, which only workflow/bin/sp sets, and bin/sp always +# launches with that profile. A bare snakemake invocation without SP_PHASE +# schedules no clean job at all, so it cannot meet the incompatible combination. +# There is no runtime assertion because the trigger set is not readable at parse +# time (workflow.dag_settings is None until the DAG is built). + +# Tiles that must not pin an exposure's store. See config.yaml: a permanently +# failed tile otherwise holds every exposure it touches (~80) forever, because +# its vignets manifest will never exist and its exposures are therefore never +# eligible. Listing it here drops it from the consumer sets. +CLEAN_IGNORE_TILES = set(config.get("clean_ignore_tiles") or []) + + +def tombstone(exp): + """The clean_exposure output. Lives BESIDE manifests/, not inside it: the + clean job deletes manifests/ wholesale, and `sp report` scans it.""" + return f"{exp_dir(exp)}/cleaned.json" + + +def clean_consumers(exp): + """Every tile in the campaign that reads this exposure — the set whose + vignets must all exist before the store may go — minus the ignored tiles.""" + return sorted(t for t in EXP_TILES.get(exp, []) + if t not in CLEAN_IGNORE_TILES) + + +def clean_targets(): + """Which exposures this invocation may clean. + + An exposure is eligible only when every consuming tile is either in this + run's scope or has already produced its vignets on disk. Without that test, + requesting a tombstone for an exposure shared with a LATER batch would drag + that batch's whole tile chain into this DAG through the clean rule's input — + scope expansion by cleanup, which is not a trade anyone asked for. Ineligible + exposures are simply skipped; the invocation that finishes their last + consumer picks them up. Deferral, never loss. + + Consumer sets are the IGNORE-FILTERED ones (clean_consumers), so a tile in + `clean_ignore_tiles` neither gates eligibility nor appears in the job's + input — which is the whole point of that list. + """ + if not CLEAN: + return [] + out = [] + for exp, raw in EXP_TILES.items(): + if not raw: + continue + # An empty set AFTER filtering means every consumer is ignored: nothing + # is left that could ever read this exposure, so it is eligible now. + # That is the whole point of clean_ignore_tiles — all() of an empty set + # is True, and it is true here in the intended sense. + tiles = clean_consumers(exp) + if all(t in READY_SET or Path(tile_manifest(t, "tile_vignets")).exists() + for t in tiles): + out.append(tombstone(exp)) + return sorted(out) + +# --- the shell every rule runs (D2) ---------------------------------------- + +_THREAD_CAPS = " ".join( + f"{k}={v}" for k, v in ( + ("OMP_NUM_THREADS", 1), ("OPENBLAS_NUM_THREADS", 1), + ("MKL_NUM_THREADS", 1), ("NUMEXPR_NUM_THREADS", 1), + ("MALLOC_ARENA_MAX", 2), ("MALLOC_TRIM_THRESHOLD_", 0))) + + +def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, + pre_run=()): + """The unit-furniture + environment prologue, as bash. + + Returned as a rule ``params`` value, NEVER inlined into the ``shell:`` + string: snakemake formats a shell string ONCE, so a ``{output}``/``{threads}`` + placeholder inside a params value would survive literally — and, conversely, + the literal ``${SP_NGMIX_CHUNK}`` braces this prologue needs would blow up + that formatting if they lived in the shell string. Params values are + substituted after formatting, so both hazards go away together. + + What it materialises (the proven v2.0 isolation-by-work-dir-content, NOT + -e/--exclusive): + * ``output/`` and ``manifests/``; + * tile: ``tile_numbers.txt`` (dot format — what get_images reads); + * exposure: a fabricated pseudo-Fe ``exp_numbers-000-000.txt`` holding the + ORIGINAL exposure name from the index (``2605805p``), so get_images + matches ``.fits.fz`` in the store — the bare base id matches + nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale + pre-fix file with the bare id; + * ``star_cat_exp`` / ``star_cat_tiles`` dir symlinks into the shared + pre-generated pool (the mask configs read them as INPUT_DIRs). + There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at + the committed config dir. + + Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's + FileHandler raises on an existing run dir, and it is how a rerun never sees + stale products (D2: the job clears its run dir at start). + """ + work = tile_dir(unit) if level == "tile" else exp_dir(unit) + _, subdir = STAGE_DIR[stage] + lines = [ + "set -euo pipefail", + f"export SP_RUN='{work}'", + f"export SP_UNIT_NUM='{unit_num(unit)}'", + f"export SP_CONFIG='{CONFIG_DIR}'", + # Also set via apptainer-args in the profile; kept here so a hand-run of + # this same line outside snakemake behaves identically. + f"export {_THREAD_CAPS}", + 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests"', + ] + if forest: + lines.append(f"export SP_EXP='{forest}'") + for k, v in (env or {}).items(): + lines.append(f"export {k}='{v}'") + + if level == "tile": + lines.append(f"printf '%s\\n' '{unit}' > \"$SP_RUN/tile_numbers.txt\"") + else: + fe = "$SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output" + lines += [f'mkdir -p "{fe}"', + f"printf '%s\\n' '{exp_name or unit}' > \"{fe}/exp_numbers-000-000.txt\""] + + for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): + src = STAR_CATS / sub + lines.append(f"[ -d '{src}' ] && ln -sfn '{src}' \"$SP_RUN/{name}\" || true") + + lines += list(pre_run) + lines += [f'rm -rf "$SP_RUN/output/{subdir}"', 'cd "$SP_RUN"'] + return "\n".join(lines) + + +def sp_shell(stage, config_name): + """The rule's shell string: prologue, one shapepipe_run, one completeness check. + + ``{threads}`` and ``{output}`` are placeholders HERE and nowhere else (see + unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number + by construction (D4). + + The check runs even when shapepipe_run failed — the manifest is the only + thing `sp report` reads, so a failed unit must still leave a record of why. + """ + return ( + "{params.pre}\n" + "rc=0\n" + f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" + f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}} || rc=1\n" + "exit $rc\n" + ) + + +include: "rules/prepare.smk" +include: "rules/exposure.smk" +include: "rules/tile.smk" + +# --- top-level targets ------------------------------------------------------ +# Only the aggregation targets and clean_exposure are localrules: a mid-chain +# localrule would break `group:` fusion of the compute chains (D4). +# clean_exposure is safe there because it is not in any chain — it hangs off +# `all` — and it is seconds of rmtree. As an sbatch job it would be ~20k +# scheduler submissions at DR6 scale to delete directories; the local-cores cap +# serialises them instead, which costs nothing at rmtree speed. +localrules: all, prepare_all_tiles, clean_exposure + +rule all: + input: + [final_cat(t) for t in TILES_READY], + clean_targets(), + +# Invocation 1 — the static per-tile DAG, known from the tile list alone. +# keep-going makes tile failures independent; the ones that lose their exposure +# list are dropped by the index build at invocation 2's parse. +rule prepare_all_tiles: + input: + [tile_manifest(t, "tile_find_exposures") for t in TILES] + +# --- report hooks ----------------------------------------------------------- +# run_report is NOT a DAG node (a descendant of every job would be poisoned by +# any hard failure — the exact case it exists for). It is a standalone script, +# emitted automatically at the end of the COMPUTE invocation, runnable any time +# via `sp report`. +def _report(status): + shell(f"python {SCRIPTS}/run_report.py --run-dir {RUN_DIR} " + f"--index {INDEX_DB} --status {status} || true") + +if PHASE == "compute": + + onsuccess: + _report("success") + + onerror: + _report("error") diff --git a/workflow/bin/sp b/workflow/bin/sp new file mode 100755 index 00000000..3424584d --- /dev/null +++ b/workflow/bin/sp @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). +# +# Two verbs, nothing else: +# +# sp run [ARGS...] bring the products on disk up to date with the tile list. +# Two snakemake invocations over one Snakefile: +# 1. PREPARE snakemake prepare_all_tiles +# 2. COMPUTE snakemake all <- its PARSE builds the index +# ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. +# sp report [ARGS...] emit run_report.json now (mid-run is fine). +# +# Anything else is passed straight through to snakemake with the same profile and +# state dir (the escape hatch: `sp --unlock`, `sp exp_psf ...`, `sp --dag`). +# +# It also loads the apptainer module (snakemake resolves `apptainer` via PATH at +# job runtime) and activates the snakemake venv on the shared /project FS (the +# executor re-invokes it inside jobs, so it cannot live on a node-local path). +# One entry point, so a fresh tmux or a restart after a crash always launches +# with the right state. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ +REPO="$(dirname "$HERE")" +VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" +PROFILE="$REPO/profiles/nibi" +SCRIPTS="$HERE/scripts" +CONFIG="$HERE/config.yaml" + +module load apptainer/1.4.5 2>/dev/null || true +# shellcheck disable=SC1091 +source "$VENV/bin/activate" + +# Minimal scalar reader for workflow/config.yaml (key: value, no nesting). +cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } +RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" + +# Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO +# THE RUN on /scratch, never on /project (a hard 27/27 TiB group quota killed a +# metadata write mid-run, live). --directory only moves state: all data paths are +# absolute, and the Snakefile resolves its own configfile. +STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" + +# SP_MISSING_THRESHOLD gates the compute parse's index build: the fraction of +# declared tiles allowed to be missing their exposure list (default 0.0). +export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" + +sm() { snakemake --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } + +cmd="${1:-}" +case "$cmd" in + run) + shift + # PREPARE failing is NOT fatal to the run: keep-going means a failed tile + # poisons only its own cone, and the tiles that lost their exposure list are + # dropped at the compute parse. The real gate is SP_MISSING_THRESHOLD, which + # the compute parse's index build enforces over the WHOLE tile list — so we + # record the failure and go on rather than letting `set -e` abort here. + prep_rc=0 + SP_PHASE=prepare sm prepare_all_tiles "$@" || prep_rc=$? + if [ "$prep_rc" -ne 0 ]; then + echo "" >&2 + echo "############################################################" >&2 + echo "## WARNING: the PREPARE phase exited $prep_rc." >&2 + echo "## Some tiles may be missing their exposure list and will be" >&2 + echo "## dropped from the compute DAG. Continuing to COMPUTE; the" >&2 + echo "## SP_MISSING_THRESHOLD gate (now $SP_MISSING_THRESHOLD) decides" >&2 + echo "## whether that is tolerable." >&2 + echo "############################################################" >&2 + echo "" >&2 + fi + + comp_rc=0 + SP_PHASE=compute sm all "$@" || comp_rc=$? + if [ "$prep_rc" -ne 0 ] || [ "$comp_rc" -ne 0 ]; then + exit "$([ "$comp_rc" -ne 0 ] && echo "$comp_rc" || echo "$prep_rc")" + fi + ;; + report) + shift + python "$SCRIPTS/run_report.py" --run-dir "$RUN_DIR" --index "$INDEX_DB" \ + --status manual "$@" + ;; + cancel) + # Kept only because it is two lines: scancel this workflow's jobs by name + # before an --unlock. Not part of the design surface. + run="${2:?usage: sp cancel }" + squeue --me --noheader --format='%i %j' \ + | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel + echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" + ;; + *) + sm "$@" + ;; +esac diff --git a/workflow/config.yaml b/workflow/config.yaml new file mode 100644 index 00000000..e06235f1 --- /dev/null +++ b/workflow/config.yaml @@ -0,0 +1,67 @@ +# Run configuration for the ShapePipe Snakemake workflow. +# +# A "run" is declared by a tile list plus the paths below. Everything here is +# read at parse time; none of it is a rule input, so editing it (e.g. appending +# tiles) never invalidates completed work — it only changes which jobs exist. + +# The tile list that scopes this run (one "IDra.IDdec" per line). +# 210/211 quad: 19 unique exposures, 15 reused across tiles — exercises the +# structural exposure dedup (the 196 quad had zero overlap; kept for the +# append-invariant test). +tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt + +# The container every job runs inside (apptainer software-deployment in the profile). +container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif + +# Where products land ($SP_RUN root). The sharded per-unit stores live under here: +# /tiles/<2-char prefix>// and /exp/// +run_dir: /scratch/cdaley/shapepipe-output/smk-g3 + +# There is no config_src knob: the config chain is workflow/config/cfis, resolved +# relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / +# $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules +# are one artefact and must version together, so the dir is fixed by construction. + +# Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). +# Star catalogues for masking, pre-generated (network step done in prepare): +star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats + +# The run index. Design intent (finding 15) is durable products on /project — +# BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and +# report live with the run on /scratch until space is reclaimed. Mind the +# 60-day purge for anything that must survive. +index_db: /scratch/cdaley/shapepipe-output/smk-g3/index/run_index.sqlite + +# Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one +# `clean_exposure` job per exposure. It fires once every campaign tile that reads +# that exposure has its vignets, deletes the exposure's store AND its manifests, +# and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them +# back out of the tombstone and reports the exposure as `cleaned`. Off here +# because P0 is a 4-tile debugging run where the exposure store is exactly what +# you want to inspect; on for any run big enough to care about disk. Flipping it +# on later reclaims retroactively — the missing tombstones schedule exactly the +# outstanding clean jobs. +clean: false + +# Tiles that may NOT pin an exposure store (default: empty). +# +# An exposure is eligible for cleaning only once EVERY consuming tile has its +# vignets. One permanently-failed tile therefore holds all ~80 exposures it +# touches for the life of the campaign. A tile listed here is dropped from the +# consumer sets, and its exposures become eligible. +# +# READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its +# exposures' stores. If you later retry that tile, those exposure chains are +# gone and will be REBUILT from scratch — get_images, split, mask, psf, per +# exposure. That is correct, and expensive. Ignore a tile when you have decided +# it is dead, not while you are still debugging it. +clean_ignore_tiles: [] + +# ngmix within-tile chunking: static N chunks (closed ID ranges computed +# per-tile, in-job, from the tile's own sexcat). +ngmix_chunks: 8 + +# The container's installed shapepipe is overridden by the prod worktree via +# --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. +# build_index.py's missing-tile fraction floor is passed by workflow/bin/sp +# (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). diff --git a/workflow/config/cfis/config_exp_Gie.ini b/workflow/config/cfis/config_exp_Gie.ini new file mode 100644 index 00000000..7c6297cf --- /dev/null +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -0,0 +1,99 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag + +# Input file extensions +INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{6} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Gie_prev diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini new file mode 100644 index 00000000..0b85fe86 --- /dev/null +++ b/workflow/config/cfis/config_exp_Ma.ini @@ -0,0 +1,86 @@ +# ShapePipe configuration file for masking of exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Ma + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 4 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask exposures +[MASK_RUNNER] + +# Parent module +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_cat_exp + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Input file patterns: image, weight, external flag, external star catalogue +# (folded from runs/p3-batch1/cfis: offline cluster, no online star-cat fetch) +FILE_PATTERN = image, weight, flag, star_cat + +FILE_EXT = .fits, .fits, .fits, .fits + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +# (folded from p3-batch1: True, using the pre-staged star_cat_exp pool) +USE_EXT_STAR = True + +# File name suffix for the output flag files (optional) +PREFIX = pipeline + +# Path to check for existing output mask files +CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/workflow/config/cfis/config_exp_Sp.ini b/workflow/config/cfis/config_exp_Sp.ini new file mode 100644 index 00000000..dd27d6cc --- /dev/null +++ b/workflow/config/cfis/config_exp_Sp.ini @@ -0,0 +1,78 @@ +# ShapePipe configuration file for single-exposures, +# split images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed exposure ID (exp split is a "tile-scheme" stage per sp_rule.py). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_Gie/get_images_runner/output + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +FILE_EXT = .fitsfz, .fitsfz, .fitsfz + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini new file mode 100644 index 00000000..0af871d8 --- /dev/null +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -0,0 +1,181 @@ +# ShapePipe configuration file for single-exposures. PSFex PSF model. +# Process exposures after masking, from star detection to PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_SxSePsfPi +#RUN_NAME = run_sp_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +# Input from two modules +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/output/run_sp_exp_Ma/mask_runner/output + +# Read pipeline flag files created by mask module +FILE_PATTERN = image, weight, pipeline_flag + +# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on +# the decorator's 4-entry FILE_EXT default (length check fails at startup) +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND, BACKGROUND_RMS + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE + + +[SETOOLS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/sextractor_runner/output + +# Note: Make sure this doe not match the SExtractor background images +# (sexcat_background*) +FILE_PATTERN = sexcat + +NUMBERING_SCHEME = -0000000-0 + +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools + + +[PSFEX_RUNNER] + +# Use 80% sample for PSF model +FILE_PATTERN = star_split_ratio_80 + +NUMBERING_SCHEME = -0000000-0 + +# Path to executable for the PSF model (optional) +EXEC_PATH = psfex + +# Default psfex configuration file +DOT_PSFEX_FILE = $SP_CONFIG/default.psfex + +[PSFEX_INTERP_RUNNER] + +# Use 20% sample for PSF validation +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat + +FILE_EXT = .psf, .fits, .cat + +NUMBERING_SCHEME = -0000000-0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = VALIDATION + +# Column names of position parameters +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# If True, measure and store ellipticity of the PSF (using moments) +GET_SHAPES = True + +# Minimum number of stars per CCD for PSF model to be computed +STAR_THRESH = 22 + +# Maximum chi^2 for PSF model to be computed on CCD +CHI2_THRESH = 2 diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini new file mode 100644 index 00000000..750fe112 --- /dev/null +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -0,0 +1,92 @@ +# ShapePipe post-run configuration file: merge separated catalogues + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Ms + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_sep_cats_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +# NOTE: only chunk 1's ngmix output is listed here; merge_sep_cats_runner +# derives the other chunks' paths itself from N_SPLIT_MAX and this pattern +# (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the +# same fixed output dir. +# +# RELATIVE ON PURPOSE (do not "fix" this to $SP_RUN/...): merge_sep_cats derives +# chunk k's path with re.sub("1", str(k), input_file, 1) -- it replaces the FIRST +# "1" in the path. An absolute path under the sharded store +# (.../tiles/21/210.282/...) has digits before "Ng1u", so chunk 2 would be looked +# for in a nonexistent directory. Every rule runs shapepipe_run with cwd=$SP_RUN, +# so "./output/..." resolves and "Ng1u" carries the first "1". +INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_SEP_CATS_RUNNER] + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = ngmix + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits + +# Numbering convention, string that exemplifies a numbering pattern. +NUMBERING_SCHEME = -000-000 + +# WARNING (optional, default is 'error'). Use 'always'/'ignore' to +# display/ignore warnings, and not raise error +WARNING = always + +# Maximum number of separated catalogues per input. +# merge_sep_cats_runner.py reads this with getexpanded (runner .py line ~31), so +# $NGMIX_N_CHUNKS DOES expand here, exactly like ID_OBJ_MIN/MAX in the ngmix +# module. The tile_merge_cats rule exports it from the workflow's scattergather +# chunk count, which is the single source of truth; this value must equal that +# count, so do not replace it with a literal. +N_SPLIT_MAX = $NGMIX_N_CHUNKS diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask new file mode 100644 index 00000000..7c185c60 --- /dev/null +++ b/workflow/config/cfis/config_onthefly.mask @@ -0,0 +1,86 @@ +# Mask module configuration file for single-exposure images + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + + +## Border mask +[BORDER_PARAMETERS] + +BORDER_MAKE = True + +BORDER_WIDTH = 50 +BORDER_FLAG_VALUE = 4 + + +## Halo mask +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction spike mask +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier mask +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +TEMP_DIRECTORY = .temp + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False diff --git a/workflow/config/cfis/config_tile_Fe.ini b/workflow/config/cfis/config_tile_Fe.ini new file mode 100644 index 00000000..9546f062 --- /dev/null +++ b/workflow/config/cfis/config_tile_Fe.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for: find exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[FIND_EXPOSURES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Column number of exposure name in FITS header +COLNUM = 3 + +# Prefix to remove from exposure name +EXP_PREFIX = p + diff --git a/workflow/config/cfis/config_tile_Git.ini b/workflow/config/cfis/config_tile_Git.ini new file mode 100644 index 00000000..72a0f0be --- /dev/null +++ b/workflow/config/cfis/config_tile_Git.ini @@ -0,0 +1,93 @@ +# ShapePipe configuration file for: get tile images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/tiles, /project/def-mjhudson/unions-wl/tiles + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}\.\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Copy command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/data_tiles diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini new file mode 100644 index 00000000..50273143 --- /dev/null +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -0,0 +1,80 @@ +# ShapePipe post-run configuration file: create final catalogs, with +# no spread model on input + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Mc + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_Ms/merge_sep_cats_runner/output + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, galaxy_psf, ngmix + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282) +NUMBER_LIST = $SP_UNIT_NUM + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .sqlite, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +SM_DO_CLASSIFICATION = False + +SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/workflow/config/cfis/config_tile_Mh_exp.ini b/workflow/config/cfis/config_tile_Mh_exp.ini new file mode 100644 index 00000000..96512df1 --- /dev/null +++ b/workflow/config/cfis/config_tile_Mh_exp.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for merging per-exposure WCS headers +# at the tile level. Input is the exp_numbers file produced by +# find_exposures_runner; EXP_BASE_DIR tells the runner where to find +# the per-exposure split_exp_runner header .npy files. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mh_exp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_headers_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_HEADERS_RUNNER] + +# Input: exp_numbers txt file from find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# Tile numbering scheme (RA-Dec, e.g. -301-279) +NUMBERING_SCHEME = -000-000 + +# Root directory containing all per-exposure work directories. +# The runner will walk this tree to collect headers-.npy files. +EXP_BASE_DIR = $SP_EXP diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini new file mode 100644 index 00000000..6e3e3329 --- /dev/null +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -0,0 +1,104 @@ +# ShapePipe configuration file for tiles: ngmix + KSB + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +# Per-chunk run dir: the workflow exports SP_NGMIX_CHUNK= for each chunk. +# RUN_NAME is env-expanded (run.py getexpanded); braces keep the trailing "u" +# out of the variable name. +RUN_NAME = run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as +# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for +# every tile (missing file -> error, no per-tile fallback); omit the option +# entirely to fall back to the scalar sigma_mad noise estimate. +BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite + +# Number of objects to batch save during processing, optional. Omit or set +# to -1 for no batch saving. +# 250 (folded from runs/p3-batch1/cfis): worker RSS grows ~4.8 MB/object +# until the flush recycles it -> peak ~1.1 GB + 250 x 4.8 MB ~= 2.3 GB/worker +# (A/B test, job 17607877). Not superseded by -b {threads} (that sets fork +# width, this bounds per-worker memory). +SAVE_BATCH = 250 + +# Magnitude zero-point +MAG_ZP = 30.0 + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +# SEED_FROM_POSITION: per-object RNG seeded from sky position (ra, dec, ccd) +# instead of one ordered per-tile stream, so results are bit-identical under +# any chunking (D4/#796). Required for the chunked ngmix scatter/gather. +SEED_FROM_POSITION = True + +# ID_OBJ_MIN/MAX: this chunk's closed SExtractor NUMBER-column range, +# computed at execution time from the tile's own object count and expanded +# via ShapePipe's getexpanded (ngmix_runner.py verified: env-expanded, not +# plain getint). +ID_OBJ_MIN = $NGMIX_ID_MIN +ID_OBJ_MAX = $NGMIX_ID_MAX diff --git a/workflow/config/cfis/config_tile_PiViVi.ini b/workflow/config/cfis/config_tile_PiViVi.ini new file mode 100644 index 00000000..246ee18f --- /dev/null +++ b/workflow/config/cfis/config_tile_PiViVi.ini @@ -0,0 +1,172 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# PSFEx PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_PiViVi + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +#MODULE = psfex_interp_runner, + +MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[PSFEX_INTERP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Column names of position parameters +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# If True, measure and store ellipticity of the PSF +GET_SHAPES = True + +# Number of stars threshold +STAR_THRESH = 20 + +# chi^2 threshold +CHI2_THRESH = 2 + +# Multi-epoch mode parameters + +# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR +# for v2.0 per-exposure pipeline. psfex_runner/output/ dirs are discovered +# by scanning $SP_EXP for the exposures listed in the exp_numbers input file. +ME_DOT_PSF_EXP_DIR = $SP_EXP + +# Input psf file pattern +ME_DOT_PSF_PATTERN = star_split_ratio_80 + + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output + +FILE_PATTERN = sexcat, CFIS_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner +ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini new file mode 100644 index 00000000..0ce9ea22 --- /dev/null +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -0,0 +1,118 @@ +# ShapePipe configuration file for tile detection + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers + +FILE_EXT = .fits, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = False + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini new file mode 100644 index 00000000..f2ff51e4 --- /dev/null +++ b/workflow/config/cfis/config_tile_Uc.ini @@ -0,0 +1,92 @@ +# ShapePipe configuration file for tile object selection using +# an (external) catalogue + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uc + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = read_ext_sexcat_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[READ_EXT_SEXCAT_RUNNER] + +# NOTE(blocker): run_sp_tile_Gic (external-catalog get_images) is not one of +# the 13 configs in this sweep -- no committed config produces it, so this +# path is written by naming-convention analogy, unverified. Uc is otherwise +# fully de-wrapper-ified; do not rely on this input until Gic exists. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Gic/get_images_runner/output, $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = CFIS_cat, CFIS_image, log_exp_headers + +FILE_EXT = .cat, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +# Side length of the square postage stamp (vignet) extracted from the tile +# image, in pixels (must be odd). Default: 51 +VIGNET_SIZE = 51 + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = ALPHA_J2000,DELTA_J2000 + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/config/cfis/config_tile_Uz.ini b/workflow/config/cfis/config_tile_Uz.ini new file mode 100644 index 00000000..fc8550af --- /dev/null +++ b/workflow/config/cfis/config_tile_Uz.ini @@ -0,0 +1,73 @@ +# ShapePipe configuration file for: uncompress FITS image + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uz + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = uncompress_fits_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options +[UNCOMPRESS_FITS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_weight + +FILE_EXT = .fitsfz + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input HDU of image data, optional, default=0 +HDU_DATA = 1 + +# Output file pattern +OUTPUT_PATTERN = CFIS_weight diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask new file mode 100644 index 00000000..18cf5db2 --- /dev/null +++ b/workflow/config/cfis/config_tile_onthefly.mask @@ -0,0 +1,90 @@ +# Mask module config file for tiles + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + +## Border parameters +[BORDER_PARAMETERS] + +BORDER_MAKE = False + +BORDER_WIDTH = 0 +BORDER_FLAG_VALUE = 4 + + +## Halo parameters +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction pike parameters +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier parameters +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_PIXEL_SCALE = 0.187 +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + +## External flag +[EXTERNAL_FLAG] + +EF_MAKE = False + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False + +TEMP_DIRECTORY = .temp_tiles diff --git a/workflow/config/cfis/default.conv b/workflow/config/cfis/default.conv new file mode 100644 index 00000000..2590b9cb --- /dev/null +++ b/workflow/config/cfis/default.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/workflow/config/cfis/default.param b/workflow/config/cfis/default.param new file mode 100644 index 00000000..09ad8405 --- /dev/null +++ b/workflow/config/cfis/default.param @@ -0,0 +1,68 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +# The following flag requires a flag image +IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default.psfex b/workflow/config/cfis/default.psfex new file mode 100644 index 00000000..a9d1a906 --- /dev/null +++ b/workflow/config/cfis/default.psfex @@ -0,0 +1,85 @@ +# Default configuration file for PSFEx 3.17.1 +# EB 2017-11-30 +# + +#-------------------------------- PSF model ---------------------------------- + +BASIS_TYPE PIXEL # NONE, PIXEL, GAUSS-LAGUERRE or FILE +BASIS_NUMBER 20 # Basis number or parameter +BASIS_NAME basis.fits # Basis filename (FITS data-cube) +BASIS_SCALE 1.0 # Gauss-Laguerre beta parameter +NEWBASIS_TYPE NONE # Create new basis: NONE, PCA_INDEPENDENT + # or PCA_COMMON +NEWBASIS_NUMBER 8 # Number of new basis vectors +PSF_SAMPLING 1. # Sampling step in pixel units (0.0 = auto) +PSF_PIXELSIZE 1.0 # Effective pixel size in pixel step units +PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values +PSF_SIZE 51,51 # Image size of the PSF model +PSF_RECENTER N # Allow recentering of PSF-candidates Y/N ? +MEF_TYPE INDEPENDENT # INDEPENDENT or COMMON + +#------------------------- Point source measurements ------------------------- + +CENTER_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue parameters for source pre-centering +PHOTFLUX_KEY FLUX_AUTO # Catalogue parameter for photometric norm. +PHOTFLUXERR_KEY FLUXERR_AUTO # Catalogue parameter for photometric error + +#----------------------------- PSF variability ------------------------------- + +PSFVAR_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue or FITS (preceded by :) params +PSFVAR_GROUPS 1,1 # Group tag for each context key +PSFVAR_DEGREES 2 # Polynom degree for each group +PSFVAR_NSNAP 9 # Number of PSF snapshots per axis +HIDDENMEF_TYPE COMMON # INDEPENDENT or COMMON +STABILITY_TYPE EXPOSURE # EXPOSURE or SEQUENCE + +#----------------------------- Sample selection ------------------------------ + +SAMPLE_AUTOSELECT N # Automatically select the FWHM (Y/N) ? + +BADPIXEL_FILTER N # Filter bad-pixels in samples (Y/N) ? +BADPIXEL_NMAX 0 # Maximum number of bad pixels allowed + +#----------------------- PSF homogeneisation kernel -------------------------- + +HOMOBASIS_TYPE NONE # NONE or GAUSS-LAGUERRE +HOMOBASIS_NUMBER 10 # Kernel basis number or parameter +HOMOBASIS_SCALE 1.0 # GAUSS-LAGUERRE beta parameter +HOMOPSF_PARAMS 2.0, 3.0 # Moffat parameters of the idealised PSF +HOMOKERNEL_DIR # Where to write kernels (empty=same as input) +HOMOKERNEL_SUFFIX .homo.fits # Filename extension for homogenisation kernels + +#----------------------------- Output catalogs ------------------------------- + +OUTCAT_TYPE FITS_LDAC # NONE, ASCII_HEAD, ASCII, FITS_LDAC + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_DEV NULL # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_RES 0 # Check-plot resolution (0 = default) +CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? +CHECKPLOT_TYPE NONE # FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS +CHECKPLOT_TYPE FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS + # or NONE +CHECKPLOT_NAME fwhm, ellipticity, counts, countfrac, chi2, resi + +#------------------------------ Check-Images --------------------------------- + +# Note: Check-image types can be set the ShapePipe config file, psfex_runner section +####### +CHECKIMAGE_TYPE NONE # CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS + # or MOFFAT,-MOFFAT,-SYMMETRICAL +#CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits + # Check-image filenames +#CHECKIMAGE_CUBE N # Save check-images as datacubes (Y/N) ? + +#----------------------------- Miscellaneous --------------------------------- + +PSF_SUFFIX .psf # Filename extension for output PSF filename +VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of PSFEx + # 0 = automatic diff --git a/workflow/config/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex new file mode 100644 index 00000000..b87275ec --- /dev/null +++ b/workflow/config/cfis/default_exp.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.001 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE AUTO # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param new file mode 100644 index 00000000..b251f5b7 --- /dev/null +++ b/workflow/config/cfis/default_noimaflags.param @@ -0,0 +1,65 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex new file mode 100644 index 00000000..ff3b2521 --- /dev/null +++ b/workflow/config/cfis/default_tile.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.0005 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE MANUAL # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param new file mode 100644 index 00000000..8ddfebd3 --- /dev/null +++ b/workflow/config/cfis/final_cat.param @@ -0,0 +1,114 @@ +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +# Can maybe be removed. +TILE_ID + +# flags +FLAGS +IMAFLAGS_ISO +NGMIX_MCAL_FLAGS + +# PSF ellipticity (original image PSF) +NGMIX_G1_PSF_ORIG_NOSHEAR +NGMIX_G2_PSF_ORIG_NOSHEAR + +# spread class +#SPREAD_CLASS + +# spread model flag and error +#SPREAD_MODEL +#SPREADERR_MODEL + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_G1_1M +NGMIX_G2_1M +NGMIX_G1_1P +NGMIX_G2_1P +NGMIX_G1_2M +NGMIX_G2_2M +NGMIX_G1_2P +NGMIX_G2_2P +NGMIX_G1_NOSHEAR +NGMIX_G2_NOSHEAR +#NGMIX_G1_ERR_1M +#NGMIX_G2_ERR_1M +#NGMIX_G1_ERR_1P +#NGMIX_G2_ERR_1P +#NGMIX_G1_ERR_2M +#NGMIX_G2_ERR_2M +#NGMIX_G1_ERR_2P +#NGMIX_G2_ERR_2P +NGMIX_G1_ERR_NOSHEAR +NGMIX_G2_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR +NGMIX_T_PSF_RECONV_1M +NGMIX_T_PSF_RECONV_1P +NGMIX_T_PSF_RECONV_2M +NGMIX_T_PSF_RECONV_2P +NGMIX_T_PSF_RECONV_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# magnitudes +MAG_AUTO +MAGERR_AUTO +MAG_WIN +MAGERR_WIN +FLUX_AUTO +FLUXERR_AUTO +FLUX_APER +FLUXERR_APER +FLUX_RADIUS + +# SNR from SExtractor +SNR_WIN + +FWHM_IMAGE +FWHM_WORLD + +# PSF size measured on original image +NGMIX_T_PSF_ORIG_NOSHEAR + +# PSF size measured on reconvolved image +# NGMIX_T_PSF_RECONV_NOSHEAR + +# ngmix moment failure flag +NGMIX_MOM_FAIL diff --git a/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg b/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg new file mode 100644 index 00000000..4e4164aa --- /dev/null +++ b/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg @@ -0,0 +1,24 @@ +-11.5 68 +-6 186.5 +7 188 +10 64.5 +31 55 +50 38.5 +56.5 11.5 +188 8 +192 -4 +59.5 -11.5 +45 -33 +13.5 -64 +5 -154 +-6 -155 +-11 -64.5 +-40 -44.5 +-51.5 -30.5 +-62.5 -22.5 +-68 -9.5 +-177 -2 +-176 3 +-78 12.5 +-67.5 14.5 +-38.5 50 diff --git a/workflow/config/cfis/mask_default/Messier_catalog.npy b/workflow/config/cfis/mask_default/Messier_catalog.npy new file mode 100644 index 00000000..ef07eb03 Binary files /dev/null and b/workflow/config/cfis/mask_default/Messier_catalog.npy differ diff --git a/workflow/config/cfis/mask_default/Messier_catalog_updated.fits b/workflow/config/cfis/mask_default/Messier_catalog_updated.fits new file mode 100644 index 00000000..6a9f0009 Binary files /dev/null and b/workflow/config/cfis/mask_default/Messier_catalog_updated.fits differ diff --git a/workflow/config/cfis/mask_default/default.ww b/workflow/config/cfis/mask_default/default.ww new file mode 100644 index 00000000..c2797f90 --- /dev/null +++ b/workflow/config/cfis/mask_default/default.ww @@ -0,0 +1,40 @@ +#--------------------------------- Weights ------------------------------------ + +WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) + +WEIGHT_MIN 0. # Pixel below those thresholds will be flagged +WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged +WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels + +#---------------------------------- Flags ------------------------------------- + +FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) + +FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels +FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs +FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits + +#---------------------------------- Polygons ---------------------------------- + +POLY_NAMES "" # Filename(s) of input DS9 regions +POLY_OUTFLAGS # FLAG values for polygon masks +POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks +POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? + +#---------------------------------- Output ------------------------------------ + +OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename +OUTFLAG_NAME flag.fits # Output FLAG-map filename + +#----------------------------- Miscellaneous --------------------------------- + +GETAREA N # Compute area for flags and weights (Y/N)? +GETAREA_WEIGHT 0.0 # Weight threshold for area computation +GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area +MEMORY_BUFSIZE 256 # Buffer size in lines +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME ww.xml # Filename for XML output +XSL_URL file:///usr/local/share/weightwatcher/ww.xsl + # Filename for XSL style-sheet +NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/workflow/config/cfis/mask_default/halo_mask.reg b/workflow/config/cfis/mask_default/halo_mask.reg new file mode 100644 index 00000000..c44f2516 --- /dev/null +++ b/workflow/config/cfis/mask_default/halo_mask.reg @@ -0,0 +1,50 @@ + 274.66813 -1.25966 + 272.54579 32.47406 + 266.21222 65.67579 + 255.76731 97.82190 + 241.37579 128.40544 + 223.26462 156.94408 + 201.71942 182.98775 + 177.07997 206.12573 + 149.73486 225.99312 + 120.11532 242.27660 + 88.68848 254.71937 + 55.94996 263.12519 + 22.41606 267.36151 + -11.38436 267.36151 + -44.91826 263.12519 + -77.65678 254.71937 +-109.08362 242.27660 +-138.70315 225.99312 +-166.04827 206.12573 +-190.68772 182.98775 +-212.23292 156.94408 +-230.34409 128.40544 +-244.73561 97.82190 +-255.18052 65.67579 +-261.51409 32.47406 +-263.63643 -1.25966 +-261.51409 -34.99339 +-255.18052 -68.19511 +-244.73561 -100.34123 +-230.34409 -130.92476 +-212.23292 -159.46341 +-190.68772 -185.50708 +-166.04827 -208.64506 +-138.70315 -228.51245 +-109.08362 -244.79593 + -77.65678 -257.23870 + -44.91826 -265.64452 + -11.38436 -269.88084 + 22.41606 -269.88084 + 55.94996 -265.64452 + 88.68848 -257.23870 + 120.11532 -244.79593 + 149.73486 -228.51245 + 177.07997 -208.64506 + 201.71942 -185.50708 + 223.26462 -159.46341 + 241.37579 -130.92476 + 255.76731 -100.34123 + 266.21222 -68.19511 + 272.54579 -34.99339 diff --git a/workflow/config/cfis/mask_default/ngc_cat.fits b/workflow/config/cfis/mask_default/ngc_cat.fits new file mode 100644 index 00000000..f51546da Binary files /dev/null and b/workflow/config/cfis/mask_default/ngc_cat.fits differ diff --git a/workflow/config/cfis/star_selection.setools b/workflow/config/cfis/star_selection.setools new file mode 100644 index 00000000..8330a1ef --- /dev/null +++ b/workflow/config/cfis/star_selection.setools @@ -0,0 +1,103 @@ +## SETools configuration file for star/galaxy separation based on size/mag properties + +[MASK:preselect] +MAG_AUTO > 0 +MAG_AUTO < 21 +FWHM_IMAGE > 0.3 / 0.187 +FWHM_IMAGE < 1.5 / 0.187 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +[MASK:flag] +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + + +[MASK:star_selection] +# Star selection using the FWHM mode +MAG_AUTO > 18. +MAG_AUTO < 22. +FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 +FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 +FLAGS == 0 +IMAFLAGS_ISO == 0 + +[MASK:fwhm_mag_cut] +FWHM_IMAGE > 0 +FWHM_IMAGE < 40 +MAG_AUTO < 35 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +# Split the 'star_selection' sample into +# two random sub-samples with ratio 80/20 +[RAND_SPLIT:star_split] +RATIO = 20 +MASK = star_selection + +# The following selection is only used for plotting + +[PLOT:size_mag] +TYPE = plot +FORMAT = png +X_1 = FWHM_IMAGE{fwhm_mag_cut} +Y_1 = MAG_AUTO{fwhm_mag_cut} +X_2 = FWHM_IMAGE{star_selection} +Y_2 = MAG_AUTO{star_selection} +MARKER_1 = + +MARKER_2 = . +MARKERSIZE_1 = 3 +MARKERSIZE_2 = 3 +LABEL_1 = All +LABEL_2 = "Stars, mean FWHM: @mean(FWHM_IMAGE{star_selection})*0.187@ arcsec" +TITLE = "Stellar locus" +XLABEL = "FWHM (pix)" +YLABEL = Mag + +[PLOT:hist_mag_stars] +TYPE = hist +FORMAT = png +Y = MAG_AUTO{star_selection} +BIN = 20 +LABEL = "stars" +XLABEL = "Magnitude" +YLABEL = "Number" +TITLE = "Magnitude of stars" + +[PLOT:fwhm_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = FWHM_IMAGE{star_selection}*0.186 +MARKER = . +LABEL = "FWHM (arcsec)" +TITLE = "FWHM of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[PLOT:mag_star_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = MAG_AUTO{star_selection} +MARKER = . +LABEL = "Magnitude" +TITLE = "Magnitude of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[STAT:star_stat] +"Nb objects full cat" = len(FWHM_IMAGE) +"Nb objects not masked" = len(FWHM_IMAGE{flag}) +"Nb stars" = len(FWHM_IMAGE{star_selection}) +"stars/deg^2" = len(FWHM_IMAGE{star_selection})/4612./0.187*3600.*1./2048./0.187*3600. +"Mean star fwhm selected (arcsec)" = mean(FWHM_IMAGE{star_selection})*0.187 +"Standard deviation fwhm star selected (arcsec)" = std(FWHM_IMAGE{star_selection})*0.187 +"Mode fwhm used (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187 +"Min fwhm cut (arcesec)" = mode(FWHM_IMAGE{preselect})*0.187-0.1*0.187 +"Max fwhm cut (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187+0.1*0.187 diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk new file mode 100644 index 00000000..e04dd76f --- /dev/null +++ b/workflow/rules/exposure.smk @@ -0,0 +1,148 @@ +"""Exposure chain — per exposure, keyed by exp base id (dedup is structural). + + exp_get_images -> exp_split -> exp_mask -> exp_psf + +Each in the exposure's own sharded work dir, chained by manifests; every config +reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a +run log. There is no `prepare_exposures` aggregation target: these chains hang +off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests). + +NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by +construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, +not over one invocation — reclamation here is clean_exposure's job (S5), driven +by the accumulating index. A temp() here would delete an exposure the moment +this invocation's readers finished and cascade destructive reruns across spatial +neighbours the next time a tile is appended. + +NUMBER_LIST is set only for exp_split (its numbering scheme IS the exposure id); +never for get_images / exp_mask / exp_psf, whose per-CCD or download numbering +would make the #746 startup validation turn tolerated per-CCD attrition into a +whole-exposure hard failure. That is now a property of the committed configs +(config_exp_Sp.ini has NUMBER_LIST = $SP_UNIT_NUM; Gie/Ma/psfex have none). +""" + +rule exp_get_images: + output: + manifest = f"{EXP_DIR}/manifests/exp_get_images.json" + params: + pre = lambda wc: unit_pre("exp_get_images", "exp", wc.exp, + exp_name=EXP[wc.exp]), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 + shell: + sp_shell("exp_get_images", "config_exp_Gie.ini") + +# Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the +# tiles' merge_headers reads). +rule exp_split: + input: + rules.exp_get_images.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_split.json" + params: + pre = lambda wc: unit_pre("exp_split", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("exp_split", "config_exp_Sp.ini") + +rule exp_mask: + input: + rules.exp_split.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_mask.json" + params: + pre = lambda wc: unit_pre("exp_mask", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("exp_mask", "config_exp_Ma.ini") + +# SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. +# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's +# :warn on psfex_interp_runner. +rule exp_psf: + input: + rules.exp_mask.output.manifest + output: + manifest = f"{EXP_DIR}/manifests/exp_psf.json" + params: + pre = lambda wc: unit_pre("exp_psf", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + retries: 2 + benchmark: + # BESIDE manifests/, not inside it: clean_exposure deletes manifests/ + # wholesale, and this tsv is the measured-memory feed for mem_mb sizing + # (D4). Inside manifests/ it died with the first reclamation and took + # the campaign's only record of exp_psf's real footprint with it. + f"{EXP_DIR}/exp_psf.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 240 + shell: + sp_shell("exp_psf", "config_exp_psfex.ini") + + +# --- reclamation (D5) ------------------------------------------------------- +# The one exception to "no reclamation in this file": clean_exposure OWNS +# exposure-level deletion, and it is a real job, not temp() bookkeeping, because +# an exposure's consumer set closes over the CAMPAIGN. The index supplies that +# set (EXP_TILES, accumulated across invocations); the input is every consuming +# tile's tile_vignets manifest — vignets is the last stage that reads exposure +# products, everything after it reads tile-level files. +# +# Three properties make the late append behave (see clean_exposure.py): +# * the job deletes the exposure's manifests too, so a tile appended after the +# clean sees an unbuilt chain and regenerates it instead of running against +# an empty store. The tombstone deliberately does NOT stand in for those +# manifests — it is not an input to anything but itself. +# * a finished tile is not disturbed: Snakemake demands a missing intermediate +# only when something downstream of it must run. +# * params.consumers carries the consumer set, so growing it makes the +# tombstone stale under the default `params` rerun-trigger; the clean job +# reruns after the new tile's vignets, against the enlarged set. +# +# The tile side reads the exposure manifests through ancient() (see tile.smk), +# which is what keeps this deletion from rebuilding every neighbouring tile. +# This rule's OWN inputs are deliberately not ancient: a tile that really did +# rebuild its vignets must reschedule the cleans of the exposures it read. +# +# A localrule (declared in the Snakefile): it is an rmtree, not science, and one +# sbatch per exposure would be ~20k scheduler submissions at DR6 scale. Local +# execution serialises them under local-cores, which costs nothing at rmtree +# speed and never blocks the compute chains (this rule is in none of them). +rule clean_exposure: + input: + # ONLY the consumers this invocation may actually build. A consumer that + # is out of scope had its vignets manifest checked for existence at parse + # time (clean_targets' eligibility test) — declaring it here as well would + # pull that finished tile's whole chain into the DAG, where a rebuilt + # shared exposure then reruns it. That is how one damaged tile reached its + # spatial neighbours. In-scope consumers keep their edge: they may run in + # this DAG, so the clean must be ordered after them. + lambda wc: [tile_manifest(t, "tile_vignets") + for t in clean_consumers(wc.exp) if t in READY_SET] + output: + tombstone = f"{EXP_DIR}/cleaned.json" + params: + consumers = lambda wc: ",".join(clean_consumers(wc.exp)), + script_hash = CLEAN_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SCRIPTS}/clean_exposure.py" + " --exp-dir $(dirname {output.tombstone}) --exp {wildcards.exp}" + " --tombstone {output.tombstone} --consumers '{params.consumers}'" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk new file mode 100644 index 00000000..92998e18 --- /dev/null +++ b/workflow/rules/prepare.smk @@ -0,0 +1,67 @@ +"""Invocation 1 — PREPARE: one static chain per tile. + + tile_get_images -> tile_uncompress -> tile_find_exposures + +Known from the tile list alone, cheap, wide, idempotent. find_exposures parses +the tile FITS HISTORY header into ``exp_numbers--.txt`` — the +data-derived tile->exposure edge that invocation 2's parse aggregates into the +index. Nibi compute nodes have internet, so downloads run in-DAG (no login-node +tier). + +Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. + +Star catalogues for masking are pre-generated offline (create_star_cat.py on a +networked login node) and consumed as DIRECTORIES: the mask configs read +``$SP_RUN/star_cat_{exp,tiles}`` with per-CCD numbering. Every rule's prologue +materialises the two dir symlinks; there is NO per-unit star-cat DAG node — the +store is pre-run input like the image store, and a missing cat fails the mask +stage's count floor loudly. +""" + +# NUMBER_LIST is never set for get_images (download stage; nothing on disk to +# validate against — #746 would hard-fail the unit). The committed +# config_tile_Git.ini simply has no NUMBER_LIST, so this is now a property of the +# config, not of an injection step. +rule tile_get_images: + output: + manifest = f"{TILE_DIR}/manifests/tile_get_images.json" + params: + pre = lambda wc: unit_pre("tile_get_images", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 + shell: + sp_shell("tile_get_images", "config_tile_Git.ini") + +rule tile_uncompress: + input: + rules.tile_get_images.output.manifest + output: + manifest = f"{TILE_DIR}/manifests/tile_uncompress.json" + params: + pre = lambda wc: unit_pre("tile_uncompress", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 60 + shell: + sp_shell("tile_uncompress", "config_tile_Uz.ini") + +rule tile_find_exposures: + input: + rules.tile_uncompress.output.manifest + output: + manifest = f"{TILE_DIR}/manifests/tile_find_exposures.json" + params: + pre = lambda wc: unit_pre("tile_find_exposures", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + resources: + mem_mb = lambda wc, attempt: 2000 * attempt, + runtime = 30 + shell: + sp_shell("tile_find_exposures", "config_tile_Fe.ini") diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk new file mode 100644 index 00000000..8b665d1c --- /dev/null +++ b/workflow/rules/tile.smk @@ -0,0 +1,294 @@ +"""Tile post-chain — per tile: gather exposures, then detect / PSF / shape / catalogue. + + tile_exp_forest + tile_merge_headers -> tile_detect -> tile_vignets -> tile_ngmix x N + -> tile_merge_cats -> tile_make_cat + +The DAG edge to the exposures is always the exposures' MANIFESTS, looked up +through the index (TILE_EXP). The per-tile exposure "forest" — a symlink view of +exactly this tile's exposures' products — exists only so the ShapePipe configs +have one deterministic ``$SP_EXP`` to glob; it is NEVER the edge. Its 2-char +shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires +``///output/run_sp_*`` into its glob, so a flat forest +makes every tile gather stage fail "No split_exp_runner output found". + +All rules are group-compatible (shell only, no mid-chain localrules). + +Note there is no `tile_mask` rule: the committed config chain is the +"sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask +run), and no tile-mask config was committed in the S2 sweep. Adding the masked +variant is a config + one rule, at the config selector the PRD describes. +""" + +def tile_exp(wc): + return TILE_EXP.get(wc.tile, []) + +# --- the tile->exposure edge, and why it is cut for finished tiles ---------- +# +# THE cascade fix. clean_exposure deletes the exposure's manifests on purpose: +# that is what makes a tile appended later rebuild the chain instead of running +# against an empty store. But those manifests are the tile side's inputs, and an +# exposure is read by ~7-10 tiles. So the moment ONE tile's chain rebuilt an +# exposure, every other tile reading it saw "input files updated by another job" +# and reran — and that rerun rebuilt ITS exposures, which reran THEIR other +# consumers, propagating across the whole exposure-overlap connected component. +# On fixture t4, asking for one damaged tile scheduled all four tiles' chains. +# +# Two mechanisms, and only the second one actually cuts it: +# +# 1. ancient() on every exposure-manifest edge. Correct on its own terms — a +# tile has no business rerunning because an exposure manifest is NEWER — and +# it is what keeps a pure-mtime disturbance (a re-touched manifest, a +# restored backup) from waking finished tiles. But ancient() governs +# TIMESTAMPS only. Snakemake propagates "my input is produced by a job that +# will run in this DAG" separately, and ancient does not suppress it +# (measured: t4 counts were identical with ancient alone). +# +# 2. Cutting the RECLAIMED edges of a FINISHED tile — the mechanism that works. +# A tile whose final_cat is on disk needs nothing further from its +# exposures: it has already extracted everything it will ever read. So for +# such a tile the input list drops the manifests that are GONE, and the +# propagation has nowhere to go. An UNfinished tile keeps its full edge set +# and therefore still drags in — and rebuilds — every exposure it needs, +# which is the accepted price of a late append, unchanged. +# +# Only the missing ones are dropped, never a manifest that still exists: a +# campaign that has cleaned nothing then declares exactly the edges it always +# did, and the cut cannot perturb it. +# +# The marker is final_cat, not the tile's own vignets manifest: on a tile whose +# catalogue was lost, the vignets manifest still exists while the vignette store +# (temp()) does not, so keying on vignets would cut the edge on exactly the tile +# that has to rerun, and run it against a deleted exposure store. +# +# THE CUT REQUIRES THE `input` RERUN-TRIGGER TO BE OFF (profiles/nibi sets the +# trigger list). Dropping an input is itself a change in the set of input files, +# which that trigger reads as a reason to rerun — reinstating the very cascade, +# now as "Set of input files has changed", and running finished tiles against a +# store that is gone. Measured on fixture t4, one damaged tile of four: 82 jobs +# with neither fix, 70 with the cut but the trigger on, 28 with both (= exactly +# the damaged tile's own chain, its two exposures, and the clean jobs). +# +# The cost, stated plainly: `--forcerun` on a tile whose final_cat exists will +# NOT rebuild its reclaimed exposures, because those edges are not in the DAG. +# Delete that tile's final_cat first and the whole chain comes back. +# +# What none of this weakens: clean_exposure's own inputs are neither ancient nor +# cut, so a tile that really did rebuild its vignets still reschedules the cleans +# of the exposures it read, and a grown consumer set still travels through +# params.consumers. +def tile_finished(tile): + return Path(final_cat(tile)).exists() + + +def exp_manifests(wc, stage): + paths = [exp_manifest(e, stage) for e in tile_exp(wc)] + if tile_finished(wc.tile): + paths = [p for p in paths if Path(p).exists()] + return [ancient(p) for p in paths] + +def tile_exp_split(wc): return exp_manifests(wc, "exp_split") +def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") +def tile_exp_psf(wc): return exp_manifests(wc, "exp_psf") +def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) + + +# Build the per-tile symlink forest. Declaring the exposure manifests as input +# makes this wait on its exposures; the forest itself is only the $SP_EXP view. +# Its output stays a directory() (it has no ShapePipe run dir and no manifest — +# it is not a shapepipe_run at all). +rule tile_exp_forest: + input: + tile_exp_all + output: + forest = directory(f"{TILE_DIR}/exp_forest") + params: + cmd = lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_DB}"), + # build_forest.py's own content hash rides here and nowhere else. + script_hash = FOREST_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 20 + shell: + # --forest {output} lives in the shell string: snakemake formats shell + # ONCE, so an {output} placeholder inside params.cmd would survive + # literally and every forest job would race one './{output}'. + "{params.cmd} --forest {output.forest}" + +# Merge single-exposure WCS headers into the tile-level sqlite +# (log_exp_headers--.sqlite, which Sx / PiViVi / ngmix consume). +# Reads headers-*.npy through the forest -> the split manifests are the edge. +rule tile_merge_headers: + input: + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + # config_tile_Mh_exp.ini reads run_sp_tile_Fe output, which the PREPARE + # phase produced. Declaring the Fe manifest gives the COMPUTE DAG a + # regeneration path for it instead of a silent dependency on a + # previous invocation (prepare.smk is included in every parse, so the + # rule exists here too). Normally a satisfied no-op. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", + output: + manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" + params: + pre = lambda wc: unit_pre("tile_merge_headers", "tile", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + sp_shell("tile_merge_headers", "config_tile_Mh_exp.ini") + +# SExtractor object detection on the tile. +rule tile_detect: + input: + uz = f"{TILE_DIR}/manifests/tile_uncompress.json", + mh = rules.tile_merge_headers.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_detect.json" + params: + pre = lambda wc: unit_pre("tile_detect", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 180 + shell: + sp_shell("tile_detect", "config_tile_Sx.ini") + +# PSFEx interpolation to galaxies + vignet postage stamps: the last stage that +# reads exposure products, and the bulk intra-tile intermediate. +# +# The vignette store is declared as a SECOND, temp(directory()) output alongside +# the manifest. This is the ONE scoped exception to "no directory() outputs" +# (D5): the store is ~tens of GB per tile and must be reclaimed when its last +# intra-tile reader finishes, but it must not become DAG currency — so the +# manifest stays the edge, and the directory rides along purely so native temp() +# fires at the right moment. Its readers (tile_ngmix, tile_make_cat) declare +# BOTH. `--notemp` keeps it for debugging. +rule tile_vignets: + input: + sx = rules.tile_detect.output.manifest, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + psf = tile_exp_psf, + # config_tile_PiViVi.ini reads run_sp_tile_Fe output — same reason as + # tile_merge_headers above. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", + output: + manifest = f"{TILE_DIR}/manifests/tile_vignets.json", + store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), + params: + pre = lambda wc: unit_pre("tile_vignets", "tile", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 16 + resources: + mem_mb = lambda wc, attempt: 32000 * attempt, + runtime = 240 + shell: + sp_shell("tile_vignets", "config_tile_PiViVi.ini") + +# ngmix shape measurement — N chunks per tile (D4). Each chunk computes its own +# CLOSED object-ID range at EXECUTION time from this tile's own sexcat: a params +# function cannot, because params evaluate before the sexcat exists. Closed, not +# open-ended: `ID_OBJ_MAX = -1` on the last chunk was the 13-hour straggler's +# root cause (ngmix treats id_obj_max <= 0 as unbounded). +# +# Chunks write nothing shared: each has its own run_sp_tile_ngmix_Ngu, and +# merge_sep_cats — DAG-serialised after all chunks — is the gather. +rule tile_ngmix: + input: + vignets = rules.tile_vignets.output.manifest, + store = rules.tile_vignets.output.store, + sx = rules.tile_detect.output.manifest, + output: + manifest = f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.json", + # temp(directory()) for the same reason as the vignette store above. + chunkdir = temp(directory(f"{TILE_DIR}/output/run_sp_tile_ngmix_Ng{{chunk}}u")), + params: + pre = lambda wc: unit_pre( + "tile_ngmix", "tile", wc.tile, + env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, + # Two steps, not `eval "$(...)"`: a command substitution inside eval + # discards the script's exit status, so a missing sexcat would fall + # through to shapepipe_run with an unset range and fail as something + # else. Capture, check, then eval — the range script fails as itself. + pre_run=[f'ngmix_range_out=$(python {SCRIPTS}/ngmix_range.py --run-dir ' + f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS}) ' + f'|| exit 1', + 'eval "$ngmix_range_out"']), + script_hash = SCRIPT_HASH + threads: 4 + retries: 2 + benchmark: + f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 14000 * attempt, + runtime = 720 + shell: + sp_shell("tile_ngmix", "config_tile_Ng_template.ini") + + +def ngmix_manifests(wc): + return [f"{tile_dir(wc.tile)}/manifests/tile_ngmix_{k}.json" + for k in range(1, NGMIX_CHUNKS + 1)] + +def ngmix_chunkdirs(wc): + return [f"{tile_dir(wc.tile)}/output/run_sp_tile_ngmix_Ng{k}u" + for k in range(1, NGMIX_CHUNKS + 1)] + +# The gather: merge the N chunk catalogues. N_SPLIT_MAX comes from the workflow's +# own chunk count via $NGMIX_N_CHUNKS (env-expanded by the module). +rule tile_merge_cats: + input: + manifests = ngmix_manifests, + chunkdirs = ngmix_chunkdirs, + output: + manifest = f"{TILE_DIR}/manifests/tile_merge_cats.json" + params: + pre = lambda wc: unit_pre("tile_merge_cats", "tile", wc.tile, + env={"NGMIX_N_CHUNKS": NGMIX_CHUNKS}), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 120 + shell: + sp_shell("tile_merge_cats", "config_merge_sep_cats.ini") + +# The run's science product. make_cat also reads the vignette store's +# psfex_interp output, so it — not ngmix — is the store's last reader. +# +# No protected(): the full default rerun-triggers govern, and protected() only +# ever forced people through a `--forcerun` detour. +rule tile_make_cat: + input: + ms = rules.tile_merge_cats.output.manifest, + store = rules.tile_vignets.output.store, + output: + manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", + final_cat = f"{TILE_DIR}/final_cat-{{tile}}.fits", + params: + pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 120 + shell: + # Publish the catalogue next to the manifest: a real file, so it is a + # real declared output (and it persists — never temp()). + "{params.pre}\n" + "rc=0\n" + 'shapepipe_run -c "$SP_CONFIG/config_tile_Mc.ini" -b {threads} || rc=$?\n' + f"python {SCRIPTS}/completeness.py check tile_make_cat {{output.manifest}} || rc=1\n" + "if [ $rc -eq 0 ]; then\n" + ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner/output/final_cat*.fits' + ' | head -1)" {output.final_cat}\n' + "fi\n" + "exit $rc\n" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py new file mode 100644 index 00000000..427e9c64 --- /dev/null +++ b/workflow/scripts/build_forest.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Build a tile's exposure symlink forest (the SP_EXP view). + +A plain script (not a run: block) so the tile chain stays group-compatible. +Reads the tile's exposures from run_index.sqlite and symlinks each exposure's +``exp///output`` into ``///output`` by exact +name (no glob). The 2-char ```` shard level is NOT cosmetic: ShapePipe's +``exp_utils.get_exp_output_files`` hardwires the sharded v2.0 layout into its +$SP_EXP glob (``///output/run_sp_*/...``), so a flat +forest makes every tile gather stage fail "No split_exp_runner output found". +(The exposure STORE is sharded the same way, for the filesystem's sake.) +The forest is a convenience view; the DAG edge to the exposures is declared in +the rule's input (tile.smk), not here. +""" + +import argparse +import shutil +import sqlite3 +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile", required=True) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--forest", required=True, type=Path) + args = p.parse_args() + + con = sqlite3.connect(args.index) + exps = [r[0] for r in con.execute( + "SELECT exp_id FROM tile_exposures WHERE tile_id=?", (args.tile,))] + con.close() + + args.forest.mkdir(parents=True, exist_ok=True) + for e in exps: + src = args.run_dir / "exp" / e[:2] / e / "output" + dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape + dst.parent.mkdir(parents=True, exist_ok=True) + # A symlink (the normal case) is unlinked; a REAL directory left behind + # by a hand-run or an older layout must be removed as a tree — unlink() + # raises IsADirectoryError on it and would kill the job. + if dst.is_symlink() or dst.exists(): + if dst.is_dir() and not dst.is_symlink(): + shutil.rmtree(dst) + else: + dst.unlink() + dst.symlink_to(src) + print(f"[build_forest] {args.tile}: {len(exps)} exposures -> {args.forest}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py new file mode 100644 index 00000000..73d08609 --- /dev/null +++ b/workflow/scripts/build_index.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""Build the run index (``run_index.sqlite``) that drives the compute DAG. + +The index is *parse-time data*, never a rule input: the Snakefile loads it once +at parse time into plain dicts, so extending a run's tile list changes which +jobs exist without touching the mtime chain of completed work. + +It ACCUMULATES ACROSS INVOCATIONS, but is AUTHORITATIVE FOR THE CURRENT TILE +LIST (D1). Precisely: + + * a tile in the current list that has find_exposures output has its ``tiles`` + row replaced and its ``tile_exposures`` edges DELETED AND REBUILT — the Fe + output is the truth, so a tile whose exposure list shrank or changed must + not keep stale edges to exposures it no longer reads; + * a tile NOT in the current list is left completely untouched, which is what + makes the index span the campaign and lets a later ``clean_exposure`` see + every consuming tile; + * ``exposures`` rows only ever accumulate (``INSERT OR IGNORE``). An exposure + no tile references any more is a harmless orphan: nothing reads that table + except by joining through ``tile_exposures``. + +It records: + + tiles(tile_id, ra_dir, n_exp, status) + exposures(exp_id) -- deduplicated union over tiles + tile_exposures(tile_id, exp_id) -- the tile->exposure edges + +The tile->exposure edges are *data-derived*: they are read from each tile's +``find_exposures`` output (``exp_numbers--.txt``), which +``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. +So the build is not a DAG node: ``build()`` is imported and called at PARSE TIME +by the Snakefile of the COMPUTE invocation ONLY — ``SP_PHASE == "compute"``, +which ``bin/sp`` sets — after the PREPARE invocation has produced the tiles' +find_exposures output. No other parse builds anything: a prepare parse or a +passthrough invocation (``sp --unlock``, ``sp --dag``) just loads whatever is +already on disk. (There is no ``sp index`` verb; the CLI below stays for +hand-inspection.) It +iterates the *declared* tile list and checks each tile's Fe output at its +deterministic path (no globbing — O(tile-list) existence checks, respecting the +no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in +``missing.json`` and the index is built over the rest. The build fails only if +the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the check +entirely), so a keep-going download storm that lost a few tiles does not cost the +run. The threshold is checked BEFORE anything is written, so a failed build +leaves the previous index and ``missing.json`` intact. + +Exposure IDs are stored with their trailing single-char suffix stripped +(``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the +exposure-rule wildcard, matching the sharded ``exp///`` store. +""" + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + + +def read_exposure_list(exp_numbers_file: Path) -> list[tuple[str, str]]: + """Return ``(exp_id, name)`` pairs from one tile's find_exposures output. + + Each line is an exposure *name* like ``2243881p``; the bare base ID (suffix + stripped) is the dedup key everywhere in the DAG, but the original name is + kept in the index — the fabricated per-unit ``exp_numbers`` list must carry + it verbatim (``get_images`` matches ``.fits.fz`` in the store; the + bare ID matches nothing). + """ + pairs = [] + for line in exp_numbers_file.read_text().splitlines(): + name = line.strip() + if not name: + continue + pairs.append((name[:-1] if name[-1].isalpha() else name, name)) + return pairs + + +def exp_list_path(run_dir: Path, tile_id: str) -> Path: + """This tile's find_exposures output, at its deterministic path. + + Sharded store (D2), fixed run dir (RUN_DATETIME=False) — an existence check, + never a glob (no ``ls`` at scale). + """ + idra, iddec = tile_id.split(".") + return (run_dir / "tiles" / tile_id[:2] / tile_id / "output" / + "run_sp_tile_Fe" / "find_exposures_runner" / "output" / + f"exp_numbers-{idra}-{iddec}.txt") + + +def build(tile_ids: list[str], run_dir: Path, db_path: Path, + missing_threshold: float | None = 0.0) -> dict: + """Build the index over ``tile_ids``; return a summary dict. + + For each tile, check its ``exp_numbers--.txt`` at the tile's + deterministic ``find_exposures`` run dir (RUN_DATETIME=False, no glob). A + tile whose exposure list is missing is recorded in ``missing.json`` and the + index is built over the rest (a bad tile costs that tile, not the run). The + build is fatal only if the missing fraction exceeds ``missing_threshold``. + + ORDER MATTERS: the threshold is evaluated FIRST, from the missing set, and + the database + ``missing.json`` are written only if it passes. A build that + aborts must leave no trace — an aborted parse that had already mutated + durable state was the bug this ordering fixes. + + The write is idempotent, which is what makes it acceptable that the compute + parse runs it even under ``-n``: re-running over an unchanged tree produces + an identical database. + """ + missing = [t for t in tile_ids if not exp_list_path(run_dir, t).exists()] + frac = len(missing) / len(tile_ids) if tile_ids else 0.0 + if missing_threshold is not None and frac > missing_threshold: + raise SystemExit( + f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " + f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " + f"Re-run prepare_tiles for them, or raise --missing-threshold.") + + db_path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(db_path) + # No DROP: the index accumulates across invocations (D1). + con.executescript( + """ + CREATE TABLE IF NOT EXISTS tiles( + tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); + CREATE TABLE IF NOT EXISTS exposures( + exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS tile_exposures( + tile_id TEXT, exp_id TEXT, + PRIMARY KEY (tile_id, exp_id)); + """ + ) + + missing_set = set(missing) + all_exposures: set[tuple[str, str]] = set() + for tile_id in tile_ids: + if tile_id in missing_set: + continue + ra_dir = tile_id.split(".")[0] + exp_pairs = read_exposure_list(exp_list_path(run_dir, tile_id)) + con.execute("INSERT OR REPLACE INTO tiles VALUES (?,?,?)", + (tile_id, ra_dir, len(exp_pairs))) + # Replace this tile's edge set wholesale. INSERT OR IGNORE alone only + # ever added, so a tile whose exposure list shrank kept edges to + # exposures it no longer reads — and those stale edges would block it in + # the report and pin those exposures against cleanup. + con.execute("DELETE FROM tile_exposures WHERE tile_id = ?", (tile_id,)) + con.executemany("INSERT INTO tile_exposures VALUES (?,?)", + [(tile_id, exp_id) for exp_id, _ in exp_pairs]) + all_exposures.update(exp_pairs) + + # Exposures accumulate: OR IGNORE, never REPLACE (the name never changes, + # and orphans left by a shrunken tile are harmless). + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", + sorted(all_exposures)) + con.commit() + con.close() + + (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) + return {"n_tiles": len(tile_ids) - len(missing), + "n_exposures": len(all_exposures), + "n_missing": len(missing)} + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile-list", required=True, type=Path, + help="file of tile IDs, one per line") + p.add_argument("--run-dir", required=True, type=Path, + help="$SP_RUN: root of the tiles/ work-dir forest") + p.add_argument("--db", required=True, type=Path, + help="output run_index.sqlite path") + p.add_argument("--missing-threshold", type=float, default=0.0, + help="fatal if the missing-tile fraction exceeds this " + "(default 0.0: any missing tile is fatal)") + args = p.parse_args() + + tile_ids = [ln.strip() for ln in args.tile_list.read_text().splitlines() + if ln.strip()] + summary = build(tile_ids, args.run_dir, args.db, args.missing_threshold) + print(f"run_index: {summary['n_tiles']} tiles, " + f"{summary['n_exposures']} exposures, " + f"{summary['n_missing']} missing -> {args.db}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py new file mode 100644 index 00000000..e441fc3b --- /dev/null +++ b/workflow/scripts/clean_exposure.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Reclaim ONE exposure's store and leave a tombstone (PRD #848 D5, S5). + +Run as the shell of the in-DAG ``clean_exposure`` rule, never by hand: the rule's +``input:`` is every consuming tile's ``tile_vignets`` manifest, so by the time +this executes, every campaign tile that reads this exposure has already extracted +its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. + +What it deletes: the exposure's whole ``output/`` tree (the bulk store — +run_sp_exp_Gie/Sp/Ma/SxSePsfPi) AND its ``manifests/``. Deleting the manifests is +deliberate and load-bearing, not tidiness: + + * the manifests are the exposure rules' DECLARED outputs. If they survived, a + tile appended later would find the exposure chain "up to date" and run + tile_vignets against products that are no longer on disk. With them gone the + DAG sees the chain as unbuilt and regenerates it — the accepted cost of a + late append (D5), expressed as ordinary Snakemake bookkeeping rather than as + a special case. + * Snakemake only demands a missing intermediate when something downstream of it + needs to run, so tiles already finished are NOT rerun by their exposures' + manifests vanishing. + +Nothing is lost to the report: each manifest's content is copied verbatim into +the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned +exposure's record out of the tombstone — it reports the unit as ``cleaned``, +warn counts and shortfalls intact, instead of "not run". + +Order matters, and it is the reverse of the obvious one: the tombstone is +written FIRST, complete, and only then is anything deleted. A crash between the +two leaves a tombstone beside a store that is still there — the next invocation +treats the exposure as cleaned and only the disk is lost. Deleting first would +put the crash window where the manifests are already gone and the record that +replaces them was never written, and the report would be blind to that exposure +forever. + +The exp_psf benchmark tsv lives beside ``manifests/``, not inside it, so it +survives this job — it is the measured-memory feed for resource sizing (D4). + +The tombstone records the consumer set it was cleaned against. The rule carries +that same set as a ``params`` value, so when the index grows a new consumer the +tombstone goes stale under the default ``params`` rerun-trigger and the clean job +is rescheduled after the new tile's vignets — the exposure is cleaned once per +consumer set, not once per campaign. +""" + +import argparse +import json +import shutil +import time +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", required=True, type=Path) + p.add_argument("--exp", required=True) + p.add_argument("--tombstone", required=True, type=Path) + p.add_argument("--consumers", default="", + help="comma-separated tile ids this exposure was cleaned against") + args = p.parse_args() + + consumers = [t for t in args.consumers.split(",") if t] + + # Absorb the manifests before they go: the tombstone becomes the exposure's + # surviving record. + manifests = {} + mdir = args.exp_dir / "manifests" + if mdir.is_dir(): + for f in sorted(mdir.glob("*.json")): + try: + manifests[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError) as exc: + manifests[f.stem] = {"unreadable": str(exc)} + + targets = [t for t in (args.exp_dir / "output", mdir) if t.exists()] + + # Tombstone first, complete, fsync'd — then delete. See the module docstring: + # the crash window has to sit where the data still exists, not where the + # record does not. + args.tombstone.parent.mkdir(parents=True, exist_ok=True) + tmp = args.tombstone.with_suffix(".json.tmp") + tmp.write_text(json.dumps({ + "exp": args.exp, + "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "consumers": consumers, + "removed": [str(t) for t in targets], + "manifests": manifests, + }, indent=2) + "\n") + tmp.replace(args.tombstone) # atomic: no half-written tombstone, ever + + removed = [] + for target in targets: + shutil.rmtree(target) + removed.append(str(target)) + print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " + f"{len(consumers)} consuming tile(s)") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py new file mode 100644 index 00000000..1a8820c1 --- /dev/null +++ b/workflow/scripts/completeness.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python3 +"""The count-floor completeness table — the single failure policy. + +This is the ported ``complete_check`` count table from the v2.0 bash layer +(``run_job_sp_canfar_v2.0.bash`` job dispatch, survey §4). It is the *only* +failure policy in the design: there is no 3-class taxonomy and no error-signature +whitelist. A stage is a real failure iff a mandatory runner produced fewer than +its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) +sits between ``floor`` and ``expect`` and is tolerated. + +This file is also the ``check`` CLI — the second half of every rule's shell line +(PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing +onto it, so the check runs — and the manifest is written — even when +``shapepipe_run`` failed:: + + rc=0 + shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? + completeness.py check exp_mask {output} || rc=1 + exit $rc + +It counts the unit's products under ``$SP_RUN``, writes the manifest at +``{output}``, and exits nonzero iff a mandatory runner is below its floor. The +manifest is ALWAYS written, failure included: it is the DAG's currency and the +only thing ``run_report.py`` reads, so a failed unit must still leave a record of +*why*. (The profile sets ``keep-incomplete`` so snakemake does not delete that +record on the way out.) Manifests carry no wall-clock, and are rewritten ONLY +when their content changes — identical on-disk state must leave a byte-identical +manifest with an UNMOVED mtime, or the mtime rerun-trigger churns the cone on +every unrelated ``--forcerun``. + +Per-runner fields: + expect nominal file count for a fully complete unit (report yardstick) + floor the fail-loud minimum (below this the job exits nonzero) + warn if True the runner never fails the unit at all (bash ``:warn`` — + e.g. psfex_interp on tiles missing some epochs) + subpath count files in ``/output//`` instead of + ``/output/`` (bash ``:rand_split`` — setools split cats) + +Counts are file counts in the runner's output dir, matching the bash +``ls / | wc -l`` semantics (broken symlinks excluded by the caller). +""" + +import argparse +import json +import os +import re +import sys +from pathlib import Path + +# stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} +COMPLETENESS = { + # --- tile prepare (phase A) --- + # get_images counts are CONFIG-FLAVOR-DEPENDENT: the v2.0 bash table said 4/6 + # for the canfar vos flavor; the nibi symlink configs produce one file per + # INPUT_FILE_PATTERN entry (tile: image+weight=2; exp: image+weight+flag=3), + # verified against the p3-batch1 baseline tree (100 files / 50 tiles). + "tile_get_images": {"get_images_runner": dict(expect=2, floor=2)}, + "tile_uncompress": {"uncompress_fits_runner": dict(expect=1, floor=1)}, + "tile_find_exposures": {"find_exposures_runner": dict(expect=1, floor=1)}, + + # --- exposure chain --- + "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, + "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, + "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, + # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree + # AND the bash baseline (both 120/exposure). + "exp_psf": { + "sextractor_runner": dict(expect=120, floor=2), + "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), + "psfex_runner": dict(expect=80, floor=2), + "psfex_interp_runner": dict(expect=40, floor=0, warn=True), + }, + + # --- tile post --- + "tile_merge_headers": {"merge_headers_runner": dict(expect=1, floor=1)}, + "tile_mask": {"mask_runner": dict(expect=1, floor=1)}, + "tile_detect": {"sextractor_runner": dict(expect=2, floor=2)}, + "tile_vignets": { + "psfex_interp_runner": dict(expect=1, floor=1), + "vignetmaker_runner_run_1": dict(expect=1, floor=1), + # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); + # v2.0's 4 was the canfar flavor. floor follows the tile-post pattern + # (expect=floor: all-or-nothing, every vignette feeds ngmix). + "vignetmaker_runner_run_2": dict(expect=5, floor=5), + }, + "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, + "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, + "tile_make_cat": {"make_cat_runner": dict(expect=1, floor=1)}, +} + + +def count_products(run_dir, runner, spec): + """Count files in ``run_dir//output[/]/`` (live links only).""" + out = run_dir / runner / "output" + if "subpath" in spec: + out = out / spec["subpath"] + if not out.is_dir(): + return 0 + return sum(1 for p in out.iterdir() if p.exists()) # p.exists() drops dead links + + +def check_floor(stage, run_dir): + """Return (ok, details). ok is False iff a mandatory runner is below floor. + + ``details`` is a list of (runner, n_found, floor, expect, warn) tuples. + """ + table = COMPLETENESS[stage] + details, ok = [], True + for runner, spec in table.items(): + n = count_products(run_dir, runner, spec) + details.append((runner, n, spec["floor"], spec["expect"], + spec.get("warn", False))) + if not spec.get("warn", False) and n < spec["floor"]: + ok = False + return ok, details + + +# --- where a stage writes ------------------------------------------------- +# +# stage -> (level, run_sp_ dir under $SP_RUN/output/). These are the +# committed configs' RUN_NAMEs (RUN_DATETIME=False makes them fixed, PRD D2), so +# the check never resolves a run-log. The ngmix entry interpolates the same env +# var its config does, so chunk K's check looks at chunk K's dir. +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "exp_mask": ("exp", "run_sp_exp_Ma"), + "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_mask": ("tile", "run_sp_tile_Ma"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_detect_uc": ("tile", "run_sp_tile_Uc"), + "tile_vignets": ("tile", "run_sp_tile_PiViVi"), + "tile_ngmix": ("tile", "run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u"), + "tile_merge_cats": ("tile", "run_sp_Ms"), + "tile_make_cat": ("tile", "run_sp_Mc"), +} + + +# --- failure reasons ------------------------------------------------------ + +# Lines worth showing a human who asks "why is this runner short?". Deliberately +# crude: the point is a pointer into the logs, not a taxonomy (there is no error +# whitelist in this design — the count floor is the policy). +_ERROR_RE = re.compile( + r"traceback|exception|\berror\b|\bfailed\b|no such file|not found|" + r"killed|out of memory|oom|segmentation fault|bad chi2", + re.IGNORECASE) +_TS_RE = re.compile(r"^\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\s*") +_NOISE_RE = re.compile(r"A total of 0 errors were recorded") + +MAX_LOG_FILES = 40 # logs are per-CCD; a handful is enough to characterise +MAX_TAIL_LINES = 120 # per file +MAX_REASONS = 3 # per runner + + +def _normalise(line: str) -> str: + """Collapse a log line to its shape, so 40 per-CCD copies dedupe to one.""" + line = _TS_RE.sub("", line.strip()) + line = re.sub(r"/\S+", "", line) # paths differ per CCD + line = re.sub(r"\d+", "N", line) + return line[:200] + + +def scrape_reasons(stage_dir, runner): + """Best-effort, bounded: distinct error-looking lines from a runner's logs. + + Two sources, in order of usefulness: the runner's per-process worker logs + (``/logs/process-*.log`` — where the module's own exception lands), + and the stage's ``logs/log_sp.log`` (where ShapePipe records its error + tally). Sorted, truncated, deduped by shape — a manifest must stay + byte-stable for a given tree. + """ + seen, reasons = {}, [] + candidates = [] + for d in (stage_dir / runner / "logs", stage_dir / "logs"): + if d.is_dir(): + candidates += sorted(p for p in d.iterdir() if p.is_file()) + for path in candidates[:MAX_LOG_FILES]: + try: + lines = path.read_text(errors="replace").splitlines()[-MAX_TAIL_LINES:] + except OSError: + continue + for raw in lines: + if not _ERROR_RE.search(raw) or _NOISE_RE.search(raw): + continue + shape = _normalise(raw) + if shape in seen: + seen[shape] += 1 + continue + seen[shape] = 1 + reasons.append([path.name, _TS_RE.sub("", raw.strip())[:300], shape]) + out = [] + for name, text, shape in reasons[:MAX_REASONS]: + n = seen[shape] + out.append(f"{name}: {text}" + (f" [x{n}]" if n > 1 else "")) + return out + + +# --- manifest ------------------------------------------------------------- + +def build_manifest(stage, run_dir, unit, stage_subdir=None): + """Count, classify and (on shortfall) scrape. Returns (manifest, ok). + + Stages absent from the table fall back to the zero-output floor: any product + anywhere under the stage dir passes, nothing at all fails. + """ + level, subdir = STAGE_DIR.get(stage, (None, None)) + subdir = stage_subdir or (os.path.expandvars(subdir) if subdir else None) + stage_dir = run_dir / "output" / subdir if subdir else run_dir + manifest = { + "stage": stage, + "level": level, + "unit": unit, + "run_dir": str(run_dir), + "stage_dir": str(stage_dir), + "runners": {}, + "failures": [], + } + + if stage not in COMPLETENESS: + produced = list(stage_dir.glob("**/output/*")) if stage_dir.is_dir() else [] + ok = bool(produced) + manifest["status"] = "complete" if ok else "failed" + manifest["n_products"] = len(produced) + if not ok: + manifest["failures"].append( + {"runner": None, "found": 0, "floor": 1, + "reasons": [f"zero output under {stage_dir}"]}) + return manifest, ok + + ok, details = check_floor(stage, stage_dir) + short = False + for runner, n, floor, expect, warn in details: + below = n < floor + if n < expect: + short = True + manifest["runners"][runner] = { + "found": n, "expect": expect, "floor": floor, "warn": warn, + "status": ("complete" if n >= expect else + "warn" if (warn or not below) else "below_floor"), + } + if below: + manifest["failures"].append({ + "runner": runner, "found": n, "floor": floor, "expect": expect, + "warn": warn, "reasons": scrape_reasons(stage_dir, runner), + }) + manifest["status"] = "failed" if not ok else ("warn" if short else "complete") + return manifest, ok + + +def _unit_from_run_dir(run_dir): + """The human unit ID: the basename of ``$SP_RUN`` (``210.282``, ``2605805``). + + NOT ``SP_UNIT_NUM``, which carries ShapePipe's dashed numbering form + (``-210-282``) and would put ``210-282`` in the manifest — a key that joins + to nothing. ``run_report`` keys units on the store directory name, which is + exactly this basename, so the two now agree. + """ + return Path(str(run_dir)).name or "unknown" + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="ShapePipe per-unit completeness check") + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check", help="count products, write the manifest") + c.add_argument("stage") + c.add_argument("manifest", type=Path) + c.add_argument("--run-dir", type=Path, default=None, + help="the unit's $SP_RUN (default: the env var)") + c.add_argument("--unit", default=None, + help="override the unit ID (default: basename of $SP_RUN)") + c.add_argument("--stage-dir", default=None, + help="override the run_sp_* subdir (default: the stage table)") + args = p.parse_args(argv) + + run_dir = args.run_dir or Path(os.environ.get("SP_RUN", "")) + if not str(run_dir): + print("[completeness] FATAL: $SP_RUN unset and --run-dir not given", + file=sys.stderr) + return 2 + unit = args.unit or _unit_from_run_dir(run_dir) + + manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) + args.manifest.parent.mkdir(parents=True, exist_ok=True) + # Write ONLY on change. An unconditional write moves the mtime on every run, + # and mtime is a rerun-trigger: a `--forcerun` of one upstream stage would + # then rewrite this manifest byte-identically and drag the whole downstream + # cone along with it. + text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + if not args.manifest.exists() or args.manifest.read_text() != text: + args.manifest.write_text(text) + + for runner, r in manifest["runners"].items(): + tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} + print(f"[completeness] {runner}: {r['found']}/{r['expect']} " + f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) + print(f"[completeness] {args.stage} {unit}: {manifest['status']} " + f"-> {args.manifest}", file=sys.stderr) + for f in manifest["failures"]: + for reason in f["reasons"]: + print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py new file mode 100644 index 00000000..9e305805 --- /dev/null +++ b/workflow/scripts/ngmix_range.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Print one ngmix chunk's closed object-ID range as bash exports. + +Run inside the chunk's shell, from the tile's own sexcat, because the range is +only knowable at execution time (PRD D4):: + + eval "$(ngmix_range.py --run-dir $SP_RUN --chunk 3 --n-chunks 8)" + # -> export NGMIX_ID_MIN=751; export NGMIX_ID_MAX=1125 + +SExtractor's NUMBER column (ngmix's obj_id) runs 1..N contiguous, so covering +[1, N] processes every object exactly once. The first N-1 chunks get equal +shares; the last takes the remainder, with a CLOSED upper bound at n_obj — +never ID_OBJ_MAX = -1, which ngmix reads as unbounded and would double-count. +Each tile is sized from its OWN count (the bash monolith used the campaign +average and left the last chunk open). +""" + +import argparse +from pathlib import Path + + +def id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: + base, rem = divmod(n_obj, n_chunks) + ranges, lo = [], 1 + for k in range(1, n_chunks + 1): + hi = lo + base + (rem if k == n_chunks else 0) - 1 + ranges.append((lo, hi)) + lo = hi + 1 + return ranges + + +def object_count(run_dir: Path) -> int: + """NAXIS2 of the last HDU of this tile's sexcat (get_number_objects.py).""" + from astropy.io import fits + + cats = sorted((run_dir / "output" / "run_sp_tile_Sx").glob( + "sextractor_runner/output/sexcat*.fits")) + if not cats: + raise SystemExit(f"[ngmix_range] FATAL: no sexcat under {run_dir}") + with fits.open(cats[0]) as hdul: + return int(hdul[-1].header["NAXIS2"]) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--chunk", required=True, type=int) + p.add_argument("--n-chunks", required=True, type=int) + a = p.parse_args() + lo, hi = id_ranges(object_count(a.run_dir), a.n_chunks)[a.chunk - 1] + print(f"export NGMIX_ID_MIN={lo}; export NGMIX_ID_MAX={hi}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py new file mode 100644 index 00000000..43ce8dc6 --- /dev/null +++ b/workflow/scripts/run_report.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""``sp report`` — the run's success/failure tables, read from the manifests. + +NOT a DAG node. A report rule that declared all tiles' outputs as inputs would be +a descendant of every job, so one hard failure under --keep-going would poison +its cone and the report would never run — the exact scenario it exists for. So it +is a plain script, runnable at any time including mid-run; the Snakefile's +onsuccess/onerror hooks call it so every invocation ends with one. + +It reads two things and nothing else (PRD D3): + + * the **index** (``run_index.sqlite``) — the units the run declared, and the + tile->exposure edges that let an exposure failure be blamed on the tiles it + blocks; + * the **manifests** — one per unit per stage, written by + ``completeness.py check``, carrying per-runner found/expect/floor and + log-scraped failure reasons. + +A reclaimed exposure has no manifests: ``clean_exposure`` deleted them after +copying them into ``/cleaned.json``. That tombstone is read as the +unit's record and the unit is reported as **cleaned** — not "not run", and it +blocks no tile. + +No disk scanning: counting products is the *check's* job, done once at the moment +the products were fresh. A unit with no manifest for a stage is "not run" — which +is a real and distinct answer from "ran and produced nothing". + +Manifests are discovered by glob (``tiles/**/manifests/*.json``), not by +constructed path: the unit stores are sharded (``tiles///``) and the +sharding depth is not this script's business. +""" + +import argparse +import json +import sqlite3 +import sys +from collections import defaultdict +from pathlib import Path + +# Stage order per level — the report's column order, and the definition of +# "expected" (a declared unit with no manifest for one of these is not run). +TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", + "tile_merge_headers", "tile_detect", "tile_vignets", + "tile_ngmix", "tile_merge_cats", "tile_make_cat"] +EXP_STAGES = ["exp_get_images", "exp_split", "exp_mask", "exp_psf"] + +STATUSES = ("complete", "warn", "failed", "not_run") + + +def load_manifests(run_dir: Path, sub: str) -> dict: + """``{unit: {stage: manifest}}`` for one store (``tiles`` or ``exp``). + + The unit key is the manifest dir's *parent directory name* — shard-depth + agnostic, and the only form that joins to the index (the manifest's own + ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is + not the index's ``210.282``). The stage comes from the manifest body, never + the filename: ngmix chunks share a stage under per-chunk filenames, and they + collapse to one worst-case entry. + """ + out: dict = defaultdict(dict) + for path in sorted((run_dir / sub).glob("**/manifests/*.json")): + try: + m = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable manifest {path}: {exc}", file=sys.stderr) + continue + unit = path.parent.parent.name + stage = m.get("stage", path.stem) + prev = out[unit].get(stage) + # Worst status wins when several manifests share a stage (ngmix chunks). + rank = lambda d: STATUSES.index(d.get("status")) if d.get("status") in STATUSES else len(STATUSES) # noqa: E731 + if prev is None or rank(m) > rank(prev): + out[unit][stage] = m + return out + + +def absorb_tombstones(run_dir: Path, sub: str, manifests: dict) -> set: + """Fill in reclaimed units from their ``cleaned.json``; return their ids. + + A cleaned exposure has NO ``manifests/`` — ``clean_exposure`` deleted it, + after copying every manifest verbatim into the tombstone. Read them back, or + the report inverts the truth exactly when reclamation works: the exposure + shows as "not run" and blocks the very tiles whose completion authorised the + deletion. + + Manifests on disk win if both exist — that is a re-built chain, and the + tombstone is then a stale record of the previous generation. + """ + cleaned = set() + for path in sorted((run_dir / sub).glob("**/cleaned.json")): + unit = path.parent.name + try: + tomb = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable tombstone {path}: {exc}", file=sys.stderr) + continue + if manifests.get(unit): + continue + for key, m in (tomb.get("manifests") or {}).items(): + if not isinstance(m, dict): + continue + manifests[unit][m.get("stage", key)] = m + cleaned.add(unit) + return cleaned + + +def shortfalls(m: dict) -> dict: + """``{runner: (found, expect, floor)}`` for every runner under expect.""" + return {r: (d["found"], d["expect"], d["floor"]) + for r, d in m.get("runners", {}).items() if d["found"] < d["expect"]} + + +def reasons(m: dict) -> list: + """Flattened failure reasons, runner-tagged, for the report's why column.""" + out = [] + for f in m.get("failures", []): + head = f"{f['runner']} {f['found']}/{f.get('expect', '?')} (floor {f['floor']})" + out += [f"{head}: {r}" for r in f["reasons"]] or [head] + return out + + +def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: + """Per-stage counts + named unit lists, for one level. + + ``cleaned`` units are counted by the status their absorbed manifests carry + (complete or warn — attrition is preserved) and additionally listed under + ``cleaned``, so a reclaimed campaign reads as reclaimed rather than as a + campaign that never ran. + """ + per_stage = {} + for stage in stages: + t = {"complete": 0, "warn": [], "failed": [], "not_run": [], "cleaned": []} + agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) + for u in units: + m = manifests.get(u, {}).get(stage) + if m is None: + t["not_run"].append(u) + continue + if u in cleaned: + t["cleaned"].append(u) + status = m.get("status", "failed") + status = status if status in ("complete", "warn") else "failed" + if status == "complete": + t["complete"] += 1 + else: + t[status].append(u) + if status == "failed": + # Failed units are named above, never folded into the attrition + # aggregate: a whole-unit failure is not per-CCD attrition, and + # mixing them hides real deletion bugs behind a big denominator. + continue + for runner, d in m.get("runners", {}).items(): + a = agg[runner] + a["found"] += d["found"] + a["expect"] += d["expect"] + if d["found"] < d["expect"]: + a["by_unit"][u] = d["expect"] - d["found"] + for a in agg.values(): + if not a["by_unit"]: + del a["by_unit"] + t["products"] = dict(agg) + per_stage[stage] = t + return per_stage + + +def unit_rows(units, stages, manifests) -> list: + """One row per non-clean unit: its first bad stage, shortfalls, why.""" + rows = [] + for u in units: + got = manifests.get(u, {}) + bad = [s for s in stages + if got.get(s) is None or got[s].get("status") != "complete"] + if not bad: + continue + stage = bad[0] + m = got.get(stage) + rows.append({ + "unit": u, + "stage": stage, + "status": "not_run" if m is None else m.get("status", "failed"), + "shortfalls": shortfalls(m) if m else {}, + "reasons": reasons(m) if m else [], + "n_bad_stages": len(bad), + }) + return rows + + +def print_table(title, rows, limit=25): + print(f"\n{title} ({len(rows)} affected)") + if not rows: + print(" — none") + return + print(f" {'unit':<14} {'stage':<20} {'status':<8} why") + for r in rows[:limit]: + short = ", ".join(f"{k} {v[0]}/{v[1]}" for k, v in r["shortfalls"].items()) + why = (r["reasons"][0] if r["reasons"] else short) or "-" + print(f" {r['unit']:<14} {r['stage']:<20} {r['status']:<8} {why[:90]}") + if len(rows) > limit: + print(f" … and {len(rows) - limit} more (see the JSON report)") + + +def print_stage_table(title, per_stage, n_units): + print(f"\n{title} ({n_units} units declared)") + print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} " + f"{'cleaned':>8} attrition") + for stage, t in per_stage.items(): + att = [f"{r} {a['found']}/{a['expect']}" + for r, a in t["products"].items() if a["found"] < a["expect"]] + print(f" {stage:<20} {t['complete']:>6} {len(t['warn']):>6} " + f"{len(t['failed']):>6} {len(t['not_run']):>8} " + f"{len(t.get('cleaned', [])):>8} {', '.join(att)[:60]}") + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--status", default="manual") + p.add_argument("--out", type=Path, default=None) + p.add_argument("--limit", type=int, default=25, + help="rows per stdout table (the JSON report is complete)") + args = p.parse_args() + + tiles, exps, tile_exp = [], [], defaultdict(list) + if args.index.exists(): + con = sqlite3.connect(args.index) + tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles ORDER BY 1")] + exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures ORDER BY 1")] + for tile_id, exp_id in con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + tile_exp[tile_id].append(exp_id) + con.close() + else: + print(f"[run_report] no index at {args.index} — reporting manifests only", + file=sys.stderr) + + tile_m = load_manifests(args.run_dir, "tiles") + exp_m = load_manifests(args.run_dir, "exp") + # Reclaimed exposures speak through their tombstones (D5, S5). + cleaned_exp = absorb_tombstones(args.run_dir, "exp", exp_m) + tiles = tiles or sorted(tile_m) + exps = exps or sorted(exp_m) + + missing_json = args.index.parent / "missing.json" + missing = json.loads(missing_json.read_text()) if missing_json.exists() else [] + + report = { + "status": args.status, + "n_tiles": len(tiles), "n_exposures": len(exps), + "missing_tiles": missing, + "tile_stages": tally_level(tiles, TILE_STAGES, tile_m), + "exp_stages": tally_level(exps, EXP_STAGES, exp_m, cleaned_exp), + "cleaned_exposures": sorted(cleaned_exp), + "tiles": unit_rows(tiles, TILE_STAGES, tile_m), + "exposures": unit_rows(exps, EXP_STAGES, exp_m), + } + + # Blame propagation: a BLOCKING exposure blocks every tile that reads it. + # Without this, a tile stalled at tile_vignets looks like its own failure. + # + # Blocking means "failed" or "never ran" — NOT "warn". Warn is the expected + # per-CCD attrition (setools rejecting a sparse CCD, psfex_interp short an + # epoch); it is present in essentially every exposure at production scale, so + # counting it here made every exposure block every tile and the table said + # nothing. + # Judged over ALL the exposure's stages, not just the first bad one, so an + # exposure that warns early and fails late still blocks. + # A CLEANED exposure never blocks: its store is gone precisely because every + # consuming tile already had its vignets. Its absorbed manifests are read + # above, so a cleaned exposure that genuinely failed still shows in the + # tables — it just does not get to hold complete tiles hostage. + def _blocks(unit): + if unit in cleaned_exp: + return False + for stage in EXP_STAGES: + m = exp_m.get(unit, {}).get(stage) + if m is None or m.get("status", "failed") == "failed": + return True + return False + + bad_exp = {e for e in exps if _blocks(e)} + blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} + report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} + + done = report["tile_stages"]["tile_make_cat"]["complete"] + report["final_cats"] = {"present": done, "of": len(tiles)} + + out = args.out or (args.index.parent / "run_report.json") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" + + (f" ({len(missing)} tiles missing exposure lists)" if missing else "") + + (f" ({len(cleaned_exp)} exposures reclaimed)" if cleaned_exp else "")) + print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) + print_stage_table("TILES", report["tile_stages"], len(tiles)) + print_table("exposures not complete", report["exposures"], args.limit) + print_table("tiles not complete", report["tiles"], args.limit) + nb = report["tiles_blocked_by_exposures"] + if nb: + print(f"\ntiles waiting on incomplete exposures ({len(nb)})") + for t, e in list(nb.items())[:args.limit]: + print(f" {t:<14} {', '.join(e[:6])}" + + (f" (+{len(e) - 6})" if len(e) > 6 else "")) + print(f"\n[run_report] -> {out}") + + +if __name__ == "__main__": + main()