From f92cb44b6747a010c4cf369d7e515e9278824244 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 15:46:49 -0400 Subject: [PATCH 01/21] feat(orchestration): Snakemake workflow for the real-data pipeline (#848) Implements the round-2 design in #848: static two-phase execution driven by a parse-time SQLite run index (never a rule input, so appending tiles never invalidates completed work); rules wrap the existing per-unit shapepipe_run config chains at tile/exposure granularity with deterministic RUN_DATETIME=False directory() outputs; in-job completeness count-floor (the ported bash complete_check table, workflow/scripts/completeness.py); one-allocation execution profile for nibi with apptainer software-deployment (the workflow never calls apptainer itself); out-of-DAG run_report verb + onsuccess/onerror hooks; workflow/bin/sp as the committed launcher. First pass touches no module code. Validated by dry-run against a 4-tile P3 subset (219-job compute DAG); prepare phase runs green on nibi. Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + profiles/nibi/config.yaml | 101 +++++++ workflow/README.md | 105 ++++++++ workflow/Snakefile | 124 +++++++++ workflow/bin/sp | 63 +++++ workflow/config.yaml | 35 +++ workflow/rules/exposure.smk | 70 +++++ workflow/rules/prepare.smk | 67 +++++ workflow/rules/tile.smk | 147 ++++++++++ workflow/scripts/build_forest.py | 42 +++ workflow/scripts/build_index.py | 139 ++++++++++ workflow/scripts/clean_exposures.py | 64 +++++ workflow/scripts/completeness.py | 86 ++++++ workflow/scripts/run_report.py | 107 ++++++++ workflow/scripts/sp_rule.py | 405 ++++++++++++++++++++++++++++ 15 files changed, 1556 insertions(+) create mode 100644 profiles/nibi/config.yaml create mode 100644 workflow/README.md create mode 100644 workflow/Snakefile create mode 100755 workflow/bin/sp create mode 100644 workflow/config.yaml create mode 100644 workflow/rules/exposure.smk create mode 100644 workflow/rules/prepare.smk create mode 100644 workflow/rules/tile.smk create mode 100644 workflow/scripts/build_forest.py create mode 100644 workflow/scripts/build_index.py create mode 100644 workflow/scripts/clean_exposures.py create mode 100644 workflow/scripts/completeness.py create mode 100644 workflow/scripts/run_report.py create mode 100644 workflow/scripts/sp_rule.py diff --git a/.gitignore b/.gitignore index db20f308..4c4b6d0b 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ code # .felt here is a machine-local symlink into it. Never track it in this repo. /.felt/ /.felt +.snakemake/ diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml new file mode 100644 index 00000000..e977803c --- /dev/null +++ b/profiles/nibi/config.yaml @@ -0,0 +1,101 @@ +# Snakemake profile for the Nibi cluster (Digital Research Alliance). +# +# ONE-ALLOCATION mode: sbatch one node up front, then run snakemake with the +# LOCAL scheduler inside it (no per-job sbatch; the local scheduler bin-packs the +# DAG into the allocation against `cores` + `resources: mem_mb`). This is the +# proven nibi path (job_p3_batch1) and the single honest first-pass mode; +# multi-node scaling = multiple tile-disjoint invocations, deferred. +# +# Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). +# software-deployment-method wraps every job's shell in `apptainer exec` — the +# user never types apptainer; the container is set in the Snakefile (local .sif). + +# Local scheduler (no `executor:` -> built-in local). Size to the allocation. +cores: 192 # node cores available to the DAG +local-cores: 4 # cap localrule/aggregation concurrency +resources: + mem_mb: 720000 # node RAM cap the local scheduler bin-packs under + +software-deployment-method: [apptainer] +# Explicit container environment (finding 2/3): the apptainer SDM otherwise drops +# the proven-recipe env and hardcodes --home , hiding ~/.ssl/cadcproxy.pem. +# --cleanenv strict host-env isolation (APPTAINERENV_*/SINGULARITYENV_* survive) +# OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) +# MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) +# --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp +# PYTHONPATH SETTLED CALL 3 — P0 pins the prod worktree src so science +# content matches the bash validation runs (which use this +# override). Production later rebuilds the sif at the exact +# validation commit and DROPS this --env line. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-prod/src --home /home/cdaley --bind /project --bind /scratch" + +latency-wait: 60 # NFS: wait for outputs to appear after a job +keep-going: true # a failed job poisons only its cone; siblings run on +rerun-incomplete: true # re-do jobs left incomplete by an unclean death +show-failed-logs: true +printshellcmds: true + +# rerun-triggers left at the full Snakemake v9 default +# {mtime,params,input,code,software-env}: params/code fixes SHOULD propagate. +# No override — the wrapper writes outputs only on change so mtimes move only +# when reality moves; protected() guards finished science products. + +default-resources: + mem_mb: 4000 + runtime: 120 # minutes + cpus_per_task: 1 + +# Per-stage resources — measured footprints from the nibi P3 runs (#808). +# threads == cpus_per_task == ShapePipe SMP fork width (sp_rule sets +# SMP_BATCH_SIZE={threads}); mem_mb == threads x measured per-worker footprint. +# +# set-threads is REQUIRED for {threads} to resolve above 1: the local scheduler +# derives cpus_per_task from threads, NOT the reverse (verified snakemake 9.23 — +# set-resources cpus_per_task alone leaves {threads}==1, i.e. SMP_BATCH_SIZE=1, +# serial). Keep the two blocks in lockstep. +set-threads: + exp_split: 4 + exp_mask: 4 + exp_psf: 8 + tile_mask: 16 + tile_detect: 16 + tile_merge_headers: 8 + tile_vignets: 8 + tile_ngmix: 8 + tile_merge_cats: 4 + tile_make_cat: 4 + +set-resources: + exp_split: + cpus_per_task: 4 + mem_mb: 8000 + exp_mask: + cpus_per_task: 4 + mem_mb: 8000 + exp_psf: + cpus_per_task: 8 + mem_mb: 16000 + runtime: 240 + tile_mask: + cpus_per_task: 16 + mem_mb: 16000 + tile_detect: + cpus_per_task: 16 + mem_mb: 16000 + tile_merge_headers: + cpus_per_task: 8 + mem_mb: 8000 + tile_vignets: + cpus_per_task: 8 + mem_mb: 24000 + runtime: 240 + tile_ngmix: + cpus_per_task: 8 + mem_mb: 24000 + runtime: 360 + tile_merge_cats: + cpus_per_task: 4 + mem_mb: 8000 + tile_make_cat: + cpus_per_task: 4 + mem_mb: 8000 diff --git a/workflow/README.md b/workflow/README.md new file mode 100644 index 00000000..4d8abd1f --- /dev/null +++ b/workflow/README.md @@ -0,0 +1,105 @@ +# ShapePipe Snakemake orchestration + +Snakemake workflow that orchestrates real-data ShapePipe runs, replacing the +`curl_canfar_local.sh → run_job_sp_canfar_v2.0.bash → job_sp_canfar_v2.0.bash` +bash layers (and per-site sbatch reimplementations). **Module code is untouched**: +rules call `shapepipe_run -c ` on the existing config chains. Design and +rationale: [CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) +(the living PRD). + +## Quick start (nibi) + +**One-allocation mode.** `sbatch` one node, then run Snakemake with the *local* +scheduler inside it: it bin-packs the DAG into the allocation (no per-job +sbatch). Each job's shell runs `shapepipe_run` **inside the container** (the +profile's apptainer software-deployment — you never type `apptainer`). + +```bash +# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which is +# node-local; the executor re-invokes this python inside jobs). +uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 +source /project/def-mjhudson/cdaley/snakemake-env/bin/activate +uv pip install 'snakemake>=9,<10' + +# Edit workflow/config.yaml: tile_list, run_dir, config_src, container, star_cats. + +# The committed launcher loads apptainer/1.4.5 + the /project venv, so a fresh +# shell always has the right state. Inside your allocation: +workflow/bin/sp prepare # prepare_tiles -> build_index.py -> prepare_exposures +workflow/bin/sp run # the main compute DAG +``` + +`sp` subcommands: `prepare` (chains all three prepare steps), `prepare_tiles`, +`index`, `prepare_exposures`, `run`, `rerun ` (forced recompute of +protected products), `cancel ` (scancel sweep before `--unlock`), +`clean-exposures [--apply]`, `report`. + +## Execution is three static invocations + +The exposure job set is data-derived from the tiles' `find_exposures` output, so +it cannot be scheduled in the same static DAG that produces it. Hence: + +1. `snakemake prepare_tiles` — per-tile static DAG (Git_vos → Uz → Fe), + `keep-going` so tile failures are independent. +2. `build_index.py` — a **plain script**, not a DAG node. It iterates the + declared tile list, checks each tile's Fe output at its deterministic path (no + glob), writes `run_index.sqlite` + `missing.json`, and exits nonzero only if + the missing *fraction* exceeds a threshold. (There is no `--allow-missing` + Snakemake flag; that mechanism does not exist.) +3. `snakemake prepare_exposures` — per-exposure static DAG (Gie_vos + per-unit + star cats), parseable now the index exists. +4. `snakemake` (main DAG) — the tile/exposure compute chain. + +`sp prepare` chains 1–3 so the UX stays one command. + +## Layout + +``` +workflow/ + Snakefile parse-time index load; global container:; onsuccess/onerror report hooks + config.yaml the run: tile list, paths, container, chunk count + bin/sp committed launcher (module load + /project venv + subcommands) + rules/ + prepare.smk tile get_images/uncompress/find_exposures + per-unit star cats + exposure.smk per-exposure: get_images, split, mask, psf (no temp()) + tile.smk per-tile: exp forest, merge_headers, mask, detect, vignets, ngmix, merge, make_cat + scripts/ + sp_rule.py the thin per-unit wrapper (isolation furniture, config copy, log-sync, count floor) + build_index.py prepare-phase run_index.sqlite builder (plain script) + build_forest.py per-tile exposure symlink forest (group-compatible shell) + completeness.py the ported count-floor table (shared by sp_rule + run_report) + run_report.py standalone report (NOT a DAG node; run_report hooks call it) + clean_exposures.py index-driven exposure-store reclaim (replaces temp()) +profiles/nibi/config.yaml local scheduler; apptainer SDM; resources; keep-going +``` + +## How it works + +- **Completeness = the count floor.** Each stage declares its deterministic + `output/run_sp_/` directory as `directory()` output (the per-unit + config copy forces `RUN_DATETIME = False`, so the path is known at DAG time). + After the run, `sp_rule.py` counts products per mandatory runner against + `completeness.py`'s floor and exits nonzero below it; per-CCD attrition between + floor and `expect` is tolerated (the directory simply lacks that product). + This *is* the failure policy — no 3-class taxonomy, no error-signature + whitelist. `--keep-going` isolates a failure to its own cone. +- **Per-unit isolation is by work-dir content, not `-e/--exclusive`.** Every + tile/exposure runs its own `shapepipe_run` in `tiles//` or `exp//`, + with a `cfis` config symlink, `star_cat_{exp,tiles}` symlinks, a + `tile_numbers.txt` (tiles) or fabricated `exp_numbers-000-000.txt` pseudo-Fe + (exposures), and a config copy setting `RUN_DATETIME=False` and + `SMP_BATCH_SIZE={threads}`. `NUMBER_LIST=-` is injected **only** for + stages whose numbering scheme is the unit ID (tile-scheme stages + exp split), + never for get_images / exp_mask / exp_psf. +- **No `temp()` on shared exposure dirs.** Exposures overlap tiles, so temp() + would cascade destructive reruns when a tile is appended. The store is + persistent; `sp clean-exposures` reclaims space only for exposures all of + whose consuming tiles have a final_cat. +- **The index is parse-time data, never a rule input.** Appending tiles changes + which jobs exist without invalidating completed work. Star cats are re-keyed + per unit for the same reason. `final_cat` is `protected()`. +- **Failure is a report, not a gate.** `run_report.py` disk-scans the trees + against the count table and enumerates shortfalls (whole-unit absence vs + per-CCD attrition). It is a standalone script — a DAG report node would be + poisoned by the very failures it must enumerate — emitted automatically by the + `onsuccess`/`onerror` hooks and runnable any time via `sp report`. diff --git a/workflow/Snakefile b/workflow/Snakefile new file mode 100644 index 00000000..04722a73 --- /dev/null +++ b/workflow/Snakefile @@ -0,0 +1,124 @@ +"""ShapePipe real-data orchestration — Snakemake workflow. + +Design / rationale: CosmoStat/shapepipe#848 (the living PRD). + +Execution is honestly THREE static invocations, chained by ``workflow/bin/sp`` +so the UX stays one command (``sp prepare`` / ``sp run``): + + snakemake --profile profiles/nibi prepare_tiles # Git_vos + Uz + Fe (keep-going) + python workflow/scripts/build_index.py ... # PLAIN SCRIPT, not a DAG node + snakemake --profile profiles/nibi prepare_exposures # Gie_vos + star cats + snakemake --profile profiles/nibi # the main compute DAG + +The run index is loaded ONCE here at parse time into plain dicts; it is never a +rule input, so appending tiles and rebuilding it changes which jobs exist without +invalidating completed work. build_index.py is a plain script *between* +prepare_tiles and prepare_exposures (the exposure job set is data-derived from +the tiles' find_exposures output, so it cannot be scheduled in the same static +DAG that produces it). +""" + +import sqlite3 +from pathlib import Path + +configfile: "workflow/config.yaml" + +# Every job's shell runs inside this container (apptainer software-deployment in +# the profile); the user never types apptainer. +container: config["container"] + +# --- paths ----------------------------------------------------------------- +RUN_DIR = Path(config["run_dir"]) +CONFIG_SRC = Path(config["config_src"]) +STAR_CATS = Path(config["star_cats"]) +INDEX_DB = Path(config["index_db"]) +SCRIPTS = Path(workflow.basedir) / "scripts" +NGMIX_CHUNKS = config.get("ngmix_chunks", 8) + +with open(config["tile_list"]) as f: + TILES = [ln.strip() for ln in f if ln.strip()] + +# --- parse-time index load (compute phase only) ---------------------------- +# EXP: global set of exposure base-ids. TILE_EXP: tile -> [exp_ids]. +EXP, TILE_EXP = set(), {} +if INDEX_DB.exists(): + _con = sqlite3.connect(INDEX_DB) + EXP = {r[0] for r in _con.execute("SELECT exp_id FROM exposures")} + for tile, exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + TILE_EXP.setdefault(tile, []).append(exp) + _con.close() + +wildcard_constraints: + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + chunk = r"\d+", + +# --- shared wrapper invocation --------------------------------------------- +def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, + extra=""): + """Build the shell command that runs one unit through sp_rule.py. + + NUMBER_LIST is injected by default (tile-scheme stages + exp split); pass + ``isolate=False`` for get_images / exp_mask / exp_psf, whose per-CCD/download + numbering must not be constrained. ``{threads}`` is filled by Snakemake and + becomes SMP_BATCH_SIZE (fork width == cpus_per_task). + """ + cmd = ( + f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " + f"--unit {unit} --level {level} --run-dir {RUN_DIR} " + f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} --threads {{threads}}" + ) + if not isolate: + cmd += " --no-isolate" + if exp_forest: + cmd += f" --exp-forest {exp_forest}" + if extra: + cmd += " " + extra + return cmd + +def tile_dir(tile): + return RUN_DIR / "tiles" / tile + +def exp_dir(exp): + return RUN_DIR / "exp" / exp + +include: "rules/prepare.smk" +include: "rules/exposure.smk" +include: "rules/tile.smk" + +# --- top-level targets ------------------------------------------------------ +# Only aggregation targets are localrules (no-ops under the local executor, but +# they keep the compute chain group-compatible: no mid-chain localrules). +localrules: all, prepare_tiles, prepare_exposures + +rule all: + input: + expand(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits"), tile=TILES) + +# Phase A.1 — per-tile static DAG (download + find_exposures). keep-going so +# tile failures are independent; build_index.py runs next as a plain script. +rule prepare_tiles: + input: + expand(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe"), tile=TILES) + +# Phase A.2 — per-exposure static DAG (download + star cats), parseable now the +# index exists. Star cats are re-keyed per unit (see prepare.smk). +rule prepare_exposures: + input: + expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)), + expand(str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), exp=sorted(EXP)), + expand(str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), tile=TILES) + +# --- report hooks ----------------------------------------------------------- +# run_report is NOT a DAG node (a descendant of every job would be poisoned by +# any hard failure — the exact case it exists for). It is a standalone script, +# emitted automatically at the end of every invocation, runnable any time. +def _report(status): + shell(f"python {SCRIPTS}/run_report.py --run-dir {RUN_DIR} " + f"--index {INDEX_DB} --status {status} || true") + +onsuccess: + _report("success") + +onerror: + _report("error") diff --git a/workflow/bin/sp b/workflow/bin/sp new file mode 100755 index 00000000..f226aa44 --- /dev/null +++ b/workflow/bin/sp @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# sp — the committed launcher for the ShapePipe Snakemake workflow. +# +# Removes the shell-state dependencies the review flagged (finding 20): it loads +# the apptainer module (Snakemake resolves `apptainer` via PATH at job runtime) +# and runs the Snakemake that lives on the shared /project venv (must be on a +# shared FS — the executor re-invokes it inside jobs). One entry point, so a +# fresh tmux / a restart after a crash always launches with the right state. +# +# Usage: +# sp prepare # prepare_tiles -> build_index.py -> prepare_exposures +# sp prepare_tiles # phase A.1 only +# sp index # build_index.py only (plain script, threshold-gated) +# sp prepare_exposures # phase A.2 only +# sp run [ARGS...] # the main compute DAG +# sp rerun # forced recompute of protected/finished products +# sp cancel # scancel this workflow's jobs (name sweep) before --unlock +# sp clean-exposures [--apply] # index-driven exposure-store reclaim +# sp report # emit run_report.json now (mid-run ok) +# Extra args after the subcommand pass through to snakemake. +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ +REPO="$(dirname "$HERE")" +VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" +PROFILE="$REPO/profiles/nibi" +SCRIPTS="$HERE/scripts" +CONFIG="$HERE/config.yaml" + +module load apptainer/1.4.5 2>/dev/null || true +# shellcheck disable=SC1091 +source "$VENV/bin/activate" + +# Minimal scalar reader for workflow/config.yaml (key: value, no nesting). +cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } +RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" +TILE_LIST="$(cfg tile_list)"; MISSING_THRESH="${SP_MISSING_THRESHOLD:-0.0}" + +sm() { snakemake --profile "$PROFILE" "$@"; } +build_index() { + python "$SCRIPTS/build_index.py" --tile-list "$TILE_LIST" \ + --run-dir "$RUN_DIR" --db "$INDEX_DB" --missing-threshold "$MISSING_THRESH" +} + +cmd="${1:-}"; shift || true +case "$cmd" in + prepare) sm prepare_tiles "$@"; build_index; sm prepare_exposures "$@" ;; + prepare_tiles) sm prepare_tiles "$@" ;; + index) build_index ;; + prepare_exposures) sm prepare_exposures "$@" ;; + run) sm "$@" ;; + rerun) sm --forcerun "$@" ;; + cancel) + run="${1:?usage: sp cancel }" + squeue --me --noheader --format='%i %j' \ + | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel + echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" ;; + clean-exposures) python "$SCRIPTS/clean_exposures.py" \ + --run-dir "$RUN_DIR" --index "$INDEX_DB" "$@" ;; + report) python "$SCRIPTS/run_report.py" \ + --run-dir "$RUN_DIR" --index "$INDEX_DB" --status manual "$@" ;; + *) echo "sp: unknown subcommand '${cmd:-}' (see the header of $0)"; exit 2 ;; +esac diff --git a/workflow/config.yaml b/workflow/config.yaml new file mode 100644 index 00000000..53542d1c --- /dev/null +++ b/workflow/config.yaml @@ -0,0 +1,35 @@ +# Run configuration for the ShapePipe Snakemake workflow. +# +# A "run" is declared by a tile list plus the paths below. Everything here is +# read at parse time; none of it is a rule input, so editing it (e.g. appending +# tiles) never invalidates completed work — it only changes which jobs exist. + +# The tile list that scopes this run (one "IDra.IDdec" per line). +tile_list: /scratch/cdaley/shapepipe-output/snakemake-p0/tiles4.txt + +# The container every job runs inside (apptainer software-deployment in the profile). +container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif + +# Where products land ($SP_RUN root). Per-unit work-dir forest is built under here: +# /tiles/// and /exp/// +run_dir: /scratch/cdaley/shapepipe-output/snakemake-p0 + +# The canonical config chain (example/cfis in the repo, or a run-specific copy). +config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis + +# Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). +# Star catalogues for masking, pre-generated (network step done in prepare): +star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats + +# The run index. Durable, low-volume products (index, missing.json, run_report, +# final_cat) should land on /project, not /scratch (60-day purge, finding 15). +index_db: /project/def-mjhudson/cdaley/shapepipe-output/snakemake-p0/index/run_index.sqlite + +# ngmix within-tile chunking: static N chunks (closed ID ranges computed +# per-tile, in-job, from the tile's own sexcat). +ngmix_chunks: 8 + +# The container's installed shapepipe is overridden by the prod worktree via +# --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. +# build_index.py's missing-tile fraction floor is passed by workflow/bin/sp +# (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk new file mode 100644 index 00000000..50403d38 --- /dev/null +++ b/workflow/rules/exposure.smk @@ -0,0 +1,70 @@ +"""Exposure chain — per exposure, keyed by exp_id (dedup is structural). + +Gie -> Sp -> Ma -> psf, each in the exposure's own work dir, chained by +ShapePipe's last:/run_sp_exp_* INPUT_DIR resolver. Completeness is the count +floor (completeness.py). + +NO temp() on these shared exposure directories: exposures overlap tiles by +construction (~7-10 tiles per exposure), so temp() would delete an exposure the +moment its forests are built and cascade destructive reruns across spatial +neighbours whenever a tile is appended (findings 4/5/6/19). The store is +persistent; space is reclaimed by the explicit index-driven `sp clean-exposures` +verb (scripts/clean_exposures.py), which deletes an exposure's intermediates +only once every consuming tile in the index has its final_cat present. + +NUMBER_LIST is injected only for exp_split (numbering scheme == exposure id); +NEVER for get_images (download) / exp_mask / exp_psf (per-CCD schemes where the +#746 startup validation would turn tolerated per-CCD attrition into a +whole-exposure hard failure). +""" + +EXP_OUT = str(RUN_DIR / "exp/{exp}/output") + +# Symlink/download exposure image/weight/flag; reads the fabricated pseudo-Fe +# exp_numbers-000-000.txt the wrapper drops (via last:find_exposures_runner). +rule exp_get_images: + output: + directory(f"{EXP_OUT}/run_sp_exp_Gie") + params: + cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", + "exp", wc.exp, isolate=False) + shell: + "{params.cmd}" + +# Split multi-HDU exposure into single-CCD files (+ headers-*.npy). Bulk output; +# persistent (see module docstring on temp()). +rule exp_split: + input: + rules.exp_get_images.output + output: + directory(f"{EXP_OUT}/run_sp_exp_Sp") + params: + cmd=lambda wc: sp_rule("exp_split", "config_exp_Sp.ini", "exp", wc.exp) + shell: + "{params.cmd}" + +# Mask per-CCD; this exposure's star cat is a declared input (per-unit re-key). +rule exp_mask: + input: + rules.exp_split.output, + star=str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), + output: + directory(f"{EXP_OUT}/run_sp_exp_Ma") + params: + cmd=lambda wc: sp_rule("exp_mask", "config_exp_Ma_onthefly.ini", + "exp", wc.exp, isolate=False) + shell: + "{params.cmd}" + +# SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. +# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor. +rule exp_psf: + input: + rules.exp_mask.output + output: + directory(f"{EXP_OUT}/run_sp_exp_SxSePsfPi") + params: + cmd=lambda wc: sp_rule("exp_psf", "config_exp_psfex.ini", "exp", wc.exp, + isolate=False) + shell: + "{params.cmd}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk new file mode 100644 index 00000000..2911c3b1 --- /dev/null +++ b/workflow/rules/prepare.smk @@ -0,0 +1,67 @@ +"""Phase A — prepare: tile staging, find_exposures, per-unit star cats. + +Split into two static invocations with build_index.py (a plain script) between: + * prepare_tiles — tile_get_images -> tile_uncompress -> tile_find_exposures + * [build_index.py] — plain script, threshold-gated (not a DAG node) + * prepare_exposures — exp_get_images (exposure.smk) + per-unit star cats + +Nibi compute nodes have internet, so downloads run in-DAG (no login-node tier). +Rules stay group-compatible: no mid-chain localrules, no run:/pipe outputs. +""" + +# get_images reads a one-line tile_numbers.txt the wrapper drops in $SP_RUN. +# NUMBER_LIST is NEVER injected here (download stage; nothing on disk to +# validate against — #746 would hard-fail). +rule tile_get_images: + output: + directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Git")) + params: + cmd=lambda wc: sp_rule("tile_get_images", "config_tile_Git_vos.ini", + "tile", wc.tile, isolate=False) + shell: + "{params.cmd}" + +rule tile_uncompress: + input: + rules.tile_get_images.output + output: + directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Uz")) + params: + cmd=lambda wc: sp_rule("tile_uncompress", "config_tile_Uz.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# find_exposures reads the tile FITS HISTORY header -> exp_numbers-.txt, the +# data-derived tile->exposure edge build_index.py aggregates (as a plain script). +rule tile_find_exposures: + input: + rules.tile_uncompress.output + output: + directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe")) + params: + cmd=lambda wc: sp_rule("tile_find_exposures", "config_tile_Fe.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# Star catalogues for masking are pre-generated offline (create_star_cat.py on a +# networked login node — the one genuinely networked prepare step). They are +# re-keyed PER UNIT here: each tile/exposure's cat is its own DAG node, so +# appending tile 51 adds nodes rather than mutating a run-level output that would +# dirty every finished mask (the growth-invariant fix, finding 11). +rule exp_star_cat: + output: + str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits") + params: + src=lambda wc: str(STAR_CATS / "exp" / f"star_cat-{wc.exp}.fits") + shell: + "ln -sf {params.src} {output}" + +rule tile_star_cat: + output: + str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits") + params: + src=lambda wc: str(STAR_CATS / "tiles" / f"star_cat-{wc.tile}.fits") + shell: + "ln -sf {params.src} {output}" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk new file mode 100644 index 00000000..84516771 --- /dev/null +++ b/workflow/rules/tile.smk @@ -0,0 +1,147 @@ +"""Tile post-chain — per tile: gather exposures, then detect/PSF/shape/catalogue. + +A per-tile exposure "forest" (symlink view of exactly this tile's exposures' +products) gives the tile modules a deterministic $SP_EXP to glob. But the forest +is only a runtime convenience, NEVER the DAG edge: every tile rule that reads +through $SP_EXP declares the underlying exposure directory() outputs as input via +an index dict lookup, so the last real reader is the last DAG consumer (findings +5/19). All rules are group-compatible (shell only, no mid-chain localrules). +""" + +TILE_OUT = str(RUN_DIR / "tiles/{tile}/output") +FOREST = str(RUN_DIR / "tiles/{tile}/exp_forest") + + +def _exp_out(wc, prefix): + return [str(exp_dir(e) / "output" / prefix) for e in TILE_EXP.get(wc.tile, [])] + +def tile_exp_split(wc): return _exp_out(wc, "run_sp_exp_Sp") +def tile_exp_mask(wc): return _exp_out(wc, "run_sp_exp_Ma") +def tile_exp_psf(wc): return _exp_out(wc, "run_sp_exp_SxSePsfPi") +def tile_exp_products(wc): + """All of this tile's exposures' product dirs — the forest's DAG inputs.""" + return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) + + +# Build the per-tile symlink forest (exp_forest//output/run_sp_exp_* -> the +# real exposure products). Declaring the exposure dirs as input makes this the +# tile's wait-on-its-exposures edge; the forest itself is just the SP_EXP view. +rule tile_exp_forest: + input: + tile_exp_products + output: + directory(FOREST) + params: + cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_DB} --forest {{output}}") + shell: + "{params.cmd}" + +# Merge single-exposure WCS headers into the tile-level sqlite log +# (log_exp_headers--.sqlite — the tile suffix downstream Sx/PiViVi/Ng +# consume via NUMBERING_SCHEME -000-000). Reads headers-*.npy through the forest, +# so the exposure split dirs are explicit inputs. +rule tile_merge_headers: + input: + forest=rules.tile_exp_forest.output, + split=tile_exp_split, + output: + directory(f"{TILE_OUT}/run_sp_tile_Mh_exp") + params: + cmd=lambda wc: sp_rule("tile_merge_headers", "config_tile_Mh_exp.ini", + "tile", wc.tile, isolate=True, + exp_forest=FOREST.format(tile=wc.tile)) + shell: + "{params.cmd}" + +# Mask tiles; this tile's star cat is a declared input (per-unit re-key). +rule tile_mask: + input: + git=rules.tile_get_images.output, + star=str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), + output: + directory(f"{TILE_OUT}/run_sp_tile_Ma") + params: + cmd=lambda wc: sp_rule("tile_mask", "config_tile_Ma_onthefly.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# SExtractor object detection on the tile. +rule tile_detect: + input: + mask=rules.tile_mask.output, + mh=rules.tile_merge_headers.output, + output: + directory(f"{TILE_OUT}/run_sp_tile_Sx") + params: + cmd=lambda wc: sp_rule("tile_detect", "config_tile_Sx.ini", + "tile", wc.tile, isolate=True) + shell: + "{params.cmd}" + +# PSFEx interpolation to galaxies + vignet postage stamps (reads exposure PSF +# models + split through the forest — those dirs are explicit inputs). +rule tile_vignets: + input: + sx=rules.tile_detect.output, + forest=rules.tile_exp_forest.output, + split=tile_exp_split, + psf=tile_exp_psf, + output: + directory(f"{TILE_OUT}/run_sp_tile_PiViVi") + params: + cmd=lambda wc: sp_rule("tile_vignets", "config_tile_PiViVi_canfar_sx.ini", + "tile", wc.tile, isolate=True, + exp_forest=FOREST.format(tile=wc.tile)) + shell: + "{params.cmd}" + +# ngmix shape measurement — static N chunks. Each chunk computes its own closed +# object-ID range in-job from the tile's own sexcat (a declared input); chunks +# never rewrite the shared log (--no-log-sync) and read literal INPUT_DIRs. +rule tile_ngmix: + input: + vignets=rules.tile_vignets.output, + sx=rules.tile_detect.output, + output: + directory(f"{TILE_OUT}/run_sp_tile_ngmix_Ng{{chunk}}u") + params: + cmd=lambda wc: sp_rule( + "tile_ngmix", "config_tile_Ng_template_batch.ini", "tile", wc.tile, + extra=f"--ngmix-chunk {wc.chunk} --ngmix-nchunks {NGMIX_CHUNKS}"), + shell: + "{params.cmd}" + +def ngmix_chunks(wc): + return [f"{RUN_DIR}/tiles/{wc.tile}/output/run_sp_tile_ngmix_Ng{k}u" + for k in range(1, NGMIX_CHUNKS + 1)] + +# Merge the N chunk catalogues (DAG-serialized: the single per-tile log-sync +# point before make_cat). N_SPLIT_MAX == chunk count. +rule tile_merge_cats: + input: + ngmix_chunks + output: + directory(f"{TILE_OUT}/run_sp_Ms") + params: + cmd=lambda wc: sp_rule("tile_merge_cats", "config_merge_sep_cats_template.ini", + "tile", wc.tile, isolate=True, + extra=f"--n-split-max {NGMIX_CHUNKS}"), + shell: + "{params.cmd}" + +# Build the final catalogue: the run's science product. protected() so a +# code/params-drift rerun cannot silently discard a finished catalogue — the +# recompute must go through `sp rerun` (--forcerun). +rule tile_make_cat: + input: + rules.tile_merge_cats.output + output: + protected(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits")) + params: + cmd=lambda wc: sp_rule("tile_make_cat", "config_make_cat_psfex_nosm.ini", + "tile", wc.tile, isolate=True, + extra=f"--final-cat {RUN_DIR}/tiles/{wc.tile}/final_cat-{wc.tile}.fits"), + shell: + "{params.cmd}" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py new file mode 100644 index 00000000..82df8b0c --- /dev/null +++ b/workflow/scripts/build_forest.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Build a tile's exposure symlink forest (the SP_EXP view). + +A plain script (not a run: block) so the tile chain stays group-compatible. +Reads the tile's exposures from run_index.sqlite and symlinks each exposure's +``exp//output`` into ``//output`` by exact name (no glob), so +the tile modules' $SP_EXP globs resolve to exactly this tile's exposures' +products. The forest is a convenience view; the DAG edge to the exposures is +declared in the rule's input (tile.smk), not here. +""" + +import argparse +import sqlite3 +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile", required=True) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--forest", required=True, type=Path) + args = p.parse_args() + + con = sqlite3.connect(args.index) + exps = [r[0] for r in con.execute( + "SELECT exp_id FROM tile_exposures WHERE tile_id=?", (args.tile,))] + con.close() + + args.forest.mkdir(parents=True, exist_ok=True) + for e in exps: + src = args.run_dir / "exp" / e / "output" + dst = args.forest / e / "output" + dst.parent.mkdir(parents=True, exist_ok=True) + if dst.is_symlink() or dst.exists(): + dst.unlink() + dst.symlink_to(src) + print(f"[build_forest] {args.tile}: {len(exps)} exposures -> {args.forest}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py new file mode 100644 index 00000000..13783314 --- /dev/null +++ b/workflow/scripts/build_index.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Build the run index (``run_index.sqlite``) that drives the compute DAG. + +The index is *parse-time data*, never a rule input: the Snakefile loads it once +at parse time into plain dicts, so extending a run's tile list changes which +jobs exist without touching the mtime chain of completed work. + +It records, per run: + + tiles(tile_id, ra_dir, n_exp, status) + exposures(exp_id) -- deduplicated union over tiles + tile_exposures(tile_id, exp_id) -- the tile->exposure edges + +The tile->exposure edges are *data-derived*: they are read from each tile's +``find_exposures`` output (``exp_numbers--.txt``), which +``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. +So this is a **plain script**, not a DAG node: it runs between the two static +invocations — after ``snakemake prepare_tiles`` (Git+Uz+Fe, keep-going) and +before ``snakemake prepare_exposures`` parses the now-existing index. It +iterates the *declared* tile list and checks each tile's Fe output at its +deterministic path (no globbing — O(tile-list) existence checks, respecting the +no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in +``missing.json`` and the index is built over the rest. The script exits nonzero +only if the missing *fraction* exceeds ``--missing-threshold`` (default 0.0), +so a keep-going download storm that lost a few tiles does not cost the run. +There is NO ``--allow-missing`` flag (that Snakemake flag does not exist). + +Exposure IDs are stored with their trailing single-char suffix stripped +(``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the +exposure-rule wildcard, matching the ``exp///`` forest layout. +""" + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + + +def read_exposure_list(exp_numbers_file: Path) -> list[str]: + """Return the exp_base IDs listed in one tile's find_exposures output. + + Each line is an exposure name like ``2243881p``; we strip a trailing + alphabetic suffix so the base ID is the dedup key. + """ + ids = [] + for line in exp_numbers_file.read_text().splitlines(): + name = line.strip() + if not name: + continue + ids.append(name[:-1] if name[-1].isalpha() else name) + return ids + + +def build(tile_ids: list[str], run_dir: Path, db_path: Path, + missing_threshold: float = 0.0) -> dict: + """Build the index over ``tile_ids``; return a summary dict. + + For each tile, check its ``exp_numbers--.txt`` at the tile's + deterministic ``find_exposures`` run dir (RUN_DATETIME=False, no glob). A + tile whose exposure list is missing is recorded in ``missing.json`` and the + index is built over the rest (a bad tile costs that tile, not the run). The + build is fatal only if the missing fraction exceeds ``missing_threshold``. + """ + db_path.parent.mkdir(parents=True, exist_ok=True) + con = sqlite3.connect(db_path) + con.executescript( + """ + DROP TABLE IF EXISTS tiles; + DROP TABLE IF EXISTS exposures; + DROP TABLE IF EXISTS tile_exposures; + CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); + CREATE TABLE exposures(exp_id TEXT PRIMARY KEY); + CREATE TABLE tile_exposures( + tile_id TEXT, exp_id TEXT, + PRIMARY KEY (tile_id, exp_id)); + """ + ) + + missing = [] + all_exposures: set[str] = set() + for tile_id in tile_ids: + ra_dir = tile_id.split(".")[0] + idra, iddec = tile_id.split(".") + # Flat forest, deterministic run dir (RUN_DATETIME=False). + exp_file = (run_dir / "tiles" / tile_id / "output" / "run_sp_tile_Fe" / + "find_exposures_runner" / "output" / + f"exp_numbers-{idra}-{iddec}.txt") + if not exp_file.exists(): + missing.append(tile_id) + continue + exp_ids = read_exposure_list(exp_file) + con.execute("INSERT INTO tiles VALUES (?,?,?)", + (tile_id, ra_dir, len(exp_ids))) + for exp_id in exp_ids: + all_exposures.add(exp_id) + con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", + (tile_id, exp_id)) + + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?)", + [(e,) for e in sorted(all_exposures)]) + con.commit() + con.close() + + (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) + frac = len(missing) / len(tile_ids) if tile_ids else 0.0 + if frac > missing_threshold: + raise SystemExit( + f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " + f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " + f"Re-run prepare_tiles for them, or raise --missing-threshold.") + return {"n_tiles": len(tile_ids) - len(missing), + "n_exposures": len(all_exposures), + "n_missing": len(missing)} + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tile-list", required=True, type=Path, + help="file of tile IDs, one per line") + p.add_argument("--run-dir", required=True, type=Path, + help="$SP_RUN: root of the tiles/ work-dir forest") + p.add_argument("--db", required=True, type=Path, + help="output run_index.sqlite path") + p.add_argument("--missing-threshold", type=float, default=0.0, + help="fatal if the missing-tile fraction exceeds this " + "(default 0.0: any missing tile is fatal)") + args = p.parse_args() + + tile_ids = [ln.strip() for ln in args.tile_list.read_text().splitlines() + if ln.strip()] + summary = build(tile_ids, args.run_dir, args.db, args.missing_threshold) + print(f"run_index: {summary['n_tiles']} tiles, " + f"{summary['n_exposures']} exposures, " + f"{summary['n_missing']} missing -> {args.db}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_exposures.py b/workflow/scripts/clean_exposures.py new file mode 100644 index 00000000..4f46bf19 --- /dev/null +++ b/workflow/scripts/clean_exposures.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Index-driven exposure-store cleanup (`sp clean-exposures`). + +temp() is deliberately NOT used on the shared exposure directories (findings +4/5/6/19): they overlap tiles by construction, so temp() deletion would cascade +destructive reruns across spatial neighbours whenever a tile is appended. Instead +the store is persistent and space is reclaimed explicitly here. + +For each exposure in run_index.sqlite, delete its bulky intermediates +(run_sp_exp_Sp / run_sp_exp_Ma / run_sp_exp_SxSePsfPi) ONLY when every tile that +consumes it (tile_exposures) has its final_cat-.fits present across the whole +output root. An exposure still feeding an unfinished tile is left untouched. + +Caveat: if a later batch appends a tile that shares a deleted exposure, +Snakemake will regenerate that exposure — cleanup trades peak storage for +recompute. Run it between campaign phases, not mid-DAG. --dry-run by default. +""" + +import argparse +import shutil +import sqlite3 +import sys +from pathlib import Path + +PREFIXES = ("run_sp_exp_Sp", "run_sp_exp_Ma", "run_sp_exp_SxSePsfPi") + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--apply", action="store_true", + help="actually delete (default: dry-run, list only)") + args = p.parse_args() + + con = sqlite3.connect(args.index) + edges = con.execute("SELECT exp_id, tile_id FROM tile_exposures").fetchall() + con.close() + + consumers: dict[str, list[str]] = {} + for exp_id, tile_id in edges: + consumers.setdefault(exp_id, []).append(tile_id) + + def tile_done(tile): + return (args.run_dir / "tiles" / tile / f"final_cat-{tile}.fits").exists() + + freed, held = 0, 0 + for exp_id, tiles in consumers.items(): + if not all(tile_done(t) for t in tiles): + held += 1 + continue + for prefix in PREFIXES: + d = args.run_dir / "exp" / exp_id / "output" / prefix + if d.is_dir(): + freed += 1 + print(f"{'DELETE' if args.apply else 'would delete'} {d}") + if args.apply: + shutil.rmtree(d) + print(f"[clean_exposures] {freed} dirs {'freed' if args.apply else 'reclaimable'}, " + f"{held} exposures held (consuming tile unfinished)", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py new file mode 100644 index 00000000..f35e81ae --- /dev/null +++ b/workflow/scripts/completeness.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""The count-floor completeness table — the single failure policy. + +This is the ported ``complete_check`` count table from the v2.0 bash layer +(``run_job_sp_canfar_v2.0.bash`` job dispatch, survey §4). It is the *only* +failure policy in the design: there is no 3-class taxonomy and no error-signature +whitelist. A stage is a real failure iff a mandatory runner produced fewer than +its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) +sits between ``floor`` and ``expect`` and is tolerated. + +Two consumers share this table: + * ``sp_rule.py`` — after ``shapepipe_run``, counts products per runner and + exits nonzero if any mandatory runner is below its floor (``keep-going`` + then isolates that unit's cone; ``retries`` handle transient failures). + * ``run_report.py`` — disk-scans finished trees against ``expect`` to + enumerate shortfalls for the human, mid-run or after. + +Per-runner fields: + expect nominal file count for a fully complete unit (report yardstick) + floor the fail-loud minimum (below this the job exits nonzero) + warn if True the runner never fails the unit at all (bash ``:warn`` — + e.g. psfex_interp on tiles missing some epochs) + subpath count files in ``/output//`` instead of + ``/output/`` (bash ``:rand_split`` — setools split cats) + +Counts are file counts in the runner's output dir, matching the bash +``ls / | wc -l`` semantics (broken symlinks excluded by the caller). +""" + +# stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} +COMPLETENESS = { + # --- tile prepare (phase A) --- + "tile_get_images": {"get_images_runner": dict(expect=4, floor=4)}, + "tile_uncompress": {"uncompress_fits_runner": dict(expect=1, floor=1)}, + "tile_find_exposures": {"find_exposures_runner": dict(expect=1, floor=1)}, + + # --- exposure chain --- + "exp_get_images": {"get_images_runner": dict(expect=6, floor=6)}, + "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, + "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, + "exp_psf": { + "sextractor_runner": dict(expect=80, floor=2), + "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), + "psfex_runner": dict(expect=80, floor=2), + "psfex_interp_runner": dict(expect=40, floor=0, warn=True), + }, + + # --- tile post --- + "tile_merge_headers": {"merge_headers_runner": dict(expect=1, floor=1)}, + "tile_mask": {"mask_runner": dict(expect=1, floor=1)}, + "tile_detect": {"sextractor_runner": dict(expect=2, floor=2)}, + "tile_vignets": { + "psfex_interp_runner": dict(expect=1, floor=1), + "vignetmaker_runner_run_1": dict(expect=1, floor=1), + "vignetmaker_runner_run_2": dict(expect=4, floor=4), + }, + "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, + "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, + "tile_make_cat": {"make_cat_runner": dict(expect=1, floor=1)}, +} + + +def count_products(run_dir, runner, spec): + """Count files in ``run_dir//output[/]/`` (live links only).""" + out = run_dir / runner / "output" + if "subpath" in spec: + out = out / spec["subpath"] + if not out.is_dir(): + return 0 + return sum(1 for p in out.iterdir() if p.exists()) # p.exists() drops dead links + + +def check_floor(stage, run_dir): + """Return (ok, details). ok is False iff a mandatory runner is below floor. + + ``details`` is a list of (runner, n_found, floor, expect, warn) tuples. + """ + table = COMPLETENESS[stage] + details, ok = [], True + for runner, spec in table.items(): + n = count_products(run_dir, runner, spec) + details.append((runner, n, spec["floor"], spec["expect"], + spec.get("warn", False))) + if not spec.get("warn", False) and n < spec["floor"]: + ok = False + return ok, details diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py new file mode 100644 index 00000000..bb330040 --- /dev/null +++ b/workflow/scripts/run_report.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Standalone run report — NOT a DAG node. + +A report rule that declares all tiles' outputs as inputs is a descendant of every +job, so any single hard failure (under --keep-going) poisons its cone and the +report never runs — the exact scenario it exists for (finding 8). So this is a +plain script: it disk-scans the tiles/ and exp/ trees against the count-floor +table (completeness.py) and the index, and emits run_report.json. It runs +automatically via the Snakefile's onsuccess/onerror hooks, and by hand any time +(``sp report``), mid-run included. + +It distinguishes whole-exposure absence (all runners missing — a real gap or a +deletion/forest bug) from tolerated per-CCD attrition (counts between floor and +expect), so a future deletion bug cannot be silently absorbed (finding 5). +""" + +import argparse +import json +import sqlite3 +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from completeness import COMPLETENESS, check_floor # noqa: E402 + +# stage -> (level, run_sp_ dir name) +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "exp_mask": ("exp", "run_sp_exp_Ma"), + "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_mask": ("tile", "run_sp_tile_Ma"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_vignets": ("tile", "run_sp_tile_PiViVi"), + "tile_merge_cats": ("tile", "run_sp_Ms"), + "tile_make_cat": ("tile", "run_sp_Mc"), +} + + +def classify(stage, run_dir): + """'absent' | 'under' | 'attrition' | 'complete' for one unit's stage.""" + if not run_dir.is_dir(): + return "absent", [] + ok, details = check_floor(stage, run_dir) + if not ok: + # all mandatory runners empty -> whole-unit absence; else below floor + mand = [d for d in details if not d[4]] + if mand and all(n == 0 for _, n, *_ in mand): + return "absent", details + return "under", details + if any(n < expect for _, n, floor, expect, warn in details if not warn): + return "attrition", details + return "complete", details + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--index", required=True, type=Path) + p.add_argument("--status", default="manual") + p.add_argument("--out", type=Path, default=None) + args = p.parse_args() + + tiles, exps = [], [] + if args.index.exists(): + con = sqlite3.connect(args.index) + tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles")] + exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures")] + con.close() + + missing_json = args.index.parent / "missing.json" + missing = json.loads(missing_json.read_text()) if missing_json.exists() else [] + + report = {"status": args.status, "n_tiles": len(tiles), "n_exposures": len(exps), + "missing_tiles": missing, "stages": {}} + for stage, (level, prefix) in STAGE_DIR.items(): + if stage not in COMPLETENESS: + continue + units = tiles if level == "tile" else exps + sub = "tiles" if level == "tile" else "exp" + tally = {"complete": 0, "attrition": 0, "under": [], "absent": []} + for u in units: + run_dir = args.run_dir / sub / u / "output" / prefix + verdict, _ = classify(stage, run_dir) + if verdict in ("complete", "attrition"): + tally[verdict] += 1 + else: + tally[verdict].append(u) + report["stages"][stage] = tally + + finals = [t for t in tiles + if (args.run_dir / "tiles" / t / f"final_cat-{t}.fits").exists()] + report["final_cats"] = {"present": len(finals), "of": len(tiles), + "missing": [t for t in tiles if t not in finals]} + + out = args.out or (args.index.parent / "run_report.json") + out.write_text(json.dumps(report, indent=2)) + print(f"[run_report] {report['final_cats']['present']}/{len(tiles)} final " + f"cats -> {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py new file mode 100644 index 00000000..cce92e2c --- /dev/null +++ b/workflow/scripts/sp_rule.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +"""Thin per-unit ShapePipe rule wrapper — the single entrypoint every rule calls. + +Under the one-allocation local executor + apptainer software-deployment, +Snakemake runs each rule's shell *inside* the container on the allocation's node. +So this script already runs in-container: it calls ``shapepipe_run`` directly +(never apptainer itself), which is why the workflow never hand-invokes apptainer. + +What it owns — only what is intrinsically ShapePipe's, nothing Snakemake already +does: + +1. **Per-unit work dir + v2.0 isolation furniture.** Each tile/exposure gets its + own ``$SP_RUN`` (``tiles//`` or ``exp//``). Isolation is by work-dir + *content*, the proven v2.0 mechanism — NOT ``-e/--exclusive``: + - a ``cfis`` symlink to the canonical config dir (so ``$SP_CONFIG`` / ``.mask`` + refs resolve); + - ``star_cat_exp`` / ``star_cat_tiles`` symlinks to the shared cat pool; + - for a tile: ``tile_numbers.txt`` = this tile ID (dot format, real data — + what get_images reads); + - for an exposure: a fabricated + ``output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-000-000.txt`` + = this exposure ID (v2.0 ``init_exp_work_dir`` pseudo-Fe), so the Gie + config's ``last:find_exposures_runner`` resolver finds exactly one unit. + +2. **Per-unit config copy.** Forces ``RUN_DATETIME = False`` (deterministic + ``output/run_sp_/`` the rule declares as its ``directory()`` output), + sets ``SMP_BATCH_SIZE = {threads}`` (fork width == cpus_per_task, so SLURM + packing density and true per-job process count are one number), and — only + when ``isolate`` (tile-scheme stages + exp split; NEVER get_images / exp_mask / + exp_psf, whose per-CCD/download numbering would make #746 turn tolerated + per-CCD attrition into a whole-exposure hard failure) — ``NUMBER_LIST = -``. + +3. **Log sync at START and end.** ``update_runs_log_file.py`` regenerates + ``log_run_sp.txt`` before ``shapepipe_run`` (after the pre-delete, so + resolution never sees a just-deleted dir) and again after. Skipped for ngmix + chunks: the log is already complete from the prior tile stages, and the N + concurrent chunk jobs share the tile work dir — a full log *rewrite* would + race, whereas ShapePipe's own single-line ``O_APPEND`` of each chunk's run is + atomic. ngmix chunk configs additionally carry *literal* INPUT_DIRs + (deterministic under RUN_DATETIME=False), so they need no log read at all. + +4. **Count-floor check** (``completeness.py`` — the ported bash ``complete_check`` + table, the single failure policy: no 3-class taxonomy, no whitelist). After + the run, count products per mandatory runner and exit nonzero below the floor. + ``keep-going`` isolates that unit's cone; per-CCD attrition between floor and + ``expect`` is tolerated. Stages absent from the table fall back to a + zero-output floor. + +Stage-specific verbs (grounded in ``runs/p3-batch1/job_p3_batch1.sh``): + +- ``--ngmix-chunk K --ngmix-nchunks N`` — read this tile's SExtractor object + count from its ``run_sp_tile_Sx`` output, split ``[1, N_obj]`` into N disjoint + closed ID ranges, and generate chunk K's config from + ``config_tile_Ng_template_batch.ini``. +- ``--n-split-max N`` — generate the merge config from + ``config_merge_sep_cats_template.ini`` with ``N_SPLIT_MAX = N``. +- ``--final-cat PATH`` — after ``make_cat``, copy the produced ``final_cat`` FITS + to PATH (the rule's declared, protected science-product output). + +Run by hand to reproduce exactly what a rule does (the escape hatch — it +pre-deletes the run dir, just like Snakemake's delete-before-rerun):: + + python sp_rule.py --stage exp_split --config config_exp_Sp.ini \\ + --unit 2243881 --level exp --run-dir $SP_RUN --config-src $SP_CONFIG \\ + --star-cats $STAR_CATS --threads 8 +""" + +import argparse +import os +import shutil +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from completeness import COMPLETENESS, check_floor # noqa: E402 + + +def unit_work_dir(run_dir: Path, level: str, unit: str) -> Path: + """Per-unit work dir in the forest: tiles/ or exp/. + + Flat (no IDra/prefix sharding): the shard existed only to keep the bash + layer's ``ls exp/`` bearable, but Snakemake never globs the forest — the + run index drives every path — so flattening keeps single-wildcard rules. + ``run`` level (aggregation stages) work directly in ``run_dir``. + """ + if level == "tile": + return run_dir / "tiles" / unit + if level == "exp": + base = unit[:-1] if unit[-1].isalpha() else unit + return run_dir / "exp" / base + return run_dir + + +def materialise_unit(work: Path, level: str, unit: str, config_src: Path, + star_cats: Path) -> None: + """Create the v2.0 unit-isolation furniture in the work dir.""" + (work / "output").mkdir(parents=True, exist_ok=True) + + cfis = work / "cfis" # $SP_CONFIG -> canonical config dir (.mask refs resolve) + if not cfis.exists(): + cfis.symlink_to(config_src) + + for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): + link = work / name + if not link.exists() and (star_cats / sub).exists(): + link.symlink_to(star_cats / sub) + + if level == "tile": + (work / "tile_numbers.txt").write_text(unit + "\n") # dot format, real data + elif level == "exp": + fe = work / "output" / "run_sp_tile_Fe" / "find_exposures_runner" / "output" + fe.mkdir(parents=True, exist_ok=True) + exp_numbers = fe / "exp_numbers-000-000.txt" + if not exp_numbers.exists(): + exp_numbers.write_text(unit + "\n") + + +def normalize_config_text(text: str, unit: str, isolate: bool, threads: int) -> str: + """Force RUN_DATETIME=False, SMP_BATCH_SIZE=threads, and (if isolating) + NUMBER_LIST=-. + + ``NUMBER_LIST`` uses ShapePipe's image-number convention (dots -> dashes), + the ``set_config_number_list`` mechanism that replaced the retired + ``-e/--exclusive`` flag (#746). + """ + lines = text.splitlines() + out, in_file, saw_datetime, saw_numberlist, saw_smp = [], False, False, False, False + number = "-" + unit.replace(".", "-") + for line in lines: + stripped = line.strip() + if stripped.startswith("[") and stripped.endswith("]"): + in_file = stripped.upper() == "[FILE]" + if stripped.upper().startswith("RUN_DATETIME"): + out.append("RUN_DATETIME = False") + saw_datetime = True + continue + if stripped.upper().startswith("SMP_BATCH_SIZE"): + out.append(f"SMP_BATCH_SIZE = {threads}") + saw_smp = True + continue + if isolate and stripped.upper().startswith("NUMBER_LIST"): + out.append(f"NUMBER_LIST = {number}") + saw_numberlist = True + continue + out.append(line) + if in_file and stripped.upper() == "[FILE]" and isolate and not saw_numberlist: + out.append(f"NUMBER_LIST = {number}") + saw_numberlist = True + if not saw_datetime: + # ShapePipe reads RUN_DATETIME from [DEFAULT]; inject after its header. + patched, done = [], False + for line in out: + patched.append(line) + if not done and line.strip().upper() == "[DEFAULT]": + patched.append("RUN_DATETIME = False") + done = True + out = patched if done else ["[DEFAULT]", "RUN_DATETIME = False"] + out + if not saw_smp: + # SMP_BATCH_SIZE lives in [JOB]; inject after its header. + patched, done = [], False + for line in out: + patched.append(line) + if not done and line.strip().upper() == "[JOB]": + patched.append(f"SMP_BATCH_SIZE = {threads}") + done = True + out = patched if done else out + ["[JOB]", f"SMP_BATCH_SIZE = {threads}"] + return "\n".join(out) + "\n" + + +def run_prefix_from_text(text: str, config_label: str) -> str: + """The RUN_NAME (== run_sp_) the module will write under.""" + for line in text.splitlines(): + s = line.strip() + if s.upper().startswith("RUN_NAME"): + return s.split("=", 1)[1].strip() + raise SystemExit(f"no RUN_NAME in {config_label}") + + +# --- ngmix chunking -------------------------------------------------------- + +# Literal, deterministic INPUT_DIRs for the ngmix chunk (RUN_DATETIME=False), so +# concurrent chunks need no log read at all (finding 18). +NGMIX_LITERAL_INPUT = ( + "./output/run_sp_tile_Sx/sextractor_runner/output, " + "./output/run_sp_tile_PiViVi/psfex_interp_runner/output, " + "./output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, " + "./output/run_sp_tile_Mh_exp/merge_headers_runner/output") + + +def ngmix_id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: + """Split ``[1, n_obj]`` into ``n_chunks`` disjoint *closed* ID ranges. + + SExtractor's ``NUMBER`` column (what ngmix reads as ``obj_id``) runs 1..N + contiguous, so covering ``[1, N]`` processes every object exactly once. The + first n_chunks-1 chunks get ``n_obj // n_chunks`` each; the last gets the + remainder — a *closed* upper bound at ``n_obj`` (never ``ID_OBJ_MAX = -1``: + ngmix treats ``id_obj_max <= 0`` as unbounded, which would double-count). + + Deviation from the nibi monolith (deliberate): the monolith sizes every + tile's chunks from the *average* object count across all tiles and leaves the + last chunk open (``-1``); here each tile is an independent job, so we size + from *this* tile's own count and close the last chunk. The merged catalogue + is identical either way (merge_sep_cats recombines all chunks, ngmix fits + each object independently) — this just removes the open-ended overflow. + """ + base, rem = divmod(n_obj, n_chunks) + ranges, lo = [], 1 + for k in range(1, n_chunks + 1): + size = base + (rem if k == n_chunks else 0) + hi = lo + size - 1 + ranges.append((lo, hi)) + lo = hi + 1 + return ranges + + +def tile_object_count(work: Path) -> int: + """Object count for this tile from its run_sp_tile_Sx SExtractor catalogue. + + Matches ``get_number_objects.py``: the ``NAXIS2`` of the catalogue's last + HDU. Per-unit isolation means exactly one ``sexcat*.fits`` lives here. + """ + from astropy.io import fits + + sexcats = sorted((work / "output" / "run_sp_tile_Sx").glob( + "sextractor_runner/output/sexcat*.fits")) + if not sexcats: + raise SystemExit( + f"[sp_rule] FATAL ngmix: no sexcat under {work}/output/run_sp_tile_Sx") + with fits.open(sexcats[0]) as hdul: + return int(hdul[-1].header["NAXIS2"]) + + +def apply_ngmix_subs(text: str, chunk: int, lo: int, hi: int) -> str: + """Fill the ngmix batch template for one chunk (the monolith's sed recipe) + plus literal INPUT_DIRs so the chunk needs no shared-log read.""" + text = text.replace("NgXu", f"Ng{chunk}u").replace("X_interp", "psfex_interp") + out = [] + for line in text.splitlines(): + s = line.strip() + if s.startswith("ID_OBJ_MIN"): + out.append(f"ID_OBJ_MIN = {lo}") + elif s.startswith("ID_OBJ_MAX"): + out.append(f"ID_OBJ_MAX = {hi}") + elif s.startswith("INPUT_DIR") and "sextractor_runner" in s: + out.append(f"INPUT_DIR = {NGMIX_LITERAL_INPUT}") + else: + out.append(line) + return "\n".join(out) + + +def apply_merge_subs(text: str, n_split_max: int) -> str: + """Fill N_SPLIT_MAX in the merge template (the monolith's sed recipe).""" + out = [] + for line in text.splitlines(): + if line.strip().startswith("N_SPLIT_MAX"): + out.append(f"N_SPLIT_MAX = {n_split_max}") + else: + out.append(line) + return "\n".join(out) + + +# --- in-job count floor ---------------------------------------------------- + +def count_floor(run_out: Path, stage: str) -> None: + """Fail loudly if any mandatory runner is below its ``floor`` (completeness.py). + + Stages absent from the table skip this floor (only the zero-output floor + applies to them). + """ + if stage not in COMPLETENESS: + return + ok, details = check_floor(stage, run_out) + for runner, n, floor, expect, warn in details: + tag = "warn" if warn else ("OK" if n >= floor else " <-- BELOW floor") + print(f"[sp_rule] {runner}: {n}/{expect} (floor {floor}) {tag}", + file=sys.stderr) + if not ok: + print(f"[sp_rule] FATAL {stage}: count floor breached in {run_out}", + file=sys.stderr) + sys.exit(1) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--stage", required=True) + p.add_argument("--config", required=True, help="config filename in --config-src") + p.add_argument("--unit", required=True) + p.add_argument("--level", required=True, choices=["tile", "exp", "run"]) + p.add_argument("--run-dir", required=True, type=Path, help="$SP_RUN root") + p.add_argument("--config-src", required=True, type=Path, + help="dir holding the canonical configs") + p.add_argument("--star-cats", type=Path, default=None, + help="shared star-cat pool (tiles/ + exp/ subdirs)") + p.add_argument("--threads", type=int, default=1, + help="SMP_BATCH_SIZE == cpus_per_task (fork width)") + p.add_argument("--exp-forest", type=Path, default=None, + help="$SP_EXP: per-tile exposure forest (tile post-stages)") + p.add_argument("--no-isolate", action="store_true", + help="skip NUMBER_LIST (get_images / exp_mask / exp_psf)") + p.add_argument("--ngmix-chunk", type=int, default=None, + help="1-indexed ngmix chunk K (with --ngmix-nchunks)") + p.add_argument("--ngmix-nchunks", type=int, default=None, + help="total ngmix chunks N (with --ngmix-chunk)") + p.add_argument("--n-split-max", type=int, default=None, + help="N_SPLIT_MAX for the merge_sep_cats template") + p.add_argument("--final-cat", type=Path, default=None, + help="copy the produced final_cat FITS here (make_cat)") + args = p.parse_args() + + ngmix_mode = args.ngmix_chunk is not None + if ngmix_mode and args.ngmix_nchunks is None: + raise SystemExit("--ngmix-chunk requires --ngmix-nchunks") + + work = unit_work_dir(args.run_dir, args.level, args.unit) + materialise_unit(work, args.level, args.unit, args.config_src, + args.star_cats or args.run_dir) + + # Build the config copy. ngmix chunks and merge fill template placeholders + # first; every copy is then normalized (RUN_DATETIME=False, SMP_BATCH_SIZE, + # [+ NUMBER_LIST]). + src_text = (args.config_src / args.config).read_text() + isolate = not args.no_isolate + if ngmix_mode: + n_obj = tile_object_count(work) + lo, hi = ngmix_id_ranges(n_obj, args.ngmix_nchunks)[args.ngmix_chunk - 1] + print(f"[sp_rule] ngmix chunk {args.ngmix_chunk}/{args.ngmix_nchunks}: " + f"n_obj={n_obj} -> ID_OBJ [{lo}, {hi}]", file=sys.stderr) + src_text = apply_ngmix_subs(src_text, args.ngmix_chunk, lo, hi) + cfg_name = f"config_tile_Ng{args.ngmix_chunk}u.ini" # distinct per chunk + elif args.n_split_max is not None: + src_text = apply_merge_subs(src_text, args.n_split_max) + cfg_name = args.config + else: + cfg_name = args.config + + cfg_mod_dir = work / "cfis_mod" + cfg_mod = cfg_mod_dir / cfg_name + cfg_text = normalize_config_text(src_text, args.unit, isolate, args.threads) + cfg_mod_dir.mkdir(parents=True, exist_ok=True) + cfg_mod.write_text(cfg_text) + + prefix = run_prefix_from_text(cfg_text, str(cfg_mod)) + run_out = work / "output" / prefix + + # Pre-delete the deterministic run dir (delete-before-rerun for the hand-run + # escape hatch; ShapePipe's FileHandler.mkdir raises on an existing run dir). + if run_out.exists(): + shutil.rmtree(run_out) + + env = dict(os.environ) + env["SP_RUN"] = str(work) + env["SP_CONFIG"] = str(work / "cfis") # canonical dir (.mask refs), not cfis_mod + if args.exp_forest: + env["SP_EXP"] = str(args.exp_forest) + # Thread caps also come via apptainer-args; keep here for the hand-run path. + env.setdefault("OMP_NUM_THREADS", "1") + env.setdefault("OPENBLAS_NUM_THREADS", "1") + env.setdefault("MKL_NUM_THREADS", "1") + env.setdefault("NUMEXPR_NUM_THREADS", "1") + env.setdefault("MALLOC_ARENA_MAX", "2") + env.setdefault("MALLOC_TRIM_THRESHOLD_", "0") + + # Log sync before the run (after pre-delete). Skipped for ngmix chunks: N + # concurrent chunks would race a full rewrite (literal INPUT_DIRs make it moot). + if not ngmix_mode: + subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) + + print(f"[sp_rule] stage={args.stage} unit={args.unit} work={work}", + file=sys.stderr) + rc = subprocess.call(["shapepipe_run", "-c", str(cfg_mod)], + cwd=str(work), env=env) + if rc != 0: + print(f"[sp_rule] WARN shapepipe_run exit {rc} — checking count floor", + file=sys.stderr) + + if not ngmix_mode: + subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) + + # Zero-output floor for stages without a count table; count floor otherwise. + if args.stage in COMPLETENESS: + count_floor(run_out, args.stage) + else: + produced = list(run_out.glob("**/output/*")) if run_out.exists() else [] + if not produced: + print(f"[sp_rule] FATAL {args.stage} {args.unit}: zero output in {run_out}", + file=sys.stderr) + sys.exit(1) + + # make_cat: publish the science product to its declared, protected path. + if args.final_cat is not None: + finals = sorted(run_out.glob("**/output/final_cat*.fits")) + if not finals: + print(f"[sp_rule] FATAL {args.stage} {args.unit}: no final_cat under {run_out}", + file=sys.stderr) + sys.exit(1) + args.final_cat.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(finals[0], args.final_cat) + print(f"[sp_rule] final_cat -> {args.final_cat}", file=sys.stderr) + + print(f"[sp_rule] OK {args.stage} {args.unit}", file=sys.stderr) + + +if __name__ == "__main__": + main() From 187950c4ec95035b0e1b40ffaf69841122c31f89 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 15:57:40 -0400 Subject: [PATCH 02/21] fix(orchestration): resolve {threads} at rule level, not inside params.cmd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Snakemake formats params once, so a {threads} placeholder inside params.cmd survives literally and SMP_BATCH_SIZE never resolves (fork width silently 1, serial modules — the p3-first10 failure mode). Move '--threads {threads}' into each rule's shell: string; tile_exp_forest takes none (build_forest.py is single-threaded symlinking and has no such flag). Completeness table doc touch-ups ride along. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NA8M1oLZLbWJxwoyTNrTAi --- workflow/Snakefile | 6 ++++-- workflow/rules/exposure.smk | 8 ++++---- workflow/rules/prepare.smk | 6 +++--- workflow/rules/tile.smk | 15 ++++++++------- workflow/scripts/completeness.py | 8 ++++++-- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 04722a73..09d21831 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -60,13 +60,15 @@ def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, NUMBER_LIST is injected by default (tile-scheme stages + exp split); pass ``isolate=False`` for get_images / exp_mask / exp_psf, whose per-CCD/download - numbering must not be constrained. ``{threads}`` is filled by Snakemake and + numbering must not be constrained. ``--threads {threads}`` is appended in + each rule's ``shell:`` string (NOT here: Snakemake formats the shell string + once, so a ``{threads}`` inside params.cmd would survive literally) and becomes SMP_BATCH_SIZE (fork width == cpus_per_task). """ cmd = ( f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " f"--unit {unit} --level {level} --run-dir {RUN_DIR} " - f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} --threads {{threads}}" + f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS}" ) if not isolate: cmd += " --no-isolate" diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 50403d38..3e934b18 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -29,7 +29,7 @@ rule exp_get_images: cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", "exp", wc.exp, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Split multi-HDU exposure into single-CCD files (+ headers-*.npy). Bulk output; # persistent (see module docstring on temp()). @@ -41,7 +41,7 @@ rule exp_split: params: cmd=lambda wc: sp_rule("exp_split", "config_exp_Sp.ini", "exp", wc.exp) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Mask per-CCD; this exposure's star cat is a declared input (per-unit re-key). rule exp_mask: @@ -54,7 +54,7 @@ rule exp_mask: cmd=lambda wc: sp_rule("exp_mask", "config_exp_Ma_onthefly.ini", "exp", wc.exp, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. # setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor. @@ -67,4 +67,4 @@ rule exp_psf: cmd=lambda wc: sp_rule("exp_psf", "config_exp_psfex.ini", "exp", wc.exp, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 2911c3b1..4ca369ec 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -19,7 +19,7 @@ rule tile_get_images: cmd=lambda wc: sp_rule("tile_get_images", "config_tile_Git_vos.ini", "tile", wc.tile, isolate=False) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" rule tile_uncompress: input: @@ -30,7 +30,7 @@ rule tile_uncompress: cmd=lambda wc: sp_rule("tile_uncompress", "config_tile_Uz.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # find_exposures reads the tile FITS HISTORY header -> exp_numbers-.txt, the # data-derived tile->exposure edge build_index.py aggregates (as a plain script). @@ -43,7 +43,7 @@ rule tile_find_exposures: cmd=lambda wc: sp_rule("tile_find_exposures", "config_tile_Fe.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Star catalogues for masking are pre-generated offline (create_star_cat.py on a # networked login node — the one genuinely networked prepare step). They are diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 84516771..8abfdd18 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -35,6 +35,7 @@ rule tile_exp_forest: cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " f"--run-dir {RUN_DIR} --index {INDEX_DB} --forest {{output}}") shell: + # no --threads: build_forest.py is single-threaded symlinking, takes no such flag "{params.cmd}" # Merge single-exposure WCS headers into the tile-level sqlite log @@ -52,7 +53,7 @@ rule tile_merge_headers: "tile", wc.tile, isolate=True, exp_forest=FOREST.format(tile=wc.tile)) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Mask tiles; this tile's star cat is a declared input (per-unit re-key). rule tile_mask: @@ -65,7 +66,7 @@ rule tile_mask: cmd=lambda wc: sp_rule("tile_mask", "config_tile_Ma_onthefly.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # SExtractor object detection on the tile. rule tile_detect: @@ -78,7 +79,7 @@ rule tile_detect: cmd=lambda wc: sp_rule("tile_detect", "config_tile_Sx.ini", "tile", wc.tile, isolate=True) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # PSFEx interpolation to galaxies + vignet postage stamps (reads exposure PSF # models + split through the forest — those dirs are explicit inputs). @@ -95,7 +96,7 @@ rule tile_vignets: "tile", wc.tile, isolate=True, exp_forest=FOREST.format(tile=wc.tile)) shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # ngmix shape measurement — static N chunks. Each chunk computes its own closed # object-ID range in-job from the tile's own sexcat (a declared input); chunks @@ -111,7 +112,7 @@ rule tile_ngmix: "tile_ngmix", "config_tile_Ng_template_batch.ini", "tile", wc.tile, extra=f"--ngmix-chunk {wc.chunk} --ngmix-nchunks {NGMIX_CHUNKS}"), shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" def ngmix_chunks(wc): return [f"{RUN_DIR}/tiles/{wc.tile}/output/run_sp_tile_ngmix_Ng{k}u" @@ -129,7 +130,7 @@ rule tile_merge_cats: "tile", wc.tile, isolate=True, extra=f"--n-split-max {NGMIX_CHUNKS}"), shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" # Build the final catalogue: the run's science product. protected() so a # code/params-drift rerun cannot silently discard a finished catalogue — the @@ -144,4 +145,4 @@ rule tile_make_cat: "tile", wc.tile, isolate=True, extra=f"--final-cat {RUN_DIR}/tiles/{wc.tile}/final_cat-{wc.tile}.fits"), shell: - "{params.cmd}" + "{params.cmd} --threads {threads}" diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index f35e81ae..d52ad8d0 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -30,12 +30,16 @@ # stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} COMPLETENESS = { # --- tile prepare (phase A) --- - "tile_get_images": {"get_images_runner": dict(expect=4, floor=4)}, + # get_images counts are CONFIG-FLAVOR-DEPENDENT: the v2.0 bash table said 4/6 + # for the canfar vos flavor; the nibi symlink configs produce one file per + # INPUT_FILE_PATTERN entry (tile: image+weight=2; exp: image+weight+flag=3), + # verified against the p3-batch1 baseline tree (100 files / 50 tiles). + "tile_get_images": {"get_images_runner": dict(expect=2, floor=2)}, "tile_uncompress": {"uncompress_fits_runner": dict(expect=1, floor=1)}, "tile_find_exposures": {"find_exposures_runner": dict(expect=1, floor=1)}, # --- exposure chain --- - "exp_get_images": {"get_images_runner": dict(expect=6, floor=6)}, + "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, "exp_psf": { From ac6bddd7f5eddf1c9e514a4e759a1c4db7660dff Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:07:59 -0400 Subject: [PATCH 03/21] fix(orchestration): carry original exposure names through the index; star-cat store naming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three P0-blocking fixes found by running against real data: - exposures(name): the fabricated per-unit exp_numbers list must carry the original exposure name verbatim (2605805p — get_images matches .fits.fz; the bare dedup id matches nothing). The index now stores both; exp_get_images passes --exp-name from the parse-time dict. - tile_star_cat: the pre-generated store names tiles in ShapePipe's image-number convention (dots->dashes); translate when linking. - profile: PYTHONPATH pinned to this branch's src/ (identical to develop@97e16d50 — orchestration commits never touch src/); NOT shapepipe-prod (drifted to a PR branch mid-run, and the live p3-batch1 job reads it) and not the sif default (frozen pre-#843). Run scoped to the 210/211 overlap quad (19 unique exposures, 15 shared — exercises structural dedup; the zero-overlap 196 quad is kept for the append-invariant test). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NA8M1oLZLbWJxwoyTNrTAi --- profiles/nibi/config.yaml | 12 ++++++----- workflow/Snakefile | 8 +++++--- workflow/config.yaml | 9 ++++++--- workflow/rules/exposure.smk | 3 ++- workflow/rules/prepare.smk | 4 +++- workflow/scripts/build_index.py | 35 ++++++++++++++++++--------------- workflow/scripts/sp_rule.py | 17 ++++++++++++---- 7 files changed, 55 insertions(+), 33 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index e977803c..07f68833 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -23,11 +23,13 @@ software-deployment-method: [apptainer] # OMP_NUM_THREADS=1 caps OpenBLAS fork-explosion (verified: pool 32->1) # MALLOC_ARENA_MAX=2 bounded allocator (im_sims/nibi lesson) # --home /home/cdaley wins over the SDM's --home ; restores cadcproxy.pem for vos/vcp -# PYTHONPATH SETTLED CALL 3 — P0 pins the prod worktree src so science -# content matches the bash validation runs (which use this -# override). Production later rebuilds the sif at the exact -# validation commit and DROPS this --env line. -apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-prod/src --home /home/cdaley --bind /project --bind /scratch" +# PYTHONPATH SETTLED CALL 3 — P0 pins THIS branch's src/ (identical to +# develop@97e16d50: orchestration commits never touch src/). +# NOT shapepipe-prod (drifted to a PR branch mid-run, and the +# live p3-batch1 job reads it) and not the sif default +# (frozen pre-#843). Production later rebuilds the sif at the +# validated commit and DROPS this --env line. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/project/def-mjhudson/cdaley/shapepipe-snakemake/src --home /home/cdaley --bind /project --bind /scratch" latency-wait: 60 # NFS: wait for outputs to appear after a job keep-going: true # a failed job poisons only its cone; siblings run on diff --git a/workflow/Snakefile b/workflow/Snakefile index 09d21831..6e391bda 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -39,11 +39,13 @@ with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] # --- parse-time index load (compute phase only) ---------------------------- -# EXP: global set of exposure base-ids. TILE_EXP: tile -> [exp_ids]. -EXP, TILE_EXP = set(), {} +# EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes +# verbatim into the fabricated per-unit exp_numbers list so get_images matches +# .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. +EXP, TILE_EXP = {}, {} if INDEX_DB.exists(): _con = sqlite3.connect(INDEX_DB) - EXP = {r[0] for r in _con.execute("SELECT exp_id FROM exposures")} + EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) for tile, exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): TILE_EXP.setdefault(tile, []).append(exp) _con.close() diff --git a/workflow/config.yaml b/workflow/config.yaml index 53542d1c..ec529c4d 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -5,14 +5,17 @@ # tiles) never invalidates completed work — it only changes which jobs exist. # The tile list that scopes this run (one "IDra.IDdec" per line). -tile_list: /scratch/cdaley/shapepipe-output/snakemake-p0/tiles4.txt +# 210/211 quad: 19 unique exposures, 15 reused across tiles — exercises the +# structural exposure dedup (the 196 quad had zero overlap; kept for the +# append-invariant test). +tile_list: /scratch/cdaley/shapepipe-output/smk-p0/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # Where products land ($SP_RUN root). Per-unit work-dir forest is built under here: # /tiles/// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/snakemake-p0 +run_dir: /scratch/cdaley/shapepipe-output/smk-p0 # The canonical config chain (example/cfis in the repo, or a run-specific copy). config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis @@ -23,7 +26,7 @@ star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats # The run index. Durable, low-volume products (index, missing.json, run_report, # final_cat) should land on /project, not /scratch (60-day purge, finding 15). -index_db: /project/def-mjhudson/cdaley/shapepipe-output/snakemake-p0/index/run_index.sqlite +index_db: /project/def-mjhudson/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 3e934b18..f39f6be2 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -27,7 +27,8 @@ rule exp_get_images: directory(f"{EXP_OUT}/run_sp_exp_Gie") params: cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", - "exp", wc.exp, isolate=False) + "exp", wc.exp, isolate=False, + extra=f"--exp-name {EXP[wc.exp]}") shell: "{params.cmd} --threads {threads}" diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 4ca369ec..530ac9b3 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -62,6 +62,8 @@ rule tile_star_cat: output: str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits") params: - src=lambda wc: str(STAR_CATS / "tiles" / f"star_cat-{wc.tile}.fits") + # the store names tiles in ShapePipe's image-number convention (dots->dashes) + src=lambda wc: str(STAR_CATS / "tiles" / + f"star_cat-{wc.tile.replace('.', '-')}.fits") shell: "ln -sf {params.src} {output}" diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 13783314..7ba865d9 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -37,19 +37,22 @@ from pathlib import Path -def read_exposure_list(exp_numbers_file: Path) -> list[str]: - """Return the exp_base IDs listed in one tile's find_exposures output. - - Each line is an exposure name like ``2243881p``; we strip a trailing - alphabetic suffix so the base ID is the dedup key. +def read_exposure_list(exp_numbers_file: Path) -> list[tuple[str, str]]: + """Return ``(exp_id, name)`` pairs from one tile's find_exposures output. + + Each line is an exposure *name* like ``2243881p``; the bare base ID (suffix + stripped) is the dedup key everywhere in the DAG, but the original name is + kept in the index — the fabricated per-unit ``exp_numbers`` list must carry + it verbatim (``get_images`` matches ``.fits.fz`` in the store; the + bare ID matches nothing). """ - ids = [] + pairs = [] for line in exp_numbers_file.read_text().splitlines(): name = line.strip() if not name: continue - ids.append(name[:-1] if name[-1].isalpha() else name) - return ids + pairs.append((name[:-1] if name[-1].isalpha() else name, name)) + return pairs def build(tile_ids: list[str], run_dir: Path, db_path: Path, @@ -70,7 +73,7 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, DROP TABLE IF EXISTS exposures; DROP TABLE IF EXISTS tile_exposures; CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); - CREATE TABLE exposures(exp_id TEXT PRIMARY KEY); + CREATE TABLE exposures(exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); CREATE TABLE tile_exposures( tile_id TEXT, exp_id TEXT, PRIMARY KEY (tile_id, exp_id)); @@ -78,7 +81,7 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, ) missing = [] - all_exposures: set[str] = set() + all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: ra_dir = tile_id.split(".")[0] idra, iddec = tile_id.split(".") @@ -89,16 +92,16 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, if not exp_file.exists(): missing.append(tile_id) continue - exp_ids = read_exposure_list(exp_file) + exp_pairs = read_exposure_list(exp_file) con.execute("INSERT INTO tiles VALUES (?,?,?)", - (tile_id, ra_dir, len(exp_ids))) - for exp_id in exp_ids: - all_exposures.add(exp_id) + (tile_id, ra_dir, len(exp_pairs))) + for exp_id, _name in exp_pairs: con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", (tile_id, exp_id)) + all_exposures.update(exp_pairs) - con.executemany("INSERT OR IGNORE INTO exposures VALUES (?)", - [(e,) for e in sorted(all_exposures)]) + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", + sorted(all_exposures)) con.commit() con.close() diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py index cce92e2c..b114fad0 100644 --- a/workflow/scripts/sp_rule.py +++ b/workflow/scripts/sp_rule.py @@ -93,7 +93,7 @@ def unit_work_dir(run_dir: Path, level: str, unit: str) -> Path: def materialise_unit(work: Path, level: str, unit: str, config_src: Path, - star_cats: Path) -> None: + star_cats: Path, exp_name: str | None = None) -> None: """Create the v2.0 unit-isolation furniture in the work dir.""" (work / "output").mkdir(parents=True, exist_ok=True) @@ -112,8 +112,13 @@ def materialise_unit(work: Path, level: str, unit: str, config_src: Path, fe = work / "output" / "run_sp_tile_Fe" / "find_exposures_runner" / "output" fe.mkdir(parents=True, exist_ok=True) exp_numbers = fe / "exp_numbers-000-000.txt" - if not exp_numbers.exists(): - exp_numbers.write_text(unit + "\n") + # Written UNCONDITIONALLY (cheap, deterministic): an exists-guard once + # pinned a stale pre-fix file with the bare base id. Content is the + # ORIGINAL exposure name (with its 'p'-style suffix, from the index) — + # the on-disk store is .fits.fz; the bare base id matches nothing. + # v2.0 copies the name verbatim from the tile's Fe output; the index + # carries it for us. + exp_numbers.write_text((exp_name or unit) + "\n") def normalize_config_text(text: str, unit: str, isolate: bool, threads: int) -> str: @@ -296,6 +301,10 @@ def main() -> None: help="SMP_BATCH_SIZE == cpus_per_task (fork width)") p.add_argument("--exp-forest", type=Path, default=None, help="$SP_EXP: per-tile exposure forest (tile post-stages)") + p.add_argument("--exp-name", default=None, + help="original exposure name incl. suffix (e.g. 2605805p); " + "written into the fabricated exp_numbers list so " + "get_images finds .fits.fz in the store") p.add_argument("--no-isolate", action="store_true", help="skip NUMBER_LIST (get_images / exp_mask / exp_psf)") p.add_argument("--ngmix-chunk", type=int, default=None, @@ -314,7 +323,7 @@ def main() -> None: work = unit_work_dir(args.run_dir, args.level, args.unit) materialise_unit(work, args.level, args.unit, args.config_src, - args.star_cats or args.run_dir) + args.star_cats or args.run_dir, exp_name=args.exp_name) # Build the config copy. ngmix chunks and merge fill template placeholders # first; every copy is then normalized (RUN_DATETIME=False, SMP_BATCH_SIZE, From 9b0456e72438ab7f6f10e304d231100c0e3423ae Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:09:45 -0400 Subject: [PATCH 04/21] fix(orchestration): exposure-name pass-through, star-cat dirs, P0 run config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - get_images looks up .fits.fz: the store keeps the CFHT 'p' suffix the index's base-id key strips. exposures gains a name column; rules pass --exp-name; the wrapper writes the ORIGINAL name into the fabricated exp_numbers list — and writes it unconditionally (an exists-guard pinned a stale pre-fix file). - Star cats are consumed as DIRECTORIES via the wrapper's $SP_RUN symlinks (the v2.0 mechanism; exp cats are per-CCD, 40/exposure) — dropped the per-unit star-cat file rules that linked nonexistent names nothing read. - get_images completeness counts follow the nibi symlink configs (tile 2, exp 3; the v2.0 4/6 were the canfar vos flavor), verified against p3-batch1. - P0 run config: 210/211 quad (19 unique exps, 13 shared across tiles — exercises structural dedup; the 196 quad had zero overlap), PYTHONPATH pinned to this branch's src (= develop@97e16d50; shapepipe-prod drifted mid-run). Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 4 +--- workflow/rules/prepare.smk | 31 ++++++++++--------------------- workflow/rules/tile.smk | 3 +-- 3 files changed, 12 insertions(+), 26 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 6e391bda..774ae3ad 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -109,9 +109,7 @@ rule prepare_tiles: # index exists. Star cats are re-keyed per unit (see prepare.smk). rule prepare_exposures: input: - expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)), - expand(str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), exp=sorted(EXP)), - expand(str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), tile=TILES) + expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)) # --- report hooks ----------------------------------------------------------- # run_report is NOT a DAG node (a descendant of every job would be poisoned by diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index 530ac9b3..d8698c82 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -46,24 +46,13 @@ rule tile_find_exposures: "{params.cmd} --threads {threads}" # Star catalogues for masking are pre-generated offline (create_star_cat.py on a -# networked login node — the one genuinely networked prepare step). They are -# re-keyed PER UNIT here: each tile/exposure's cat is its own DAG node, so -# appending tile 51 adds nodes rather than mutating a run-level output that would -# dirty every finished mask (the growth-invariant fix, finding 11). -rule exp_star_cat: - output: - str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits") - params: - src=lambda wc: str(STAR_CATS / "exp" / f"star_cat-{wc.exp}.fits") - shell: - "ln -sf {params.src} {output}" - -rule tile_star_cat: - output: - str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits") - params: - # the store names tiles in ShapePipe's image-number convention (dots->dashes) - src=lambda wc: str(STAR_CATS / "tiles" / - f"star_cat-{wc.tile.replace('.', '-')}.fits") - shell: - "ln -sf {params.src} {output}" +# networked login node) and consumed as DIRECTORIES: the mask configs read +# INPUT_DIR $SP_RUN/star_cat_{exp,tiles} with per-CCD numbering (exp cats are +# star_cat--.fits, 40/exposure; tile cats star_cat--.fits) +# — the v2.0 mechanism. The wrapper materialises the two dir symlinks in every +# unit work dir (sp_rule.materialise_unit); there is NO per-unit star-cat DAG +# node: the store is pre-run input like the image store, and a missing cat fails +# the mask stage's count-floor loudly. (Earlier per-unit file rules linked names +# that don't exist — one file per exposure vs the store's 40 — and nothing read +# them: the configs only see the dir symlinks.) When star-cat GENERATION moves +# in-workflow (P2), it becomes real per-unit rules producing into the store. diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 8abfdd18..1bf7d424 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -55,11 +55,10 @@ rule tile_merge_headers: shell: "{params.cmd} --threads {threads}" -# Mask tiles; this tile's star cat is a declared input (per-unit re-key). +# Mask tiles (star cats: pre-run store, read via the wrapper's dir symlink). rule tile_mask: input: git=rules.tile_get_images.output, - star=str(RUN_DIR / "tiles/{tile}/star_cat-{tile}.fits"), output: directory(f"{TILE_OUT}/run_sp_tile_Ma") params: From 6fe141fc01221eb06b3016a883999195369282fe Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:35:07 -0400 Subject: [PATCH 05/21] fix(orchestration): {output} placeholder inside params.cmd survives literally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same single-format trap as {threads} (187950c4): Snakemake formats the shell string once, so the escaped {{output}} in tile_exp_forest's params reached build_forest as a literal './{output}' — all four forest jobs racing one garbage dir (FileExistsError at 50% of the P0 run). Move --forest {output} into the shell string. Swept the rules for further escaped placeholders: none. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ac/code/session_01NA8M1oLZLbWJxwoyTNrTAi --- workflow/rules/tile.smk | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 1bf7d424..d36e91d9 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -32,11 +32,14 @@ rule tile_exp_forest: output: directory(FOREST) params: + # --forest {output} lives in the shell string: Snakemake formats shell + # ONCE — an {output} placeholder inside params.cmd survives literally + # (same trap as {threads}; all four forest jobs then race one './{output}'). cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " - f"--run-dir {RUN_DIR} --index {INDEX_DB} --forest {{output}}") + f"--run-dir {RUN_DIR} --index {INDEX_DB}") shell: - # no --threads: build_forest.py is single-threaded symlinking, takes no such flag - "{params.cmd}" + # no --threads: build_forest.py is single-threaded symlinking + "{params.cmd} --forest {output}" # Merge single-exposure WCS headers into the tile-level sqlite log # (log_exp_headers--.sqlite — the tile suffix downstream Sx/PiViVi/Ng From a110d89faeca7f0ac6bb2bf6a0ac79862ea044fe Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:42:12 -0400 Subject: [PATCH 06/21] =?UTF-8?q?fix(orchestration):=20shard=20the=20expos?= =?UTF-8?q?ure=20forest=20=E2=80=94=20the=20module=20glob's=20shape=20is?= =?UTF-8?q?=20not=20cosmetic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exp_utils.get_exp_output_files hardwires the v2.0 sharded layout into its $SP_EXP glob (///output/run_sp_*/...); a flat forest fails every tile gather stage with 'No split_exp_runner output found'. The exposure STORE stays flat (nothing globs it — the index drives every path); only the module-facing forest view carries the 2-digit shard. Verified by hand-running tile_merge_headers on 210.296 (1/1 OK). Co-Authored-By: Claude Fable 5 --- workflow/scripts/build_forest.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 82df8b0c..86dc4354 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -3,10 +3,14 @@ A plain script (not a run: block) so the tile chain stays group-compatible. Reads the tile's exposures from run_index.sqlite and symlinks each exposure's -``exp//output`` into ``//output`` by exact name (no glob), so -the tile modules' $SP_EXP globs resolve to exactly this tile's exposures' -products. The forest is a convenience view; the DAG edge to the exposures is -declared in the rule's input (tile.smk), not here. +``exp//output`` into ``///output`` by exact name (no +glob). The 2-digit ```` shard level is NOT cosmetic: ShapePipe's +``exp_utils.get_exp_output_files`` hardwires the sharded v2.0 layout into its +$SP_EXP glob (``///output/run_sp_*/...``), so a flat +forest makes every tile gather stage fail "No split_exp_runner output found". +(The exposure STORE stays flat — only the module-facing view needs the shard.) +The forest is a convenience view; the DAG edge to the exposures is declared in +the rule's input (tile.smk), not here. """ import argparse @@ -30,7 +34,7 @@ def main() -> None: args.forest.mkdir(parents=True, exist_ok=True) for e in exps: src = args.run_dir / "exp" / e / "output" - dst = args.forest / e / "output" + dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape dst.parent.mkdir(parents=True, exist_ok=True) if dst.is_symlink() or dst.exists(): dst.unlink() From 28c10d8cdca5947184fb5a2d2f831e84468b2c95 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 16:47:15 -0400 Subject: [PATCH 07/21] fix(orchestration): carry a wrapper-scripts content hash into every rule's params MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default rerun-triggers' code trigger hashes only the rule's own shell string — NOT external scripts it calls. Bitten live: the forest-shard fix changed build_forest.py but no rule text, so stale flat forests survived a rerun and tile_merge_headers kept failing. SCRIPT_HASH (md5 of sp_rule.py + completeness.py + build_forest.py, computed at parse time) rides into params via an accept-and-ignore --script-hash flag, so script edits propagate exactly like code edits under the full default triggers. One-time cost: every rule's params change once, re-running completed work — the same tradeoff as any code change under full triggers, and the P0-scale rerun doubles as a live test of rerun semantics. Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 14 +++++++++++++- workflow/rules/tile.smk | 3 ++- workflow/scripts/build_forest.py | 5 +++++ workflow/scripts/sp_rule.py | 5 +++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 774ae3ad..3c869482 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -70,7 +70,8 @@ def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, cmd = ( f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " f"--unit {unit} --level {level} --run-dir {RUN_DIR} " - f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS}" + f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} " + f"--script-hash {SCRIPT_HASH}" ) if not isolate: cmd += " --no-isolate" @@ -86,6 +87,17 @@ def tile_dir(tile): def exp_dir(exp): return RUN_DIR / "exp" / exp +# Content hash of the wrapper scripts, computed once at parse time. Exposed as a +# param on every rule (via sp_rule() and tile_exp_forest) so the default +# rerun-triggers' *params* trigger covers script edits: the `code` trigger only +# hashes the rule's own shell string, NOT external scripts it calls — without +# this, a wrapper/forest-script fix silently leaves stale outputs in place +# (bitten live: the forest-shard fix rebuilt nothing on rerun). +import hashlib as _hashlib +SCRIPT_HASH = _hashlib.md5(b"".join( + (SCRIPTS / n).read_bytes() + for n in ("sp_rule.py", "completeness.py", "build_forest.py"))).hexdigest()[:12] + include: "rules/prepare.smk" include: "rules/exposure.smk" include: "rules/tile.smk" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index d36e91d9..04cc4c6c 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -36,7 +36,8 @@ rule tile_exp_forest: # ONCE — an {output} placeholder inside params.cmd survives literally # (same trap as {threads}; all four forest jobs then race one './{output}'). cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " - f"--run-dir {RUN_DIR} --index {INDEX_DB}") + f"--run-dir {RUN_DIR} --index {INDEX_DB} " + f"--script-hash {SCRIPT_HASH}") shell: # no --threads: build_forest.py is single-threaded symlinking "{params.cmd} --forest {output}" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 86dc4354..e945820e 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -24,6 +24,11 @@ def main() -> None: p.add_argument("--run-dir", required=True, type=Path) p.add_argument("--index", required=True, type=Path) p.add_argument("--forest", required=True, type=Path) + p.add_argument("--script-hash", default=None, + help="ignored at runtime: carries the wrapper-scripts content " + "hash into the rule's params so the params rerun-trigger " + "covers script edits (the code trigger only hashes the " + "rule's own shell string)") args = p.parse_args() con = sqlite3.connect(args.index) diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py index b114fad0..8ee71c69 100644 --- a/workflow/scripts/sp_rule.py +++ b/workflow/scripts/sp_rule.py @@ -315,6 +315,11 @@ def main() -> None: help="N_SPLIT_MAX for the merge_sep_cats template") p.add_argument("--final-cat", type=Path, default=None, help="copy the produced final_cat FITS here (make_cat)") + p.add_argument("--script-hash", default=None, + help="ignored at runtime: carries the wrapper-scripts content " + "hash into the rule's params so the params rerun-trigger " + "covers script edits (the code trigger only hashes the " + "rule's own shell string)") args = p.parse_args() ngmix_mode = args.ngmix_chunk is not None From a273e76369119113d05251c3c8b33beb3e1ec654 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 19:59:57 -0400 Subject: [PATCH 08/21] fix(orchestration): keep snakemake state and index off /project (hard-full quota) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit def-mjhudson /project sits at its 27/27 TiB ceiling: a .snakemake metadata write died mid-run (killing an otherwise-green invocation after all 8 ngmix chunks of the first tile passed), and by end of night even git and file edits on /project failed. bin/sp now runs snakemake with --directory -state on /scratch (state only — data paths are absolute), the Snakefile resolves its configfile relative to itself so --directory can't break it, and the run index lives with the run on /scratch until project space is reclaimed (finding 15's /project placement stays the design intent; noted in config). Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 5 ++++- workflow/bin/sp | 7 ++++++- workflow/config.yaml | 8 +++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 3c869482..5d93a181 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -21,7 +21,10 @@ DAG that produces it). import sqlite3 from pathlib import Path -configfile: "workflow/config.yaml" +# Resolved relative to THIS file, not the working directory: snakemake runs with +# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project +# (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). +configfile: str(Path(workflow.snakefile).parent / "config.yaml") # Every job's shell runs inside this container (apptainer software-deployment in # the profile); the user never types apptainer. diff --git a/workflow/bin/sp b/workflow/bin/sp index f226aa44..d1f10554 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -34,9 +34,14 @@ source "$VENV/bin/activate" # Minimal scalar reader for workflow/config.yaml (key: value, no nesting). cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" +# Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO +# THE RUN on /scratch, never on /project (hard 27/27 TiB group quota killed a +# metadata write mid-run live). --directory only moves state: all data paths are +# absolute, and the Snakefile resolves its own configfile. +STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" TILE_LIST="$(cfg tile_list)"; MISSING_THRESH="${SP_MISSING_THRESHOLD:-0.0}" -sm() { snakemake --profile "$PROFILE" "$@"; } +sm() { snakemake --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } build_index() { python "$SCRIPTS/build_index.py" --tile-list "$TILE_LIST" \ --run-dir "$RUN_DIR" --db "$INDEX_DB" --missing-threshold "$MISSING_THRESH" diff --git a/workflow/config.yaml b/workflow/config.yaml index ec529c4d..f92cbb69 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -24,9 +24,11 @@ config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis # Star catalogues for masking, pre-generated (network step done in prepare): star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats -# The run index. Durable, low-volume products (index, missing.json, run_report, -# final_cat) should land on /project, not /scratch (60-day purge, finding 15). -index_db: /project/def-mjhudson/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite +# The run index. Design intent (finding 15) is durable products on /project — +# BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and +# report live with the run on /scratch until space is reclaimed. Mind the +# 60-day purge for anything that must survive. +index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). From 0010def6c3928185b3c84334983d6625538fd749 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 17 Jul 2026 01:45:14 -0400 Subject: [PATCH 09/21] fix(orchestration): drop exp_mask's dead per-unit star-cat input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-unit star_cat file rules were removed (the store is per-CCD and the mask config reads it as a dir symlink), but exp_mask kept declaring exp/{exp}/star_cat-{exp}.fits as input. No producer exists, so the main DAG became unbuildable the moment the tile list grew past the prepared set — MissingInputException on every new exposure. Dormant for P0 (all 19 exposures pre-staged); caught live by the append-invariant test (196 quad appended: DAG build died before scheduling anything). With the fix the append dry-run schedules exactly the appended cone (185 jobs: 30 new exposures x4 stages once each, 4 tile chains) and nothing for the finished tiles; restored-state dry-run returns to 'Nothing to be done'. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_0188niYp9MmhLmZou1KXWbye --- workflow/rules/exposure.smk | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index f39f6be2..82cdb4cd 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -44,11 +44,14 @@ rule exp_split: shell: "{params.cmd} --threads {threads}" -# Mask per-CCD; this exposure's star cat is a declared input (per-unit re-key). +# Mask per-CCD. Star cats are NOT a per-unit input: the store is per-CCD +# (star_cat--.fits) and the mask config reads it as a DIR symlink +# (see prepare.smk) — a per-unit file input here has no producer and makes the +# DAG unbuildable the moment a new exposure is appended (caught live by the +# append-invariant test). A missing cat fails the mask count-floor loudly. rule exp_mask: input: rules.exp_split.output, - star=str(RUN_DIR / "exp/{exp}/star_cat-{exp}.fits"), output: directory(f"{EXP_OUT}/run_sp_exp_Ma") params: From 833aa9369232d37f0ccfeb35fecad1ef223fc00c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 21:50:51 -0400 Subject: [PATCH 10/21] fix(orchestration): nibi-flavor completeness counts; file-level attrition reporting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sextractor writes 3 files/CCD on nibi (120/exp, not v2.0's 80); vignetmaker run_2 writes 5 sqlites/tile (not canfar's 4) — both verified against the P0 tree and the bash baseline. run_report now aggregates found-vs-expected at file granularity per runner, so warn-runner attrition (psfex_interp) is reported instead of hidden behind unit-level completeness. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRBKnHV6oUXEqRpsjqJpex --- workflow/scripts/completeness.py | 10 +++++++-- workflow/scripts/run_report.py | 36 ++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index d52ad8d0..b2246abe 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -42,8 +42,11 @@ "exp_get_images": {"get_images_runner": dict(expect=3, floor=3)}, "exp_split": {"split_exp_runner": dict(expect=121, floor=41)}, "exp_mask": {"mask_runner": dict(expect=40, floor=1)}, + # sextractor expect is nibi-flavor: 3 files/CCD (sexcat + background + + # background_rms; v2.0's 80 assumed 2/CCD), verified against the P0 tree + # AND the bash baseline (both 120/exposure). "exp_psf": { - "sextractor_runner": dict(expect=80, floor=2), + "sextractor_runner": dict(expect=120, floor=2), "setools_runner": dict(expect=80, floor=2, subpath="rand_split"), "psfex_runner": dict(expect=80, floor=2), "psfex_interp_runner": dict(expect=40, floor=0, warn=True), @@ -56,7 +59,10 @@ "tile_vignets": { "psfex_interp_runner": dict(expect=1, floor=1), "vignetmaker_runner_run_1": dict(expect=1, floor=1), - "vignetmaker_runner_run_2": dict(expect=4, floor=4), + # 5 sqlites/tile on nibi (image/weight/flag/background/background_rms); + # v2.0's 4 was the canfar flavor. floor follows the tile-post pattern + # (expect=floor: all-or-nothing, every vignette feeds ngmix). + "vignetmaker_runner_run_2": dict(expect=5, floor=5), }, "tile_ngmix": {"ngmix_runner": dict(expect=1, floor=1)}, "tile_merge_cats": {"merge_sep_cats_runner": dict(expect=1, floor=1)}, diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index bb330040..683433ec 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -12,6 +12,14 @@ It distinguishes whole-exposure absence (all runners missing — a real gap or a deletion/forest bug) from tolerated per-CCD attrition (counts between floor and expect), so a future deletion bug cannot be silently absorbed (finding 5). + +Attrition is counted at file granularity, not unit granularity: each stage's +``products`` block aggregates found-vs-expected file counts per runner across +all passing units, with a per-unit shortfall map where files are missing. +``warn`` runners (psfex_interp) are exempt from *failing* a unit, not from +being *reported* — the P0 validation found 46/760 psfex_interp CCDs missing +(6%) behind a unit-level "zero attrition" claim, which is the failure mode +this granularity exists to prevent. """ import argparse @@ -52,7 +60,9 @@ def classify(stage, run_dir): if mand and all(n == 0 for _, n, *_ in mand): return "absent", details return "under", details - if any(n < expect for _, n, floor, expect, warn in details if not warn): + # warn runners are exempt from FAILING a unit (check_floor), not from + # attrition reporting — psfex_interp is precisely the runner that attrits. + if any(n < expect for _, n, _, expect, _ in details): return "attrition", details return "complete", details @@ -83,13 +93,28 @@ def main() -> None: units = tiles if level == "tile" else exps sub = "tiles" if level == "tile" else "exp" tally = {"complete": 0, "attrition": 0, "under": [], "absent": []} + # File-level aggregate per runner over PASSING units (complete/attrition); + # under/absent units are enumerated by name above, not folded in here — + # mixing them would double-count whole-unit absence as attrition. + products = {r: {"found": 0, "expect": 0, "by_unit": {}} + for r in COMPLETENESS[stage]} for u in units: run_dir = args.run_dir / sub / u / "output" / prefix - verdict, _ = classify(stage, run_dir) + verdict, details = classify(stage, run_dir) if verdict in ("complete", "attrition"): tally[verdict] += 1 + for runner, n, _, expect, _ in details: + agg = products[runner] + agg["found"] += n + agg["expect"] += expect + if n < expect: + agg["by_unit"][u] = expect - n else: tally[verdict].append(u) + for agg in products.values(): + if not agg["by_unit"]: + del agg["by_unit"] + tally["products"] = products report["stages"][stage] = tally finals = [t for t in tiles @@ -101,6 +126,13 @@ def main() -> None: out.write_text(json.dumps(report, indent=2)) print(f"[run_report] {report['final_cats']['present']}/{len(tiles)} final " f"cats -> {out}", file=sys.stderr) + for stage, tally in report["stages"].items(): + for runner, agg in tally["products"].items(): + if agg["found"] < agg["expect"]: + pct = 100 * (agg["expect"] - agg["found"]) / agg["expect"] + print(f"[run_report] attrition {stage}/{runner}: " + f"{agg['found']}/{agg['expect']} files (-{pct:.1f}%) " + f"across {len(agg['by_unit'])} units", file=sys.stderr) if __name__ == "__main__": From 20cc50630b0a9c3cfca922a1e136b19ffd89fb34 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:02:25 -0400 Subject: [PATCH 11/21] feat(ngmix): env-expanded chunk fields; position-seeded RNG becomes the only mode The three module touches named in #848: - ID_OBJ_MIN/MAX read via getexpanded, so the committed ngmix template can interpolate $NGMIX_ID_MIN/$NGMIX_ID_MAX per chunk. Unset vars fail loudly. - SEED_FROM_POSITION is retired as an option: each object's RNG derives from its own (ra, dec, ccd), so results are bit-identical under any chunking. Tile-seed path, Ngmix.get_prior, and self._rng removed; a config that still sets SEED_FROM_POSITION=False raises with a clear message. - RUN_NAME read via getexpanded, so per-chunk run dirs can come from $SP_NGMIX_CHUNK in the committed template. Tests: tests/module/test_ngmix.py 24 passed / 1 pre-existing failure (azgauss needs ngmix>=2.4.1, absent from the runtime image; fails on clean tree too); test_ngmix_weight_validation.py 14 passed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRBKnHV6oUXEqRpsjqJpex --- .../modules/ngmix_package/__init__.py | 16 +++- src/shapepipe/modules/ngmix_package/ngmix.py | 73 ++++++------------- src/shapepipe/modules/ngmix_runner.py | 32 ++++---- src/shapepipe/run.py | 2 +- 4 files changed, 58 insertions(+), 65 deletions(-) diff --git a/src/shapepipe/modules/ngmix_package/__init__.py b/src/shapepipe/modules/ngmix_package/__init__.py index 230606bb..fca6ee0b 100644 --- a/src/shapepipe/modules/ngmix_package/__init__.py +++ b/src/shapepipe/modules/ngmix_package/__init__.py @@ -40,15 +40,27 @@ (no batch saving) ID_OBJ_MIN : int ID of first galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, so an orchestrator can + set the object range per chunk, for example + ``ID_OBJ_MIN = $SP_NGMIX_ID_OBJ_MIN``. ID_OBJ_MAX : int ID of last galaxy object to be processed; not used if set to ``-1`` - (default) + (default). Environment variables are expanded, as for ``ID_OBJ_MIN``. BKG_RMS_VIGNET_PATH : str, optional Path to a ``background_rms_vignet*.sqlite`` file produced by ``vignetmaker_runner``. The string may contain ``{file_number_string}``, which is replaced by the current tile ID. +Random number generation +======================== + +Each object gets its own random number stream, seeded from its sky position +and CCD (``position_seed``, ngmix#796). The output is therefore identical +whether a tile is processed in one go or split into object chunks with +``ID_OBJ_MIN``/``ID_OBJ_MAX``. The older tile-seeded mode is retired; the +config option ``SEED_FROM_POSITION`` is obsolete. Setting it to ``False`` +raises an error, so a stale config cannot silently change the RNG. + """ __all__ = ["ngmix"] diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index d4f64a74..fc9779c6 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -335,13 +335,12 @@ class Ngmix(object): (robust for galaxies); ``"wcs"`` uses the catalog sky position projected through the WCS (better for stars, whose HSM moments are noisy). See :func:`make_ngmix_observation`. - seed_from_position : bool, optional - If ``True``, replace the tile-level RNG with a per-object RNG seeded - from the object's sky position (:func:`position_seed`) inside the - object loop, so metacal's ``fixnoise`` counter-noise and the fit - guesses cancel across Pujol image-simulation branches (ngmix#796). The - default ``False`` leaves the production path byte-identical. See - :func:`position_seed` for the physics and the seed construction. + Notes + ----- + The RNG is always per object and seeded from that object's sky position + (:func:`position_seed`). Results therefore do not depend on how the tile is + split into object chunks, and metacal's ``fixnoise`` counter-noise and the + fit guesses cancel across Pujol image-simulation branches (ngmix#796). Raises ------ @@ -364,7 +363,6 @@ def __init__( id_obj_max=-1, bkg_sub=True, centroid_source="hsm", - seed_from_position=False, metacal_psf="fitgauss", ): @@ -418,20 +416,14 @@ def __init__( self._id_obj_max = id_obj_max self._bkg_sub = bkg_sub self._centroid_source = centroid_source - self._seed_from_position = seed_from_position self._metacal_psf = metacal_psf self._w_log = w_log - # Initiatlise random generator - seed = int(''.join(re.findall(r'\d+', self._file_number_string))) - self._rng = np.random.RandomState(seed) - self._w_log.info(f'Random generator initialisation seed = {seed}') - if self._seed_from_position: - self._w_log.info( - 'SEED_FROM_POSITION on: per-object RNG seeded from sky position' - ' for Pujol noise cancellation (image sims, ngmix#796)' - ) + self._w_log.info( + 'Per-object RNG seeded from sky position (ngmix#796): results are' + ' invariant to how the tile is split into object chunks' + ) @classmethod def MegaCamFlip(self, vign, ccd_nb): @@ -461,18 +453,6 @@ def MegaCamFlip(self, vign, ccd_nb): # swap y axis so origin is on bottom-left return vign - def get_prior(self, T_range=None, F_range=None): - """Get Prior. - - Returns - ------- - ngmix.joint_prior.PriorSimpleSep - """ - return get_prior( - self._pixel_scale, self._rng, - T_range=T_range, F_range=F_range, - ) - def compile_results(self, results): """Compile Results. @@ -809,7 +789,6 @@ def process(self): vignet_cat = self._vignet_cat final_res = [] - prior = self.get_prior() count = 0 n_empty_cat = 0 @@ -843,24 +822,20 @@ def process(self): n_no_epoch += 1 continue - # Position-seeded per-object RNG for Pujol noise cancellation in - # image sims (ngmix#796): the same object gets the same fixnoise - # counter-noise and fit guesses in every shear branch, so both - # cancel in the branch difference. The prior is rebuilt from the - # same per-object RNG because the guesser draws its initial guess - # via prior.sample() (ngmix guessers.py), which consumes the RNG the - # prior was CONSTRUCTED with — so a per-object rng alone would leave - # the guess drawing from the shared tile stream and break - # cancellation. Off in production, where the single tile-level - # self._rng and the tile-level prior carry the whole loop. - if self._seed_from_position: - obj_rng = np.random.RandomState( - position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) - ) - obj_prior = get_prior(self._pixel_scale, obj_rng) - else: - obj_rng = self._rng - obj_prior = prior + # Position-seeded per-object RNG (ngmix#796). Each object draws from + # a stream fixed by its own (ra, dec, ccd), so the result is + # independent of which chunk the object lands in and of detection + # order, and the same object gets the same fixnoise counter-noise + # and fit guesses in every Pujol shear branch, so both cancel in the + # branch difference. The prior is rebuilt from the same per-object + # RNG because the guesser draws its initial guess via prior.sample() + # (ngmix guessers.py), which consumes the RNG the prior was + # CONSTRUCTED with — a per-object rng alone would leave the guess + # drawing from a shared stream and break both properties. + obj_rng = np.random.RandomState( + position_seed(stamp.ra[0], stamp.dec[0], stamp.ccd) + ) + obj_prior = get_prior(self._pixel_scale, obj_rng) try: flux_guess = ( diff --git a/src/shapepipe/modules/ngmix_runner.py b/src/shapepipe/modules/ngmix_runner.py index 3d98356c..fcb50be9 100644 --- a/src/shapepipe/modules/ngmix_runner.py +++ b/src/shapepipe/modules/ngmix_runner.py @@ -86,9 +86,12 @@ def ngmix_runner( # No batch saving save_batch = -1 - # First and last galaxy ID to process - id_obj_min = config.getint(module_config_sec, "ID_OBJ_MIN") - id_obj_max = config.getint(module_config_sec, "ID_OBJ_MAX") + # First and last galaxy ID to process. Read via ``getexpanded`` so an + # orchestrator can drive the chunk bounds from environment variables + # (``$SP_NGMIX_ID_OBJ_MIN`` and friends); ``getexpanded`` is the only + # accessor in ShapePipe's config that expands ``$VAR``. + id_obj_min = int(config.getexpanded(module_config_sec, "ID_OBJ_MIN")) + id_obj_max = int(config.getexpanded(module_config_sec, "ID_OBJ_MAX")) # Centroid source for the galaxy Jacobian origin: "wcs" (default -- the # catalog sky position projected through the WCS, trusting the astrometry) @@ -99,16 +102,20 @@ def ngmix_runner( else: centroid_source = "wcs" - # Seed the per-object RNG from sky position instead of per tile, so - # metacal's fixnoise counter-noise (and the fit guesses) cancel across - # Pujol image-simulation shear branches (ngmix#796). Default False leaves - # the production path byte-identical. + # Position-seeded RNG is the only mode: every object's RNG comes from its + # own (ra, dec, ccd), so results do not depend on how the tile is split + # into chunks, and metacal's fixnoise counter-noise cancels across Pujol + # image-simulation shear branches (ngmix#796). The retired tile-seed mode + # had neither property. Old configs that disable it must fail loudly. if config.has_option(module_config_sec, "SEED_FROM_POSITION"): - seed_from_position = config.getboolean( - module_config_sec, "SEED_FROM_POSITION" - ) - else: - seed_from_position = False + if not config.getboolean(module_config_sec, "SEED_FROM_POSITION"): + raise ValueError( + "SEED_FROM_POSITION = False is no longer supported: the" + " tile-seeded RNG mode has been retired because it makes" + " results depend on the object chunking. Remove the" + " SEED_FROM_POSITION entry from the ngmix config section" + " (position-seeded RNG is now the only mode)." + ) # Check PSF vignets first: if all are empty dicts {}, the exposures for this # tile are absent from the PSF dictionary and no shape measurement is possible. @@ -161,7 +168,6 @@ def ngmix_runner( id_obj_max=id_obj_max, bkg_sub=bkg_sub, centroid_source=centroid_source, - seed_from_position=seed_from_position, metacal_psf=metacal_psf, ) diff --git a/src/shapepipe/run.py b/src/shapepipe/run.py index 962953ed..fee0dad3 100644 --- a/src/shapepipe/run.py +++ b/src/shapepipe/run.py @@ -84,7 +84,7 @@ def _set_run_name(self): Set the name of the current pipeline run. """ - self._run_name = self.config.get("DEFAULT", "RUN_NAME") + self._run_name = self.config.getexpanded("DEFAULT", "RUN_NAME") if self.config.getboolean("DEFAULT", "RUN_DATETIME"): self._run_name += datetime.now().strftime("_%Y-%m-%d_%H-%M-%S") From d1c0afb01edbb3d4eee59bb76c38c73e8c39315a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:06:58 -0400 Subject: [PATCH 12/21] =?UTF-8?q?feat(orchestration):=20committed=20workfl?= =?UTF-8?q?ow=20configs=20=E2=80=94=20no=20wrapper,=20no=20runtime=20rewri?= =?UTF-8?q?ting=20(#848=20D2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New workflow/config/cfis/ (example/cfis + p3-batch1 deltas, rewritten; example/cfis untouched for the bash flow): - RUN_DATETIME=False committed; every 'last:' INPUT_DIR replaced by the fixed $SP_RUN/output/run_sp_//output path of its producer. - NUMBER_LIST=$SP_UNIT_NUM on unit-ID-numbered stages (getlist env-expands; SP_UNIT_NUM carries the dashed form, e.g. -210-282 — rules do the transform). - ngmix template: $NGMIX_ID_MIN/$NGMIX_ID_MAX, SEED_FROM_POSITION=True, per-chunk RUN_NAME=run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u, SAVE_BATCH=250. - merge_sep_cats: N_SPLIT_MAX=$NGMIX_N_CHUNKS; module reads it env-expanded (same getexpanded treatment as the ngmix chunk fields). - make_cat from the nosm variant (what the P3 runs used) — no spread_model dependency. - p3 deltas folded: real INPUT_PATH/RETRIEVE=symlink, exp_Ma external star cat ($SP_RUN/star_cat_exp), PiViVi RUN_DATETIME fix. Rejected: SMP_BATCH_SIZE deltas (superseded by -b {threads}). - tile_Sx from the nomask variant (plain Sx needs an unlisted tile_Ma stage); tile_Uc kept but its run_sp_tile_Gic input path is unverified (Uc is not the production detect variant). All 14 configs pass a strict $VAR-expansion round-trip with the workflow's env stubbed. workflow/config.yaml config_src now points in-repo. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRBKnHV6oUXEqRpsjqJpex --- .../modules/merge_sep_cats_runner.py | 2 +- workflow/config.yaml | 8 +- workflow/config/cfis/config_exp_Gie.ini | 99 ++++++++++ workflow/config/cfis/config_exp_Ma.ini | 86 +++++++++ workflow/config/cfis/config_exp_Sp.ini | 78 ++++++++ workflow/config/cfis/config_exp_psfex.ini | 181 ++++++++++++++++++ .../config/cfis/config_merge_sep_cats.ini | 87 +++++++++ workflow/config/cfis/config_onthefly.mask | 86 +++++++++ workflow/config/cfis/config_tile_Fe.ini | 76 ++++++++ workflow/config/cfis/config_tile_Git.ini | 93 +++++++++ workflow/config/cfis/config_tile_Mc.ini | 80 ++++++++ workflow/config/cfis/config_tile_Mh_exp.ini | 76 ++++++++ .../config/cfis/config_tile_Ng_template.ini | 104 ++++++++++ workflow/config/cfis/config_tile_PiViVi.ini | 172 +++++++++++++++++ workflow/config/cfis/config_tile_Sx.ini | 118 ++++++++++++ workflow/config/cfis/config_tile_Uc.ini | 92 +++++++++ workflow/config/cfis/config_tile_Uz.ini | 73 +++++++ .../config/cfis/config_tile_onthefly.mask | 90 +++++++++ workflow/config/cfis/default.conv | 5 + workflow/config/cfis/default.param | 68 +++++++ workflow/config/cfis/default.psfex | 85 ++++++++ workflow/config/cfis/default_exp.sex | 133 +++++++++++++ workflow/config/cfis/default_noimaflags.param | 65 +++++++ workflow/config/cfis/default_tile.sex | 133 +++++++++++++ workflow/config/cfis/final_cat.param | 114 +++++++++++ .../mask_default/MEGAPRIME_star_i_13.8.reg | 24 +++ .../cfis/mask_default/Messier_catalog.npy | Bin 0 -> 4209 bytes .../mask_default/Messier_catalog_updated.fits | Bin 0 -> 8640 bytes workflow/config/cfis/mask_default/default.ww | 40 ++++ .../config/cfis/mask_default/halo_mask.reg | 50 +++++ .../config/cfis/mask_default/ngc_cat.fits | Bin 0 -> 201600 bytes workflow/config/cfis/star_selection.setools | 103 ++++++++++ 32 files changed, 2418 insertions(+), 3 deletions(-) create mode 100644 workflow/config/cfis/config_exp_Gie.ini create mode 100644 workflow/config/cfis/config_exp_Ma.ini create mode 100644 workflow/config/cfis/config_exp_Sp.ini create mode 100644 workflow/config/cfis/config_exp_psfex.ini create mode 100644 workflow/config/cfis/config_merge_sep_cats.ini create mode 100644 workflow/config/cfis/config_onthefly.mask create mode 100644 workflow/config/cfis/config_tile_Fe.ini create mode 100644 workflow/config/cfis/config_tile_Git.ini create mode 100644 workflow/config/cfis/config_tile_Mc.ini create mode 100644 workflow/config/cfis/config_tile_Mh_exp.ini create mode 100644 workflow/config/cfis/config_tile_Ng_template.ini create mode 100644 workflow/config/cfis/config_tile_PiViVi.ini create mode 100644 workflow/config/cfis/config_tile_Sx.ini create mode 100644 workflow/config/cfis/config_tile_Uc.ini create mode 100644 workflow/config/cfis/config_tile_Uz.ini create mode 100644 workflow/config/cfis/config_tile_onthefly.mask create mode 100644 workflow/config/cfis/default.conv create mode 100644 workflow/config/cfis/default.param create mode 100644 workflow/config/cfis/default.psfex create mode 100644 workflow/config/cfis/default_exp.sex create mode 100644 workflow/config/cfis/default_noimaflags.param create mode 100644 workflow/config/cfis/default_tile.sex create mode 100644 workflow/config/cfis/final_cat.param create mode 100644 workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg create mode 100644 workflow/config/cfis/mask_default/Messier_catalog.npy create mode 100644 workflow/config/cfis/mask_default/Messier_catalog_updated.fits create mode 100644 workflow/config/cfis/mask_default/default.ww create mode 100644 workflow/config/cfis/mask_default/halo_mask.reg create mode 100644 workflow/config/cfis/mask_default/ngc_cat.fits create mode 100644 workflow/config/cfis/star_selection.setools diff --git a/src/shapepipe/modules/merge_sep_cats_runner.py b/src/shapepipe/modules/merge_sep_cats_runner.py index 927c79b7..6e99cf3e 100644 --- a/src/shapepipe/modules/merge_sep_cats_runner.py +++ b/src/shapepipe/modules/merge_sep_cats_runner.py @@ -28,7 +28,7 @@ def merge_sep_cats_runner( ): """Define The Merge SEP Catalogues Runner.""" # Get config entries - n_split_max = config.getint(module_config_sec, "N_SPLIT_MAX") + n_split_max = int(config.getexpanded(module_config_sec, "N_SPLIT_MAX")) file_pattern = config.getlist(module_config_sec, "FILE_PATTERN") file_ext = config.getlist(module_config_sec, "FILE_EXT") diff --git a/workflow/config.yaml b/workflow/config.yaml index f92cbb69..592ecf98 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -17,8 +17,12 @@ container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # /tiles/// and /exp/// run_dir: /scratch/cdaley/shapepipe-output/smk-p0 -# The canonical config chain (example/cfis in the repo, or a run-specific copy). -config_src: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/cfis +# The canonical config chain: the committed, wrapper-free configs (#848 D2 config +# sweep) -- RUN_DATETIME=False, fixed INPUT_DIRs, NUMBER_LIST=$SP_UNIT_NUM baked +# in. example/cfis stays untouched for the bash flow; runs/p3-batch1/cfis's real +# divergences (real data paths, external star cats, SAVE_BATCH memory tuning) +# were folded in -- see the config-sweep report for the full list. +config_src: /home/cdaley/shapepipe-smk/workflow/config/cfis # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). # Star catalogues for masking, pre-generated (network step done in prepare): diff --git a/workflow/config/cfis/config_exp_Gie.ini b/workflow/config/cfis/config_exp_Gie.ini new file mode 100644 index 00000000..7c6297cf --- /dev/null +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -0,0 +1,99 @@ +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures, /project/def-mjhudson/unions-wl/exposures + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = 000000, 000000.weight, 000000.flag + +# Input file extensions +INPUT_FILE_EXT = .fits.fz, .fits.fz, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{6} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Gie_prev diff --git a/workflow/config/cfis/config_exp_Ma.ini b/workflow/config/cfis/config_exp_Ma.ini new file mode 100644 index 00000000..0b85fe86 --- /dev/null +++ b/workflow/config/cfis/config_exp_Ma.ini @@ -0,0 +1,86 @@ +# ShapePipe configuration file for masking of exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Ma + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = mask_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 4 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +### Mask exposures +[MASK_RUNNER] + +# Parent module +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/star_cat_exp + +# Update numbering convention, accounting for HDU number of +# single-exposure single-HDU files +NUMBERING_SCHEME = -0000000-0 + +# Input file patterns: image, weight, external flag, external star catalogue +# (folded from runs/p3-batch1/cfis: offline cluster, no online star-cat fetch) +FILE_PATTERN = image, weight, flag, star_cat + +FILE_EXT = .fits, .fits, .fits, .fits + +# Path of mask config file +MASK_CONFIG_PATH = $SP_CONFIG/config_onthefly.mask + +# External mask file flag, use if True, otherwise ignore +USE_EXT_FLAG = True + +# External star catalogue flag, use external cat if True, +# obtain from online catalogue if False +# (folded from p3-batch1: True, using the pre-staged star_cat_exp pool) +USE_EXT_STAR = True + +# File name suffix for the output flag files (optional) +PREFIX = pipeline + +# Path to check for existing output mask files +CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_exp_Ma/mask_runner/output diff --git a/workflow/config/cfis/config_exp_Sp.ini b/workflow/config/cfis/config_exp_Sp.ini new file mode 100644 index 00000000..dd27d6cc --- /dev/null +++ b/workflow/config/cfis/config_exp_Sp.ini @@ -0,0 +1,78 @@ +# ShapePipe configuration file for single-exposures, +# split images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed exposure ID (exp split is a "tile-scheme" stage per sp_rule.py). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_Gie/get_images_runner/output + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +FILE_EXT = .fitsfz, .fitsfz, .fitsfz + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/workflow/config/cfis/config_exp_psfex.ini b/workflow/config/cfis/config_exp_psfex.ini new file mode 100644 index 00000000..0af871d8 --- /dev/null +++ b/workflow/config/cfis/config_exp_psfex.ini @@ -0,0 +1,181 @@ +# ShapePipe configuration file for single-exposures. PSFex PSF model. +# Process exposures after masking, from star detection to PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_SxSePsfPi +#RUN_NAME = run_sp_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner, setools_runner, psfex_runner, psfex_interp_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +# Input from two modules +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output, $SP_RUN/output/run_sp_exp_Ma/mask_runner/output + +# Read pipeline flag files created by mask module +FILE_PATTERN = image, weight, pipeline_flag + +# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on +# the decorator's 4-entry FILE_EXT default (length check fails at startup) +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND, BACKGROUND_RMS + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE + + +[SETOOLS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsfPi/sextractor_runner/output + +# Note: Make sure this doe not match the SExtractor background images +# (sexcat_background*) +FILE_PATTERN = sexcat + +NUMBERING_SCHEME = -0000000-0 + +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools + + +[PSFEX_RUNNER] + +# Use 80% sample for PSF model +FILE_PATTERN = star_split_ratio_80 + +NUMBERING_SCHEME = -0000000-0 + +# Path to executable for the PSF model (optional) +EXEC_PATH = psfex + +# Default psfex configuration file +DOT_PSFEX_FILE = $SP_CONFIG/default.psfex + +[PSFEX_INTERP_RUNNER] + +# Use 20% sample for PSF validation +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20, psfex_cat + +FILE_EXT = .psf, .fits, .cat + +NUMBERING_SCHEME = -0000000-0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = VALIDATION + +# Column names of position parameters +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# If True, measure and store ellipticity of the PSF (using moments) +GET_SHAPES = True + +# Minimum number of stars per CCD for PSF model to be computed +STAR_THRESH = 22 + +# Maximum chi^2 for PSF model to be computed on CCD +CHI2_THRESH = 2 diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini new file mode 100644 index 00000000..a033f28d --- /dev/null +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -0,0 +1,87 @@ +# ShapePipe post-run configuration file: merge separated catalogues + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Ms + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_sep_cats_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +# NOTE: only chunk 1's ngmix output is listed here; merge_sep_cats_runner +# derives the other chunks' paths itself from N_SPLIT_MAX and this pattern +# (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the +# same fixed $SP_RUN/output. +INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_SEP_CATS_RUNNER] + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = ngmix + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits + +# Numbering convention, string that exemplifies a numbering pattern. +NUMBERING_SCHEME = -000-000 + +# WARNING (optional, default is 'error'). Use 'always'/'ignore' to +# display/ignore warnings, and not raise error +WARNING = always + +# Maximum number of separated catalogues per input. +# BLOCKER: merge_sep_cats_runner.py reads this with plain config.getint (not +# getexpanded) -- $NGMIX_N_CHUNKS would NOT expand here, unlike ID_OBJ_MIN/MAX +# in the ngmix module. Committed as a constant matching workflow/config.yaml's +# ngmix_chunks: 8; if that value changes, this line must be edited by hand +# (or the module gets the same getexpanded treatment ngmix_runner got). +# Chunk count comes from the workflow (single source of truth: the +# scattergather count); env-expanded by the module. +N_SPLIT_MAX = $NGMIX_N_CHUNKS diff --git a/workflow/config/cfis/config_onthefly.mask b/workflow/config/cfis/config_onthefly.mask new file mode 100644 index 00000000..7c185c60 --- /dev/null +++ b/workflow/config/cfis/config_onthefly.mask @@ -0,0 +1,86 @@ +# Mask module configuration file for single-exposure images + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + + +## Border mask +[BORDER_PARAMETERS] + +BORDER_MAKE = True + +BORDER_WIDTH = 50 +BORDER_FLAG_VALUE = 4 + + +## Halo mask +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction spike mask +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier mask +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +TEMP_DIRECTORY = .temp + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False diff --git a/workflow/config/cfis/config_tile_Fe.ini b/workflow/config/cfis/config_tile_Fe.ini new file mode 100644 index 00000000..9546f062 --- /dev/null +++ b/workflow/config/cfis/config_tile_Fe.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for: find exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[FIND_EXPOSURES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Column number of exposure name in FITS header +COLNUM = 3 + +# Prefix to remove from exposure name +EXP_PREFIX = p + diff --git a/workflow/config/cfis/config_tile_Git.ini b/workflow/config/cfis/config_tile_Git.ini new file mode 100644 index 00000000..72a0f0be --- /dev/null +++ b/workflow/config/cfis/config_tile_Git.ini @@ -0,0 +1,93 @@ +# ShapePipe configuration file for: get tile images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = /project/def-mjhudson/unions-wl/tiles, /project/def-mjhudson/unions-wl/tiles + +# Input file pattern including tile number as dummy template +INPUT_FILE_PATTERN = CFIS.000.000.r, CFIS.000.000.r.weight + +# Input file extensions +INPUT_FILE_EXT = .fits, .fits.fz + +# Input numbering scheme, python regexp +INPUT_NUMBERING = \d{3}\.\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Copy command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/data_tiles diff --git a/workflow/config/cfis/config_tile_Mc.ini b/workflow/config/cfis/config_tile_Mc.ini new file mode 100644 index 00000000..50273143 --- /dev/null +++ b/workflow/config/cfis/config_tile_Mc.ini @@ -0,0 +1,80 @@ +# ShapePipe post-run configuration file: create final catalogs, with +# no spread model on input + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_Mc + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = make_cat_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = ./output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MAKE_CAT_RUNNER] + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_Ms/merge_sep_cats_runner/output + +# Input file pattern(s), list of strings with length matching number of expected input file types +# Cannot contain wild cards +FILE_PATTERN = sexcat, galaxy_psf, ngmix + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282) +NUMBER_LIST = $SP_UNIT_NUM + +# FILE_EXT (optional) list of string extensions to identify input files +FILE_EXT = .fits, .sqlite, .fits + +# Numbering convention, string that exemplifies a numbering pattern. +# Matches input single exposures (with 'p' removed) +# Needs to be given in this section, will be updated in module +# sections below +NUMBERING_SCHEME = -000-000 + +SM_DO_CLASSIFICATION = False + +SHAPE_MEASUREMENT_TYPE = ngmix diff --git a/workflow/config/cfis/config_tile_Mh_exp.ini b/workflow/config/cfis/config_tile_Mh_exp.ini new file mode 100644 index 00000000..96512df1 --- /dev/null +++ b/workflow/config/cfis/config_tile_Mh_exp.ini @@ -0,0 +1,76 @@ +# ShapePipe configuration file for merging per-exposure WCS headers +# at the tile level. Input is the exp_numbers file produced by +# find_exposures_runner; EXP_BASE_DIR tells the runner where to find +# the per-exposure split_exp_runner header .npy files. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Mh_exp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = merge_headers_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[MERGE_HEADERS_RUNNER] + +# Input: exp_numbers txt file from find_exposures_runner +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# Tile numbering scheme (RA-Dec, e.g. -301-279) +NUMBERING_SCHEME = -000-000 + +# Root directory containing all per-exposure work directories. +# The runner will walk this tree to collect headers-.npy files. +EXP_BASE_DIR = $SP_EXP diff --git a/workflow/config/cfis/config_tile_Ng_template.ini b/workflow/config/cfis/config_tile_Ng_template.ini new file mode 100644 index 00000000..6e3e3329 --- /dev/null +++ b/workflow/config/cfis/config_tile_Ng_template.ini @@ -0,0 +1,104 @@ +# ShapePipe configuration file for tiles: ngmix + KSB + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +# Per-chunk run dir: the workflow exports SP_NGMIX_CHUNK= for each chunk. +# RUN_NAME is env-expanded (run.py getexpanded); braces keep the trailing "u" +# out of the variable name. +RUN_NAME = run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = ngmix_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Model-fitting shapes with ngmix +[NGMIX_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/psfex_interp_runner/output, $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = sexcat, image_vignet, background_vignet, galaxy_psf, weight_vignet, flag_vignet, log_exp_headers + +FILE_EXT = .fits, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# BKG_RMS_VIGNET_PATH (optional): per-pixel BACKGROUND_RMS vignets, used as +# 1/RMS^2 inverse-variance ngmix weights. When set, the file must exist for +# every tile (missing file -> error, no per-tile fallback); omit the option +# entirely to fall back to the scalar sigma_mad noise estimate. +BKG_RMS_VIGNET_PATH = $SP_RUN/output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output/background_rms_vignet{file_number_string}.sqlite + +# Number of objects to batch save during processing, optional. Omit or set +# to -1 for no batch saving. +# 250 (folded from runs/p3-batch1/cfis): worker RSS grows ~4.8 MB/object +# until the flush recycles it -> peak ~1.1 GB + 250 x 4.8 MB ~= 2.3 GB/worker +# (A/B test, job 17607877). Not superseded by -b {threads} (that sets fork +# width, this bounds per-worker memory). +SAVE_BATCH = 250 + +# Magnitude zero-point +MAG_ZP = 30.0 + +# Pixel scale in arcsec +PIXEL_SCALE = 0.186 + +# SEED_FROM_POSITION: per-object RNG seeded from sky position (ra, dec, ccd) +# instead of one ordered per-tile stream, so results are bit-identical under +# any chunking (D4/#796). Required for the chunked ngmix scatter/gather. +SEED_FROM_POSITION = True + +# ID_OBJ_MIN/MAX: this chunk's closed SExtractor NUMBER-column range, +# computed at execution time from the tile's own object count and expanded +# via ShapePipe's getexpanded (ngmix_runner.py verified: env-expanded, not +# plain getint). +ID_OBJ_MIN = $NGMIX_ID_MIN +ID_OBJ_MAX = $NGMIX_ID_MAX diff --git a/workflow/config/cfis/config_tile_PiViVi.ini b/workflow/config/cfis/config_tile_PiViVi.ini new file mode 100644 index 00000000..246ee18f --- /dev/null +++ b/workflow/config/cfis/config_tile_PiViVi.ini @@ -0,0 +1,172 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# PSFEx PSF model. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_PiViVi + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +#MODULE = psfex_interp_runner, + +MODULE = psfex_interp_runner, vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[PSFEX_INTERP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Column names of position parameters +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# If True, measure and store ellipticity of the PSF +GET_SHAPES = True + +# Number of stars threshold +STAR_THRESH = 20 + +# chi^2 threshold +CHI2_THRESH = 2 + +# Multi-epoch mode parameters + +# Root directory of per-exposure work directories; replaces ME_DOT_PSF_DIR +# for v2.0 per-exposure pipeline. psfex_runner/output/ dirs are discovered +# by scanning $SP_EXP for the exposures listed in the exp_numbers input file. +ME_DOT_PSF_EXP_DIR = $SP_EXP + +# Input psf file pattern +ME_DOT_PSF_PATTERN = star_split_ratio_80 + + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output + +FILE_PATTERN = sexcat, CFIS_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner +ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/workflow/config/cfis/config_tile_Sx.ini b/workflow/config/cfis/config_tile_Sx.ini new file mode 100644 index 00000000..0ce9ea22 --- /dev/null +++ b/workflow/config/cfis/config_tile_Sx.ini @@ -0,0 +1,118 @@ +# ShapePipe configuration file for tile detection + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Sx + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = CFIS_image, CFIS_weight, log_exp_headers + +FILE_EXT = .fits, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_tile.sex +DOT_PARAM_FILE = $SP_CONFIG/default_noimaflags.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = False + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +ZP_FROM_HEADER = False + +BKG_FROM_HEADER = False + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, +# MINIBACK_RMS, -BACKGROUND, #FILTERED, +# OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = XWIN_WORLD,YWIN_WORLD + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/config/cfis/config_tile_Uc.ini b/workflow/config/cfis/config_tile_Uc.ini new file mode 100644 index 00000000..f2ff51e4 --- /dev/null +++ b/workflow/config/cfis/config_tile_Uc.ini @@ -0,0 +1,92 @@ +# ShapePipe configuration file for tile object selection using +# an (external) catalogue + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uc + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = read_ext_sexcat_runner + + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[READ_EXT_SEXCAT_RUNNER] + +# NOTE(blocker): run_sp_tile_Gic (external-catalog get_images) is not one of +# the 13 configs in this sweep -- no committed config produces it, so this +# path is written by naming-convention analogy, unverified. Uc is otherwise +# fully de-wrapper-ified; do not rely on this input until Gic exists. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Gic/get_images_runner/output, $SP_RUN/output/run_sp_tile_Git/get_images_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output + +FILE_PATTERN = CFIS_cat, CFIS_image, log_exp_headers + +FILE_EXT = .cat, .fits, .sqlite + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# File name suffix for the output sextractor files (optional) +SUFFIX = sexcat + +# Side length of the square postage stamp (vignet) extracted from the tile +# image, in pixels (must be odd). Default: 51 +VIGNET_SIZE = 51 + +## Post-processing + +# Necessary for tiles, to enable multi-exposure processing +MAKE_POST_PROCESS = True + +# World coordinate keywords, SExtractor output. Format: KEY_X,KEY_Y +WORLD_POSITION = ALPHA_J2000,DELTA_J2000 + +# Number of pixels in x,y of a CCD. Format: Nx,Ny +CCD_SIZE = 33,2080,1,4612 diff --git a/workflow/config/cfis/config_tile_Uz.ini b/workflow/config/cfis/config_tile_Uz.ini new file mode 100644 index 00000000..fc8550af --- /dev/null +++ b/workflow/config/cfis/config_tile_Uz.ini @@ -0,0 +1,73 @@ +# ShapePipe configuration file for: uncompress FITS image + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uz + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = uncompress_fits_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options +[UNCOMPRESS_FITS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_weight + +FILE_EXT = .fitsfz + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input HDU of image data, optional, default=0 +HDU_DATA = 1 + +# Output file pattern +OUTPUT_PATTERN = CFIS_weight diff --git a/workflow/config/cfis/config_tile_onthefly.mask b/workflow/config/cfis/config_tile_onthefly.mask new file mode 100644 index 00000000..18cf5db2 --- /dev/null +++ b/workflow/config/cfis/config_tile_onthefly.mask @@ -0,0 +1,90 @@ +# Mask module config file for tiles + +## Paths to executables +[PROGRAM_PATH] + +WW_PATH = weightwatcher +WW_CONFIG_FILE = $SP_CONFIG/mask_default/default.ww + +# Indicate cds client executable if no external star catalogue is available +# (e.g. no internet access on run nodes) +CDSCLIENT_PATH = findgsc2.2 + +## Border parameters +[BORDER_PARAMETERS] + +BORDER_MAKE = False + +BORDER_WIDTH = 0 +BORDER_FLAG_VALUE = 4 + + +## Halo parameters +[HALO_PARAMETERS] + +HALO_MAKE = True + +HALO_MASKMODEL_PATH = $SP_CONFIG/mask_default/halo_mask.reg +HALO_MAG_LIM = 13. +HALO_SCALE_FACTOR = 0.05 +HALO_MAG_PIVOT = 13.8 +HALO_FLAG_VALUE = 2 +HALO_REG_FILE = halo.reg + + +## Diffraction pike parameters +[SPIKE_PARAMETERS] + +SPIKE_MAKE = True + +SPIKE_MASKMODEL_PATH = $SP_CONFIG/mask_default/MEGAPRIME_star_i_13.8.reg +SPIKE_MAG_LIM = 18. +SPIKE_SCALE_FACTOR = 0.3 +SPIKE_MAG_PIVOT = 13.8 +SPIKE_FLAG_VALUE = 128 +SPIKE_REG_FILE = spike.reg + + +## Messier parameters +[MESSIER_PARAMETERS] + +MESSIER_MAKE = True + +MESSIER_CAT_PATH = $SP_CONFIG/mask_default/Messier_catalog_updated.fits +MESSIER_PIXEL_SCALE = 0.187 +MESSIER_SIZE_PLUS = 0. +MESSIER_FLAG_VALUE = 16 + +## NGC mask +[NGC_PARAMETERS] + +NGC_MAKE = True + +NGC_CAT_PATH = $SP_CONFIG/mask_default/ngc_cat.fits +NGC_SIZE_PLUS = 0. +NGC_FLAG_VALUE = 32 + + +## External flag +[EXTERNAL_FLAG] + +EF_MAKE = False + + +## Missing data parameters +[MD_PARAMETERS] + +MD_MAKE = False + +MD_THRESH_FLAG = 0.3 +MD_THRESH_REMOVE = 0.75 +MD_REMOVE = False + + +## Other parameters +[OTHER] + +KEEP_REG_FILE = False +KEEP_INDIVIDUAL_MASK = False + +TEMP_DIRECTORY = .temp_tiles diff --git a/workflow/config/cfis/default.conv b/workflow/config/cfis/default.conv new file mode 100644 index 00000000..2590b9cb --- /dev/null +++ b/workflow/config/cfis/default.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/workflow/config/cfis/default.param b/workflow/config/cfis/default.param new file mode 100644 index 00000000..09ad8405 --- /dev/null +++ b/workflow/config/cfis/default.param @@ -0,0 +1,68 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +# The following flag requires a flag image +IMAFLAGS_ISO #FLAG-image flags OR'ed over the iso. profile !!! REQUIRE FLAG_IMAGE !!! + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default.psfex b/workflow/config/cfis/default.psfex new file mode 100644 index 00000000..a9d1a906 --- /dev/null +++ b/workflow/config/cfis/default.psfex @@ -0,0 +1,85 @@ +# Default configuration file for PSFEx 3.17.1 +# EB 2017-11-30 +# + +#-------------------------------- PSF model ---------------------------------- + +BASIS_TYPE PIXEL # NONE, PIXEL, GAUSS-LAGUERRE or FILE +BASIS_NUMBER 20 # Basis number or parameter +BASIS_NAME basis.fits # Basis filename (FITS data-cube) +BASIS_SCALE 1.0 # Gauss-Laguerre beta parameter +NEWBASIS_TYPE NONE # Create new basis: NONE, PCA_INDEPENDENT + # or PCA_COMMON +NEWBASIS_NUMBER 8 # Number of new basis vectors +PSF_SAMPLING 1. # Sampling step in pixel units (0.0 = auto) +PSF_PIXELSIZE 1.0 # Effective pixel size in pixel step units +PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values +PSF_SIZE 51,51 # Image size of the PSF model +PSF_RECENTER N # Allow recentering of PSF-candidates Y/N ? +MEF_TYPE INDEPENDENT # INDEPENDENT or COMMON + +#------------------------- Point source measurements ------------------------- + +CENTER_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue parameters for source pre-centering +PHOTFLUX_KEY FLUX_AUTO # Catalogue parameter for photometric norm. +PHOTFLUXERR_KEY FLUXERR_AUTO # Catalogue parameter for photometric error + +#----------------------------- PSF variability ------------------------------- + +PSFVAR_KEYS XWIN_IMAGE,YWIN_IMAGE # Catalogue or FITS (preceded by :) params +PSFVAR_GROUPS 1,1 # Group tag for each context key +PSFVAR_DEGREES 2 # Polynom degree for each group +PSFVAR_NSNAP 9 # Number of PSF snapshots per axis +HIDDENMEF_TYPE COMMON # INDEPENDENT or COMMON +STABILITY_TYPE EXPOSURE # EXPOSURE or SEQUENCE + +#----------------------------- Sample selection ------------------------------ + +SAMPLE_AUTOSELECT N # Automatically select the FWHM (Y/N) ? + +BADPIXEL_FILTER N # Filter bad-pixels in samples (Y/N) ? +BADPIXEL_NMAX 0 # Maximum number of bad pixels allowed + +#----------------------- PSF homogeneisation kernel -------------------------- + +HOMOBASIS_TYPE NONE # NONE or GAUSS-LAGUERRE +HOMOBASIS_NUMBER 10 # Kernel basis number or parameter +HOMOBASIS_SCALE 1.0 # GAUSS-LAGUERRE beta parameter +HOMOPSF_PARAMS 2.0, 3.0 # Moffat parameters of the idealised PSF +HOMOKERNEL_DIR # Where to write kernels (empty=same as input) +HOMOKERNEL_SUFFIX .homo.fits # Filename extension for homogenisation kernels + +#----------------------------- Output catalogs ------------------------------- + +OUTCAT_TYPE FITS_LDAC # NONE, ASCII_HEAD, ASCII, FITS_LDAC + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_DEV NULL # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_RES 0 # Check-plot resolution (0 = default) +CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? +CHECKPLOT_TYPE NONE # FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS +CHECKPLOT_TYPE FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS + # or NONE +CHECKPLOT_NAME fwhm, ellipticity, counts, countfrac, chi2, resi + +#------------------------------ Check-Images --------------------------------- + +# Note: Check-image types can be set the ShapePipe config file, psfex_runner section +####### +CHECKIMAGE_TYPE NONE # CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS + # or MOFFAT,-MOFFAT,-SYMMETRICAL +#CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits + # Check-image filenames +#CHECKIMAGE_CUBE N # Save check-images as datacubes (Y/N) ? + +#----------------------------- Miscellaneous --------------------------------- + +PSF_SUFFIX .psf # Filename extension for output PSF filename +VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of PSFEx + # 0 = automatic diff --git a/workflow/config/cfis/default_exp.sex b/workflow/config/cfis/default_exp.sex new file mode 100644 index 00000000..b87275ec --- /dev/null +++ b/workflow/config/cfis/default_exp.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.001 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE AUTO # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/default_noimaflags.param b/workflow/config/cfis/default_noimaflags.param new file mode 100644 index 00000000..b251f5b7 --- /dev/null +++ b/workflow/config/cfis/default_noimaflags.param @@ -0,0 +1,65 @@ +NUMBER #Running object number +EXT_NUMBER #FITS extension number + +FLUX_AUTO #Flux within a Kron-like elliptical aperture [count] +FLUXERR_AUTO #RMS error for AUTO flux [count] +MAG_AUTO #Kron-like elliptical aperture magnitude [mag] +MAGERR_AUTO #RMS error for AUTO magnitude [mag] +FLUX_WIN #Gaussian-weighted flux [count] +FLUXERR_WIN #RMS error for WIN flux [count] +MAG_WIN #Gaussian-weighted magnitude [mag] +MAGERR_WIN #RMS error for MAG_WIN [mag] +FLUX_APER(1) +FLUXERR_APER(1) + +FLUX_RADIUS #Fraction-of-light radii [pixel] + +SNR_WIN #Gaussian-weighted SNR + +BACKGROUND #Background at centroid position [count] +THRESHOLD #Detection threshold above background [count] + +X_IMAGE #Object position along x [pixel] +Y_IMAGE #Object position along y [pixel] + +X_WORLD #Barycenter position along world x axis [deg] +Y_WORLD #Barycenter position along world y axis [deg] + +X2_IMAGE #Variance along x [pixel**2] +Y2_IMAGE #Variance along y [pixel**2] +XY_IMAGE #Covariance between x and y [pixel**2] +ERRX2_IMAGE #Variance of position along x [pixel**2] +ERRY2_IMAGE #Variance of position along y [pixel**2] +ERRXY_IMAGE #Covariance of position between x and y [pixel**2] + +XWIN_IMAGE #Windowed position estimate along x [pixel] +YWIN_IMAGE #Windowed position estimate along y [pixel] + +XWIN_WORLD #Windowed position along world x axis [deg] +YWIN_WORLD #Windowed position along world y axis [deg] + +X2WIN_IMAGE #Windowed variance along x [pixel**2] +Y2WIN_IMAGE #Windowed variance along y [pixel**2] +XYWIN_IMAGE #Windowed covariance between x and y [pixel**2] +ERRX2WIN_IMAGE #Variance of windowed pos along x [pixel**2] +ERRY2WIN_IMAGE #Variance of windowed pos along y [pixel**2] +ERRXYWIN_IMAGE #Covariance of windowed pos between x and y [pixel**2] + +MU_THRESHOLD #Analysis threshold above background [mag * arcsec**(-2)] +MU_MAX #Peak surface brightness above background [mag * arcsec**(-2)] + +FLAGS #Extraction flags +FLAGS_WIN #Flags for WINdowed parameters + +FWHM_IMAGE #FWHM assuming a gaussian core [pixel] +FWHM_WORLD #FWHM assuming a gaussian core [deg] +ELONGATION #A_IMAGE/B_IMAGE +ELLIPTICITY #1 - B_IMAGE/A_IMAGE + +VIGNET(51,51) #Pixel data around detection [count] + +# For GaaP photometry +A_WORLD +B_WORLD +THETA_J2000 + diff --git a/workflow/config/cfis/default_tile.sex b/workflow/config/cfis/default_tile.sex new file mode 100644 index 00000000..ff3b2521 --- /dev/null +++ b/workflow/config/cfis/default_tile.sex @@ -0,0 +1,133 @@ +# Default configuration file for SExtractor 2.19.5 +# EB 2017-11-30 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_TYPE FITS_LDAC + +PARAMETERS_NAME default.param + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # min. # of pixels above threshold +DETECT_MAXAREA 0 # max. # of pixels above threshold (0=unlimited) +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) + # or ABSOLUTE (in ADUs) +DETECT_THRESH 1.5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 1.5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv +FILTER_THRESH # Threshold[s] for retina filtering + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.0005 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +RESCALE_WEIGHTS Y # Rescale input weights/variances (Y/N)? +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + +#-------------------------------- FLAGging ----------------------------------- + +FLAG_IMAGE flag.fits # filename for an input FLAG-image +FLAG_TYPE OR # flag pixel combination: OR, AND, MIN, MAX + # or MOST + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # +PHOT_AUTOAPERS 0.0,0.0 # , minimum apertures + # for MAG_AUTO and MAG_PETRO +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) + +MAG_ZEROPOINT 30.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) + +GAIN_KEY GAIN # keyword for detector gain in e-/ADU +PIXEL_SCALE 0. # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.6 # stellar FWHM in arcsec +STARNNW_NAME default.nnw + +#------------------------------ Background ----------------------------------- + +BACK_TYPE MANUAL # AUTO or MANUAL +BACK_VALUE 0.0 # Default background value in MANUAL mode +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL +BACKPHOTO_THICK 24 # thickness of the background LOCAL annulus +BACK_FILTTHRESH 0.0 # Threshold above which the background- + # map filter operates + +#------------------------------ Check Image ---------------------------------- + +####### +## AG : This parameter is set in pipeline config file. +####### +# CHECKIMAGE_TYPE NONE #BACKGROUND_RMS,BACKGROUND +# can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +# CHECKIMAGE_NAME check.fits,back.fits +# Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#------------------------------- ASSOCiation --------------------------------- + +ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +ASSOC_DATA 2,3,4 # columns of the data to replicate (0=all) +ASSOC_PARAMS 2,3,4 # columns of xpos,ypos[,mag] +ASSOCCOORD_TYPE PIXEL # ASSOC coordinates: PIXEL or WORLD +ASSOC_RADIUS 2.0 # cross-matching radius (pixels) +ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +HEADER_SUFFIX .head # Filename extension for additional headers +WRITE_XML N # Write XML file (Y/N)? + +NTHREADS 1 # 1 single thread + +FITS_UNSIGNED N # Treat FITS integer values as unsigned (Y/N)? +INTERP_MAXXLAG 16 # Max. lag along X for 0-weight interpolation +INTERP_MAXYLAG 16 # Max. lag along Y for 0-weight interpolation +INTERP_TYPE ALL # Interpolation type: NONE, VAR_ONLY or ALL + +#--------------------------- Experimental Stuff ----------------------------- + +#PSF_NAME default.psf # File containing the PSF model +#PSF_NMAX 1 # Max.number of PSFs fitted simultaneously +#PATTERN_TYPE RINGS-HARMONIC # can RINGS-QUADPOLE, RINGS-OCTOPOLE, + # RINGS-HARMONICS or GAUSS-LAGUERRE +#SOM_NAME default.som # File containing Self-Organizing Map weights diff --git a/workflow/config/cfis/final_cat.param b/workflow/config/cfis/final_cat.param new file mode 100644 index 00000000..8ddfebd3 --- /dev/null +++ b/workflow/config/cfis/final_cat.param @@ -0,0 +1,114 @@ +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +# Can maybe be removed. +TILE_ID + +# flags +FLAGS +IMAFLAGS_ISO +NGMIX_MCAL_FLAGS + +# PSF ellipticity (original image PSF) +NGMIX_G1_PSF_ORIG_NOSHEAR +NGMIX_G2_PSF_ORIG_NOSHEAR + +# spread class +#SPREAD_CLASS + +# spread model flag and error +#SPREAD_MODEL +#SPREADERR_MODEL + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_G1_1M +NGMIX_G2_1M +NGMIX_G1_1P +NGMIX_G2_1P +NGMIX_G1_2M +NGMIX_G2_2M +NGMIX_G1_2P +NGMIX_G2_2P +NGMIX_G1_NOSHEAR +NGMIX_G2_NOSHEAR +#NGMIX_G1_ERR_1M +#NGMIX_G2_ERR_1M +#NGMIX_G1_ERR_1P +#NGMIX_G2_ERR_1P +#NGMIX_G1_ERR_2M +#NGMIX_G2_ERR_2M +#NGMIX_G1_ERR_2P +#NGMIX_G2_ERR_2P +NGMIX_G1_ERR_NOSHEAR +NGMIX_G2_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR +NGMIX_T_PSF_RECONV_1M +NGMIX_T_PSF_RECONV_1P +NGMIX_T_PSF_RECONV_2M +NGMIX_T_PSF_RECONV_2P +NGMIX_T_PSF_RECONV_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# magnitudes +MAG_AUTO +MAGERR_AUTO +MAG_WIN +MAGERR_WIN +FLUX_AUTO +FLUXERR_AUTO +FLUX_APER +FLUXERR_APER +FLUX_RADIUS + +# SNR from SExtractor +SNR_WIN + +FWHM_IMAGE +FWHM_WORLD + +# PSF size measured on original image +NGMIX_T_PSF_ORIG_NOSHEAR + +# PSF size measured on reconvolved image +# NGMIX_T_PSF_RECONV_NOSHEAR + +# ngmix moment failure flag +NGMIX_MOM_FAIL diff --git a/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg b/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg new file mode 100644 index 00000000..4e4164aa --- /dev/null +++ b/workflow/config/cfis/mask_default/MEGAPRIME_star_i_13.8.reg @@ -0,0 +1,24 @@ +-11.5 68 +-6 186.5 +7 188 +10 64.5 +31 55 +50 38.5 +56.5 11.5 +188 8 +192 -4 +59.5 -11.5 +45 -33 +13.5 -64 +5 -154 +-6 -155 +-11 -64.5 +-40 -44.5 +-51.5 -30.5 +-62.5 -22.5 +-68 -9.5 +-177 -2 +-176 3 +-78 12.5 +-67.5 14.5 +-38.5 50 diff --git a/workflow/config/cfis/mask_default/Messier_catalog.npy b/workflow/config/cfis/mask_default/Messier_catalog.npy new file mode 100644 index 0000000000000000000000000000000000000000..ef07eb032b08de4fcf508418524692e27f92ad75 GIT binary patch literal 4209 zcmb7{dvKIj6~H$S0z_V+Av_zF=Wf!F0D%A@VJ{fCEf+Q}Ge}K(fi%99nrA zDunWq77DdgDz+6YRTR<6s3U8&j-3Igj&%@bimd}oA2U?zlbKdM_uP9w(m(ve`$y*c zzCFM5y62pGb0i#Dx@=XB?^WN{Ks?dak_puMR|f+dThsACg+H)u`P4vJg+CbRYTc6H z4~^zMo@mjpb;;T0Jrn)RYq*17Tj|%eB$n*RU?!Sg*O7@Qw1q{{wyp$y?@C2G6Z9UO zJ*|SwZ1?}atFSw_CB3P=b8}@&M!PQkRQogQ9WJxR1Qh>397tE&bDwclCSdu1(AH6 z4(J=nqclLjNNy$y{Uf>I4k$$U+;$icOckc`Be|`!t%iZDp)MQ_Fa8fL{-b;tQzLp( z1zK|XR4X~i+6ssD2bS^mr21)@q|-L|e`)L2lj@%utcLWY`f8xj0Yf~cz08Nmysu}I z@{^g-7~8uRih{w2Z!HWBN-@I6ONt@pAt6OgeuG|8G~c*H>$1v0xQBsaDf~%ZQ5pK+ zj9#xu(RJZAt&dbL!Y~#qk)qmYi%L-!8?ToXsYiGI;Jwj}Di2{ei;a+?!V|0AWsRl> zym+Os9#lTUNERCeCHYt zJUw6aL%5fLQZM+Z%ud_-6Iwr&qT_?AH)5Y0Q~eRfvY5YD%-O+3yrf8%k}`2kQ5lS5 zF$!?}vw9tm%8M`a(@7E=S+VZ$>vz-ugb6G*QHqIP%nRl)Kw>5JofNeih%kx8CQH%J ziDe7S6Z!TxZ4=cXgaC_C$oH}-GAe=*UV1+mP=gW5Sgf4nY({85j5bP5FwOf$skxcg z`ew}uH3Z>41}X@U&gz10ej=di1-q4`b^hq(-Jdoa+e?F#{-Xl{Y#R$_Fm@dV%C0vB0w`Q0Pu=9N?r10aa z1fiP6W{_UL)7z_%e`NK-A8(x7p+->k!%PNhq!_BzUb;+rXK_@tft|MwtqK197itW` zEEbzBMTsX?^IN<3T|d4!Y5&KoR4Kw77ON%ms|DMaqS5$4ebivn6}t8M9V&xgsj&!i z8K@&*oT+IkM%@MMhIoJxJ1e4yYTsCB3-YHfhMU}t3w%~Us)i{I@1A>5M z-PR@B7L@6soDj-%^Uc>N)_z{GGL!si)iAfUj; z5chgC? z3g;oLX0bJ-x6I~7j4@k7dKc!~37<-tWQGvdGVllib}C9S%nNnqSzCL_=S?yN!a4@l z6EN0g|6g7*%bn5qnc^)}C2BrGlz|ul+q+-jSP-A>&68b=j-t$X$xCiGfE6@H@}#1%&`}s!}C4|3xRHS%ncEW8jNY^fzTg z8JXpd^5&~XtPx=|i*1pj&>|L+d5Nq!9g?sS0L0-4w@4|Wsp3qZx=0U6~aLVzCyqRm{F~3&hHc>Jfya3>+iC-sHUVhk$^KK6XW4(DKwegl8D|DgmX=&0@1}8gR!+><@H9 z+WAAZ9^qLAjuT*e#Csq>?XndVKW$FmD8h3Le2oCVEeJOk2$!g6{6`KDwc_*QyqTuQPCx0DJ5Z>D8!r%S;}ikXzqWafEL$@B#sK&c5wX z9YclwMt!znzPT4B5ME^9B?1bpOTA8Mbw^hpRnUe7=8}*^_$C9V2=Fdu-YK1(=oR%* zt|gU1IL*MfJ`;oX=kB5NT zfba?f-uzXxoJl@$H4aq@J{Kw+tpw#gl=-AJ5(Csd;(*Y{{>v*ks<&9 literal 0 HcmV?d00001 diff --git a/workflow/config/cfis/mask_default/Messier_catalog_updated.fits b/workflow/config/cfis/mask_default/Messier_catalog_updated.fits new file mode 100644 index 0000000000000000000000000000000000000000..6a9f00096170565aeb7fab3429c2256ce56f3533 GIT binary patch literal 8640 zcmeHL4NO&K82bxk?Xcg61^5lKgn(d%puWZoviN(qOUV;yoAdgbQ9S23R5C zGJc%mUqCidTB!L;nwUE8!IV-Emzk_l)0xbfKNvc7R`!19<}PlRM&quncFy*U!+YNM z`JV52-?M$^Ch6l6Vxt8?D|on!!gOJNj?I!|&(0GZIf71aOcL@OCY#x0H+#s(5YrnI z^eKJtg(9EHZZ{PQW|PAtI0|zm&n)h!0aEly_^(HOFCm{TKRZLR3ptjD%+_qlmS@ee z<+(|E-9@JuqYY7g@ELn0_VZRD>u0W-raGWHpgN#BpgN#BpgN#B@ZWcUA19Oa@dm9B z$n*vyu-NFpUb~tuWLRw`d!gVkWn@W$RN!zOPJ5+1di$@QpJ)8@^Rsdu5B7Y14xWX4 zS&}W&v4Ee)%@0rF0_OL1DSR_L^MwZY!H4;MT?t}*vH{QW{rFzUwCCjK3b`h`DO++# zcF*4nulyY!*XREJQ20EvxYZhU`smoGB+vRn9^rG1jKwNtnfqonipd7O(vLv1l&P%K zd;Z)D_w!ESQ~KdWJ`L|jpvgWz+iDYpf$N*;MLuI{LbS_2yUFFz!1U?j6XRTb_*Jc3 z1LNcR9`<9t7x=h77oYd^x%!)DEt2Npr~knA%k{bVyr<7KFN(fYZ|HOJc~4)6GS3UG zHswc^f&1s;18>Zq`uoOTE~2Q~q|Mla|%g za-h=JE%N?j_uw*?bnuLd%4718+1H`mV&Nd-x*iGk9UB7p&}C4$PlbH^_CmVeV&Nd- z=89JNK<*p#3k)C#Ou1FXUhvhhA8^?Mtb-}voMsd9Z@}Xk4L!{R2OH5KblZ)dP$=zM!+^*|xtz<#td_j+GQ1-&6ZGat%$AEx3wsX?S2kT*O z(P(-PbvTh(obuFT5Z{>yK3MWT-;4Ig4*6`)85)KG1QE-xCN>W;=W>HMh*)Vk)ja6M z@syJyu$7@FVSU62dK5DdMBK3RxExb`isC~K0*QVOCn9^$ zbp(OdhJ(xyr{#o-Fcz?5&!N=z7J3m$omj$lnD%Km2LvbsqXagOsg}PA-bQ25LIe>V zY_+_lgL<|@I z07LL-iY?$qV<2@4R4q?dbnI3#Dr+G=U=KWr-9NxdBlb3dj*Wz&SaOD!L_8P^mIHNs zdT16GHv7tAi0Nvf^(agkZEeVSNc<=n#-hoUN=9c0WOjG+Cb?dqlKPu$YDEYqwBwk(z^jfXBY2yG-{#>NlWyeBp( z@5@r2p?zSM!|o1OUcz)9@V|8#^oBHrL)mTU;l)flYcOm;Vdr^9f&XuELZ^<7!rdZ> z7d$Tc4;qc|FXe?ZLlMI-okBI`F@A;7=F| BE!zM9 literal 0 HcmV?d00001 diff --git a/workflow/config/cfis/mask_default/default.ww b/workflow/config/cfis/mask_default/default.ww new file mode 100644 index 00000000..c2797f90 --- /dev/null +++ b/workflow/config/cfis/mask_default/default.ww @@ -0,0 +1,40 @@ +#--------------------------------- Weights ------------------------------------ + +WEIGHT_NAMES weightin.fits # Filename(s) of the input WEIGHT map(s) + +WEIGHT_MIN 0. # Pixel below those thresholds will be flagged +WEIGHT_MAX 1000. # Pixels above those thresholds will be flagged +WEIGHT_OUTFLAGS 1 # FLAG values for thresholded pixels + +#---------------------------------- Flags ------------------------------------- + +FLAG_NAMES flagin.fits # Filename(s) of the input FLAG map(s) + +FLAG_WMASKS 0xff # Bits which will nullify the WEIGHT-map pixels +FLAG_MASKS 0x01 # Bits which will be converted as output FLAGs +FLAG_OUTFLAGS 2 # Translation of the FLAG_MASKS bits + +#---------------------------------- Polygons ---------------------------------- + +POLY_NAMES "" # Filename(s) of input DS9 regions +POLY_OUTFLAGS # FLAG values for polygon masks +POLY_OUTWEIGHTS 0.0 # Weight values for polygon masks +POLY_INTERSECT Y # Use inclusive OR for polygon intersects (Y/N)? + +#---------------------------------- Output ------------------------------------ + +OUTWEIGHT_NAME "w.fits" # Output WEIGHT-map filename +OUTFLAG_NAME flag.fits # Output FLAG-map filename + +#----------------------------- Miscellaneous --------------------------------- + +GETAREA N # Compute area for flags and weights (Y/N)? +GETAREA_WEIGHT 0.0 # Weight threshold for area computation +GETAREA_FLAGS 1 # Bit mask for flag pixels not counted in area +MEMORY_BUFSIZE 256 # Buffer size in lines +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME ww.xml # Filename for XML output +XSL_URL file:///usr/local/share/weightwatcher/ww.xsl + # Filename for XSL style-sheet +NTHREADS 1 # 1 single thread \ No newline at end of file diff --git a/workflow/config/cfis/mask_default/halo_mask.reg b/workflow/config/cfis/mask_default/halo_mask.reg new file mode 100644 index 00000000..c44f2516 --- /dev/null +++ b/workflow/config/cfis/mask_default/halo_mask.reg @@ -0,0 +1,50 @@ + 274.66813 -1.25966 + 272.54579 32.47406 + 266.21222 65.67579 + 255.76731 97.82190 + 241.37579 128.40544 + 223.26462 156.94408 + 201.71942 182.98775 + 177.07997 206.12573 + 149.73486 225.99312 + 120.11532 242.27660 + 88.68848 254.71937 + 55.94996 263.12519 + 22.41606 267.36151 + -11.38436 267.36151 + -44.91826 263.12519 + -77.65678 254.71937 +-109.08362 242.27660 +-138.70315 225.99312 +-166.04827 206.12573 +-190.68772 182.98775 +-212.23292 156.94408 +-230.34409 128.40544 +-244.73561 97.82190 +-255.18052 65.67579 +-261.51409 32.47406 +-263.63643 -1.25966 +-261.51409 -34.99339 +-255.18052 -68.19511 +-244.73561 -100.34123 +-230.34409 -130.92476 +-212.23292 -159.46341 +-190.68772 -185.50708 +-166.04827 -208.64506 +-138.70315 -228.51245 +-109.08362 -244.79593 + -77.65678 -257.23870 + -44.91826 -265.64452 + -11.38436 -269.88084 + 22.41606 -269.88084 + 55.94996 -265.64452 + 88.68848 -257.23870 + 120.11532 -244.79593 + 149.73486 -228.51245 + 177.07997 -208.64506 + 201.71942 -185.50708 + 223.26462 -159.46341 + 241.37579 -130.92476 + 255.76731 -100.34123 + 266.21222 -68.19511 + 272.54579 -34.99339 diff --git a/workflow/config/cfis/mask_default/ngc_cat.fits b/workflow/config/cfis/mask_default/ngc_cat.fits new file mode 100644 index 0000000000000000000000000000000000000000..f51546da7fc88ae1ca919a3c13882fe1a7df6768 GIT binary patch literal 201600 zcmeGF2~?NW_CJoJC^(`d4mctzj+u&vn#6MsJc>gOJW`TaIFXu~17=1VYN#lVC{9`A zfF^3U*7~iL>ptto=A6A> zdpOfR16KEE`t~vd0zw1+=3oDSCjy4PGB*B|31cP(B)k&PwR?XnU}D1HvBL&W81^^i zL6Y73_w7F5Kk(}aeuF1W7(69l*x-c00SQya4gY&CZfb8apu2@1{}x{x@Ebd6%#h&| z0$z!CaUMBl_}GafUl}{m&EzltWe(_X_Kx@u{QCc;;Q#PHnc4ra>_5f+vA{nT_{ReO zSl}NE{9}QCEbxy7{=aVlnoe5X`}7VCXezt+?k{97v*};7dLm%R$gzVbObJLBJY@9n zfZ?wt=*j6{iu|Ad`Ty$tx!wQ&^JkuqxBdJ1bK4N`8$EpNO9>-rK0eGf6AwW7|MZXf zwfh@>ZQHi*(EdO0L;3&ouWxvt=-!ye{}0v+FHLx5(zt+egC`6gGdyAVguh=ebo>9s zukHU~Kj!!MT-<2;ckOQWim?73Ux)vLpAMP$k;6w1`_Ekahe!AB&i-gRZ1_to=dbbS z7W_Z_FXqSo_%HGcCVw;?JYndVkz)e_{`dIW{}=i7kLhb_`%M_EZS=px*R@a7GnyY( z)%@>&$B*#2{4w;uz>n~0e*Zmu+P@P=zCQd#EdBp?`Vv0P@4tsn#|z_&`ETIU{Qi6R zI&eI{JaQ~2RsQ$()BMDLBmVx$ZXxzixrB3$GW6Q(0ez6|7tpwsPev zE9_$Zz2elE+^{4Mk1&5CT)25il=*6ESZeMuO}OIgE6V;pk$!lZtK9R-la(jFLG{Ce zgM&?@R`xVmw#zfjG&lE{X_QTFpo;r{VpIYT4-XH^@R^e$4$l1-Ges54DBmR2@g6(K z08_?cRP44*R{!E(VYY_{OED^5k1^7gTU-U;EThuiC|UO7iAJ^HUxETctHr1ZQx=%Kcc z>e6aXiYz-~7y%<~Eu(H>f;j#D?66GD)H3QnxkmhJSO7Bx5rmtE)MtC1F!n(?CNwx6 z;tccqBy1<3L#$~8U8p0z+%$}G(2~t@Ljv7JmJ`7|UZzeZ5xO&1b%ycChkTEkj*#08u z1Gi0Q8in>3eKv%}$H!}6b9P-&&wXxi37FF|7NCiRz)1zJ){t5u)GmYsGGQu}SpIc} z+EXEvLmO30W5q_H_DtR9YJ`@t_>XjPZTfC%JWq3p#cMppwecNDq$VsG(^*_wn#fha z)HIeL=&r$0umx2Efq1;Ph{dwS(gFj)v=LVUeA7rfGL);JKIvl_OP3rL*Sg+(pRn$*MJnDbjw~neYf%DumDycd@N&)SDq-I(u$P>({Ll>$u(-1_>-aM z64G_!)`wXOI=OASZgW;xbaXUV2`1ledNEDxO4&kfMYost6MV#u?>}Z;n8*9hLb21= zmuwLVf@NeOaCbtijrhQ48d>r2VwZn8cYAr1P!uV4Rj$FVgtp*uJVfk5UkD6Zx%baz zJ96glRDy`zjm=k!9l5ZYyS-$Ej(6nyVQf>NwTx|Gy0iVWVK;Q|U>RGUIO51R_b`Et zydApiG)Hi&NH!l@A( zO!(x~Y4u{Sr`&$|!YCBI8<}p;efo8Vy&gSGyf%(1pUvGVWI1sCEHIs*7+BJ^!xPi^yWs?kwXT2H@Q-yKogI z{OOlyufM#Bbs^}Y%xBxA^d2rOj170uDT(r4dFs#ZHw}VLRfIR9@aB=544n)l%2&Lu zPLEq}OhMa+2`o_#eNdfl+l+~@M=n))giyX`E!AUMPeuywo1((zEY(W<$=y~a!a-E5 zc}{sfJH?2@phe1w3bD$hiy(&F$z`8CnqVixkC|^XXUAn9*lk1qQqlM z4y#=`dq{mEb3}!vUvc2F)&1vh6qk++A;*}glV3rKn0^Af7(b%z3;%6?Mi1~H=*+af=O{HQ zX$d`9*~>RgXeRVxl=9&FRPrFN^Tq2o!cwnyFb z@nM5Hdoxnjw%$|wjcCl2NGz(DqF%=+L+Xxzh^DANI8s%s(}yEjS9#({*hsV}*g_+% z7ZwyY3=N2+;Il-%S$Re<)CWWQEK$Gx1tapCpP9fV>h}#XI--LuK#3$$uD+EK7K`5U zJeb-{fVM-r&U43DeN1^`(a{5g_I`a#zAX)#nusMDPH82B4(to`9x^iE7cd8~e-|5o!87jd<{ax_ zY>LL4Zi$)d&xc)(XmaYbIQr))HZtqdNG^2ZzU$*|$W6YSp*F>SY{bVu}_=hRQ~-uHgymaniOq8qk1M2iPwFW{DQR z`N$^K284(Duad8l`Yo5Alcl5Da|HRXT-8^!7`Hy>5Y{h1Nxm^f%e8@GzgHeR3ihx> ztG*#(f3*SJIYO)@n&%!<2fS~SE;^E1qGQ{SUJ4?0IJ6#HEPmTjm3c7J)=h;Nj`%2% z0vt0(sr{JqXQB#+L(B9wYJcp<-1b?@<7iQ#eLrU&NfZu`f8YF|N8}0agIKx8cU^EI zKW<@o06JG`m*ioOTbIv%0nrle1I8GKR{!Y2VTq0vuZ!ZXGu`13;=f8w!Hf+kbrg0S zbUNpI48Q~o3R+fY(`cr6!Ek7sTK0J@-{5L3IjC!8p z`#;jTMKScJ+h0&$^g1RkAfyVYO6WJy>i4+D+!rz-tf^2T(|Q_RF+*uZVvRa|M46l* zcb8*eJ$EX+q09Ry@#Gr2Efx};blbOtv@*o0l-lG1?S}A1`QpK*Ta1)-3GeK!`eEFM z#dQM`!?K-P*B)b6t|VaHo;V^))opITsKnS3kr`v`Ow9Un!IXLQXm(3g7}NRT#98z> zS}W(b`8BvdWnFr9_O`#rkmLmu`V)^{_4nFE$3xsbOs`(Y>@zi#I~@9=qu)KggFD8a zX_BhSVS4 z*Qj|!VNG}wa|if@BES^=e@IhNYdo0;J1QEJyeLfa0GkYLis(PmRn($4*2IGJmEHMIesT(7)XvvquC6tF#(z=(k#z zSF8;C`kQ0PFFYF;uZ9KL*U=4sjY!TQH%vgke)IVR9->*f3H>hH-=RZ+Z*|a3#MJt+ zdwp3fBw6B7LaR9S*q^6dNW;ZM@c8z_imnD^ijm9DseNc-2sn*&WcD<*FYgm}WiMhn zY4T+Id(2=-IVOybj&>G3e?Y_`)-iE4#jDR%u=A%JWaV-hM}m*j|6UOrnIn1f;7Gdw z10P&Td`vO<)4O&7G9>t|Zu^r96LM~Ux11ftQeIo#$DUo#k2`uMOj(zob8l)h4hNPp zwMHAULoYS7J>Hmf$o^uW7rC;80O&uw)09mJv6h&c;A4MLn853(0wN?W3dsrIv6jWA zQUgjHTkOmXv8glB?e?wYyXq}0iu`piWr^vr&2yfa@hwLoGfjGDuTu%YAXa;S=6jJ! zR^G^s(1To=^ih%Xcc~>v>M{ZFO)$G=g*$9{%70VuAF6igpKhW#TH{6a}MW8N&tYNZiFeF zi)8(@0X*RBB&2gr@0Dc`QYB0+F()#`SoB9ai$yg{q?AjMWyTG3$5;xcaHTP8gC5$q zDR-h|={YA{Vl6QjQ(DhbyWMSW?zvRy^Ftc1_~J<0SxDQ?vzKrW1NAL2KV-5jlhmE6 z=%AZdr;k&m<5@OxDp_p)%|o)x+s!$WVfCS6am89T^02HNw6(+n_@zw3vxHRhNNrv0 zRK8Ww-5M6OE>?@%40J^ym|{8m>nA4-5A7&REb4bzdQ24SyHottWzz+9fgb6oIbb|u?_g)9qafvV zZ)e2)W<1`q=N6aR?TpC#gbbvarv2H|C^NqXG1LZHda$Q)VEsMj0W6kCYn<)WTn zz`!?_SiVt+I}m%r<)~#?Cfr0K(!Qd<3Sdg`&9 z<48l{x5U^KD116Kf~pqeF? zVryW~s+Qb5Y_iAFuQQxM>B_Ck@=M9a24tQH80^|tq+w#TWLdNrhi)97Dc(lzA~lH< zOu8RiwY9V3J*XF3j6L`EAWviV(@VLD0lq2LTw5<@Kx~pr{Z$i#otM`waN~IA**1pN z{U#F%D1zQy<0(CT*STZ;UH?^L%nh9`1yhv3)cpA;j__PtS3wGT^b*}$vKumPt&yd_ ziFU;ps^le#w_=;~1Rly+BBN6)lA!5vtp>F6ISVktb@OOJ|?bgn+ z;cw4!n`3P=dR|vMv73M&wD;F0-!&$O*xdDTiT4_3tNPX7VIJ&;wU|3pi~Ep)p@byFkE)v%m}`E+|(U-O)ig%lEsYu~Qj z66cgdNI@id4;E{Y<#@xDH$Y%r)=vyJwm$c^yUP0BGr}fxooB?LPfjIAZTL1vmf3#A zfMUAJhEu0y*)xG0Fw7%!$bIQmS0~)rV(X7y&w24mt~>pc`C^(Z3tfl@K1*Z{36y0| zDh`-TGU0|kvBnlmN5PPwn<6tONS57S)>Q>1uv)8%IUPW0T{5r7$g)`QLL|XQmEt3^ z%<<3N_So>V5H*qQp%P$fiH*_GvaDAgwJ7>bv2kD@S++qbW(vmOacrZgakZG+t*%n` zu4unexJzuru;!idfJ$hqZ^G!|-L9%z%BF^qqW6|xjnQzFLE66GBUG0CX@R>B+JyOf zEp&b47q-XylMczUCFx@(Y+7DHRL6V_QZ)2Uzg@`r3K4|g>f!bMW_64Sm`akP3P`#4 zv4TGJPE%U!>b&Lqv$u%`=n!BWEoJle^|GA*8SYR(L=~*FYoQ8$pbrFWmDItK*oLE+ zu<>89P9+dlLJI5wi=9h#Ds{~xN0IsU$68_y7+}Y{k|OMKw zu=L6V(o|x)JvusPBla3W&~DgKD@T?>XF$O40eZCz)fh(;;06u-0|c(m>Al=Zqnasp zbV(QOkb|WXFrg)~Zv^K|C|R1Lge7)j_F3Aaki{|~`^hyad9=cCXKtO}vM* zwk&o>?F7dM6L_H14QThg>mn<8JNI;4BKzWTr}watEVdUR-Tiz*&I6;64aD}?b!&~< z9UX0~MDs8R*q#M5q}Qaj959UIkt%zxt(Rrb-(lt0Q9ECYQcHARN~{0jwqpA`yE|=C zgC_!J&xkrXW51Z?9<=sQk_kgnfu4<9Vqe|nvNDFHO3;Oo?(3hZ3?xKSL5L>abRm_ku{(-^8h$Y9K`71mC@MkhL11DIlgz!ven~B?0%%O zF2(etoF#39S|aDbc~U*-WU;L&Ei{mV}aIOD9VRF775(bS=sjW_5UyvHmK2UH0c>b*$mQ|TD@mR!O{1U5hOGPf5d91NKZ9)}DMFty2*IXG4jQ0Xm3 zs!QOIO%vOL&!M2*0od?Vr|zGA+Pesrun*@s)gx2fCj8``y|VJ8*WGE#PgdlpJzz>m@mb<< z$YlE(PRsnTwN69MLN{Ec*R4s#g%R$J^u>tb2s zh|X(6A4~je_BLa7Nd}&!oWOQV5LTW5rJ3dh2g~v}ngCt2qfVT;ul68wMJ1R=o?R$? zOU_)eawlT;y5)=3zYAE--@S8hPBC1p))`fZGBcJ@;tNjfdp5rRtVz;{E zb8mB%0y5BPEM|6m-<;N_TK&Sv6ww!3&mfSIEm3fxjOB7C?Ec#-T+e3$g zDGI;N5Fgw4>h zLEw^KQmMpu*uZ=Wn{5zDpC!J3ElQkf*@ax7Ln?nvXUD4a6lI;rCP0ca69v9S$dt3F& zG%uuy9;A$@Am z;yZg8DT{r?4c{RfSkW55lL?I@!ABXG80G4zf+c>$QLm~9+=on02aHj5j6!Z=tnKM_ z`S$eB>bi9)Lb{{s_JPb4I80F_cFU^M|8yfQiinU^-yG{|gyEtHi;=1r#a8OsmL-1J zzTP;3yyXozicNSLBTzZaq%+}LxWW{tvB6!g?gZ)?=mU>=*Jah!KT?a*%Y>p2j>xKK z^_cBJHZ96MCaXTVhIyb0l)D@!Qjz?Fw%TH6a7wLOx8d%VeSUA0^3?~IbRTpEJ%80R zySS;NnkCMRJg%O>{tFDG-B9%Ve$f+E;58kmKOs+7^->oStGlf;7@lW|KWPR`?zF^D zJ!^?~KJs>jC75Ezd*F%ZjdTQ^Kh00i%^W4`%`w=ONu)e4I<*h0tpPwTUtDx>2lzWxK@+$m30wRI+og=kY; zxEv>|b$FejGq5wiT~M=eP6?jZJ9-qlbZ5jzLCO42}N;0f4 zqa`k$xGc*e5dlYQ92a52GF7gU6zzr!J7SDo^_H=-J28%98;xCFd9H9U#aZk$4>_z4 z95U0hgCm_G2lf##Duohaw==}%0gL<+E-qoaB^dJ{C`J*`U0i}2YT_6Ue$cvH_{iH{ zaN`Zm)4@nM+BC>h6=SM!1KI+dB`(!Kz7T8VT$m6JH^rse39`(;V%f+n_7XBiRo*|& zJwK{h;%xRbXDH4p_-j|frgow7MHE}QmqLl#lT|V{Fo9Czc*G6=e#A;#LdwMV#UAVz z00WMVLYAFAnh72hK^L$3*;_F1;kN|NdQ|L{RSDv=My7kAc?J7oWfycZP@g_iT)~cn z|L|}M7#;jqkm>OTflLUrmMCsJP25IK`XThI6NANaWF=9Bh^Dxj8822;i8at(j1~+iaEnlm{5G;vN+IGxWdvB*RW6c`?$xbBhW~% zZAcaOa5R#{0y==boNOP&egme~TK!*JCW|8Mo5E`pc;JCn*#nygwrh;K^lTgFrRK+} zMbgk_iR&Gs!~>sB>};R~0Sn&qyPl<;A|hJiMs&3J6Ip!lAkn6{4y3PReH)At;A4qj z-`T6G;-qWPF%M>X3jyQNM+Yt} z2BstG95{w!!1#d-4W`4g50*vOmjpsQ;kf-~kn}IVh35`nYKg1ueVo|`mbn}9ZTKZA zY!3%NOS$vdVQ~wg4NY~+zJu=bJkmcvNx;CgLa3^bBWFOWW9+tHqN@KxDIPs~k#ctn zR*9d_-{BqxOigk7OrZMc&9SVUKOwz?ZJ&zHLMAZMdpMTVA3MCzgKT1odoU0P)DeW{ zl7a4ecqrfOX|7gjiTnFn$#y+Lc>rdyH=x{{AN4|$n99-_W7HcsQv+Stx3cu6I%;aE zMlPh`(xXYL_@#0UCO}HKcz`4CDQlLwIwebbV0BWb&X1g7g~lMH1&fP6en<>ied#eO zR19D7>ci$CJz`dgCop|N3DUbBeK;_oce`GZNgI6 z@DIcg{jfx-Jrz_S5{TqEmVzqA@4mhqQb}YFPhI^|B}T?x(va&hZhLrcD^`9bOHn9@Dl-;3Rj_P25BUDX)T_!7 z@dH-#K_GySDa&F*Hv}sjvMHLxQ%4_oQgU zu5y(pMsZZ7oH0h6MXwED+Q^SoDGzJCi(~UJpGN9^cU8`ludQb2Nu+K0pgb{RP%nZ` zU$`v)cxN#J-35tdVBXj~SRh8bk(Nt2t~y}Cnu`)_k8gU|l3(&(zpVVWr`OpR8Dd1opm9R?MdtobK z(ArkKea>m}wmz_&N>VDX-fwS5!W|4L@GR;3>Kgf4wTmJE*$sTAtc>kiuQP$XjASEM zdA5yg3TK~pHRQ@?7OG4%4-m)=-DQ_DJAuWrwpF}^+OYkK zyG2z&BJ+Km$UcNAek;<}H?B}N!(ugFwW5^rrR_K@Q(1}=C<;oBHnIA?*ljAntt0Tz0rOZu zp?J{iDfS#B0T|Borrn;*+vrdbOuOcZ0_cJW(pCHpjglegKOjN7^05WQYHCS04l{y> zPq8|TbF8SZy=NY>P8HcdW zL)#?A@otciZ0E;E7ZSxZapG3IF%4*MaRsype6xXp{?bpj1&X7Eg*%sWWEmpl*ih3MVrxF5~V)8u#DQJMwe(}Rp zvKewpFfy+zWsPp5*8oQ(IE>Es;6KWxCT=`xR2MoYjvgY;><%fL_m{5mh!}3GzQ#iAE1JJod zt<9n0$Hl&^3tPR$TiN!kbKg+F(AQnotQF-{MPkMaz&t3%YQ0=bEo-1JqA;+UpWjmN zB1uik0hlRk;cin3Qa7-L_FV1S-m?7Vn(Qc>Tv;3ElRe?v0_|H%)^2-Qdj9+s$0;+d z-7!b}k~opqbKrv#(T8MdjJKn~lyTG^vr&4UxXe5N-ID%sg~A`_9a2$(dH6p_6B|!F z&p2RHl<=9MS_OSzpgs}7^z#(yIXIH7p2WZ&Asp*0Jzv<%eIYZgxuJp6t`=uD39TjT z^e&c-cDH35?3Y>(g47cj=fF@mu{v12HhR4;hm=1dt$$^stnkJU9NJu^_MXY&<380G z80%8|-85AR-39SLi74s6c$!nUbuph*VH~w~20Qgy58}4Zj;i~0uq@lB8f%+M%GF6o zRMl}WfL3d)t^X$x>aChO13WpHkfosHYr()K?-mzO;l}=yC57awjQ?8$|L0pJ!qBFSJuNWb6MQ= zq7opQvVPz7Mm6MucLIm5@(7O&sj3a;Qz)*(p~2I8#jj6Ru^q8LSHcw9 zww;!Y+g06nTg|OdB zvb1m(m>%#neDqOKjkJDloKYFGVB!JBC^0w7{ssX9hPp2dh|iOCk+-~z{CWVzN5Go# zvTo=;_ARP)mkrv*h{qSb$%BPWJnHtp;0(msFNzOl+MwTM;{ zErUF|Gj#B=WFss_f~v&2JA(iuM7F`V1zI^6X!u>_g_|V^I+~J$^X0hf#IXX;sfK| z&Tgn8pFV@>`3=_;F7V44;_Ql7dA`n$YE%(NG;dFKx9mo5rpfAqmQj4s2P5PBSg9Sq z@F)hmnz9j&oRvB~j-{~rfh|Jq9XKbAfYC7)ICPY%j&TkWwD$vRC#dmQi@|GT6Y$8p zAWI?E!H^6MLOw3D;ADln$~M3K{bJ7W5i6T;1U^VpWpLq=q_8U+4LWVSgYA}TAl#s^ zY}7DPRT^@ib@5>6$*E7T!Cf!Z*T5QOT@Tydc%<<*8W82_WFu^1%)sRh6xXex5$1){ zam|E~vT}_O4x@2{l1c!aC4)*YlmSQ)5>k``p;DX-K*|Iqw6hxr2CAB9>Oe^)pxlAe z>H@|}+Q02eQP`wqEjt|@>_Z++YF1EFP(0Tqwh6YcOBY2l4_FLUc1NqyS8^$6=?j-l zwr*6Du+fexI_R1_z!7gu6j7Xv1Yj>G8}XPsoCo7f65A9z&5d!H#fC0PWFV9{_+ZIl z>MqEPC{cC2n66LUY4w9zWU28;bU)N3XtD3Xsd&!-RrDmU$%{cMH6nsbu*I58J*`ro ztKhEEEFxQ$-{!}jW3f$NU2|{=QsDrlaWu^+G?rqKNCYSczkC(ryn*t)qubn_I!66gpVwL6)~b_+}IHfbEq>z!7!0gR-8F>s8I6a6uP2Pj>o#flse zjlkUr0^nP+B~rgp7;gzW+q7k8Z()^c6bexEh0B&96@-QUl*FQnDVyI*P!iMgO0WlO z+bl9gE&Q2p`Rd?rj{CPS&3d?JVagVnc{!&b1=i9fT4ATT8BXjH(o8bpBZQV{mB+Bo zPC}frE-kaJtK|0jun+j4N}nS+8!&OkZ?$r*PI-#{xDQmKizVA;`x-Wm69c+dt}X7G z+Iyqigw_d(GN|7=GBVAkP1$-xiVVUD_Y!TxW$VShGN_9_^9_5L5>j&31w5dRk)VTs zL%x+ThXE)ZFpqx_EBa#ho=PxMT7dlkOvlQMiNSbFd4J zXc92!1H%8cMC%1J)Uq0Fc-t6NV8X{@?3*w$A!V^`ho>HV2k%Vy>##Is+p?)@9QI8~ z3W%tJL2KNTYsk?$VA?K>weRScD0GQ-Et8Gskm`i$I)d6^Gqwo^VCbQxw8IgaCb;?d zkPy5xLmWBx4X5@BXmZuAZgVvr#~LA0M{?UX>s7qIO{%4||MaeGh1(0zLkC@lj!`lg zcZqFC;lS1d5Y+QxdZBJq#;WaWv6+Yc_9%* z-Vgb`k9-{a`!`^9%>%7r%#i!;0l3r5eAyD~>|WpjGg`7^%5m8oxg4qzOnp#CoQG*X z@eOuEE+Kt#eS})xI+NG?6RAXpf>ml9kivvyBvW?4?b>lTg9fI$MTJahrD}h)f_$q@ z7@;ENd+8Nf>;n<5^Bnw z^4ureOc-0NUdPq|iDl(NaJ}?(lz50VBrQX|j%PNYsgALb6Vr0$;~MmNS8E8Fm#mD8 zF@LFo8>s@j<~v~^e3tCoF-JB<+Jy)JX37wx)2HBwW;GBBsHY3LkfOrA|CoCR5KY;! z;cj&tcOwBzcNd*1gxdeY1)&mPmh6P<$MNn%i>B<Bq#ZYhDiiad5(3o! zcO!RhK2%lK52R?nbiu&`|JlPC7`w7_+smpJ&eYrhQec5DmEzQt&+2mjkxIIBR*~vh z$x#hm;)k@Hp1o&q&H&Xc8H%uMv}CH=4WZqS%i1e)*n~UTKv#=-{gUZ&!&A5`>W_dq z4Mx{b5B1gZg8tsJK=4x_`-IzGC*5^K!F4NUx2hjy3A z=L|3G&?LFqeiyuFFbCZQ8Hly*a{aI!iYXivBW+FDrT+!{Ev!9If~pObVVzdVCU{7r zga=9-Id05hj35VfFQE19@?FoFe&Y?`>*5mer7;Tct6-GnUqQY#;^}ni+4q zc4!#KBPGHnR}ii6oCERTDo_77MzzF&c0#(6;`Hfr4eUl3)$xNa5%zRX<%1^>Vn9GC zs2>%hd@xu%EYbO`2q$@dXNn+gJMn-`ENLQeo8WRW>T4sL z3CK*ms}I&Q^l&^0F%5GV6xSufH>IfYxbHz#{3$-n`=@0PF0w%}id9Q?ZImyYVrR94 zV9JQn_w72pXRy^lLH$KKIn=u#t~V& zNCv&79|XyzDv>yX*JQHI-e*Te%0SsiaVGllR$C-8W~+CfFDPLlO<;aLx5a9QL6SU`%3tR9~D# zpb{X$>gd;3z$euZFbpgbhe^E=$!;FaZrPs?E^s%oZdjbH?lH?ajKKo=K1)U}ub`HF zvS0|Bm)1AIK z`gj8~?TQzd-nzd`P?hs!kI0%8oaC2=leRj>dVCOTC&Pp`8W7N0vS*$BYBOGt$ptV? zcyG5;?XPD9dTvT_s;zyQ+^Bi|S%C1No)%*1II0>;|*u8^ftKh{Sy zA=;9?angP>W^BX*%9*k^)?%B{<1W%}Fl9fyQ}H{_^TRLHBBErJw~$TyC$gj19#K(; z(OuNyNQM;fz_mRe94R7QP|cKm=~1F5*BBpaf+hQ2sG}r?lbz5dlft2Io&8R!#v_?& zDz&J-SeoCNH<+&xbfwO*&z&5{Z)G`d9w=eSn7l-}xbb|R(IW!D@0Yk!^kGE?=0Tl-l^Um3VRPxB*#4C}t4vG-W8i(5z>@vnNLLf* zq;T)R@ew^N!hRR2PC(ZkY}D3`PWLY9JjabEi$zD?QgnR~#dRb{zdzJo`)zBsIx~$v zdR?u)(wSEl;A6>WdnVhTqanjuWJXg)<9rZ3J%Asyfu8+3Sk=REQ%KMTdiMAIsvdH% z&_l<1^xNr9Rh+q`7Im0$AZ2oQ-gOqsz@Ed47@tj^YyblVFT~_W%4 z>~{7GBYiG(yjttdDVg#S9s~Z|tEOS5N%VqpH^}M(GUL_8CoA&Uz$FG$4RI@%df9JGzqW5rYH%IA6$Zj&*syXt!EY@&pt7Ysu%aeo4clss%171M8-% zB{-x9kva;W$J-ryLu`8efqDptMZ2x+bR)5PG#&W)EqfV`6%Y@I1=CZf)mNAXo`){r z143dSr*iIF#`kd4apEy>#(lMHdLxQc@(l<$ny?#FIH1%4GjRHDwKU`jZiFCOav-MG zJ4-AE(O~+0y5sX=DPAKR#55MS!Cphc%ytFtz#)O^UfM+pN$t7Vb}?${`jJK&;uLvY zQx0ksmy?ML4@s!Po*UHKFDDc2mTU^K&^9N{X%nvx*zKhFh{Xi{lnI z(!PWTO56xmTd{`&9J)`A#kTRTnWMPZE+Ab7RlKft+lAZ+S2B*@eC+ha>$o!j!ICeS z4eaH30F;;lhb3dNPgnydW#bVr_)Pi2)bZ-MW7jx7H1y)oT!{*XT(9A54?sd_@d;C@AP!4jLM?!*WeN_mb>Nn*3<{NY^pN& z0tl~kWaupRkt&1HeeT6>6Y7&qEE$)Vs6N00T(AYTIa3bCl6Xl&ebFG7#+Si+CfhS` zIIh4IL64M(D?Xy?Vb==`sUd?0w$ZC%XiS0>5FjNv)oF*TNH<_H@PTrbj@fVNJgzP= zc=di&tz=3IEa0&!K~>u~iH0Nk!sU?qJ?$BAC4MN8g6#RSs&8^q#b%4)c+|Ag-M9fk z5_tT3iAQieqRWum6;v)>)gWzCN&O+6y`9JK#@$Xphd!1ZHZeoZ#72nWGGYACeB%df zq?V|UDl*V{rDSVyC29N8h&t+3%uonAl(XbWY%G_$vzn!_fnNHtP*uiuEkx>OI-*0g zeH4p#)Te5u9EamkDNVQWJctP+njcqHfg#g1+j>;Wk>jd7a^u0eAzO0jD@U9;$X3lakkSAi7HZ`p^rklEsy0ae$Z|h)gVZf!*&Y@ zx~Y#!?4#C|v|ETa<){sJ)q2n0*)QN@$q`8YgtV`>nn)NVyxsIhdE=aeUV zp9vH2%Go{~6}17KtBh}vMOUgM+N>PMQgF++U*8ZmGT@tX^a4NUEi9Tz7p^h^<4biU zBJhK@>4a59syed84@->0bCywPQ7d6}=8=F{uk=`Z?oX2_EGOZp*HM!nMAasoc&V0Z zhhyRoc}$#}(9?{3VFxJPHd^zb^)3U+3%=|%mz9{9#xNO5L&>c<3# zw&cX~@y_GJ!-Ws}pq=EG1oS}z@WU2FP$n^*_)WAkVB)9luJ5$~4^htVJWZG->XvQ{)@wbefwIUBWe?7pRFJ^ zmwfwbz!)bAPuh`zdQrRmzl)vJKGB(py)_iq7JDr|&k0L!!(%1fY01er0g$LC-#WBs z?i(u8)0L5mK|Fv;Bw{}1`I2z^<&CN#j_0(}q7(Yl7_!)uQ{$aD#Az_b+iGO7*FJ6F zBw~xAz=br?qtF?A)sK~9u~V@EzTd*1*FZrOFw>Axw6N*tuiKm@r@!SRPU32p2Rw%_ zTux6uBu=i~#S{3IbcXfKR>#y&$n*lKHu9S|Q1+Ai^0_6GlE#EZ<2?g-jd+0Yj-Rm- z^V5ffH};%!;;;Y010Ud9a``iz<(3J5Ng++cv^XwwABWTpDv`7*R4w>kzaRs&mV6V< z^abpR)3z~I^QN4BBGRbc{#jnlqsre9X5b~?ws@!JJb*zLFs-r1d9i*|ntG8LLHHro ziQH7j-F?nzIL&F_;Ct>rl3157dpe1MyWGbb=M0a=ji7F#F6?1biJ3D3)l_V@k%15a zrkrsuRlR{HL&(SseddJwDiLWHh$Oq3a_02QDiIqYAZUxt#5yVwIoNn01rJNkDCO%6 z%-JJ6;q2oiWyX?)jwo}IZ3?@Z@~xKFbGjfa35)9{_Ev|xIbpa?S%Slovq$8}={wr< z=`fg}yPWyheW!IvK1kzugfOs{6Zq0x?h8@Ek|}d;$tANs;K72$F6s!vwSnB@f&gIm zJ)NQ07=pE^4>4tO%l)e1vdUx}-H?-;p0i)VqM3j}-UtE z>yHx#I#?(eC1)X#xdA*NK^y3;7mqjtOYUeg(^;*Wt5gukruZy5tNJ08S~9-C4G?=c z-f4y_B2Y}1nB8yM!MF7jD!N0NjkEmC@!JA?I|=FRL7l@|lniUkV|H9&*dWhQ9H-D2 zNS7Tle#5y70;a{PY3dt1?MqVFm9tLfsqezw53|ZSl|93JaqB1-_5f4x7<1a*H+wk8 zDcf{bYdOvFxcyWj)|9il2?r^4?i^X{?5_^nv^t>@ zn($2ndo8Z~2f!~-5Xyaf%YF~XaDYK~O37Fh#g;r}n@Xel>|YY>bqFccr#M5EKaSa1 z2wbSHwVmT#!F~@-9lzC)JjZK|y#RwJO6ahBMB6!W9`-yOJAs4}9;tD5O41U8c(6U@ ztiEn1m%P&pK9-!b`>s6)&z<52t;-yoLn)8*4w+Dn?J>8Ek9c>reo_}`O*toKsM8w# z$9Y$Y*?AGp^CO1wDI_m4;and-yZel9Io44Hgg&u&C}0c^7A$ryrcUo7Z;4PKv8J5c z+t2P*c_X`lot^SYwi*Ck@LR3^-1*~eYr-$w1w$V&eT2vHLi=q@SaVUu<`Ty|?eq(E zsEKKd&CMuO%duL6gc7Fj^i<2k_R#D#6xA#_FL;#N5d0}OF;;)xAIDV9l6f+lFtyq! z;}U|OEXE~L{Vo{4eA}8N=sqa5YGF<#%;`wEOtMF6^dVIqz0^tsuCB7bx&0(swuL^Z z6^TtfwlU{t?3|s4O;O5{^XnI@b+7m3oIy2eb18_^nJGg>RdfbWu8*8QsXMxA7D{Ta3jnXcwr;Asi~fKmcaS1?>*2atmYKL(+nfbE@2; zH(8fnC@TJY;` zdp%ylq7q5umxWc*>@xbkSV@V}F;0`h16;y`OBf#ZN?b&tDq5GC+ngqNPa5?#j)jZH z*q-Qd$;hnSqQ(*SRvd026LOVB=3V=%zaE@k)VtXJ0tsfSqH$D8ciLUmZ!>_0Dd#77 zI5RT}36u_)MTJFn8*~P+mTpmtKCkoibS#=tT(_tN-}P}u;{lA5uG!YYe}}&KXtLcD zlRSXZ+Ag{kuO96Gfql!`F6ws92}7GB&oN=~FGcn~Jgi7!*|$qtc-vjk_DL*0Q!Z%| zZHsx=Ne_*5iHxzkVqdHhq!dsev?TJD-393%;(;obT#}HjCSflLztx@6;;o*Jx6-fj zXdLaZC_+g31IG;tx<@RQ*1T2Ni4r$Jv?**wxp{L>U z`BH2x;sTPtQvCRRLSAu_Xza_bOB0j zk7e5nRXgN^NEa=(V5ofuyIut#KoX>U+()&=-;p7v!9+-J7->I?GXm}shceVuyvbC} zB_;i*V6|8-WLyMk*hA}*j=MQ%a|9&QHkX@g2e$mu|v16z50 zp_6c^A&JyRUU|EMlYlGk{;s6`${&s>4@``5(M*}g%G1U6gOa%=N&(oDZPbIcy9_WS zKHcSty(vytQ|r=;EVi;R%c+5}60=vG#$H~rk25duK2M#v#H#1B)GtUM!^paKST%0F z`UOvfu5`7i)n@{YOn4vKY8)V_30Tfj3GnDHSK^QTj75SOKWN{sE4PUF+NT@JA3jmTJUeS-JOm_E8}?3h1wuGvzx=Bb@1%HuLO)OROE1 zsNTdfAs#qjMoX?Q<72;%#9c1LLP1Ne#g|BPS7$%$F*tI3&7fAwHPtyGO4(t0%p~JXmbzq*OJe1KqZXI55SB63$_J)52Jh0m3*Y;6TVNbZmqpm5^=> z&sVow=@+Qr7fWvZ5Pw*rWI=;crp$bAnj=5E?Y8MAY{s6%on{XJ!v`KyH#$9WM*^PL zZGYpOI!@11y;uq}eXmE6^X{&8B$7&)a`TC4cGdJYA^^bffl#n2=L(()y#@k4F#R;z zncebB!+;cgrrcCD#F>p%<3mDLMu@uh?)Uh)WOlJ>`8H=>YMfXE9$Y006V`H`zteJu zz7}$8Q}I*|j+-XwDp7K4M8wlxc)SL`CB`7cd4#URUjQP4O@#NK&vNDlXE3k~!m$<8 zrt~$Q^!G5xQBiUWvgaTCsnhywv8}@!s9E6+U9&AqzMp%{nb-Hthi`_;&6m@iq*Z;n z5pq~=35;=Gz>DOF9&N&{SH`PUtO{=EIEC1mqnz>{g}hdW2`#x5f7+;TEWgP_e7ei6 zb8b1kpX_gVK)HAdx(|jNR$EePa~=l3EV&)edSAr}L~<0wn(~9qp-zvI85=w5gRSG8 z9>~B$f^I-N_6-$bcvJ+%$!ezDQJkjmJSQ_{DIZ3r$oLrxT{AXQ?i`k_zUefHoeeDA z<*voP>cHRv_v~WxYjvFUU34}OAz{e_{dUW@KWxJWCte5{ zJ$*DYg^?_|8*9Qk7;jMm|AQN{&)j$BH3(v)3~U$jg*|SqaaY-Mag|tx=dm7E*@Nbh zi}@Jp(-$T8Cao76kgAB&1XJ$)Dn_is)6D^{u-rX7+IbVtNS;1@3O5k}-~SS7)Y6nK{W5Ucqm7{Gx+cjF)7 zMp^63I}9EEwd6+*au>MSM$l2plKafN&g)He?*OKz{P;nvI@zEHdoGhgYJY1S3BV5C z6ulz>N8y}OAIy_;!GnPvFvh5*80QdhdQx+sS97%#(;nM};~+e@9dleN!Br08l2HL( zi%v%e%eNN}Vm~{+79u!D{Aop`)5veGq!wR^>>4|l)Lt_&hz_V3ARJD#`jf`L%V zk_X?r>?B>PX_RI%h|OCv1ii6-$0Twx?4H<6Hs4A(QM;&?H)J!h;X3;1}j`WL%;8_)C3O5|%gRk>}2- z@3zil-{z9G$9yl#i9-S@<5a@br&x}|GjQOcL+WUFupA#dftj+Ak8Wrny)Z+;59CTf z&uk;ThVwI$IxK&{yTw^E7P2lks1N#LM3g#JcLF^xO!`>zC?+GDaCr+q&=)1YxE7)o zZ_8vH5N*k0WB1Ck1w5hAemOQFP?kk%4Tksh-lypSvs_{6}&4 z=^HF0k{~6H#eUaEokYM8p%+V8QAc{!`H_lKd_Wi6;r#}04|*Y_{3)bzahPLv2OV@E z0BQROXBtk+K{0LQGcEhbte0zXBlI9w9&ge>cEBY88=k{w%G^F_PSU*1%v2LbA9Iqj zlG$?*YsnLRJ!E?ve>v}p*aqJSi0Pbo4}i1^dnqrC3{ojQj{8uVAR`5%rhEOy2#CvL`NEBUp4Z{l;Ux z>`{UL@~Cb=Un9G5ZdqjxOV;INO(A>WXf@db@GW_=dn?&vRvveKY{I-Mv8n@@Cb=T{ z>t{TjW~R<-GxU>-3Z;BiKLiGSOnGvDoQ!;1|J5DD83>=(k&$>L5+aEYq&(hPR>EEZ zF%6~|%f}x+i^NH{c$g4`aQVaRZ8Bq^7hk%XLaw|T5F+39`hcxoK!p5yp0e%N7uhe+ zxVt>rvX4l|g@;zU3q^_aJQ=w|-%ZdsR<083acU2+wO>wVt`QsXzya#(M)+;0r|g28 zI)oHeELm7CMNS|66#EvwvE;WakBbPr2253e6eaS7_^99v1;6f-zcV~kf1K6_5AvQV ze+)Y!=MLHBT1a%0r=Ca?+c#%X&=DU?{)or-=8nx2rF0YPE(`mPai-ulSR8P`rk4CJ z_L!51ds=Xk2KIeetn(W3V}z7>oSqph4n~}0S53A5o|YOMRp_&Rjtc z?S>zErHj2dVMZnJS@QI2w^VIp1PVZ4ZGYHvPI`xpWxv2`mOS-ag!;bo3bMMvOuxIA z;JgtrpY6do9x3r7?l*Wp&u^z|i~Wesz#I3Yk^r5>{x~;OdXJw*47Dyr6%r(#qvKwR z#eV<(Drc_8heiNOtR$r0FDi8AKIO#?D4zWC{jEGFCHvuyW_MW>xKJGWc`gOMZc#

4h9YH;BjCK1aktyi9r? zzQt$C^H#pBd$rgdmOtYKs(xsJeBM`jmHUSF9~8T`SZDG*>9{Lg_Q0t9wl6z zA^Qy*Nt%Y{5|0aS$I1RBmwVU^KaDx<%*C-2s7@mgO4Lmk_e(aYSjr<+F8vlEmw10g z7zjFaX?NI}SLFydpbU1yzY{Kw_my>#=LZHdk|i%rik5Ye%qY--8zq0SGUW4kK?cUr zv3@ZtS>)p~Z6*u^aH#T0w)hrPIMk;x%959QB#0Tm`A{fm-(IPaCkOrBf-MH3DX$Ek zCSSl_BarGUSMbc&ko<=)E}8P`xb<>;$vqtQ%hd=UIpi;YFi2iqNcQ2=1j@o@p~ zurBy2hIQFVt}HH-EeF4H#ErDrD^Cu=+0i5zD40rI!Of)wd!r2C(7F^4xFv_+nky0T zp~M$ia>(_n?l>*RP=J)YJJzq>A1^H&(|U;GYTOvv0PjoygKiC1)7OZDnDdjxSnSm` zodxD~T!N((KXqN!#NrGlq$=p7Upi+LW3Mq6eqmj%-boQhO8zPWN~y68J$I`P#3(vInkocS5YUBK3>W^AE=#=OrEh1~*(e?ey#1m&d3~ z*0%QyQSw_|qPRHL*a;l4Cu!SPUiaxKpKbS-I}+Fa6(kZ%GHBrlwinEpV7zKtzf5*`fw+PX=I&f@ts<0}Jp-Br^LKt+FJEwcfC2POsVGV{hcA28zYX4)Au~#C{)4CwvgLTaF@eUZsl*2A(%P( z>y~ZFo4EKn0+)L6TW!-@7;wJAB@!|+9unAg@E*jW*k^#NV)h_-}eLE%2N2CsS z5439&Uf|KPEqOYMP3L&wDIvCUVi!=Zl&8Z)FQlrjm*bAM*UiWfNI?T3*;i!L<6?QWccHeFt* zIxKcrBek=&)h|p6cFI_$Oahp$^5?cjqJ$j@B&dd%F8|z-D7LTyUnlmponk$UD!|rl zUwll?zgM@0X{HK?y*=Q7_tn$_-PGZeWpA4m?0uCHiU(jY9V!3J-Qcw3>Tih8QGez5 z2`_mjz+mk0*MiaFAhRS0M*Y7gtQD`(Nh^zi1z@|1oN=U&4#FPVgnx@_v8MFVy7>5; z@hNYfXsfSR_W0|Gqn;x7))DHb%fFrr_H1<9i$W^@dgG*L1D&*zg2l4r-w&tlYoFCh zOVP?T7@r`YY0*wQN>`~}H9~w!_9&1lXQj(phE6No0X*7f$=c!t;?sbsb-iV6`DC$` z^wCH#5mdWppIE^#P#H)w{kO?Dv6W376w?ITkB2p1yML`VGLo`vKavn<9@FDO6WV`s z2z(ivk7d{+ALzEq)OVb7tZ|)f`+XZIJ~YRG zU0`!(IhhA0>meLc(gBv~yuky8hp-1W&9WQ5S}Irlx21(Wm4Ya@d9L@Rf9MXy_WNO>XVIr27sJ8!yAYf@DE?<(N1tUkxTRE_Wcf!qTk~jeQ@r?~;07%P zNtC#C0^3Hbl^)v2c7sQY#0RlD|Cmcyzgv95OdQy{$F&=*ND&(ls|be2x?-_`c_ju? zM3k!x_C~Y#SHuU0U%K5eEJExZd!t6GaWo=V)+!EhdeD{#NVgj|J}uV_D80C-VmFRR zkOgOkXwQ|=8tleBi)F#_)k+rwtTE}N3a&TfFNAM}Av&W6u-U$oW=-|iD>2{;qWV!028XcAzX~TXKJd0V}@dGK<;BiZ|wUK+# z{%!-Z8$TuF>gwNgl@eWITZPy_2AK>TMK#-Q_|hCPW#t6DWuuukYFaDPNZ*^SN@%j+NNdht7FHyU$PbYML>53#ulX`=_{iIMa{U{LTOaO8a!ckjWVoP|9` z9Q6kD4YhJ`C0HPky|Qp-vgHR4kBXqi%(T|^oT4L$YX6JCv}s-TSY1LRZG3~DY;)wK zR$t?2e7z9s$PWWo)kIJ(d$(BUUfqEyglj^bDXdC@!8F1qO~%RV?ro-dXrzr#UXa7N zm_u>(1II&?#X9Z*U#!x2QKc9(y+%YKh2Lnq@pob3+)qoi)wTLfZrLiYwpQy;W0q=_ zP2yV0CM3@bbw{vv6HYOv6-O!anby$cWU`$8>v(-90H#!VcaKb4Rjs_HklJ<=&vFQx z)gP)efbjk-nYQtm`Xi-Gd#*HI6WkK;_`ge(#mck?X8D*>=&S zlg78Q{GyRxKIkc8PKf!`j}rTYCzZ=^)HhvDld8j>QFM+T0>1Zs9c*iAZ_B zLS9b>{~!q39{vL=Wv55B`vN!1_HPmC>Bc}nF%4I~^=~@Ub3Iv#;;2IKsuEi~quCeB zfM8106mcD))Iy}eI9~`s{j%(U*2S_V zXV74%oHY`{IoYzZTy(n&%nrX#v|c(9Sr|O!qxLM8mX|utsri zJOiZsAc$(&Do7_pG*tpf(nMAaUnf1rZaQqOyp!{Vh&6txXVZY&i$s6=Acey)y=jqk z2Yb={VX6c?aa1hj)J-|S431^{L}R`U7fS@*xK%#O+7_ytF2}zfbf-`(W#L~b=cg5_5s8V-}@`TD|u&kDtA9;7S74M@qdn|?Q4&ib)NkApl) zO0x%M$$S6)R$C0Tl;9TDhfSB)8K`OOZc*_123-R9l!zT63Q9UFhAz|CYC#U7OQbwF zPqrXC5MoUfHqBZh3RZktS0Zp`g_&Cqc_Z6eW zEi1)pmO2q@da0%}S*UtLw@SsB5+CjpAMwprB`_g84g`yjJV$l^0dxqX&N}Nj74DXx z%0KZk=F4YwX8;jkOJsigVVH9l6HWVE zn9ty7c3*%T`EpKOx7GBuG-ux5Yt=x&0vJ--tQ1=xouvDZJVj_)b;io(Xp;)5e2rtk zO7TmzdHKs!$zRJ2?7HLocC%9_72kUs@@igg{uD z;BWTiEUS_M8hZumR8^K8*lM0k_-wQ4gVZO9Dlf0Ks_3;Ln7|F}5G{jP9o+>XPeBjt z94~|1vyy;=M?WE(bNp6Gp&C5C8RyKTb3{`UQh~&J6k8NC+|jrl$O6Eb-#*X*1ALg) zBTNosPkpuWtqI(~(4CGmX@pKD7No4s%W|^o%k6#YkF*1)R){Ov^HexAu;yPB$raNq z9k^AhmuhA&@jmj6PE>2G9>9az%jUFHlQWlal4AwG=%+XgA#mSz@$->qw3HIXBhXtZ zHmuQk6x|Zu0Fh5Vlnm2-X4%dA@0QI7?e%yLi$Q2`NaVlKPZJLSI zEB+vl5uP_DY~DUbJVtgP^ibX(V>iEHrtE1(C&rb{Ck4wMydhKLi}mKu&yty~LbRhW zLAu?%UyRtu`47d^;E0PAPqFBx2r#BZ-#y}~O|NN!sWK>8JjWWna<)bq)ONqj8@N%g z85ro+K2gNZF4RvEuuq-@^y~QZ(kxd^7s)_+!A-q6dPH_#wQAXmK~Il#zLodQ~^WmiU9G@lh11PVGPoGRR+{LJhP;$ zP>pby8|i&?Qa-1-ogky=Ps;(8q2puA1aDZg-!HCi*g=7FoQGNXk3w6IVj47T zr{`JL=oJp#DFsDkI*ZSlCk;`b5}(eKQ7=r<-v%^>fajtPn)EoD(4tY8Oyhl117r76u+5om~%NZu;GN_bx@XP8!oEL_w(1e)*=66T_V%BpBZY)#Gttgq)gpR9DeBTJX&tT>XOn&S@SkX73e= ztf3&mL}81&YsFx)N1?3|+v1u%p0C{drq~4Ncc)k_-OmZo$3R$VH6cHLhA-AzPB|^- zpR2O?uj+Er?H2cCJE4 zZHtWubUHdD7!$VGo8y$SJFBYTsC2s(+14HzW?stEgqEur2WLK@b1beKf#+r z#)Av^7UCGY7HWec6JlWkOC30f!Si~KcmD*1_Hp>Vx1Zy2p1@( z=;FUY1h`A`2FIyDXGw*5LxxgWb}*YM8*er515pL0vkFBtN1MQ)zPS*X@U~}-9t=h~ zMd-q?7=|X+TRgtVyPmU7${t!ut9QnUZJctALb-A!r4=*2C^kV$e7$9>U(Sfy;@fm5 zjU2_2260n@8W#Ut){sc;3o z_08MGn}ihgO$XbW<*+x4{;hLm>uC|sRs^NG=`LE|zSr47|50|ainCiRGdELQZT;h&E8MIayHsmv&5BcnJ0Q{V(VBIs zrSJf@Yt!0fv2@f+x?|J21as)<_FF&KQ}Zeng{^;_^X@E`!_Y%j0qIXm-o12~ngg|gKnf9R%etp0g`vyO=-|1~(;9LkidF2wn? zO}E>85hho$uTF6zHaIRv3} zl87x-wf|KjsnB|!9CtD95ed7^?~!s9-KTB`eoT69OZf9@e=)s&Q{8`X?@OlDQT|WO~BCjc1EoA>?Kbd*==5}6+1{th51aZ zx4E&>S?g!z(oo z!(&R0jK3|_*Ti7Ti;n$y7J%ZeO^Xl8{-Y+37&zNu#PFOC`OVFMwKw(z_ae72s%KyJeHmjSCfdtLQ0vkN60%I zCrq5D-C)GtcF6NQd*b}pEpcyzx|FA@7$eW$o!aTwHma?pP|CJLHbjaH`qMh3H=S{| zae~N$6iOlGtvP}>F|{kfGSUwHu~t4YZMnYl?lL{(tao!EQGzN-lN~>%k^EOF2;tEL z$0pkve<=0S?a=S`$`#SKiw`(-G(?pMZCWgT;O0^}>X)dH3cXM$AD#H7z!v)WtsU~` zO3zUytEz+%TYkv-nvW;NO(cgL%Jep3V>t>bxe`xwR@$@B!pX^Y( zNIbp#OS_)RD%Eo#7ioZOi2{6VUd(*^XGU~uk`GykjREWha z5K-K;hS1jzicmK z+Q`pta1IvztFP-DT|Te1mT_jHuF9rL=LGkos(eOnKRsF&t!}M*ZNoHok#i->4E*7$ zKXsOK<9tt`!MwYXJBOblI009$*E)YvgUeP?X*tS7Cx}66 zuWOI$SkK4w{E8{REW6!(rSb{#o|oenU8UWMt?~&LL-Ps#JY}G^YtP6AT-||(X>)B~ zDix;~OsEpMDhlfpUMLn-=bLHhNU`m<_b(7JW~8KC0nx0^6FLVmOny-f}hz4omc6k$1Q5Y)GRAJKGf;{jcgtotOL)*PAMJ`h>RU$)6nZ{vb zP9UR)f!^-REpqw2nHEM?aYhxU8b_FW#1W_PNw?cQkS}j~^IbK+HO_APSCJUwUJW%& zzn$m2&U3dxs9&Xd?3wADb#G25XkGq$+B1Y9e!j2EXM%fQ>jRc3pCu@P81lS}Ar7-k?mFQl`~s%zuK` zpv2}WSGJ2v^rUdlvmE8L)!VfSbT;smAR?5ZkuqSV$8xWihHI-Q1$%~bof5^AJ1H@~ zNZe1>Y9uJMF!c|0{$@)r*Efb~*LAeBfrrl!Y246m)gkddsRotB4AYV`vf<$Y>JNn$ zrbXvu!~Z;07clL}nNiEbRvA<&*HWfdcv5*|8IdM%+jlu6@1mnetck+KI3kr%A`3%*|dGb z0MDn~6Ibd34#FcRWl9T^vm64t{By=B5&tOlO_$T2yprEN{;|TLLV_xz13l+>lM4*0 zRi)eQcle3(2PW%Y3dRsPYnkk(2Zj#>HtZVnI%SmaOW2+v%M`MSV?On>?e;fhdYiIa zt(3z^NO?nusSJ!TpK*45=)6w3{{0JjFacw#Y@Y6%Azd;kl}*y^u(lV(Kc~0pO)ij9 zqA=Teh_M3_T;(2G=;R><*rwYGYY{0TNn@P{FkK}qW`kJ8DJ1?dCE6d%cTzcCsX~fs zS$242ptvI75B-VGP=v6z0%YPrV^bq`_H^%3XM~!Us-}*Szf17)!jy&TDjY`IVPy7M z+Hj@rP~ZXdD^EJ7Nt;+ol_1S@>~1G@%!eAf78|x?mNN_-9*~x(a0r{Y#aY6ciCj!* zaEyy|mL$#9eldilhn)a!qbuboMU{tRosn*)c=SP+6De}dT|X-+u02+)bgtm#VxXF~ zANJ84r-YvFA`f1vlB>&p)7NftbTj>Bl(&ctJ3p$_Xr|-O$_~$&3uH#hO=o0>1--Pz zv@YQc;UaE6rU*uC_@D}L73aqNP4fVlh!e}0dI8&5y~Ffux#rc@8V8an(ff!sg3VY& zy2|~W?9`HPD+(N@#M}xQ!6T6Tq2esv4(Gx34$tn<+QJyZeU;#7pzLXyY50SIKz~95 z!bTc26hMxaPDW_ zau-#!lnz{G={EO1-EOs!JA9WQr@5z0JjyQ-vyaF%H@>5nvZ$72cUVm#V@`)v(7+X&LD{;lMD!^N<9J(S-)c5O0y7~ zZr2OCuL0%BMHiy?0`F_;aNk%z(EkpT;b0 zkL*J4t*;zV8@l=;tlgpHu(R?%PwB%g+Ak4ZcFVZ7r*xcZrX9XJ;(Ym3R$a#y!TGTY zUPGd|3I2#d<798Sc-2Zd zjVxy{bgLYL2d95vsECL?K2to)6Ml;!%vUiM!D`@0-YE((M$WQ3^03u$4wGUF*B%dy zb6(}Rhq7s|YGNHzg*Z;y00j&JV(Srh70J0kCnQzyQsRzLvh!L;Yg?juY}+rVoHAD- z4I#ZkEFWR%E8SEZX?Kh|=WM_BJ^dxjQ|fo*&_Dz3Nbq0NUOKMbL@0w6gOYwj%Vh1yS&@5ZnV|$=SXi9pT=^e zPz9#7TfC8EOM>9Wn03?}b?t(>{-e{_LUD-HM-(?7i_T9SkOhx))XoNNmffjSq!`8O z`FfaYq@Bhki=Rlv@C4pBx^T;;-^K=dQsl4ganB-e-`ZK)4Tgt^^Y&*tTZ0KTuufOR zd;8^@i(4>83aLw4^5M&aX|8g1ry=LOEtyv)vHlUILaI|}j<>~Uo3*11kHS)KOQDag zxyqH3B9sjXP@0hH^cVSHIfPXP0j61Y=MHnc&HrhrqS#dF-0h%T#M!TMl+z_TU9rg9 z@)mtX)x{wedIvl=LLY2PP@YpIR7x2o#SpHE_r^c)rv5f)dvtD_EOW-4(*1`~?gJsJ zIMa&+RVCf-OkV4U9z3Q)%9NN*vfHbcY8;pVDS4UR7FXY-1{1FO_aF2IowRiX88;mF z6Q%#%s2g&Iil9!2~X{t2~_JZNq|lp3gv?4(#?ueqfGRg)2vO9vUXT4{ur*{GCo#cq2bA)y~!??7VH3 zNZ?bDWLhOv`tEVAA`uuEl=o3Tez_*_hjIgi{*#@ny4|L~K%m9he{A^{hh4bq4TP_K0 zGC(Bxt=3XJ%IcBr_Q>gswv?k_3b5RG=Ra(7#9=~RqI1Go=NO3=fNf%|^OhLr8~QQ+ ztBC)L$E$P1Hqx;8nb4*LM^>kEvtCItetF~o-v6|&A$R_KtFyxWSWpcUe!tW6K8F|b zFGlk5bDsBE{-M5!0ou{`-$NK-j)jP*$I-^+bP&Va<`WXwHt4Ldvfmn?r z^6Ex%#H9v`3|}ZljrmYRFW2hxCA@_y{Gk*?+o4OuD{Q1@!8A|nLXwpRcbw5-X)Km{ zQmo<>lB!aot4y0I+kSC_CWLFIK?@x9!nRVMU%DO19?n%fK)=g`6jIt`JI97<&$;$! zf57<65RoF@~6B2CB$8&atIZLtn3P zj8AYr;^R4=P6?YJ_Ue4+8!YC*x5hv`FaM2PsUw*x@VFM}Y=SE2s1M=R?andJ(Ym%D zIc>RftZb-)ZTj`dF@D|{26^BxA=SBZgg54#e(l#pa^&PC-tJ`PP-0;W4)p@+zBlw^ z*x4Fs)ft(x*4zm|EG2eVILVXB^#=z@|BLX$Zt)Q(W+-l2ROGd5z0*lq20`R#=wI%C zH-ljh^<6hqR$H5ynukN+2h-Xu!oy)frL8tlm)JtN{@FF!>KbWOvqI-x9@$YmbbLf{ zccl%7Yn3bcVW8g|d0Th;Q}r-L+o(x#vaNN=z$o(j9A_W>sgh!N1Xnv1BoFrkdYQ6E z)ZJUHjm3W|P$nFrmPLs1T+4bLXJ;#I zT#YnCBh5?nhP?ZY?t=^=X|gw%YXr(3`Km-0uI;VjhCj-wA7X#MV0}%VJpM327j-_- z`kLT}ptK!hcL|#*erH-8?(<6&ei_OPA`yOiva_kX`r zuYVX_f=4(Be4;6CtllLu&`G#p&iwRuriNb`~1YkRkiWHuAy?|d}4B?gS z&H?UuDyzdI-R|;UnpnXRycl?Rls%$@4~eTk4@gybE#Cn&XBgM_E)Zirb?FYw^ z?P8L7h}J|43>s%_ExIagBJENLd~AQSS0CCz=Bg6X|k5X*z@GiL=B*o{`GxCO)EfEEnHF zfT8&!M?ZbUxq3;YlA?a;cGvmUvVdca7V4L1d#tW?1lb$-uW5wQcf<>om;}XK9(M}y z5Gxa|6qNgAgUsNnTP{_Mwr2`tMhAUb&a~UEZEEG*ht0it_#ieYP2T31G^561Pv$$z z#y_QdDHx;vg*chv|EQLNUm#o#J$lbkab4j%dWR-RSNSSbHfsG~UGs=$zw%pph8%(} zMl1Jovg}Lx0;ODuCe$pE{W!D;Z1atZb`FUE9{5xQZT0B#h2l7`x#A?#ZAE`xD{f}X zm;Wlu1MJI4k;4myWxn}K^okT`IXCQ(;8uBujSwDRtb*XtRX%4)jqeL3Tq&PUu->X1 zp+CS`D)^&+8zCF5KcNR`#vb*;vU-6iC?BhR>y|hx8oz%*6O6^aohL%wkGL`N8vr~AwOps*X1q(>)It)wweE~I)Z1uKwTTBiD+)T! zVCEc@Q^4U*6~m)Tuyx9Wr0HS0lISUb`Aax{F(s@Bk?*dM?9#weCy1_OkIM6HFLWgx z%w+dfH&Y_JP%Mr6P0#qWJ-X&(ilv+l^w2y&I@0d?;$)dUQFm1~`2K^0A1SAG)jd=`0?wnu_i?xi`-3y=JvhhLOJ zY$a*V-HE2QN4M59MMpx4Ka_7M(JxSRWI+U57+~EdjgVarbkOZZGvzp@_FVR-7 z$KzP6=QoC5$}dK2@d9u1b@6pQe>YEr^BwCjiic*}z2QnRb8dh2MK=ro-8l|)inmJC zcuY&_-ha2q-=!D1jib6}$H`pI($?_^DH1m^K+7cxgS2uNszqC#MZ~x2mu`3OzF$a= zHc>+TAf!Z!o*c7K7Srl?AGTKXy!4z>_laA?H#Ad97<+UdbJUtfLNc;y7!$^PG1__h z{b|}jI4jHU{&A7aVV6fq(du`4@d|$imcDYTvXYwlbdcc~b#a><| zrbB>LNNOw?x>bJJAWl(bv9AGjFCXFTVHXg^RH-aGreUQxo}rIen0~$cuScCXxx|VD zGal>y?@4Fv@HKV7VtO19m8^|}(%2*BRYt*Gr`Lr;%&imTE#W70d&y9qi@CKx-a;Z{ zKU$?G)Y}G&IWWR`g7*na4D@t6W=_j@xYI1KxEj~NV{4cwVn<>2_IAm4ZTON zS~=yXPZS0dLG_t_=;saHIjC-3yT{N3ak#R)?vvDG*gF^%Tdlq z8N5M!$KkkqbTNdEX-+rytdrKZN5_LsH~Jue85})EP8KI8=`}WEakaHb5L(9+wjVZC9Nc48E06+j(J*0cr%5jg~q*u^TkPHLWR0@L~+;I^#}(m#7XuE6;o}op2J7URbF#18pcRDS}Bfjur2>0k9vgi z5#k8Si%m;5c-ygxuYyi9jX7KE6!lr`8-Kv`i?h~SoDflB0Y{ZDFIaEU<0>BE z+NQgTy!6TX<7t4AfUCtPAJ=tTJsT|%$2jDR3DplC^+M{4(cX|eeP!2_=*cYU8^6`s zFJ($>z5U7_Cn~*j$;D{-{L=Hz15&Wxpkf^pAm!fz@0|1Biz45CRL`-&)^sLCpft|z z`RHCL()DX!97;nm(|^R~L|Cid`^r)!v>RgiWG1d#HBz=6ZJ*=gQ& z5&At00~r}qVZss(Y|~2dHWSTU2%x3PS7+p{O_%$CL1;40yN>t8Kx$$vmZWBlhOZH6 zKrdIG>(!!>T)N=6Rt~FCsQ97N=mq{(q)9amGOr%vLQ4& zB36_5TS-AVn6{1(qe++>Os9nRAh6t#eOIv<3*ic-ZLcne#F1U^>s84z1-)0B04wFq zxtgigrI)Of*KRA*YZo(giF#5Z&RSDB_?VCP4G(VnwC1d}G9lp7kuuq7%^`3!b&+=1 z>U3kx0P`6W_Ue`5bYp?23Ie6T5xCN6Ko*G*pBs8{!LWejYG`VLuGf_@)@aV+=MmZ< zMasy@RtCo!m4Wy{SiM!w$T+BPE$ZW9y>2AIEJ-xh!-FbFnM^*&|2(Cm5VZeG{X0U% zb2rY>bJHMQ<(9415RO>6IFRz(WEuI;KUx>Vbmal@97uNool-D;e!qBrbGm8?rvK>m zOiSVA@O8ck2Y5VvR(Nl^OLtp_$BTuseX9w2C~1HdkwJzOT`*Lo((T^OxnIe0y3)l+ zc`I8y%q?FJ;ebsL`$L?(-hHu8SLyXvn9OodK^vz3P%!U2ZpZL-xltB4DVCiwJD>%G;R*~fn%4d(z`>cbqyB{@~NVM z^=7bnEf<7Ti5gvkbJ7tlZmPQ(+dJZ{;2A)DR>4$h6W~mASFYd#(|UwND3GVLb%sY&SKS?|0*Nfc5n6yI=<0Q z#h5Y@CDOaU(;4%Zq025+B`&&DP8a`J ze!g&s3k(pCkc*N3nsA5Iav{-^{$s8@cvv2a;RRkM_lF{*^;R_ z7+h`p9(D#kqPIAW3F8Ls7bRriR_Yrb^$5drMG2tzujx?YQV)pd$%g_`We=d&lX6?I zY{`?T3Ws5O(;ng3_l7o*vB%@1oEA(pOD+N%S?C0q`M%5a;nU6#H}k?S6%KJ`(tI&F zqaC+#o@mEDAq-Otjyo2KkdeQu`sVW*cV~<@_=yj-#i~?K7nh&r454MCt#M`C2NT5W zF*-AtiJ&+(x%yvvek^YMUZ;QG64fE8pKZroh!ZC&?y76Zan(CTCDTa$>++~qg5F^= z0|E4K!{Mj1*3BeoQZ@w(CgcHwvpnuwiaoGemffd&BYADwSNZ~~A+*d*9ZSnDM+#P> zMEk(ht1+Q6a)xq4pJka&dsgxk!B6v8x6kRoWCQ~lV11&NJCUq>s}c~??LM67?!YVV zKsQ$JbNh5Tk0p7f3ngHBUA!E|sc;Wp5B+KEb9<3<6In(SQw_b(n0;dZngtdjO*8E? zXMw!-p&5Ejx< zv@?lK(2|P=;@InB+?-ZCjQZ6Jw2j^RU71Z48(s;6!u{$e7t#Y%{5TXba`Xg zJ$h&j(vfzb(>&a``##0Pe1ZA~otAgPNaC-f6=P0|L9X2 zDYkK7NZErQMrQkI57%z6mI=$gcg_<#dJpr(X}aB)z}*q@gN9BO)Mv=D#`#CVc5%$f z5jzGq*Mu@{kMqmD?Q3W0vtQa`edh&=SIUMektVeJ&R-~AS@?}6fD|0jbL5kvTt^U$ zv+Ta}CyQ6gPiTS`+n0-dubilR_}%W?b-D~Lj?z8ME?s3nt<3I|tj`zvtN7?^hsoPs z3fIuF8U#+czFhjKPkjhUljZoUbjoE_3PMVSoKVtP$CxJ6+a&MH;@=x=9$z)3FCqD-e1**p~wqsuc3oL-FjR24%g2=iP=J`}3B0TUpc zk*rLp62^0D4|}iTqO@WPIK=YeMJ$WMHNKBU-y;*`B+><0v|EGYy;$*il{TNN%a;{0 zxrKSm1;%LW?Dn=}7f|siSBToU#{EwV;}>4d7%^T?jsuluNFT>=SY1QSapummNcWge7f}Gvz9~J z_aM~wD8KXzh>_E0>+2A%#lG4jZwt|l(0up%E{~J9H?29E?3+EWtjGB887BCxbnkM+JyartI2m^F6EUei}edRD;{T|`y%vy zZRUu_Z#$&br$oA4A3FCdH}NG@%2ACH{SS%9nKcHXT$Kja_l($eSDU)9?ALF**kux= z7z6bib57pLIt*?wU0=W9i$o$L1C=oaxBwelD2J0dUoq9L>^CA)lyba*KNMrcX6==2 z`W?{9!G|gn^F=A=r7(^w<+f^3${MVyVk~xdttjF;Lydt>m3uQqH@+_lhbi$uq~|?O z{QE&zuhuU5EsYo5IVGQA0vE!PND;$jCyZ<&sNeEJv59+p${r0@aZ6 zM1(VuaPYvC67X0P>rCVt0TN8}=*Pl;R_8t1w=ky4`pKdjpI3@2a1gi$`~};Spj4va zQL;g--QPtMjFfld#bT~8Dx|=t$~TcR^Xq%Ga^WhH>+|^Hh;vQQ!+IKNN>nZsyIHEJ z0eYE+UU67N^W>${#Q-}^k~h9UilBkTlc4Rc1-;@DcphW$Z)sMp@^utm?(F!f`PMabY%n#@n6j-NYbJ<*DtmF=Gd14}O#=-YFZ;T{`!@)7`c1XuCk$$x zsQpr-d5p=FIoFvNrDHUXhYyM`++3o zzm(y*63v4gRqcY!abzQw%#%MR_EW}D3W78}*@@sH8yJj{`%kZwf~y%-SWHXlpA)g~ zQLZ5=BO`?>^$6L)`&O8<3MPX3-y0$1U$cEJiYgBZ@e&u66fpQeSUO64FyZaG!0lg8 z+ZWEsiPLXUD@N*vkvElEpPOrprg`+wKIGX<=jic`a{9l%K~5-WbqRsPlcPCqT7)a} zRXFtjWRx7!z`X2~s{*F~%W0yB8{GV1BB=lM1>ytJ(dAKvA5}gOa>CdfbrhCp9Pk*y zw-V!+HrL6 zR1Tvjp-Z`HQ3J>lkh&s8TMVngbOujoxL-m)&2)OH=Rx-Of$dssc99&*qAWL`)c(9; zxS9-H^d+xRr1 zi-Hf~g$U1icg@jAd1stl@xo`i1f?j!*^Qgm-J{z}khTY@TW{mV&p6-G-he&l#0icB zP~VlZuGs0rM;r;J-6r&lk$!DY)va6&_^8(NCy6FOsj6U)iVY(5!8>$EU!##8*zG*U zF^gU3Vt{>f!Lhi3iTPaUReQvlKjvLx^dvxf@agOGuOesw>7?%Ne3!ytkPfJAWF6t8>};Pa6B?FUU-B-G(j`MxNeDQ=dAdX$L4bqM ze6+lqEv#~wdX!BQ8pL_}c6w5Wl!jivO29m4W1q$P&|;LX(upZ?o^DiByp1IF6tRM;kr@c0Pi&h!J|w=&psr+os~~k6CeI3MWJq4lEB3a&~>xYpV`^Ls(EGCm#Ax$5@Ciaeu7%@X@o{ zl>np4{RhPXCK7oU{qjhROu0VG=gKK|0^f!=nP-F2w5WuIXT+QAi+Ox)KY@fo-v*fX z)iJ>syFTHbJ)+`lwl9*g*ov*ZKcF4_*QI@I&ho<=D}Ta%b}ZLtFMv`Z02(z6tGw63$y*+B_1rjT0Y|2n>t6 zQAmF656O6dwcJMd>=~gh2`KfA33u!h|Jit~ZVZ#Kb+q`1%Yo&-a2Pl$K}_OyEf`!L zd)JB@j?mQjdYA++_SkBcvakutIljEmpg?~GgVJ#RyXR|V<4lghAdJm z90`i)z_4@s@_9XMDc!hMe_)r%;$McN{MQ8Tz^)vBA)Pj$RJD=z!0wHl$JwDlq#~r- z18**nE4bHK?o)1HWQ_A5mwWPkUCzJ>2j#pcpH(w)Rf;Ovlf`vpUVw*LnHYF)rnC0P zTlA6|B>*;mvh!5iXLYY_2ut>h$Dhj5210#!W*d|W96Yu)O09J#t47MU{L~94Zr5$UOtq-`C3atslA}EQ z*NplG?w%!{U{F&h<*jJ3mL)aUFYtJCp|j|pQF^FABtqb7#(=2TE%Z=db_i^vZ{T~o z{vy(Yk@8mDKyede13%dyZ++rJ6{E^Gwmqmpp?Hp7N>xD;y0D7lNwiuzH)e~);M zORQ+Bng=ER8*5GGNPwymuEY)sYAH5zM3euT5`!89i%qXxS0}cfltE-$`#1KHx;?Ey z{-aW>r)^VG4AMc(_lg4MrUgWDxt4M{jzR3IQPJQUCTs^nFL_^0&@aT9t^(;?-@`oCrAtiFV*tEQZ);3%N z>zQda<~p{*VF=Ns;yE+DH=V(tp376ecei{EumP1Kma97oDMm)hpkzn+X>)_dWQrGgGlk;nhbyl? zDMxZ75zo88ZWiJ>cc@^1P2J)g=HuxPuySqFDW~OCIBFN20YrdJ^Ap8f-av}_VULLw zVw0Ow#UPztsY}!W|={Dsam&RwtxVmjDC^ACw)_quJK z3c5jg=VZz$pSbdOFKWk>;oYq51lW#*BSdbEXskn-XPQOVKB7%Wz%J;!FsFWg54l(HDmcPw{) z;n^zwYnu9?dftCU$fa3DN~Gze2hHUm#zSUZ3iwDV%aKd(eOd=hzLxU(BH5dy0Lp~n z8pr-3vBph@YwW=dh&b*@6i@>A@HlYD8$}8xg#+Ix0tpQb4>w>AHgawupEL$`wLLY@ ziX$P}Y~S>L5INoA9-5&`7#`K-l=G&&N$6#4OL$76M43Ir3%sii^q;)v5vD}%3??%9A4WgJAHqfR^Q-wZ>Jpj z>v-!c{9*_^IrSVXSLmUf1ygicXT7egK!osJn9QxwI|;5E&ie@u`7RXFT?$8HvqIVW zx#gDKQk%B9*Tag?B|!u0xKsSV(MUuZBPVtV7C&&R z6CS1!CiW0c8}`;HZd{qz`>d10QQ~rdX#*wpA0hgDQKXqdL`q^oY6+VuOMRi8XeU^e zoDB8&qA+n_tjuC59I>iX0{Yl$+4rMQ^&+GJcKvqoJv(PH3|TM|OqU)Ox4Y@+s0QI? zo`_}US%Q>2<;wbmoA$_20UdooXWNMrcFL=cWNE9HX^TB_(izE}0Hqu_V0w49xXwN1 zX>_@Bf^4(QydpvsJU4fXRYjVw^{8)fupx8Dlqh}l%|IW2R?Hwph60wak=BEL?>Uj_ zZVZ_cE3)N{5&nAOZKM>8mUnPEj6d86+E^gIoBX&YAR44E##wKIG#eb@n!wvWInB@N zrfvwkj>tQUx+!g4&%F^N@7xsavj-M?MF`0aZVx3gG}4!MI!W9+$pyM53|St$Zy6t8 zM7ja<{y5qAqZGZCLkW2B4B+!bAyw202f*H{l@D_1s{A6b&kl((goDy0UrRY2A;R6G zURp}x4~N7}cTChnC*$lxN9E%4-&+bg)nHl`FDl8|r*N1m^$4dptxqZ!io1bZk159r z_L1H~0pkbsA12FSwx5xrE=}QEwp+@0hqns#jVmt{ip+@_IzbymaQuEj4&$Is2_-x_ zEPo4@7Ms%wn65IUQ?j?uRVjM?BVR!u@?e3tFG<~1m2hRCAwg?pDrcSMQ$_P2nWB97 zp80kAghP0P>RJv^T|pJVQDyuR>npyMQP^XbYPUl;NB;{qmlPpSm*{!aNoQUO2DeJb zmU0;DLHySxbS3ROpH~lrT&>IH>JNz>A^qJO5ZcH?Xr?J&w$REMT?VGf6vw2}H`vO+VTVGRoPY*$D@`)&|-)5jh8EazDc~oP@zhf=$k87xaZg~ z<4Ai51J=1Lh5J**XghSTe4yCmW%1M6jy@?KCKojSRT`(;L$a%#NS0hnr~(fN51zCR zyXnOZj~qWa_`U#(5^6F5uzFQ)J}KJ|?Ij*2PHVK0d2p=ySgZyGWWb{ zKyEk`owIgg50IKZdB`)d;umIGs)X^&YimXB_E5cYRiKf+eZ*SNtwpN1&fc~~F8=sz z-F@UCrDv=+*f><)Hzs^%i%dx#Bi8WEuF`HeQeeHzrCuCu8qkm<+pYcPz_RPguZyj> zCY7sCyh4kSj~9zOnorX0rA&1OLw;`L*la$cx~uKU3)b70u1eM~QGMFl$+09bxME35 zlfm_4i&mMSeS0>=deJ@djv2A~e-~ID-aAp+8hiYAr?s6g5Q@1mcA}DJX3d5Sr2uxe zRGcMSV>!SK;jg{cOaFYMUFp%`a4ud{lYNd7Dw>gUK0+L0XBB&xCYIEcgzQYb(bR-x zQllf*OC(lQB{YttfEe-P@P>L6JX=9e>afyDX2z#Vpi7pW)IQL9nZ6LHrejMAKV-ei zBGY=G#gfQ^_ak`&0i}NMXt703ySbole45m2j+1N-#~D{9#hn&Ea(h8lF*p*6tk+09 z0aG)3P3pN^CNWzJ0!NwZl#+N}KB>txUvG&Nxzf6WgOBrlgYTpPTjjKy_FPh>%^bOw zv85kU%2g9f8oSUcV0sTj(;AY-9M}V@-1gdO&7(%!^mg*mvV{dG zSGgH2t(VD@=K-lnG}4>H6i71ELkX=1P5jl(>Jb z`1L8B#oP2_No0;YnbpeoiMH*e$0DuoNoiMwfecbQgCAZB2Zr18Cq1`N+{^r>fKJJX z5uw47R-tjiQ zRMID!3vK-4xdQalRh5qF@@c>N>C#jI2#8;ut2=|2ttPbJxXfcS4evJ1`Q%@=hmFup% zR=Gh%^GG{s=OMXxN230Ec(ii#V-C3(GXREIke2VZ&d{+@#TDxfkhihdSOX5KAcZqm zPcsMdyC|ie)lVi{Ue>i$m2wU2WWHGPp{3)4YDhU>Y<4+D%5Nr zIwUCHQsQ!^NfnFalc%QY2r|kAwUoEbTB!VLJ}1eYL#-csp3^&cMas9y1EQ^;cxnMq zhH28}VVN9rm^c0YP!VJy8tBPetzVe(pqOz~G8w3bvSgA+c;h`vWLA1313%TH;20IS z$iezu*=yP`9OU4025E5X`8?~uf2El#^(^mZ}d zy)r`;c(5lP<)+A}(aoctE3+4hRZ9`SPh&A-c;k1U6DL&Q)n$@->dT1H@H?l+xCtnNA%i2fs5*o6uC5 zTOoR|4F+t}p(I}$D<7&bDX9#Ptg|w4@)6}E70K9xt)pY4u+LzWV`-)fMvGOiZ&1@F zMFiNKmeRl90g*+L$9P@h!5DEL>$nhHU6y5vv7d(M@j6nd^4J#7?>viG1`a=6WmTN# zckZjN$1kXlO&=)`53#zTN|-A7X<~G@+jO%1H+8IYhj%1Xz!0SD(h}0>k92kkCw7~ zzX;Dhp-pI%tJi<*jTLvHe%3{?^|WPAYaYX%IK+Y$=w$~zZ>=)d2MGt1JFwf@%Z^ou zZ$&Y=ELOB*gpz}D<+=fV5-&zMGt@BBbtT)^4e4_DuiKE1M>q{RDVsrOzz-?MT8jRA zGWB{P)gb)mto0#VM@VsTe3LBFIQ*n|*eD6^H1yMVVKed z$CIa~lDShL`FqXpOkP@yuH?)7Xb%;A> zU@(3eI_Sc_^Ne7X8~ja`Gvf7Cn>A82#vXrccYbwuc?^%?hpj1`yNCe?)nJ;EFS3qz z)2$NG>Gn{|Pu$FMSPtqVniBELWdi9OmZDtA|F+n5xz3v`{9i!{l%vXZ8$?5{GUXb3 zz?8>Z{n!*%(AR5Sa*B0k4vPB-nbyDWeWe<)X{JMS&WRibSC~&Rg~!ATB8X&BvmqE9 z(`H(KPR!69ERqmA>ySW#=t{TMSXY^SR$O8xXNgY*hSIeMm|Y1h@dQOvDI@#y7)(z@Mxw5OGGwzy@2ilduD<4$Tt;r zW2vF!gUus{NP_P$$IunEB97*v#O7%pY;d=Bd1Po44oP&9SyDrSiS?muw8xO^MnMN0 z5%f;JNaJ}|iYvw--4rdZH0MuMz(9Cyr??{Z4P^_pmIL9vwT>TYc@e4FKNfp^k7zULTANZ041K6lvPj0P9;lz*G{E+r7K!fdCqvk`T{Hl?(uFGU`0!|IC)OR1 zqQJr9dRNahBZm{M*b4&^DRh^%#*hsi`Aaf zC5HZUM&xo^5M7kRslsOc5|$bGuW5wC*aRKGhY#4sV#C_3^qloStb3Rst^Vb}hBZ$R z*KxGE3?A@->8U_(vgchI$aq40Vd2NU}AKMj0M(T>C=>N&G&X#J;JC2>rYNo!qBwrVL6E+kZehSGF|Yn z8?!}o9va1KrXL$NH`((O$#0Z$5Ye`ta?`U#8}2p2)O=rt%}x+^)6pwuS7~7P1?U4* zC6qAg-?K=Z9g?a0LWn50sMb2iGQ)9n!DxfdgIO@#9J{g86L@p5vEpr!-|E;armI zX4p@A#bJ(kD{V~`Mg~XHFVu}bhp}>XgtgBM6JtnWPWbJ%#iBPx&R6XvWe9h8t8{!B zBc~)y7T?tN)VknXs$3pHDLnR5$k-BFn4W*w_Xou%Y(OYiCMYAPw3sE1a8|*!2Lv{k z5=n}P9;zC;w5<@IR-G63(>(mO`j_(8n^f?=4SStlH!qhKO-e7Xf zn1-AZT`2~VSl|k*9;8cjTObk$%L24DghAs(=Ekc;7&gs!)wfRZ-pLFT$uJ#8RzTV< zQrsx~b-7%#@RBUeDR*q}_T|`X6uMw}s@#$5ZP~`8f-+3WI2|3M+lm=Ur%c{!J<0() z>}i^5%E4sI!=R=Nu^AdiPO%8%YPo_QVwi3aDdfz+TIz#=l-WzHtsit$)-s`;vSe23 zJA81!p!yJO`tVNKgu{aT;o4($tr*K>WEUm!RN$sO9x6_;8kpc4C#O6cZ><>6T8}P_ zl;^ity|^2xc=+imJ7!v6bCRjV*RoS?JiMGH;><`tR zt&Cyj^LuA{@D!)a z(xYHXQ6+h+x9h1DTDd3<{gm)V|9+n~a=0!LS>Ww<<#r|3bo9gfRCv3#Oj1luoDN?$ z(@R=A-8>AiekVOIGS$d4y~9X5Sr zdpcvFX@24m*P+UAwvn=`&>OufSeM9E6RY7@gvyZxMH+`;IypghV`hQZl#u|&u4s@o zQV&Oo5eOXf9L=Dn$`_}@7iEj&@&0 zzu=ALB3u4NV2_TrelPx0k1aLQ)GHQQHu)WLOy>v>QXq5>>#KFyr7iY!n5U9#jr`%N z|6qY^IO-)WHc$0I!wYLY-!eW`IE-?eD)+t2nJXnWLj!v*&Uzr*T*zIozH`HuMaYJ| zwu;%fLE{*{Y0f*vT$!;jG9|#Vr9ez2vmgvjBOJbUiI~D7LK0H2DJ3}lvytwz24H}L zu)C4fn-8BlGf|*g)bKYCiA3|-jSGxyYl%#Xf&fMpO1yDET-)hM#Z*ax2aB?4WGUjm zF7$c?r%x@4@uI*33p(d-eLIXrTe;bZLj8{#lD=CTFWHh3Y;CKrEHre zC#*5YfsELCggu#Z0wIN_<~x@fxY8T*uX*v)K!5*&xBFdf^i13sxiZe%eYQ^2;6~va z1#-Ome3mJ3Vyj5XYNfRe(H0}<)_mPp#~ARj$B%Qw5FYyhx*FJ|+o_>Dz0p4}*2Awj zrETg!6Jw}u43s+hq_-;<*r{*6*~7VKu{fr)9`)%e!@q1KZ$EaoZnp-V7OMSwKeeYH?WKqlMI0{`Z54zMz)%%*UDa?Oe6ZR)50Mi78)F_u3Aj%u;)+N?! zZSUUdIl*2udMGSFf1=nM^LeR`WK(5oBk?smc*5#mU9Xv%a zRaTUW-gl^w>R8X$eqmke={GzUB2^Viyu@)2PE;vExVBjNWZ7cc6x9+GffBzJS}Qp8 zoIwO>T~dGDCnoUC=CqvZ{y_Ry!oDxun;4Wzx2=|=Jma`{5l59MUE)vX`|N5cVEkZu zpin-3V{biTLo^0zbyDZlQl;SW{|GzxxT>o3 z|5KJ0sYzmLmM9vAk=a;z$;)1Aa})(bIijMdc%!`F6-6@+;-=Ag@7Jw_A(b`8O~%S0xOOm!}37@}x2tR28H=6D5vu79W&m*kk=O+aboT z@!Ww9j-QpQYO!y{Ib&I*Ry;5=JT}COG8U@T+~cBl_O1wLOoD!8*L*CjcWXIgIg?j* z)j%7Mi_hmw*M8}sL#hU0+iCGRx8iY<8Gc#Y(#7Ybg8{aO^rr;3H7Rx!Q+_^MgcZ82 zSI^c=sRl47B3Q;2#m4|!q>WsDIrt&V`dfjHKD}*gagOx!2Nw5qc|S(9W5iY$@AHEP zi&X1LCj~>#m2MsJ&BX1hxQAo!7TNmAz5;s)p}oI1OWs5-dCx$2?oh6%;#ua}SKG@s z#ZGOqYAC-LXV)-LX}awG${}3>1Ht2B2d9n6Ijp)av?XEP-3;Ir4{YMMMP#|#y8HD? zib6}3&nv{|q{2?ZK>Yp}p(00ozHFP;#gw2$2`^DT3R2~KmiRkE1wT@@Diex&Q=Ng_ zCsa%|Fj4P-Gmv2q7(7Rn*OEoPDD_G647BK+Txs$rs~!R#4_$WKK3}1mX+RtKM7T&V zh*Atq716l36vOCQl96)TLGciMGR4jJR&?Cy%*1O*QNXarLtUKK+?~ZZ28U>ORLm{@ zR!f0LZ(GEdI}N?t*Hi;X&q${shZYqN7(?i_(`obFC<|Rw1Au9tG^ZhVR?$`ggYd{^ z(c#sh;uh>-c%+|kmT)GI{~9BUUNKH&^S!FL3Vy027hB6&yFh|hBB9iY;oN6lD3Zmj zEVnrenK333sUk+ZQG?9W-|zw0E2rJ4apr_srgp=qG^Yv2PJ*GZ6ay1`JV&fg2bd`_ z{G1ie`LQ%(BzR;dI^pymTTOf*g=c5NIf+naX5tWfh0BI)#x^xWrxaUExfJ~PhqCIv zu%-!fGo8BhK^GB=w*SkfGb2T_!KFGZt2Fe2fO9iz7yO~%Q)NN26VB_DILY&VNt|;J zo>S0w>wwwXUgu>|RVuX`c#(}m1G+y&AE0l`abihQ4Gz!y>1SNY_%sXy5~e_{9^uGC>@pWuu2jST#p{_S8SLat3L^aZ$-xNreL*gw&hqPAj%J z`LAd7oks<0(%Q(GTH6B^PA`_^tCia^UVJmZmG1cs)5A@i-lZ`*3Mqv>+$-Wb?g2N% zruYWJduPOTw>_vGRigGh#oJfJ=WGk2m|~3jH3&zttj?p%t8Tax9@|r7mlkGc1Rjde zOb$qYNB2@j?9XQ$i|qx9dt$%3AZk^Fhk9Jwmi4+gXT6)`h=#`rA--TZs&HU6aGX3Q zzF=Y%Mra!X7laB~f<}VZ#K;;M%aElKDG)K?8yl@yF7;whRjs!zJ05b|wBJ%~LfNUA z+eWlfP0aXK#$*PM;Rde?I4Xo3M`8g#9X;^vys9LqAG$mk z>)ga(MBPmYDqSQGOH`b;s~a+DUYwyOiI@3`$tWQzxE)7*6(2CQE1Wnw6#g)7kVBK5 zp)6IPm{+B5EoaD|S6Eb0>QklNUh(qGD$#;r&r8a;vfV~enN?Q(DUkPV6DPT&sVv5i z5<{XL(rbmh507C7oQ=EyQkM>8xAMMxEZjMk`i-_JK?h-4ksDR_5gn(1gU1~QW&d{P zl(S7}%OxG%&g65<0~ka~^gZr&<~bmMc}5;H$4R{QsRc<&DsjJgvp!5cttcOgk^+?Q z@T$zIaQe|2$_y|Cy^i?&_fuNRMU8aS74ZcN|DJLn9TiP-=ofSq_&^vFFTSDy@rNn} zfzRsaKfSNjhp{3Y6`!AMt@JQ4Ci5-@A0xSIZ4*I=&AZg&5T~-#cxGuf&=dae{WM zbTL)d2yu!GcJT1LzlH*Pev0tyz#myEcxfl!v&kc-FJ|#7l($+A2FJ7GVMtu-q>YSCg zOnXjmf|hBqCyzO!NOu|ln5ptZs*}Nfb`=3*O8h-ioWN0@w(#H}*U6VMbpL^92w#+o zQA{-`?zN~Rh2jK@e;zPOFgs~P&d?wx+^s{qGD4gnTO%Spdwg_7Msisp^J<)Oko?3} zoyW*CGN-~%us){z!VjijH4!IBTcmgx-+sMXobU!D#$x2kI#K;v7A8DLXTTBx#g%eC z+rBf_$zCLNi^@~pw;N3j9^lZTGGRa&$Y$587c+~VbQvvld|WvoPVvG6#nlfUH=GeC zzHXsI3dR8Y*9CEsMf)(*UBKg}W6Fej{sNIIt?LIr2Zt&F4oduyhaG#?8BdN~Fc{_R zn2kY~eGf!>RcbJe%?%cN=Phv-Olwt$3fe8Ycy0f`r)6a6o2tGSr~fUMkq4J*0u}>U z1Q*!JmjkeF23W%m;uKdiszMFOCW+T6?jvb%oxavfNK`}H!5y7nY3eXk8bj!wx$g^}(1C{uOFLtWbBU&|bO~syx6KwO%t-{L zF~AIF4LU^Qu?D|!OZXQPL%*f!18Q$42ue9cNE6?2?*!whYKR?q zRvaghASgZMmK|^=^Fqu;ni!?1viMN&60hxfQXr7>@C%+bQ#cHdC0oRo#26TqJ1LQO zNqpI5Z74n<%$+JudfTLwKy0wkxsE$DibtlF@=A_4$$%s&gd_gOsqWn`*m}%wO7P(6 zN&G^6#TcgB!(?w~kG7aW`tLbT$6pRuVb}x4l&C@YJ=*C+?sU%{K0E%p1gGOM^E?ua zsq*1yagzLWC}x_6{pm)L3G@<72+?oH-xlrs{P4z5Xrs%!U1Z;$M>TZqm-t(Sb2o`e zI7??1cjB#%Zr2OXX%iYA^^OO3vF11%N5v_z@saVtpXn`?az>Z)VdA8>`G{zM{gNt9 zF=-B>T$si&w8HgO{!?tCQ!-4Szu@+KFJARfiVsXj#JKH-9oJ0Zfwrjuw_ch#GNPHr z-yZ4KC2c?@${Fa76a@dnf*BG#LeKWj^(?6cA!uOn4U@&0NA^|wHa_#H+b+!H8#g=> z!h-)`ac4RaWN?t%_RR06LTwHlPhD|uXU_r*#{2P0wz#*C_GzZNioI&JT>Xzu+NMpF z`tb!(&W|_W5eiFqhzMENJF*HmO7u+?Uofp!O!HJ|$7jbnVH};E4Pm!xFYy!h$s5Q? zon&GR0$01fJW;F-6wnr{L8t*eCtb$;^Yv?Z49S)wy~LXa>7gUeDK0DUhZUxQameP3 zmk-tuGLOoEj6e9p^o8*UGo5$6S3`BI$Gb&hh<9EaKIn3wKoRs{qD0$+H>ZPM+V?yy zh4b29bAgOMOq|v<;iztMAV&dllKR1v{2v3o1tP=rmu!3 z2-6Na?_hOWf}DkIPK7|QBKRp_(k%L#WDlFo8B>VpFw;a6lI zX7l`EOz0auzQ?C*3^lxJplLhAH(ZB;p^0Q48+NI`*HIWukL%lBDsL+stxKQ}V(&?j z?O3+0R*s#bAG~8pRHMXer^F8o;gax+F%Sp*&vN8l0fR`1cPeCL*BkX&$nUo%Nk3*)>-tG=z@f zZXDa3E$A(4D|_?|Pjl~kWO{YL^h`SA#xa5gl+m_l(rWiU(yx_Z4}(rA-j#D=6Svy~L zA`iR*X4KCpbvAICvPEc2*Rw7OxY)6JE2LacPA~t*oZL~N36JH8?!9EN^M{tgdk##r z9B;1xZ7DJ3nA6Y8*p{h*wK*-jdJ#kkpwHgpEa2jWQm#ZZWgRAuP8xz|sRO?MV@K$kx}rf#$|&t%jwBiNpA<^&5^L8FRC7r_L1@+t8& ziS8|vv^}2NA%5Y4q0%;}OEBdr_43rz)g>x(odpdCtMx+l1IPI=C!awt^_3(D7dJZ# zz3XyXO0S!aIbU(n0EVg*B{%@Fz*`_{s%?5U!|Fu(#y}`=do@aR{z(<3zA14!Rs8UH zGu;|6qu=h;y}eVwOIOchLd2rW&++n}mX)G6F;=BZa6tE0PVDlBX=1(Z3W(nr7;S}6 zqQv&tCeRM1mRA^y)v_a@cPGn1W6fmgPQu&7^SC>h~&%bL)Nb zjqZaqj$W(gxb@!uRXNExs@MAQZoP4fbqND380PkPe!2ccBMGJ)fBB7(qXK56Y)W)m zd0!x7!ndQH?S0Q%-=bWZ#!;h0`BW!?6e@Y9z4+{2+ls9#q^3ZEX+XVdl=!s0lf|{0 zG}JfH*B6L?d2@Ab^wJ1}5)b2?!0F$Qwwg3`+HbFreJ&$CS95z(bO^($NE3@j;g! zIpS7QVnXo5PK|IrpO&UisuZghmB2Iiw{d0vW`ZtGIVzz=n2h7}@os>j93=*4b$^8s z8=yQM*}MTk5Bc}E+ejckB@e8IhLcpJ?W#z^VZOlC5)u0%OgVqdJ2 zoYh53fe5e~1Tx0nasCGth3Z@PKX~-MZjXCiM(a@T0FPjn7{UEpr5tvp$~)slMdd%W z6hq*y#y_uJ|48_MS2`Y>A9(B-dw7_hp zglWR9r=7MeA<;}tC!KJZEHsSyRTB%S!0r7|z_~E9N;eOr`0Uc8BJb3FkYCf72Q$uBuUG_&GPO!8}o3vQBK)V#9Zc5ng&vs!`&uT23pNg2OZN zzstq1Rr-{y0oJ?uQ8AJ=0|ksyN^DjzpK(wg&NjgO`<%bfC-a8_M~O-2oF8{|)k_ss zy2`^x#ZZ#Hs1i+;=X!Sv6C?R(@rNg6O^jP-_6+?sg$E@bJ}M)4=v;|K5==#cjG$F| zVikcAN>F3)qVTNRt~z7Y#+8atrmOsAv=zyaRzc?nkHS6jR(fsz>)E5~sB<Ye6Ja3b7xam0?lE z*SO(A9kJXyWn!vC#97@hK3N?OeIDKD-oQr-re=|;&(O`HJxfS&K!*s^xAsW?4f+GD zfT4cbE;-DbiDxR$^@-gh9tIS8nE2?^AWVjFQ@CL$=xP#LCCf18`@lB7?bG&@NE}(I zW6UrO-yyraI#k<38>mlqeWz&e*P<4Z@|3oH28D}97>))IO9`LdXF$AIVqS4Kp6heR zDcOZ{^b}W&De*_;`V1)&UAdV84)vo1nJ2rN@0|I}_ZcDdH3H0M{L&}mn0SO6?byPE zRG;)+V$u3!?H2=V;w~|u`ZW`fzLepdBCqvDpl{U{Ta;g_G8`&`Q7#Z$den2aaTI3* z2Qs0l3WY--5^%}#^Gc*=vFYQ*`=Ush zM>@-hv!O$J5RRTH{X3Vx5NQIY&#@h)pLy$+M!7(&?6UKmg~e58fYo0P$XmQ6S0urs ze}Wv~CGpa_B!(51Rx%ac8p4rSKhden>5eLNp#(fK^W`{hP;EA&Pk1aymQU0z*T-Kd z<+l?j?RBm7YxDsbzY6Wd`bWiZZ~Hn=g;e5Q2dwY7bOk7769`O3zGEg1Y-L8Do%rA` z>-3Z@)r~OGx2<#`hiqztqfFT&@sSGa+m69%MRIHb_Aauf+S|?u}>75edWeZ)xt0JSe4bU^RHW6fbjr`&toRvkw#u^&g4hr{o;(H1<|ykEEAYyB#NVt&X6?f6sAm)~5qzer~ zR;R>}Z1FirJ6yHcic8i?HZhb5jgkLV;9QB0R*lfOA?Zw^)A39*-3WmX4}TLmX39{F z)Bt}O9)@t~UD?i&$dJ;$Ow42=mBzm1TqS;nS{(gz;@N)})JN$1BkCHk@{d@k`t zJwgF`Z@VuU${J47>G@3W(APQZS}%9eiwF%BkG^ekTrqq~^(I%}wmW1qrn6u$uI#&X zv$X`z2SUpA&FUh4BIzCGGf~)=w2AdMKBEU{EtJLju01Ax^pftP4^m3v#EGei_tv7bQOC3bI(QVR!J3jj4zVI$;7!8DS5iYq>XYo7p73%)ApSX;z!=p!9ePV zlw;ZAytj~P2poA|Jb1l^u6aCqFw>oQ=8`zi(Y4KCVafKR&Bj}G_~2aS@hC&MAI(<0 z3Cl-&C&-vuyLGHLRn6nk0U0vp*uDCiiD6oK*16c-WPs3OAN_2LbJ5&uG)N!)Zin+L zqrNJ1DO1Wly0@9g=Rm3=1eJ1+`p3&*u|1Uf%If&`FK6YZ?@jVws`%_ji!0>)tQjb) z8yssd$@@7{s2qin{C0B8UgzSZbJ}ydL~?^m)|1TUr}N=6RZ5P?PLm?lL_%pym4~97 zpPS6oUk{_p`HoUOCx#RcdQC!>YaVRHJ{Ewl-7Cp;c0kWm*JsK7O2x@2C5s?GNWlmg)K~*OMWIG?`AT> ze-&d&(Dl_`V(D-&z>-hPDeh;XC)!XhGgFpq*Dc!s>sPyd(BU9gLyTNoOL=RZY})kd z3(7F9LD);u|4X-M9L9uuj?3nK&6J^7)jTnc6sqKf1+RF=f$$9m>TyV}B=a@JO_$Rzx|?hqzf3C^rb~Rj zSx)8~1$0lBz?|R(a=KC9*dsYmE{o}LtFshX2D_sna7q-c!Wsh8AQ;w|OEm~cS zQM7+9gH13fSHP4JY}B_$EQ+gNirw$oaG92JPzRk>uKzt9gRhunN`)5Xn2Jt&qe{nV zK&$`V4%w0$A&ZE@G{v;vtC!^X7cW*jyMM3K)^i`+b_``U=|0A2l zx9s%z(bgDQ0mHmqjSs$>+mr3(5JbYbbn^6;Vz{DwC`Y+pN9pQFf0Nwbuf9<1k(yuxCx?JFY+N1xmqF`TUjU}OgNr{ejzCnV*p|$P*`~}(Q z$UNP$jkepfWFubQ0F(*-8U{*9?BLd6XJof&**-gEP>g$n^>p=uNPpgZyoJpr#2Tb2 zZFUA9K9!)ALp1jIF;h0UsGmpjbXmDhPDVK}c!W!dvI#NPGlVpyPqcN(40}xQr!<-> zoBgGSh7K0Lo$^qsypz=)Y@v({uun7HyBC;GG-m8=r|@1<<4UvCsV$b$C0e#JI6S8F z!o{uEwbIHZsi5o6OC-fzZ>(;&{bxtW`5)=0^OPBpvS6QR?!CxHm40^ruB+v;{387Y z(#}o^#EQnuZuO!vZ;@FnhBibw!{hy( zq7kcK3WrgSL<`r4FVcZqrmHZqcYQF$D-qlxZy}#ST}tF>MT9|GNDW^uux_s4x=wggq1IL~NGXB*qy0_F! z`<0xP8>akV@rSDBxBcx)MJByAe<*z*bl)qwaZp&{$kRy6yU4V$pNJ}EYZc03eyf>C z;T|9MFeUs;lSQMg=7CQbQ=$f8S*mDM{e+*-_P>|moF^#&UQ-sM$_uH%os5*QhZ$|9 zMx7SnESJL2n9%=R9V__bIiht=bGBO&&blnd3 zzNB(}49B=(cd|^6x=x1_jA2?9E7Px=sACaT;km@P}CLI;m;Y$861{zSS ziF+eQz0yLiObsh_JMmRlRq~Xh2F#5T0kSuML74zxH3$pqiva5ml<=4?P8R_dRh0L& z`UBqWVjbqioZX=wX281v>+rWdlnd05*0u+{5D*(U(p(m5Km$&s2meg0S0WZHemk{k zg7eEgW2y(NsZArC&&ZswkirL~XHrU6kvj~pdD{MYoorTYo_xSSlz4BKC}$}P#Y`g{ z@Y*SP>$ayC30BxyoqH_*cLkA&0)rDCpk zY%QQl)FhCcU@pD|l(AT9gG(aUEFgM>mV3ki`cs8eLr+aU6zto5m2T=_foW<~u(x+f zK>Iee`8n~|M&|ClDbcn_JnuabYY2}OJ4KuzQf>f#Z#&g@K!o4*j1p;P7O4rx#LWhn z5^MjD68Bb!OWb=6qlv)-1N9y)o@YhzB1{7s*!ViuFLf7crZ9%bxCHALIs+xv;J9at zyo+ONTd_J)(B=1Tt~KL{>S;pi!+T|Yq)#fs;_8Rk)rr=5uDT&Xm7+wG zj%7# zj>*SvcT~`s?jkk&jC?HePdXe7^o_M-#I=`<1~!TkH*hR^w?T^7uj6HdGh@_T%xF7s z!x{I!z$5xdGg$ogz;=G ztp_9*l}!hZzY_d`b*W0!->RjIj1#r`pVe&+&)N3CrU|mScgH(V8ED{RIo7FQZ*6sL zv4QOGog+^#5=?vye7vK${_{23x9~u@w~ojuw=@Yg51&1-4_A-!3QS;$# zs$*>6tWxVILK_LjrUMtoTR$`7Qv{=nERwfzDK8N6?Lc3Yye(;=0_IgI++(S2V-!=R z=#-{3lN%qbuS6h*Qvr^$178tdkt^CKODJ4cwjVl-yNIBh@$twT^eb{c%`m zq=eY@$ui>djOzZhrj+tjagam4M3B*C&<(q+d1M->j*r19assCqv6vA%h-YV7b74y1 z;ODmo)sM7JG2Gw}&mMn7I;c^qHHYgiA2Wn&r~~GsgR(9+>;f1FNg)07sI2QHN5ZZ! zjZU+2=?g2dm{xA!>20zB51%PuMr>TH^(E&|m4UDtRUX-FC3~5kVeGR9_25N&@5G&! zGRVGQ^=Bn-IymBViAN)?pkqU~<2Ewk6UJk?SK4t|L3pCI1h zK9J%8FvPZtl+6z=4NU?dOz0r|q_b2djB+KT_aA(vlRh2=4%ELu+GcLpy~S|DAYED@ zhVP8j(@3fzWk`{%*QlPhSm1vXY5;q_Kn!J!yA`BH>A_I_ zkWz!NFJ88M@uG!d>gTry-?uH;m-ib0WqR$w31LnO-DePOwJrm{m?MksyIqGAqG3u{ zwz=b8JwPi_AFV+b3*;>$t_w9&m~ub+J?|=H2c_-c8>8e692mN4MOt_RH~1*=bF<4oevO?X;Hb#OJ=tdcbN3dShSH>uDz0h6{hQ*`H4IGUOkmBvB840f9!81L641TR^oT*nThd1!m2*R8*Q6}h20 z(Oyu8=Ehyrv<%;#hVt35$@1dPrNh>!bHbPwSu6KlKdz{=Ndwrs>s(8BO?6h|B-AAOc>5#T*GUA~F z`ul<;aFEWy_wl!Sb)!nzW5}|p;xOkO6m%_i2#-NMw(B9aeqww(2d{oGacpgd59BEN;N+KO9$iDX+otYBMQ! z4$~cqanzfqrQE7}InOV@q{`dkwpVx3LmQ-sa~u`LTI%PshhD!!-o|bd^*sZ?SB*S0xmY%(8&^DF z43C$)h>LI6(Izy?i9J%b8>T=Rn+`o(VeKG^1{ZSGxQ;4s?+{z)J{41(1%au?#izPe zH;~hJAS=OuUuiE|0Oo#n0%D{A1#0md`gUE$2)bCcD4alL&}E< zVsl!5?H90M_0Ph@v&_VSZv2uqF+*;A`&|`ZreCkop$rY2kzJMzS54h$o9;XAEajbM zP@2whDA%#OFlnZQ0hWH>Hfxyog0dD{qq`XT{CJrtKhrIHx7K!Vq3rV7bD_XRIUeci z@?z6#l;cT|E*xkFscQQ^JH30Pbqn(bR5wOWf2Y7HX27}?&!HeCnpMa;WGZP0QcBTn zm8^4~x?!2_wtx>|ak|rh^QSRXfiYr#EtYNB2FLqK+unBi&1dDkcj^6XPyN_jdB?Kh zy4%uK(qBEce>dG0B29^5z4tg|xy=jcrN=D?_PlsyL9vZF}rfL z8-m(EKWE5P_SN~rtHRZcJHFeYP%23<9Z+93>%3ogD2B&?Gx84RQT(BtMTsHNvKgyi zO4~fG?N=9^EVAL?BoEk6qn)*+>8VW#6r_sF(G@P$U@g_uvDiOOi$O_`Xl)VO+aC5q zEwPFl{=txehLZr}d2hiTNq&1cPmuQ_rHm?J2;DcTkQy9uB=Je zbKH8K&HxOi#E3r8&JNt5fKd%8V^+&61oJwSQGWz4fIP#=t`cxi3Si-NWR|BVCy*+p_O* zkrIaR#}XO)V%>YZ)D&O=`jITzmS+H!E{KM}GjILwD^uq4aKuE4pBNeNUxOnfGf~{Y zbfua=x}bH@uj@W}t5PmwZL`v)%R8&xdlO?B$I={LywGldF;Xta$)4W`Ef&#!J0mh) z`qt_+fyVn8mFt{^SMJeFt2EQ5xw0o)ScH@b?Tp8>#JT+@?HO2l+oQ%FcUOG*j0T30 z{PxIZo1Nh^9?%vGQ&L7oop#i*69wH6TJLg(lZXRQiU4#{sPuekyLOcJZAQ`@*>k~e zWeWw2+qPjHdw3a^6&7^4xLMBQr9A4ZpKXuGKP8{Gmuo3VLM*v%MtOrg1MKBgXSugm zkfeBwWOd{p_%;t7rogxq%W&xo~F@ zO$>&{b0cqza`H;wRG>VYeqZ3sX9TO%*HSX}x0k)xenK%tM3+B8&v>y&4qDnodoG}m zW|SV1gD~OlP*{#)H*y%g)KwlMXGe&CykYWO8RgnvlD)|6;~5#G?Nem0A5DMyf0THs zT)fRCfl7b{wQ@5vWskgL)dm_>NA%c0-s0Nma#7hMV@jzkNeQ2iqq0sv6QA77xwaKTn2p-_@*`tPa z>G2_lIkrMTV@mJ@4_8zbQ)Bf}9gD^Gq91j@lqnpezTD1|X^(ddcU;B2V`e==MmPrH;!G#&mL11IKj91*L;z_9?cF$#J92-NK087$RJ_zf< zlj7=dof>Z=cJDFK_wBVxITH?}_SF}Cy(3me+gYWud1Jrs#^FPi3kR*%oG*+EeZ@wR z4rbnsuZEPfQ7%esAaJpTvDoMbGn@|IZBozz{d53sgAd z98vGQe!Hco)}?=(9Fd%^7#gdOsk2j#Xl<0csJe?$?C}l%p-c5wy#~|K*}J;)fPo%M zm1!gYG5ANnlk9Y)5eAWBO!#ZM9D4m{)m27w4khKzcQnE1(rTaV&)s_u7|M;zl|yPT z)gvWcV$76MXAF5ZGhm7dC^7Y{9254m4$G}dxiL8$*CnlBn|D9H%9(cfeH5KOnTvlC3%9O9|y)pFL*h(H_MtRe-@XpfSBp%e%R@ zhe#7+WBQfKyLrY-RnbVt&dhfL9BNf1@|2XZ^&_2*?42kaM(p%tCxb@}nGKVzk@PT0RN<%W(E%R6hYUoCM>+)WW;|9F^6l828)ciB zx2WQt8)^_-#c^^P+~J5PjOOs#7!Z^Sr8uG==A5x9se~_ zGDqzQp5>s9!l6ryJ$6ubEO=axKfpqj$AtB3onN(BL<2oXSeLxPW>e+;%qQzx&oQ6l zzsjx5i+Cjh zDS$F2d~8XfjCtM6_cbu}THggubt2OE<+0u4rQ}e&V#?2FKjy5Hl2d8~i?P__^`^>u z_ui~STSI?5VvoG{{joZNfbX{-&pRfPN!+^|i)~fVAK#S_e8^k43n*aYNvm1#=4oY9 zHLw9``(v_j#wLvvNmOCEl<$5B2ZZlPAN^T79Un%@H^Tb%%3IX|^Z2B5OwP>1BStK} z_P3mpM-LDBF`ao-879;?#<7Y&y<@OC_{WuZv7W1K)-9VV zi;df#DY`O|P*wOr;82OhqeIF-G0KGFUe6GxIgbk{)1izXbjVpqvZyc!l`5phH{5LP z;Y^K+Q^S<`{w0oA0^0=4c(#of^2iQ#l!7oFwoQ&Ay|_OlWqjsFtCTMg>lTVgiTr4( zav`CIVhmx{B{}NAZXL-mhVbi78AV3nDoO-Yz+^>4xN8Ql*WXHj`R$4O_quDI{!Swe zDw~e~ce(TP#Oao?oNkt~;%g<^gDTYJP3 zdWSsJ*T5z;*y7wwtIP~_!Q)9M^`C~{DqDE!UpX#zaF~=T9)Uz)S!+yoLKp*#FX1Fs zt`t*#etSZz&7zQ50iZm>hBJSfnGX9%lndSdhXI-(3#dgWZ`#)d(otxOYzG%%p zay76~MRM|udfH_ApouGUtvk3|tt^I3a6|ti;;-z!DY1rd`#ITqka6YmiRde^eB4<*pLwrplL7#f?l>!^}sEaL5RA zzUIpTL5Xf_XJzklzUO$hvKZh1cK(u3SvB~t`XOavx%k@bR~i#$aXWvlEYe~Pj2JoqDydAayuuxc85BpNi_%wrDFT~WTlwW z7^Z1^gmvxK^Q2X=J#Ha+XBbHN0sXlmxnkf(P0%Ksv^-yoWAU#x1`4VYlR9O|Xim#1 zu^RfME2-AIESM=^T1wXEP2_Ao^DmjtD^-3k5*3{6$_q)E+%L;2<;o28jnyZQI3(sm zP)v>3oA=21JIzx*F!kA!6Sj!mEQYEQ@PUv~D&{f4^Kd|z-bCcHrbKbEDe^}Lm}90d#_gu@raRu-zNc^uv>wlW|AgYn$tven`R(m8~M zz8;gykBhC1>Z$(N^vN}tP8ks{*0GYe)pRI+drIzJ(Yrc3p>0o@xmOfzzog?7K9nGz z(zX`@I@UF?sdpU+*5N{30E>Z^690@Ag{vJ6ohm+i3U3{4BWH$+WF5FuTa;QkT)Ux) z@&44PLsm9ouc|*~M-$nID@2MZKfgV-ZM0a%LKO^EDW5&{&N*T^#!=7>jyvPUGKP|> z1V5_8v=__BF6JpmmAg~LvVA8)7K2cOsaPkLKX^tl^nh8k%IX93$?Q=yekBHDxG>#fZp4xkzSjM<8V34YS=_r?xin1;QVijO%5n?%m zUaaLw8FWZ2zxj?3bd>9wAxc?hD5DB|sM2?;*g`%>RmIqYS0c7}mq^Rxlf}!WAm2u)} z{!l!UG_bWP4Av*-NHFoRRyGedwSQ^;w}~t(bSl5I#wUW1CC%& zk|?oqj}C36984kHJym?iCPokfd=%lb_@49Hl`w?>(wDo)v0o->9=gP|SHk631`~>@ zE-}6B6?sGJ7S*9WJuXAu&_<~Ls-JC7`+mIafAl>)71c9vjbrMT`f|mU zsTw*8V$;L>8Fp2 z^*pJpJjc&xXLmf*V-V-*lz{@;9^a&kQ|xJh(ge)(b2;L_<_#)ipzIb~WPLXLi7&;} zZ)f*DB5zr^OgmfKH2bw^>jy@CU{F#hk;*+!-bPn8MG9hvc93;h{o)VfsO$+4xQ(um z@}mmR|JF^@{g1L3ZeZ_q?kz7WktX=F^9p6%sYW@l!1Qpk7-d#!j6JeT$b&|&t#knk zOy4Fo7I*OKA~sV4`*^&J9JxZ5&{DE%5Gul@dWV<#2FHx)7lOC(uBbZ3cTuZ71(X@n9o0pLK^r%wCH%^cp2T&6JI9DzpIQ@yLpj>4Rr#<+N{Sa;~ox z>7X#A?PXI7pS8+el|K)}EU z9y7Ygmespn@TfuHsg{;3%mB;`NM>fRou58U4dE18cyJR`%u3NXG}2jno4M1TsH4Ag zc%Kr}3*<1?lfj|*AcduyyLexW;))1?6@j*U*H?FaGrzBpt@a$y4LPW)Fem#f_{i`~dgo`C`? zK5{so%~F=urA!f?n7G<~YX9EqFHjEmqoNrn1Ih&MtduLVo_8TV7 zg6TKm96b!iF8@jz_6Gx*2|>VL50H; zx2B8zQGG&4A$(RUbI$2xm5PWx?uruo$(iA~0Rm56jwf|1^-VAJ#DNR4A(K_)8ujPC z8*o>~WoR6R$FQmLvFO)zd(j3`Qlh-1s+6$pCmuT_A7g?CQxj6NPo0$uBL{}G_1jOr zpW{yW?Q8WZP>lWd+zuDq35QTANR*aW#@>}9=X&_b1%!%kuYRs^mho%8p|{Q=xq zNl0m{&wU`yoyZ}(43-&ml_%btBWL!TrwK9Yd;4U=eb%8XZ)qHP%5zUP4!7R-wimF; z|FGDT*IyAIGCoz3C;{|cXRTs}l1UJZ)j#PXm)RzxUr-@^a>W+0kDPA1u_;m@T&ZOR z5}Sk~*=Nsb&9y9EMd%FlGUb;!M=r@(YxEAUanzFq0onGKC-g@g(HN-vUaKjO9@Y+Z zpHD8`Cm&)p5DBJ<&Aqd}c$@rE5Ii0?#L8L37!r&do}5)Ehu>>f@r^Fa&&gqhzgS_R zg_$QKcgZM*$^(I*Yk5|_;4KgZl?i7t+IpeQQ`OF%%P9D^ z>-Oj|E+v2-wNp;>rW)E}PyVt^=6-%sn-DYl?c5ew;sp7VkziV7?#l<{n6Uw6aidG_ zeV4`YRdaOP$JstRw||^C&N&nqdL@RR6UUwZRJZKh!UXZA_g-V3!jU`oxHw)BZiQhn zYyyv0!o_i}%T@DugKXk&1~m^vQ)NGdX=(a%qSep+LjQ7xA*!tJh3tb4zmNgpUqtn^Fp0{=7#{2%71v z5u0{aF8QvT(pG`P9xSL9vH=M}>EpBKb&Zu1{u!--X%o(SWFI_wS6gh}fqgPN?kC+D zPz~ruQsh*o3Fu+`lDFoNHJgtG1XYa^^!&5#FvpTm@P82Uq7&upPf`^IgJb@)74Do~ zzv@F+nn&KssWNtGa`nJDuVt<*J`(Ha$Xak(|yY_=}+!%d6-hW7fT^?GeySE4Rtm7=5`%DMx+wa+GJWHw)yPugB>U z8v1;O3N0g^*RF&BkECPbImV}8Fi7Xe2IT!M{;LQckNa}v>LVYh|0)wwMa0Q=YiFwe zs-JD=&p6`d-|)NcVK5RU66(u#@0ux3m1<)1nRtKy$UfaCYaa8yUM)Au35ucOL5Y!5 zWxFjcs=c4zbe-E^+!0xqPvd6QJIr5FF7M}c%FK}W=TEFJ$G%`|rrPTBIZE7$G^3qy zcBMj^S5_)lY|tBJYQY>}+j+_8;afshx9#~8OXb*1z58a`-291`+?ASpHrPGj1r$5k<%t^)l(k$`0f1E1lgMB8Um)N`|SMX zMdEur7es7{F2S+JS${cF?c4l89po&gkyJ5A^S6}B##~|rwoxv>DpGVJt5-(I+4)(c z<>)GXdPp?@q{CMU#yTHPt-lPzgNpG69TVY+~9uZP(8P%R3F`|SlOJ48NN6OdqRn*VJrIi-H* z>Nb~uWvU!)X3>g|&t4EXW1YH?rW5F}Zvl_6a+!JiY+VJ$5SkUs6s`|~(lA|6w^U}f zGxL`u71|4)i4zMtn%7+}Y995k$SJH$poeK53&_$u;V}^kOP{^)e2Tl2%z_HKkrH1h zC$)G(Dds7cm@X%AtN?o&9{EQKqPYIo+_{geq!uU*jms5_0%d~L)6juL5I#2WUJ_(Ms8aNALt*x&>0Y|UfAPow2@ zuDB~6AjM)GcFHx6>!$=27R36C<*MK@Z6Hd)orE@#hSXp~#%t0l*OF)h?yC~^H2 zxg`0J`mfP;;Z0lQk~>c6;4e|1f`zx9mPyU)iy2saGm?&KYrAETJT7* z3x=kPt?V~c88`Usf{4q~V(qiUDCf5e;=^S9Uiyxe(gi~AC|RF`xk|ZA6_yKsJ0?eW z%+XfYVhiSV5L6ryQL@U!LE?;wBslJ{c_noXLv;N2ON(=owMSEnnQ|DOaaLF78%K zM^KUmR(mhX&DZS(eL#A1u3X$Mzq&;gT(?dxmal7qsWR%E9LxL`)jg)S!n9MVhHiMQ z%92UsRZt~N1T8I=gG)?qN{C3gd|B*$rk-w5Rmy~m&d0mC9DPH5&!)d!5(ij7R2Jh$ zm4C;If4ycB=I&O6MZD_vV_>0fa{zSg?1OT*1}Qtr&i>{y z9pYLRBF$%Q5sPlq`Fc>?#8^$jkc09GE^H&g#Ob07iQ@gHdfC?m|Kfp}&YOR|tS8=^ zmDoiu=gK~3E@>$m$0F|FJ@N7a%_B)``>POD=Wo+~0gG)fnteb%!H8Y~Gf2sl_C!ul zna_M)7thRe4up4;5AvBO2^UhnIW0RbGn;K%%HjnboC8Y=b;c!2l~~+qi`dA3Rk?u@ z7?}<3j33O4Y?&%7Yck!wOV0OBxCb;+I+Uj?@Ol<&4Sm6-ou)Z!cMV!JT(U zwvG>&BKCz;afriDS4%7|Ef%Y|xzw43JM=;9+i7Bz*(WqSY7%PglLe2|)?!VS-=js~ z)7iQKRjPJd_;Q?E(sq=#m`1v!`(7t_@(lq>m68q|+tTGD%z=nv73+xIoFzxrJFlAu zOnIs0WQOcdmer<|;ODcKj6N+!vORebiy5Xv62)J>y?U925_6KpWZn}G5-_I3@Ezhe z^3zp?Vru=$>C*pCh)vU8lO+TJla4v6`)C zB?ag{yRccYv#IlpP!`P;dx>*dY|rndy1s5@+P7V<$Qdl#VpGKwV9r_b8Xp|QdIWhz zW-s}$+EGg$i;>xH&A6t-U3=tHtxS?g#G>t`Q{sK@1SlLx`d@^cVmad9X>yoWWU1>U%u?=*q$y z8Cv&`2$ityr&^tMpJXd-w+RPuaJ*pvvql9yQ#GK%hmMO~EbOWh_y(~~K+eEGXlkSs zmP9!7$??CLb~{CdW#O=O;@$2h=^dr8>Benx$y?^_YwQY-XD`Slds3AIyBnVgeY9-5@Mx)+!>u&01i%g#i+8w>d=X#}{r1w^GTpgtFY1871U~zz zmfOS^y+`XMHQ@X0r>5@{Sxs-!Mn*I|SO6&K8mrozk_5rsC=0rq*E%)fsf8)>=7$bw zZ8h|#HWi6C+4rEnr~a%I`AqZ`ov^M-F@5UwEi#G?h%(dDL1=eME~#~OpAf5u=gXo8 z2kH_a_1l3}tKFv;y`pCYL1n_HZm2Jx9`=m-?kjL8_d~8MjvcLsP8!G37UlA$?q(BI z`(pHK_lPyRFd!ED*z-JTx{J2)k*yH(hcXs>R^oK7`*<YRv`BI(S@$B(3z{okQblIv6ba#|2FzFmT z_1kxpi+VO!@udh@{f|hOb_kb?I_ce66NO8E%@MCH{#7>*t^Ts+8{PcDGu38{>f8PH zvcPJ0{$F0wA%*DP_R{~7d$2k&tIrP9+9_M`(aHoE%GvhP9sA^@r{`7&ZlDoqz3(v- zJOh2mM!BqRhJ~h{E;R^MnQ}Pq;h>8$5(auWTHcb873wY!JMgG%dF@kR0d5Os(OrPS zd|m_FE{HOYiDL^@&1Wy0U&|evU}>uxrf#CVu6dj4uW2B2ubj5QWPi~}1IPEuhgmPY zNSmVyV!w-&4?kF6g{2x#`RwJ*I=D~1I7%}mw2|`tPIpkxqx!RkSWI}~iaU@YoWg;T zAf!Z#PZ{J@8ekAw%#q#yoS*~7lo(Vj>s0tduJqf>Um5Q%>@-s&ML`H(opuXZ4pTfd zVcBbY|UofKICE%JO$}-G)cs(+d%sm9|Aw&$@-!qY7sSwfc3B$^PX1DglR9e|f#j z5PEe(toqq@k#CPXixcju3Z?vZ(PKN^{O>R5_z0-pvS{*VH~;hLA@w25zUo4N8v7D@3)zNnRu!#OI73se;P?G;_a+>u>R=y@fj&~|txIc9c+LYoq! z^4&h0^&*02F%nGA+qYU-+<30227$fC#CN7^SC%NGD^^Z*TX2|zKa^d4_KKnkx5d{j zR3Bz|tSF0hr-dEURbWh&kN3G#|9zt-pgx38YPr+ypRbXIsS+z+Z12t^p&|b@A+_@P zCT{LKZPg#94_Z02i93Rk<3+U9JXYqOawoH5gjh@SSW%KHJ8`@Lg8JF^$|GCc1suJ{ ze9H1Zd*#RB?t-qd)r6uicFCtdN(;pYzzWvMsXGd^a(Noqv3z%E`Eb3or`@n(bvHTh zE0gAn620w}&5LB5_h#%?6);ahUN!`QA>JTMSd0_EZuw_7Y)B|n&cvd+1iH^~*DgMz+_C;YzR zs28AAXF!R#LOHwe4&ARC9`_14dr6_{7}d|Vi~oL3?A?8J&A?}`s(U7QjO{0kqeR2B zd3$GK)o)@Z>DNZ97FGPqI+?*VLHSl&toYpsndhC^M6_+M8pC&v?6+k|;Rlc0d^wp< z0mV&~XZq(ly?NIQPzs#iekL{7N#rbkC4``I1cPPU^Bp6q&>00iJf+&46MN3(Idy!n2{hp3> z&0~!(;0oT4ia|L|Slc|)T~>I9&SDkRNVgWbop|yR3C8&?XHMkxr{yWFnLPU^2mK=nauv$JFY?Y0uJ5GXP9khAK%d3(VSI7l8#))mWi zY$SP$Sa)wh@Oo0oCBamO)Vip2ahLa~9egP9aHR7PbJM)gKwy3RYPXodL|v+& zF(v-C!|g|&O~tfIL(i?_{Ef{e{!rBr`&pLTuXwN;y{b}F2^0pore=kOps$bY=%zh% z&0fL!m6zSZdDp4&x{^edvR&2*a0ICWbeR72vh@XrCo_>^pr4EnKFfkTpp-t87PvDH>KmJ`FQmI+2CR5&Robq9a*Igdc=kmI{2-J}weD_XUbP3U-`;Th zUbpd;xAbx;C4Bb!uPVBK!P*v@n$B@Oxo&=WXP*pU4@>}$(}C`9bB~V_%HdS`F0%U; zvdwp9*i%<)d-hECb(E;21UOWQbKQSsR-l+_vFk5HcmMd>bJ80k5}X^IiQ32kWx@^h z_gcSld4oR;jtvF1T>H*o^g}8B2op@5=B`5=XK5lTu{b*suYCHrOpG~t_7u0ZexQKXVqeJvmPEkdm~>D z$qoPACLFNG?~&pjE`U%){g9HI=??EPNlEYsH?I>{emo~xrBREBK;KvH9A@uTmB`ao zI7ff^%j4>JmHNTs^=8gVhNFm7Kg5aA%#CLYD+n+o!DDriSngc`(AsXgF2aq^KBJ9{6u-Tx-BCB?p$CBARoT?2 zmg6ImFTylWh190O@oopw@+x3Df;RqiR-B9bNzbAY4Uakd+^*ybucQP7s&qIl9wwQG zvU-^Uw(0RL&S7q}gF*c$G5(x$xJvgw#$ubY0?y${=QJ==qVECcz_r1@>AQ_?UfrMc z9}PNK zSF!7ho}e3{(dC;a&hxaFt8RF9Vn=sLv9`Ez<)+=qVj7#m)K|J7<@FRXjdXOFPjyNV zK8ttW;?w965L?;Bd5hiwo2o|Wv!5Lk=5{@`LZ3p?HhuQ-FtAE2jT7bP5R``}1AFDQFlfH&Ej3Y*%oE22dunpY4<((pX`EU^?k%B{@WH zd{tW=a8wy~+@sLKnvl+b>idV zZ}q1Dg3sREn)5-tA_7At$+kDghlyu6{SyQS`uOe5&vtNQEBv|yIH>Y{u{-6~hU!wE z3M)$ND|BPG>_Vb9cG&!5H#eU4^8Xln*SM;R?|&Rc#Vd+p;uS+fK~#)P#qjKz%~2G* zu|*?MQHs=}BrP>k5k*Chn%9IvQMsrnN@Q7H&fdq;C@eGzv9$1$W~HH+rk3`9ubF)~ zyuSS&{2%NGXPud~-s`ew&t=Vw3CybQeEhIO!v$<}Q6+@b&-pG*{}Bh10D~#zQiV?E zTS9_KIE{>M%qfvgC62S9jAKC;^`*sjy8GDAVQ&*)%fQ_^I@u9~TXWzO*^KX8=;i2x zVG~fwSl_7~a|FF7hmiuu&d~*qAnfFda=LP7-%G|IER&L05H)otZio67NKLV-x-;44 z=(Ipw(?q49oaLC(Y4z8F2TFmkvCt80#SsQhd+cQH>{}#XUj|sZ0$+H5fWV)il5!k!4C;_Hcb=T%W zj)334a=i_^n`St+4z>%X{sev3Yl+MgiA$s{9#!2nA9;=#0+u;_yDKHmD8Xz`3A`_v zE~=UXZ;GKk=jR^b?>K2BqOldvb(@mf`u z=r|(+P-5>%$7^%iiW2$4Q9sV+%ZFSRpXi_HMc%zS($V7UJmDx|)7^7JjA--N)l@R_ z?wNMh18aPQ1Cl^ElHq8vw1;rRRkHf-6*a6F3n67F0gX{2Cxb;`m_XXP5s%!ao{p&_ zUJ(0$mH#&Zoh#PpJ6b%T7SpQc2G%$%E9K$x?Z_{???)&!6a^^L9{>6%;pSzOr~)ZX zf)x!rmZ7N(huj%{j!`?S#PpA_SgvKh@f(f|Az-jPszg^9*Kn%nZD)g$`x0}sU@wc# zCX_>o^iap}#}?A80+5p~<1Z=C;KNr19zxsP?O_Hbm5_mCiM0hRw&4hpU=sc;cJv*j z3pW71rslQ7#oimd#HvP1a%Jvs6~@0vumcY1%3NE9r3l075D+BO8`q8HP#>DgMk&uL z+0hp%yd(ugtD4s)$@mPr^avPC04eGf#~7RsLrj4bC1US8hJ6v{e8oWM=b;}!!va?N;j=cKcD0j8|L_!qAjRgq#=%@i~!F*?D~IzVQa3FY##{2Zx1a-5A)C^7Pqk&7ejfk6gc-XMSb z0}LcmL69Yert2-Sm;wy)CCrN)YrKQQJmxx0m=_aaLNH8#Sm~F%@5efphR+hg-%+T4 z0sWwzRl@35gyf4HQ`k=}W(Z)R}=8|$T#<-VMCYTE4^1Ti_Uii#cwEU9CEicCl-;Z^M19%)R zXDhInl3M4vBhhRH&i+AtIpEE26=YnSWf3M6ObV6hs(>MVbl{gSbBqS~G`e_H z2eBR)YGfklB#z#gr$bvf9Lr$BFZn$^jZ2sgYYANt>zbP1Hqqi2e6H;jY0#%0l8OYfb!Ed7|%+IwumSO{06~v+x2y;@5k1#0^f8OQoF>Bop2C5>a1)gDS z*=P5uz64hA^bunrj`Rm88Tc)WAP_zVZ=z#dczgjbLlTYo97V2&I6T~3C|cfa79 z5aYm4AG#b>@N>Aa04b~aPMiL>#5eQxWf(=1SipyrOAn39a>|vA1j6Nq#xJCC+_Ib4sB!M$eQ6%9%mP3sq&4&h$71)!y~g(>)|khiJS zoc&~2{ona|8n#lye3I#&7V{lV;0Dx}NcRjZFz#dgVgo@S-Sc#saX+h%lVDZ%-~zU2 z%xxjQOknq8cmu4P5>u(%qe9*Qd*4Ew-`hQHF0kcT^Q00{V&sm;?d2U%H=AU(Gy z@ra`(a%nT2;jm}S5w;XBVH4nh6o`EWCz7Lc)g8K|3c`kDwgwCSBo;P7iA_GnDMMa> zA_xJ&H*456(}d6%RkFN|;_bbtkvE~QtknnO)W~)S2U%s`5l1W60u`#P z!$||U><1D|9P5kN3god}1=1=Cm_2I`v)X6-3Fs&_Qr+7q!1%KNV8sp4AqgdL9Bzj( z^|spYx$w|Pz;Vg>;9(QG+{$uv2(c^e;*D1Vv z$9oxLZnhPEfuT`l)I;M5B$Pp<)ON3yW0cgNQQWI;Qr^czCRCRZv^P3i`2+(A9aW-8 zxxL8|Mme@35DxGGVL^&fo_J4WGf2Xm3%mu^=ZFU$lvs41?ZORYHV}XVgp?xgar!Uk z8wFsC!?*{MIALJ&3kXYmK#&v6sD>&_FL4i?xKskMlCU9&d-NYHctnX3MGv_L66;|c z84i0_S8xwpLQRCSIv!v8h|ikjv96l!zb{u4B;n|6hxhec;sruAh?8Q!uS|R$$ z0@z+`L}`YNY;b`YL1-KgPpG$Ve(&k#ywxL_rj;*%RRr*+BNOT|z0LG=+*G62g+pa6 z_Y7D`z)Y*xRVYWPBwL; zklNQTR$qs%w;NO+IeB1GErhSPVFT|S`rwJ3TuWu#}a3J>?5R8fgXj047W5TUlRxUh%JNKlc5Hfc(?G|63=JMhR`tda*tG<0}W9)(fGj&@1Q zL`cCVDDh+s8!S(omq-i4eAy68W0Nj60#?Yp*ifXdTyFn6OSH=|K{h?g+7LL?fLF9AFH>u2{=Pe1KghEWPgNii|rQCBPKA zobcAqt>D7yvI=gdbz3e^D3kVh6y{ibTHZ4PEMWR^7TbyGV-z=W{1olzhHF*xVdOHQ z+`08^r#V{;9*}Zxt0NG{Yt2Q&CTzOD$1z7AOsT~KhlF08#k7q5POGEJ^}9^_d#*Fa zAolxBY!_yWVLsWQ6#lT4?ZS~%s4v@(!n?uxK`iM(G3mMep}~$c9IytY64?HB{toZ1 zTSzQ*(KU5{`}>Y|9W;UCs__0TFZKr-CE_9MvA?UYqdoSmK?=1HsM2J;l7ea5RB1vG z0?Hjd5k)?jUvEGL+TT0c=r6;`3OGurq2K>RC~FE`U<(szV4{6Rq>n&~ z65!FllC}I$-cLY+4gsE zC?fPA1EEBiJ3EW9DSnt@!|t=QPwpb+=%Et-AzY^tf``OmO<>QX?*MDj17Ow&_AAE7 zsjvqmfiOOXw|-jQDr*zaAM@6)18F|0NFJ{Qvjym};a~YS>|dU1L}4-ms>^WLzr5HO zjm6k1AeBsCs^wi?$tT#P9FV>`n|F~3hcc4)FTKu|m{<8gLExC@$-8WcafQ_W%w%H> zCJj|Sex;85aNFf>~v1wZw4s&7>fbje`p=IpsF@#7}899P5= z4oCshYvHUF=08ZeR5Ia#Ub7t$*myw%lmg+Yt%{06L`{pK%ALgNIMgV}Q`0Up57B$(ykBaRX>k`MSNI)a|BxA*=)`@_nJji=~e8T<#j`6)f` zoOJ2}9Nog073bvur93hAK(7#`2e$0yqdi46g8qLA-LENr81EBwS>pfda$v||Wd^c; z;5F%}12eK6UP$-23IkPH=h+~2qsY4P7dBlMqMtw&=pw`6zz%=If#cSrfCKRX9^12-0;>}afP-=$yF2kO(P2`cpi64jmHwFg%RweUl?P#LEEembzVu5`i#Rq5Ggl~1k5w)5^k9>bEJy^1 zCguFueJoGvz=3Kg(QB-6`C^o7y0oZO7H@$sA=v{8qDsf>Y!tTcKy_)tqRu&NCytdM zrr?1Rz2bDz6rLj-VCr9?e1UEt@sK5OtQ8i@oE;KW=@M-mMFvqmsznLqo~*E6#N0S| zm`I;YW7DvPgZk1QMT5_X5-4tJi-QH?%`KUz2C)N^S^SKLF3%M`9mY<>gd_#Jpv0&` z)&qBQlt~^SumaW^%HdxV$M7(H0FoDo;3kxdEqC;LL8T>5G6A4RR_gsQS|jD)3L+fS z8zKV$KTKd_{PZAsRwofhszt*qSgd)of-aa2f5;{xZ|im)kLY>I`ny*}z(}!Ah4VLB z)QT)Xk^q6R1CDZZ%w#6)gQV^R}!8Tz=0I*G_>pYBN?A6EzQ=0(xA1Q_Y%3-wpZguU) z{?7V4*yT%Aq?EnB%I_asbD9uR@?+VS_v+n%b49V=Eajw$7$ucP;$*IcvK zMW5#?=g^+s1`b)}%VYXlbb6_ZG|-7yHW0fwQ{kwNLdxmctS8?6k~;5~cV^wN#0ix2 zAlCB}^^4{ViG=)nqi~;?JV!am2aoxc*RWPmcW$mz_ z>jG8@zo6R#P$sZT-oxHBx7wo$cpNQKZaCIErGW4st6yGk;J$g+;jTs30$5`l2%^`; z`b{_HgU~i(3`p+=Gr!w!31F}qnBJbx8kl#bg$Tl5>8u&TvJARJ2{$}+XH9VQBJ63t zK>wG=!A5r0u>MHA=zb{ggGC*pSYx~% zz#zSU@Tm;P0!$85H~@=PJ=nUIHNoeEbP>Ki*wT|-z)^#+mW+>sZ920I3`J2~y5Zns zN0|G9cPU^@vE!21FpNE_oL{knU1QkS!6I9Zm_kY*w*ARti=_Godl$2(;{ph!RR3VV zFy@ZlIPnk`JE){kMuZp?0WEZyUf@_cCf^nO2l1WrM8M%+6S|KZpLgC)8v~ge#mB+% z1;%d{fBIL3!@;k89J|jg5+5AlsDtCP*g%}c*bzMNKne~J9*EZi^-Tk*;^-3A8}_(L zAB%(k(Kar{as>TT;(=-?@mw{tAi1T)`6B<9urR{-9sB2WFa>;6Syp6(;RSLbU7XEU z;3G~vQiU#WU1Oavkx6a<0aaEXVY6`vPd=2BB?__Y2n`!mfn8BzZ8@8U#HA1r+yoD- znonCOPXvt;B|h+Uyl4|cNYmgbvUBa?Fvo zM&3>-4fJgg`#g7+Xr&~uU#=LBVbY$I!vk$?(Z?@gj|4%Nz^t|_u@CDlP9J;}##$nA8ud*h|2vkoT)|z<#UXbe))EW4q+A)jK!>_tbG%~C zNyB>}xHEQ48R>G=A_a|hlz6xjKV;w@YI}+|!0T}vy5IqiHfMN) zLpR8L^r-3~T%5QXdE6b%(9?A^NZNWHFTM*bF-yd8b;6vlR!t(Lmhqa_%4voj0 zB}T6#MVRo=6n7<|e(KvH9Nb$NxsQrVM;+Q@cRU-PB>FI@2IxB@9Ahw9-UK)R41~45 zY>{~s1FC_r?V4VKTe{t(*pcd?Jexfatpg#IRrYMv`{GUv6esmjqM%mqhs>RhuqjG` z$0w(hckva=cfPlWzQ5u~`{HY=VwTX)u+zxvONC7()2|ZsA&}tWQQlb(!|oR1fl^>v zenx-#NVMPqJ|NV@IpSLm6}m{V=kK!AcgKoe4&bB2PbtcKuI~%{f~&JKv1S^h<5#IfanD@GW4KrXKj@tS!6RLvd)~>nlF(w4(i~I2 zL9iIK9jO+#DIT_M{B_p^K=B}EB%=$5;)De}cs4s@JaV-`#Vo=22y?KBkP;%Kuo&Z` zbBib>!@v|(#^$i|xCakE$k0|0jhAp@rPpS!F)s}F?(tp;f7`%U9n5WZq$ zQ#>HFV65A%fnv}sw0+-qlcNvrp-1`G! z*af_V042q~KP1MW{%IL_$PyC*l)2`*X_Tlk&0qQ6>=&kzw(n01VZYnZTrW6@}Vn5Ja=k)RW+rpLfw;GF{ zfz&oT!8kNvvCu_I$@VptYdKCgK+5(=5Lyc8O&xBaR+842VjA={U6{2-9+*dbum|SZ=|Z`Z#szE;raK@-27gH_cXqX(x3Gu6 zQ4(>+h(&f7j3fP0;$O%vBgV);vP58_@;73gc;KS7oon?w&WSR@-Oc6Ybo0gl&~fY`!rnc2LgN$h9?d)FkeU(0}n_U5jPB5 zBAhJ@2-bpOrR9SNrHS-K#)>efgyQt5YRPPG)wjUW0sUQe9lwf~^ zC-x2J<>N;Qq@WrIw~w)x5mN0yC_^JXY^j%*avB3(8|*>q58Jn| zkd4R9fy6@)j@&uP{>I*WXe*cw)1S^}e`CKXRUxEdm;c#nY&|yGX)(=$>tc*P*xE%% zNvx%LlM3Y-EQ7h3pJ6sl$|k_A(xg+=texYC51W`;M+jz3m&_1 zg4IQI;Q(Fo*o}L@^UWnaQKbkQrV0=rb=rP4&(oL)SE4>KjkbK9?rTgmzxyE}A<>rK zB7JznS4#<))b{HqYL$*Fya{{I zT}gU?JWpDxzYD7~cd;X+#Ppo<ZBDMliol*XEVn60DHmHw|QEeQgg> zq>s_?UNe_2{jrndy15JnJ{lW5J;LyTa;8nKY)H1dF$v=ah!hPQW1CZyG|a*Qy3}^m zhuJIuX%>(|kClx%dtLbpqXO`dV#m0-8KV(n#1!B(HWt2(vd$FRW)fj?s0RzfLaHfN zV{yIj8qZ-Hl4(;bdjTh=%-htOb%$?qNJ>1c9s|EXq{Q(&+OeCM{sBX(rn1?8)Uaoe zssW2ju}Rn}H#hfz;GqzYq?{yn9wR;*VjZHbZ0^BQKI{_@N}`aCdad_ecDLR@g1x@+ z6uXBrnTRQ*Xlx!|9~Ql+H)$%G3$_$_#oJ^?H-XdGvWxCY+O^J3ZLMs@3_sQvy{gS``gbW0RkeWa;pa{Hd1kLFZ7U1Z4Z+z=aKybh7!lR zsw~Snxo#?T*;pHEEJgAYl@Lrfuc)wun{R{Ac1sH%%P7~ht;V+bZc;K{?x5gbdJ#32 z*>Ju7uX!SdfS!%J=6k^VC?QRVtR3!-WaBy<+uNQpRxRpX_l;uPk>xtF?=oRiVW6E| z0@(FGrZ@veV>xXGv7}OY9*QjSO^z`VTQb3v9xKbecGnn-WDWv`M`L+k*6N#)%waQI zZMFwV2i^X+P6U5PiqkzmC9!TYZCx6>p`HyfaohYQK-24Z?k|_*e4SUzNo& zGdj7{->>RZ{tiWI#e z)UV@lC@Iu_2MNn{zz%&Jq;xlTlnIL+dbQL@LW_$Z5_<86*-97l)*TTa@4pvdOu+V1 zQ;Nn)-f3=sh_5=~zyreO6#d88%c7Bnqcry6m$k-7tO8OA7)WCuH4igJ;go<9curLL z5L=lRqW@9H^b;>HdoWI&H9e=Xk3SAFlF(1i2aiO2#b}`#YH(uqH_gFBU#ync}5!k#Sa+{XMRstR@BR< zW97}m*!bWZ!ckEezp$!K^VrC9i^Mwz_!>L68Y^dL40NXn&)th=z0vZ6AUapk!^GHm%(#n2V5X~<3c}8Ua2zy+@>v1qjjF7~X3H#x< zLZc~;2S$BqkBejDj8SklAr;~9$+pexvdgnq8D zn5gp0FU*pQgduVRJg2d1841c9=#dWscz}R|O!uvi6IIHH>93z>=nV$GNj*ntvELf| z9{(`g-Np1qMG9|QB0gy{q;Age<38}JFD?3nol5;$(Q%VEc@_nlT(Sq>;4v{Ib@53d^;q zFRR?FykbAK;9X%LA@+7-oTX--ydc!SjWhngLAE}ANSofq0Xh}Pw(J@IylX>#eQqC9J%IAeH-TPiOWL=GaJV2|4OsK)LcHN~AhV{Da{4Hv9ZQtFeD@ z`$s!$5ziBU>rM zFo6t&N8@e}JoIXev&kNU&|rbx=!9)I?EpOl+H!Y4Kjn=TToA5OmBzasvgKIgO9T&5 zqKVgawhS%IZB!xUcoUy!B@gQ3U!fdt(yK_XLZbwq2v>5C-)nU@j0!T%_k}m@8)^*0 z@+6Q7tMg`PpmXJfyRZjup69FofS@C*!zNbV;>lA+6nX~0A(?s>d9%x&L!7a$anG`N z7K)o#qnu1V8-*Krf6W(tDS#Wv+vpdJk(kDIb6TwJf@sTT*TCD#+d2GLHHJK7AfcSM zXBw+Uj~><{##Y|8c(E}P-+fZ94DT)XE=@4{j!B~*1kB3Y-yCGwTCabK>!!RcyU_b2 z9>8Jc9S#&IA+7`}jd#eoaDrwPps7^f2a8jA=7q^%I*$AN;eDbpR-32<2paE*F3Sdl z19YLs$~(v7=5uUw#lO-GUFJ9Eo#ER}m=0S(uIzfPv*L|qbU>G`?6&Hz{sS6msuD%I z_|8gD`pHC4sa!V<-k+E6z7*SiVV2Sj{SOqEp6mXEuQ3#zJmQf`;qW-d9_x@u4o68G zk2g!uFJcUI8}U&>G1mPTX2f94lS;rIs1jOjyo&B(-C7fL?Lj1a4f(`K=-Q(Hq|LU`aeuI6wIO2`6akhwK2oI{zfnz z(6>MtfZ#`Q>D#An6z~Cw2gzE}Vo&{%#s|!Ji6qdY@~0knpu{~F;mM*JBSpEOz{rwE zNacKG06IowAc)rZz@>+I2OMD!r1V(%z^6Rf(*5yHduV*{_9HxC?puOTM!JL!w)0-* zggeAq`Jl@AC&q(moq<9F!+5}3Tg84;paqXk^9%*uVn~qo7<@O|U}$AfU)pr&npz%! z5l*TM2aSh)n9X}!mHi-;()iG1oP&lrr4o2w7--m_=DbH6nQJWxHF3Nrc)$h3)XFWb zaMKlVG=a7-qxS!YFvi=m2YH_mNm8P@MGLX)!6tb8kkBp73ytx1f2TdHoU5~Wz@EEK zzi6C)iJRbmcHI%el?&C%Rt&4JI&mmP-o{XK&Hx6oa<%b&-eINdz+A4jF6AAP2MQd} z$I6-81!E}o6T+1wN#jhvaN-R#235}HB3#A$^6?4r?u!!Po#yc#gPVyM6V49*Xg+@; zYPr+d8V^5>gGd{Rja#yH7~Z3j_d;U;H;^PNkNB%O@3kpM@PJLMe0W8Q{T~>abg_|N zM)WFDmZAwJ0|~JsahiCq53f@FFt|bEg7C>Wm*<9Gk2fMQTmdNQ%7{;@jae9sfFOB{ z{%0%ifrA#nP)dnfbjf%X*~cB7jnbHge#%RjAR-DOlWKywjTKoC@xD(c>M6N z4=^bUU=rzs@m|A_Brn9Gl*T8Z?I^~S34X|TZhFLBo;f^Gq}7Z?Hnl8x|0c zxNecgCisPTM3J^r7G2V}V4|uH`V{{PVgqy013gDFvVcA#CWhIOLjupy z6V2l>(s~B7fmA{;P1v!?XyMAFw(?mGBG^|r7{z%YJD>HLjqUi-$H_zE^FOokH9J0{ zz2>y_M&omjjumGj5L2P;T+H_DKmr7QNF%?sFo<2mUgCV15Ra8FL}o}iS~y4%=LhlB z=jItI2F+FAVWY5Iij6(I^&hULZ|OR3CCc{)@tOiUN^Fl*CgQUODbj>X&u1tdJijD2 zQZ*}2n>62e8U51|*}_=)vdKQiNHb1_`peLaKZ7nDj6;u=FL$pr9>aVgRRIo-uWWi( z&p|`qQ8EQ#vXA~QGVQ5^F!CzveU^?R?Z8lKyK+T^F%~JB_*Vj3nH_2j$5@PblnJK0 zLyQq}6N%)p@@Tpdj&xJAgq5$nzE%GPtANl&I%?(l&c+x(NAyS@>z>)l-x;x5OhUq@ z8ee@k*zm#XN;?1pIE}AKImWt;C=wQfSSw$XU!gz18&xM|jpL#IKXV2i>Z1y-m+yf2 z(>j~lE=B5>kq?9Vghk`Fno|AOqsN4iK~TBvcBTHSYYqj3fBp4e%~f-tweq!1JL^~R zd9^tWwE3Vve{XsZB1j{zAJkll$3TZ|G7a-V&_x2v zc^IlpHK&lkN8>rz!We_WI~dAlE9bx^V}VS5lIrig7j3M@K~E(RE2`wS+{&B1--eEE zWZg+@ep^5NA@X4~^k9dOP4j^x!(4qs6_hBdv3HFcsz3@oR=$V(o!Dmgcb=&d&G-6O z=+#j}SumL5iRSydUa}v>(oYVmpbiMH$Jq4?^C%IFvi?Efu|_DeTp@uTE8myurgwFp zElw!wPEra%@NTr0ic1~`zushwlufWy?og_mu^!jnp*~q&0@Gm_o zubk@5Bff4&|H_c6M4U!w^}eQ+ORs~+I8h~vtX`Rs#UpT9G=7*&(f^2m)nQ=rt;)ZP zeyFd;t|M5B9xJbY;y&MT;t1`2kPXU>M`?WD_<`aDLJ5uka;AU}-`|1Mr^m{F`Q3*P zM^6dBh=<01ZM}(AVB!uxB+_3u<*?i47DP8u0>kP`9Gi$Avcz@IBkbsi^2e|8?*87M3vR^vB{BiOenQ6&we@tXAe%#g=@$o8Y==WFZ>%sZe8sju;y z4gJ`+NGm{n$@FGm82c9M9+`E7zV567-3`+~V7fle-UA}*2)FiCv#;>cBIttYtvfNS z0!wK`fT1;BiyOz!PHHZ^pGw-^{-m0fVk#VBWk}r)skAId@U;9xMN6dAPj;PQz*Gd>bD1-^4yg6gpk0y15@? zI@&_wLC{sV7CCxza{&`bHMP+@jBmUy@1lS{n(FaqI4^X&AiBkZ;AV|6thj^;^@%Zf zRF>-Qt|=rCsw3>Tk(uEP2i5Ivu)U_ZFPsA}Vv zuknq4%VTJTSoeeeiXVa>stez$jq`l@^Oz!l=Y@gP#*wwGE#{M{gaFoLZGn-w@-I;W z27)fny4h(G5>4F+SD77QiRwhaqe zf6T*>a%JS(wn{Yn9lLBuEFP=cw%-xf5%W?MFhXqG1$R$m%y?MWT&QgqS12RRo}i?r^l+cuW8Q4V&b24K?xA<*mUPMHGtF9$3C;^ z9g*lqB_v_xd_~23Tlac&=$N1%M#q`r17_6J4)d;A{4kY8CWL6K+Trz+mdDK@AQVK2 zi^(h;;XoxsoOXDWt5N}NbLjt|)O{z0r%zsu)?STOPnp$9!ywaZ6LpMrNEKPVic)h?BBYyehQY)*@HdvZNngzcrN z5Q|4syFD4p7GWNjs^C%8ZvBoJqp;{_df%#c3ywA-T}c!mydJ~GU~QgwpbB^-iqLP510}n~?#ks|eaWZL;GaS0_Niyun zy=dq3t#6pS{u9QPU`W-hs&9<1J`p1vh^0qUeG{{l5~QucYclA3`8?x?6MKgS}iGT+sjwc%vKlIUgi=^gxcH?dM{GO;4cObD@|19)b=SB=nveBA7c~4-=_X?YZ1j z{~Yr$C6EFmf$(0ur5H#dlB$64VTI+J`Wte4ev1=|u$z=hfR9yuqE{*NM`HkyG8~== zPGY^$fzL+?ltKyLE5;P$(h{&T()Nk5rK}fv(&Pq!v#S32H7q$P)M+tQ_21{g=3sjm zB*-fMXS0-2EIyJhAZn`rrEKLpgf?IkWALaxY>B|7b|e<1G_{xeY-0q{tVjwr@5rw@PsldI!d`Hpne*0v_O_slDHdXW{r*=#bKpnD$r2_r!e@ll1=GOkO#3bZRc^z^L5kdni`0dfG*}oE0drH zUb|pSkuRY{+Nb>`qcO5O2$(3*r>KU%jNSz>kfd>HpM_q`SH?MMtEqiTD)k&7ty3#LurXAgG9s-W6u6#wxz;N+HK=E?y?n_lLKgu&NUdaEkdo?r_k)Jl zu-33TKuKUhxQaB_++inh1nIeKH+FLXsRS1ENtm83_nONpeV_5<5onYkfh4JF-_glN zj2u`=V0~ZR#0qhvG#F9|P3^m+IS)sCl*o6^s`gz!pN9v@;|m}Oglu2-b;A)Z=zWW` zxcW_N;z18ed=SsWv4UrlR-&kn2Vh9PV0Y!hWIij3$_r1qOny zs{Kx;pFm=ptF`R+VXzT^<^0?7H3g4*gIFspgP}Ny9jW$jcZ{_`8-$>hy7Ygil(obT z2oekEn%aN+5u-8uLM2cQgraQL8oNoU1h9Z`JjQ4&7ayq#m|`T}+B-#jUl1cG@nf9* zTVzt#Y1{v125-FaDPkyX8r&_L1>q$mU7|?a;DMLyWf<^6ti%zlrSry78oc<3(Qt93u&ETg+|y`; zBUDnQZ>?(Z@&Ka|Hqw{LaM08NtqPcihK-ypstjl!!y++!t5a?O_IXAkM*y}UqE#Ib z8mn)S8AQZqq&lEFft{$A7Of80bC*w>X@{vj=%T41SmX-)xw}|h5mG`vy21l7dyXGcY)H8q z4-8-H;|0pQc*U;5`y$2$ zHk`+!<;L`7tT>>2)>6jVr1Gl;mGceHuw=qu^Uap;^N5kTF;cOr>w1t}SF7 z-9QwVNC)}FGKSk2yW`CU7EK+*=jj^`O?F0*RULHZK8wa^jkFaw29;l6cC1AwLR(?v z!MI`fQ#7zq&R1;khuQjhnFKDy4#t6XW6ZBQs%h%rlRm5!E`TMAp&AHPLF_W1lP*9D z9GFyzZ&52+ONd5^Un5uz#vUYA@^}=j6EaE<s=)dIgU>-|0TPXqknI7rvjRNL!_A{@C>c5)@q@apbeFkIG*Wql!0i>Gx%&Vn* ze53D#2_aTf!+vSbmcx}%vMn5^4jXqt&#AW#qz;?x!zN>S0re%YVXN*MtFb2$K9R}| zTN`Su3Ed_LAX?R784>ImbL*c_ZdgVF8;LfVa6my#wLIx(^gvRp4Fre=;ps?yEm|r3 zpb}Qq619~b!3l@Z)GYCQywO08U*zjy+18m2L(V3OOE*|BZQE5IZy}@5ax8^)#mRB7 zg~{|_9J_%@CNPz{SXQ56cd(1I#HovAO)a}~x=73@N?_ktvOlnfLrC!e>CFswbH^yB z)iqT~^3^wD{sWpy<&>9>=^JIxnS{;pdLe)ThlH+t8_Mos2R8x310D|wS(;iRv=s(o zZE}r<7y$!=tirnb8Vm65*I8XTyO>Y>JVTTyBVepw1*4V@C8Ryr;u`i1zUp8o)#ttK zdL_nlnbMV3mE((i4t?!96Y@b<*f&?|u4M+(`-k~r94Cc1l@{aE7qbd;eIMX7m9K+X zbXiO(AiV9ysvC+EbtH7Y{|pO2nkR}&ZFy}hD?$rHZU7cdRWVt9&A*!nOJNT+p_2XD zV!E)pBz!a5_%pDOB1pDl;bAe1u}4A|!6STP8asj}*aj)7wC5~*Mi`%n0UlWlk5vsf zvREnxymdIje~qvw<8uP0q@bpT-?7=JA}y&dKCC@)=d^r9O|ifD^Eu|hMkr-ftwHfz z!$egb59{lp?9=TcA4Y0xEr`?C<0>Bl23=5Pe>}^L+(#&-O^3gp%vNE|0t^YMrVjro zj%8pbuLRIh4TR(FECZ>pR6>Xye)1Um6icm9U<%8l#OH3j8PcQb2qW4gv7G4<&N#KI zBSyO$tDEf?Mg|W}9T9Pt<>4H7)R#C$U^3$qd~k>$Y&rthL+4>W85m4Lj62KgC_Y#O zA3UZevRurtqKd>ZBEB=r{Y%7HSQ$Yd5pQR?vbm5Z9I-e}v0=nlh2RHTl-N|Md?8cL zOeq^8ly%j0&{cJWo?%?Xs}4}6*a$bCh&eVgkmQkjO<9jv#}8@35x+e&zJH}gl#sgo z4kxJmPsu2%EbsP4J zfgwFsb>!0(Y%&f)M{()fk$c=&*zf7i5~@0?vyYLAy@0T|RBqIg(qZpms9{>ns*bvO zLGKI2beK@sbX4tny_b2YyMP|q*qyCHnwHH8eKamS^~41z5KG^FO^w9N_q#|&M{$WX z5|h;HaDhQbXR95#+K)}Z=^jKVBdbT+*iVkQg^!Lx~`k9#tLPypVk& zZ*iAF7j=FRk9ul{tA!aoAzWXB-WJ6=s(=SJ_MQCvEQyq2M@K}nPjJ2iOlT6GgOq<7 z*DEo#hJAc@g76DUfycBW_HoQFE?q{yG?smg3xiD0p~OXBV>L$eC{B-69la-)oy6CJ zO5g$XW8r-2ta7Imt2*YnSYssS*j!aU&*f9I`jBGufXCNysA6%30|;jm_*9&aiz-t6 zF*64lBVc4`D&u4Hx4wMp^WEv?p$Aod^WanE)Si5!#>_7@M%3HLrjEWjh)-R0ww{!m z7x>gI+w0-D=f+yJeF`v5hk+2x5Y3i68mz3wQQbfL$Odp<9;?UB;z*7COPuyoZK72%g z`l#}IC7t=O`LYTKUghiq**}#ysKkr-ie)-OdtB$S?8S;t zUG^AT8_OfH7ELAaK_rCMfdd&(73<6m3PZ1TG=Z90BkjdB3P zCgLH1ZA)S=VxA5Jscp1>jUI{&04gDvM)ywAhhP~DKcusxlY{j1`lp;l=f<;{NRESI zW{G=)48Lbai{U-YsHqyd*00NjAu3^2C-e)pZ^n0D$9pSqObEKBD{`GswjUG1F6qPS zB`m8GX3S$rm|%t!v&uWA`fu{=Faigr*_ZTmbkD(1;+TNG)LQiUsR|xVov?f>TZ<&{ zg*ax^pMXtR)tPPlwOq7?-~qz_Li8~h^5BQbS#2 z5H?k^kT)g}1{n?$x4E$pykf8?NkWOoBlwG0N&`dbxrvh)Pn5T*nN=>F($`}Nyu=wF z6Orn*4jZ?KAgn&g%}rm2jgBrJSWH2pjw>7{{#MP_!M8vu#ZC$gQx0Gi5Ckc9QeR)? zAchHbR-ZJjRM~~S)pbZG4RmK~k!L`<;89gUK+u^MQ`Lz-r1159#f>>6T2m*Ts9`52 z%R>@`k)Q2_(+RKMb0P(g8_Ddlx%vjC<3Mm{mp_|Hs?(#Y&u%PaRxB?NFes?0&whK- zvJ4S)8|q&rtH(AfwxpUzatmP3e&NUd!e<^oB#visy4Y7Z@t{t**mjjHZc%SqDp7P@LVm${fL z{`W+DbR=z`+k2Q>ew2BifUm00?cd5RIJ^@`Nn@)zc}yCc4ZjdURGGZGnzc7i;skt6 zoxE)jd;90^0(z7nWIbdXWCDaV;pB^4rETH`W2G#9(L= zdM7E*)}LUG3y4sr;#Ic+2OjXiK^&9npPn5zwSql{>>nxt=$blpp9c@Tn@t4Cbn*f_ zU+wWiJ+WJSX%{CIm&(N*KdHQj8x2}Ic})Ia2w&j-xUiUDIt5+Yx3R>DACl>mmf>uh znPmbKTGc77{aNmBw}n^`G0QJez281h8puWGmB<9R^#-64Ta(D%;G&dRb!a zeI*N0Odddr6B09VJv1ax73lKNosY(be4EqR(>_RHZ=aW!luKPOh4gk*z2$}YNzIK+ z9DYLvLN!f|?^eUwV{n%0Z0O_t(^z|SH^>brVO8Uo6zP>1H&B&C8k@#{QfOr2lSBl7 z1CNicC_iEG0RNiy_}?+U>g+%fFhHuQ)5n(THRkdp;9J${MZs)!6Yi=q{m15P6-GGt zm;9osGm4HV$>wUc;4$O-9OZxbs#6I;Nbrm_o`hI}F0`F-HIcPP71LtiaW$CD#Bwc_ zkWA48&o&Qc1rJr7fsfz9cc8K1=R8BOKAZ@>azTki9e>m8hd3#Laam5;FM1yB?c zRyEjSKh+CWkkI2+Irha$WG)NoV^v@DiDvV$gawh(l`mT2*d#Nf3d(8fi?0^xv(bekivb4_n(MP%NByYk zi)(V&*8BAiG6L}vcFr{C0qM=h4Rh{{4hH@K8tt1vDtm@3vyX-icEr60z zW)1hYm!pec1tUWrO`V1Lk7JE}MXZCUsk0YE+b@|DM6fGLxLx3@u)1P{8w6qYD>W6{&zmdp|B4csB=%KxELZmL5d?wsm2QdLhHo|)%AlL$@6Fz< zp9?n!YxCL2J$IEDTFVx|`v65&)6_Z7r&unSr_NRh0?T4=Ak~XXl#xB=@KbC)j-zu` z!OYd0^{*$GGd_W>#i?&lo&16-@wse)dHNiPR&@@vO)(d^QR4qa$OvL7?YEOiauiBr zEN15<2I;vu8CmQcPR<7odLZRq5_=WNf+Q9X2!D@duV8t>B!I_%3AOAMbKfhZXlk-& zB%6%B0~ts#O@90^TZ{f_9gbwnCbkaK9d%-pN88w%>GDQ;p>1+pF`Hwi&q=1!gV=h6 zHljz`BYDCZwlVz>{g5_IP6%N(WHOOh5o5`*UTht{gj7Z1m{iKvHR~jBWRkYY&j&D@ z{Ug_locy9cTZhj)@sK!>9kvcbb6_x~V1awBJRZj^@v1*tjTuFhkc3yiIK|g?O?GMv zDJx6anxg!Ag!P-)2IR8ArY6(*SJ(!O3G^nV)NIa{|FyRBlv~)NIRxGpJYMrZ z!mOCIrz*lgujLlA?&z-rm?@=OHeVAXQ{Pf4=#pQ?TNx~po@JG}&unE~u^;(roz=^u z`I?LJa96=}?%Nlbj}$4H&OLL9J%tI?I$(2aAM&}+APUr3OV+%0gr4mrzs?N=E z<8StS;F?33Hz2r1$6kWap`KMMsgOsOsyhtL+AMl%c-lF|TC+UsfXz1A@ky zI`>KpU-hM&#TUUp_rXKH>V1(HOiVR(UawNV>cx5!%x~b7)pwDtPrx8WQ|HgLGY>e5 zaNx133-*WeAsf7e2~kQ@7nEl4!55CZ1|th@gz|86L%$IF=FwV~WlmK9n5r%uRLWO! zKiAjv&EYkCu>0R4q=Zf1Y?{PpFKFg`O*Qq+6%~9Ly0m2grW3G~VcC52(7CQEDQqzx z4JlAeDwkq!&SxFEA^PipuBnUM$MUz{xatb+#XE{vj+P^ogBdk-ahtn5uJb!$Tq&w7 z3C6AElbg7tESYkjFGgqsB~{bZ)UDOZQlv68fWiL`kzq{pf*9Xp@DQ230%Axfm4s^)K> z6}N|(Oy4ZzFU;5Jhm5hM4}BRf8Wl)|8&Yqb=m?T;`%yUpjW*E9`tpR;~B=On0V+Dq7n zgAKWWP3}aMq)U7qcKMUV1Yy}uFW$e|9f4G6yKMI{-XDvPghPm3hMb9h*xHo|dkA35 zz3*}ykfH!K@zK=f&!+ICMajbI&_`3#E3UB&Oqrm*RBna4Kiee%d*L!rHD%zAduT;>o=SIr#6+jta-K{J4%O2$cc2J;^PMvtnlnitJmW62jdWOI?x z`JE*K`#y~G4JH97dg?&lbn*@SK5ub{Uuq94lt8ZK~f^dH=FtF~}%A~H| z@5Z}*-$cBGLjBc75cmE5C(=bqS@Ttmq2ts8HyKia>YAao+=|;wuEM5Ztf^};zHOxb zEqFwcD>qC`WO+CM$E-3|-LSNPPh8?Jj4ZU>aMqniq{KNz7P@!)}{^aGP*!yB2N$~s%(u?awlLbECre~1xyI(EavccpH8peeVI>qv@u^T>_$4#_ zlCcrX3pNCd0G9b_ih;w#2$+PP)g;YmgVY2aRfK`EIvru%a9b836+E(h<5*Wr+)<1P zDOqoL9Nmvbnv@ev-yN1{WVcyDKP1z4PhTVcm%y^ScpA%5f(U|XcDGZ;GVIiW zktL4ouqNDJ2_;_4j5xbZ(c^)a+FkjA_^(1qm{wve#xAFQN-a%0VBL$cd*D z%3Q5O_MS7wMBH(hiYjH~{p`Y0VNWlkUkB1r){l)1GVUouQ4`V`H0M$23{D+6# z$B4$A2PjUDRn7h(+<41O)c{kgx_wxh(FePI^5H0$QB${%tu&VA{41)U1PEJe*a%m` zG6)wclwC;GAgc?DZLdjJcA~Fc=j`qOcq;ADVKsqS)g72&9e^W;8WN>A+N zvB{PWCBg?;{%0<739&n@zKSLHfs==(?ihYVX(7);kp|lFVvf=hDH3HeJ_f2eO{4Yq zuuG3{0Gw6LY38e!#9kJ@MJY|qX;*4l^K0F@7zmFiSiZ;YlyxS|@vT&rU=f3mf`_W+ z6h$hl&DDEhpd5XxGJNpQq6(@(%9V1Z@1HxI1XQ_o-M-D7b%OUGCESf|!5(iNOaTsr zJ6`s+t{y!I|AgBQnrGUh%1Cu*XJ%h@eUH;~nz}Qj*4T@DZQ=o|S=F6S#@hG)*g%vJ zICc&#wLgxvHE1fsVdwI23(N$Ur35DVKX@u}-(i69GsyHTSDXeK;~zfYKzCW{Eiz)R(TzYq80Q zMrh|d4U}i8Rcu&os#7j+TAb0%+}k9W=8ZeUwxd3@C41mD`5E4ZFJ3X|K_g?_Q5bO{EbV=_3fJ zf)aPUjAyah1Psy*`P^H1xwkz21Vr4go}Ohyp>d=*g~sF5`~%mGL!ZrbwjU^QG|uRO znQt?aaTWSswMru}wSg(n;->G`^NqsSVrkAmt^^dcPc{nCJ%`1}NGRcT*!cfAd;2h( zsy6T+0ANJ%<_NtvQX16h$hfQXVShnUqRRDay-~CrMOF2{o0> z>@7-0ROBtg5UG?JPl{5%&suArIp_QMUf1`!et+yg&N}z|z1G@muXV5cE@=0(d5anK zm;aEenqASzSe*tSu0P?f;5^EglsevJs{_ky-axGnWtmeG-Px5kY7fFRES*xaAQGfU zdFYa(r4$~Cy!g(q!c5lthHO)!sU}LT){T{Jd#-lTV4eN?d4*RGkpGnXmA?8^$t<4T z|1DmPC}v0Y%J0{?Irh8K_MkV;tH#*A(2Lz$6hLe}h$zsPoE&&A`UU((Zid`tGp1O< z-Z)zBUg)2i8%f_dHo?{9#I)$fVsHF+Tj}Mz#j139v*nc1q3p6?LJRTcMMq1AFn<)< zS)1x|!@9d8qsPme?F-eN94!{0J`Z)@7FKfykyk;}Fl^s^d6qoh^n25j@h>zu^!w@J zvu05P3T1c_M+e>~y+sw5RavX$zY}jT7Q<>lEdSrKMCVm9+Oyj2z#2x=FmkN=DKD^| zE1Afl=}=arWq$v1w$zZpc8Bwe%H(B9Pr5c@TQ{6vW#<1~-DT};`4<8H_|~WKMUtE4 zw2Kg3b97zl!;FT#jBF#=+g)PanZ$lo3;X5mQ1?Dwg*{fzrAZ* zX*Q`#P~Qr+x_+@*kTXl$)IzK#bHk$O1?%e9rvwU+LvukmN-eBeUpk)sG1S*INXMVB zF|dG`D}LDS@XqB$CPA$3$p+<}jBye;J3PP-zv1S6?i50J5!)*F&KG%sn(RqK!;Y7C zxIzt%u&9@?Vo{Uz6M3hLBq01GBEN+rs1PlcPw&L2}W}H(r<}_MHh_*M8%H&-;lVbAne;U zN2Ue#`|A%&^nbg0vJ`7&r89&Sv>cJ`=PFVIAtf^s6PM zM?Ps~wp-kT7kUSDo#2t-j_GXp=G7D`4eV<`4D2=tOpD zpSNe?=o1Qz3G7n=4*pFkK!)`K580<=zmDPxS^|=>5TUS zOENR%ZPwZe{wg@|%B&<=3tOK?D6ry|9kPn~R5$R10E4Q0hwCROXgRJIdwuX#RUr3kg{=%21b_1+K(kLgsv=^&S?R_n& z@LlSD_V)pOCsF$*BuzP8De;3AH5X|~l4_4ba(-wu8c3Dy@v_h+EY%IIs;_e!`>$6+ zqZKzNOX*_;{&yQ8z3 zCl6c|*poP{YTAlMdK zAL5W42rF3EjLD@B`L9CJI8RGz`iGpCUi|XnT=!F3)6ZJHXZu2TC%Y_I z+$xvW;;6fw3%T0+unkG?HpQ;j*OZ&HD1Av*Sxf?*YW^UUj6Un7oQ$uCZThM7CD^!_ zKv3{Clq3B=&Xjllnjh|CjwrGNt;fh~!jk6fDRI}^%|FX!`0Xd%b(|ETnAIik^O=ES zMgt9|C(G3_6TN=y!qH(Y#MQ~k-W3}TXq#HmN9`$*H!ic)kB#R>T(sM}c~Y64U@ezX z#GM!On%6|D>HaaQYn=NbM_7gaOG`$R)qFiu4!T$_*9}i|UEavrKxiD2F%(ir;$`9Y zkD!b@TjBLs9x47T6K3Ri4ps+TOB<3~)tvs#nfjoipO8#i9OKoy^5p)RPbj_GHV<^a@qe+hdGO9|=l9dL3^tp~SGX zzJ0K~m;l{tBaSL~YNj{1U6vtg3`AyhRUOg6xAu6Xq=Tx-Z6B?iG0-Diw;e+OH!@f^ zJY2fZOFw3`wf1Ki>8qKt}Y( zMtAahjlI~+9|mIfb9wH!go+jd!4s6ni>3rFj_lGp^4OfwUZ0wo#{1A9A^QW9z1|3> z*%}&8-o<$XvG{|nZ-es0-XbsU{SA7rPfsX8c`Bhuo*Zl(t#zTHO|d-5p)dsX6O=jq za#XEbZLl$PLh=-;uupN@crO;i&>?yH{S;Npo{U%n%_-bg(uppm4%vpQ9q#dye zz8sD`ef+fC?_VD@Y@g1WZLU%Sp-~|EzYv_g$aFq=bcZp1nLB;GN__kg&C+)5c{eAj zng{Kr2}BRcoYgbDG4_PlLOe}bvus{Th;1c(?r^gF!*a9`0~wV00g`8N!%oWq;gHN9 zzgm8sW7EGHhy|o?WY1A=xa%h*&(i5?P^KvXd+>y0!3vQ-?2TpHKNenbTq;?gg0F27 zl!dqKP&MTu&B5w&9*q~tbX1){Pz1A`fAP(U?qd`FFuf2OAz8fXjH<~kEQngsm*~+z z5IFtRP6mrfzV`@Y|5SL&SxgCxM{Y9}*sWma<+Oi^TE0Ko8OE}G>4|t%v+MFOm!K@= z2)r&402s)!eTnPomE@uWdMaf7ma*=kjbgAgq`W^`ovFUwTTt3Ec8(h!sti;7guUb&_x z8(mf`pP*_kvG?YIARRyRN3A83!YCnmEqAl3`O8n?hM>IKbV-T6%%Zs(r7jNGowHY{FGn#>#CFEesFBH)rIOj%AUj zYulFLtObyrlw(L%3Ld9gWb#R5o=-#a>AQ>F@0c!1p@BXj`D{m;)TJ;Dwy;D$Z&BZ?TRPW37%rcaT)t7S zZ}nL@UER;L$dq6 z3a|Ffwkne$x_fthcX#qyBPn(b%Ko3@yfL?gvhj*Ka zzDx^t;PyEBX}7Q|gF@hfSA%#5{not-XK;^Q|ZDj*v z?M(O0**acbwxFxzICx~Y`^aE3AKN~5_`W)_j&H&$h>AbgOAgZ>XJxqUiXrg@k{SMO$5zdhxI0FCH&nI1`2hjKUq=5XET67B$ zS|T?y2$91Y=!cZC8%{Dw>joe~^1}_u8!EYcuKO4b!{vMOT0e}oenUup)K*K=%_mQ2FL_BUa8hAqT+obfs@3?_~_2bANO*YHvSIu=OYq8&dpCqmcsCG0h2(EOthRE4YA-VkoPw%G0)FuZ2DC1NflzgOE><-b zmY7ZlYZ~&4RE^h9nJxv5psL$qT4325zZ#+z<$sgh*ZDftJ@fn_Gi?=WqV@JwsOpqN zZRn3ERqv&BlEpQo0)Nb@dT*!6Kz|>G#*k`IbG52RL?-y!1R-_7>~X3_mvrMwq@|%? zlH6~ryI6?Wsf*P;O~046Oe{P>)o9BOul@A(hQoZJ>2`3CYN;Sh#@3);~XbfgVsbu+1fi_OT9Pg~l{KAa%R|C`=AZ}tThGRXiHG(DTtw$z>xNd`ZXxu(>NT%gO`aj6 zDa)+=f*C`q&Emd2R$r} zwq)Tg;(A9Zg3&l!wdH*~-k+r5;;8dzw4=7*LaUq>jtu^!`l%pwnxqRxq#9nU#|9# zNkMZ|$2}z_ncQjCk^%(;l}K*kaJowUPHg|29sa5DKCFRgoaUY(;!d-*a?IJ{o_Y95 zv)bCLCGFC>T(%NQTm^9{pXRkjchc`lYoM-sa;0IUQjMd!yqf6MrRcIo!LHOA;Qev| zM--*F0pUWbYsEVEK3>3DQVUUDEOC4UXb8|kDmA`Dnm4&3V#02hp9p;5pC%YdFaI{P zv^jAo+H+-^Xd0Vt3GC(gL}_4sny_(=Hn4sim45AH)tG?x0=R$%#IG~F)~x8)*l`i$tpv&~pLU0pQyfr#hMt45H% zveJv0a@>e+Q2Ji5q7>>8<^F$^C&)OHB8yhquoGjTe(dzVuy0CuREAVvc4~_8+$k&u zw~*>LV1m4G@Mww~2ZLGY&YaXGj`Wb9hd`5gwY1sRHD%N=;3Dkju?LqxE?sij% zECHNzH^yoX;GXXXQ{aV+-bNNwwd);s<^v1MI4oR#c>t) z?p4Mw20|{*^al06T@MW{7!?hAZ1e^V``Mu2EQ%n+>~??SrG-!I0Lo|-`N_XP}6XlZS<}guru;HaX`5y`wb%)XdxBM%kye`d(3EnH8l^4OT0l`wMRK? zpn>zcd({aC!lqWafw#tc1IXQz&j7M)55#zbK7KMX_OpUXUS?;TWW^92bj5V^Gz1)2t83<59YVdp0yg{cML{J7FPWA>x z#gM2WZL4`h37*uxH7G-1TLyY=mn#+~JGhXh4CW^p1`^sy}yB3LpNRLedB13j(LmXVoyWcY3?QVmqUSPR`u}Ltju<&YP4d0$4?F8#=+pRRnRnHL z7a1C$g;XB3NduB47tZ~r`7g&tn*4O?8Ur*Sc#VmQ{)xw^R9aPC!?H^Bd_aB?*>0I)Mz`R zQJgmskH|Tk5WJ6M5CNfv~sh%_MwSp`rNB^x#sIsukTQM=4ptIo}cRVxNU6YWik5lSZ{RS z`Nqg)x))Nc)EnKaURZQTjlQChH@f$~q8hq)_eQgK0YUvZ>e}OpUJk3RLfdhJYV4*n z-q84|T1t%;Qsd&Ld$mc$(3bD)hPt0#|6{y2lz0aX0gaHFF!ZF9y;A7ZVI9SjtFCXi z&btv;@~?#$dtkD6TcPR^n=6Jz3(~Vyew4{?>ovoU7Fhew>%%I#;;$2vNr+M+w5LCC0 zo#a-KJ)0h6iJsSR%e*C?bNZNZ6zXTIoBqi4?x550!wPoukCom;76P!eqv6)k^Sv90 z5zvAeXWyQ>%xm|Q{Rl8}O#Ww{yyQoA4c3C)_DYgBiIt;9G2LMjZU`mW{KR;5%;7?b^(=42YrmOS0s4g0jI&d`{E<^5Hk~&4g!jUOdn0lbTyfmH^Q)2Jz8+FDf1eh3 zkU%cFgSBbFe>1%?l#1*f?zl74j=JaA{aPM2P(fv~cLTQuO6{tTS~htb-TUaLh;D8A zQ1NN^zSGCF>iVJl&4-oV7+!0O!bBhXc(pf*oIB`Y+0Gg|)9X0q8{;_|L+WAr$4#8g zYpWaeXSJE>UG;ozLlin8HEW=FSNRD+j0qo|R9^Z%!7az|9MXo=V;98qk0llk87!Aa zWQI55)TW3WkDlyMTI{cbfC{Nc9t2{_h_D<%^~lr9ysMc|AgHCK9?o@f-qk0@M!f$h z?}q%>6Ld-l>+DBAp6-oUP(Px6AsaH=ercjZ8ugQ_9=o;DTM%asQmt}Nb9}v$gF`r2 z6FzI(XKpVpec^GlFtTzyv$v>J z7gE?8w)!)_Z7ltd{eH1*pE$bFTf{9rEM{pu+cDR@f8!c+v{aUVz-Yf@wY5mF29?p}JF5l#HMuB?)) z7Tq@AUHWmRF(Dlv3y9atedMIclQRK#!&;bw##4WUa=e!r9&ZE?ulp%hOz3pmYwr{PN<|Sl(VFt31<86wzKQ-7yWmreO^XoeI zeLl``rREe;YwKrtb>6gb(x41iYoCdAKe~ReIa)lWZTjx=RJVM8ZG(cdLTdF#>E5E< zmxW^{Lh6IUZ8C?5e+1DFb!hsmcURWfV^)ra%X@ch@#cSEZ)(y7f?(A{Ky+r<&C5#)M zpxU(~P1U%-UNbQU+WYZw_29lsO%E#6qjL9aTy*^1Y<5{U?0N2tEa2c!bE&3#;ocV0 zWWk)#`iE^ev@S_KEqgV%mE*wDd{zCd$%kU)IB-p+ypd>vjqUt#VDwm7>E~uB(`?U! zICN^aYWBHJjAqzYv`-R`_idH7hhNW-zoODMtHbZ)%b%RzmLeEXA$6D{kY|ayK`}cj zkHoE(KiOqrGSH1db);jioPB4hmeNXkq*IZcU3**)K>axCNQ+fc;^*xF3Y4jf?PMoyD%UjxArT>YlJ!G>tCbPsx<8iC_lB()i)TSXGDU(yE3NTtD7VTuwol z=+A9()MVbbh4{tL__-oW)#zJGJIZQ%^4=Y)#@!~tfpz7{C(C6&Cq+P5N1d$PtZH@Y zAMx!k6EozSYIF6!mc}oXuGu){MbiKp6>~nHDH|X9Q=2cNOgH>G1fm>sXnPoo{XB{F zX+gHW8)fcCsmf{X&R3s`A51b>S9!QQDaB z6pqUBHxO;ZoDQlnuSS08=l@n9=bqWJZTj8Xp4QpF%{&)ait5^gJVAAO`Y;t6v&Kxp zKv5v)T)FD`+kLu3thLzbpX1a8XHpCeAcE?5F7VHoA4PIz3|D`U7;R-%!01v<3--tLv9h_J-OR8wrZf_l zb7VBN{o_nbciD8rY{~l#7nIB9>+Xzn?KA9-7V}zzpmo%l&UNJDUb#jNh&t-bOIfmc zq^Y~2dq7B?d2610L{<&l=o5Wuh7`@TIjN2MXFi@TE7^jk!)aMU>g+YMo7seS)OaQ6ZAZGUc+C_^4$4X95?T| zV+P{p*|L=EFMwN=zb_jtr#QqwC@bl?PCMMI$cuj}Y@ol>R>>*8J4W4aWqDQwmDw9K{J9hOE6d08s1EjNzBNI@s&lMYhBjnqtLm-Og%mUwGa8(lE%RmWvRkSq6sKnE z7+KhHe3W2q(_~@(m8Jp0(D+xL%pciR%b=g2Q*+-+ne)`vs_3P40&l|J0pkTQoCD)YW=7U(ksv?wUAi*=%Kq64eMg$OWxUlYlxszcib>{ z5rKXT9?dD})a!U!)!;Y|JuK0B51x>BiT?4~0x|QPyi3>#_|~TNW=)oNxc~x$et;ld z;jzWfnVyVIsi)O5Reohc@r_Kw5TXc(+CvVFLc{grI`vO+%e^RggI(5ZhePd@xKU<9)eXOm?G$6#=;#ua0T3f|ZO`4g8wZ&T^r?Up4A>~ch{M5Sg(u`(&?}LKH;p|o5Jk(}8D90WhXeC*c7e9~ z1)LIpz}SzV42dZvoX2!JOXKDBveFN*Gm>7g=d_$6$`yio4uQtjN+}MWiP+ zE_5p8Br%GmbSYzx3%7Sr^|@SJ2wQ`4;jk@U3)mv1CEB>jO4W?mK!oxe&MThw=utW3 zr$8_!Z2Z^fM-mdB-L=)EfL?wGSnM>H8PAsX|#c(Wgk~EQB0eF1~R7XBg2rkhY_p^C%k`i zR}_0_ZL^&wTpX`mH*&h-H2Hag7sGbP@-Rx%{2gA*HN(uNJ<^7p^BW45c`>tFnlC;J z@xKjm+?#Jk^s&~(XdAa?mTF#VTwtvp*E7zm&G7>G*86dPPf&3rcP+E(J5Jo+#j0+5 zN83WvpsZQ56|(n>6Q(DFHr#18yhPPGK3?PdG`7s}Y8Lh}AKs_*@QC}Yj#rafdRjRn zN3(J9s`m8G#tm?yidoSd6?b!0v2?gN@m}>ht;`3nF=6xCCsq90KN&fY1(eM*-Mt)O z9)qZ*ad4XZE|EJ8=~6UO*6jLN701mstsD<1e@=8y{MsY%4G^ZGd996J-S2HuGt(`b zw<~hbung2hkv8ZwAG5`)d$);9wEr`!IVo z+vb0b^%|rvjLfGkVq@L2$4pWIYq1uqcgUYCC3O#iFxAxV?(RO}FY>T!$Z7FgxvHo3 zM21*=Lab_Wt-bROO2~<)GHr{oleCkpqvDh1saiMO8tJ$#HsyJ>8k$XeD|-CH+fOtUj6TPL?Bw$ zPF3-@pA7fHkdyG%Xs=#=-N?cy;q6?nE+3kRZO2^8rn|k`B+wb)N8LCrx5VsVbDno#mm7vmzf4*k3_Ch9*o>22IAS( zs)?U0%5X`1V54m6S0geRB$6Jd$%Lo`^@-HP_;|nF_+O-7Yl-9Md$qYl0teea5^tn5 zeC#L_I-8;;Z8dO}9Af`;0{zr-Ih5vIz~*o%v!&6cb&I3wv0Zr*BHH#1gOXq5)nIwf zXtF4+X6*E07Mox}Yq8eVXQ&viGbDs(?N)ozygCm}G(-_B^EkKE9xd9gofKOnyIBVnVnQCu zymd-Z4Y^AUxYf4Jr^~#qo0S>rgAGLE({4^t;2zCi2bUxaVrcty$ zKpf8%VHzK7C+VQm@siOZgz;eutsrI12INRO$|0qGfEct{di&v~7UH^8xrERrjBLkC z$MKnx=I6_?5L1pz0@H=&!UK(IBAsF;8cB`SJ5KK|Eu*i=cbru&mqlOe?D*mqN%=pB zF4-sjxFnDeUZHe+HBqwub$Pfuz~${_auNO17ad(z&ya3ZVTPqI={s|z8wIv}2*>I8 z-ch;MPYO`Rjx!Cb#M$*-m?$*f+ad$m5bqFw=-+V@MxO)F#o zr(O}8Zpo5sT32}=8s#}cx!W)e8p!vV&+MQ(E_5LJ<1Fc83tL%NcKo?mMssbCe{GlQ z`145-{}!ip)Zq;G252m&(^(q-Oq0Hk-WcW*ayr!PvsCz7?z!k#2!V`K+MRDY3h>B5_`oww-#P5SgE9jBMoSB(ZX(y*{X+ zGM!EoxERI<9zU#dok)u}jCf1mw~o_k+%QR@vbA;;G-$XbRz}o(GD@(Cb!0@$ME$Q1 zarbVy#!nz@?9plJOzFo_i59F(`?k})>m+dK;xJJ}zkjt{e)!gK0|w%Hdh_QGg^2>e zcjc4xa|F{5qCeb822v3Pf*J~xC8wnY+WHNkEZZhIbbi?Chlb)bX-NVTc`@=M_$I&w)1f=?d5TZZtlrdcI(Arv} zr{X1m_aj_REs;wJzm3RoE?q`o4=p+m#RfT=&?u1xl6Q`HqB0^_=L^zhfS=~vaOs>d zP6pawQY&fa%topntDJ)HgwlEBY*mjh94#pi)dz==3pz4hU`w*>hP8JZNf8c++?CRV zi{t$8iH3H|B^TckCC8-6s$SG)xkKSZ89^|I9zX~NjR`xY&z5mvqEx@RqrNnZBoBw} zooQ0f`qoF88Ye9un;pgF(W9!~G?M{S+ZCcSljZtzX2Z-nsxt-Y+C(SJ?L2di+(2#@ z?FOsMgA`7Tc4g<88B*Jau#yt`a+b6Cgz&`H`KdVZX1@{Th9?VU5H+X(_leF;mK&I* zd;_7&ym-msqd8)+c}wI=F!FHI3WLUeJtWDR>Is)3}4S=|~(H>{LAVw`#D zX^k;5d%*$Xf;9~q@EirD@~mGp8W3xSNitbiv^{uWyJn&!-|?Wa7!bpq&g;%dka8Xo zi+va`S2HIUGSk5c8muiF5iS;SR9U86$*loc`doJ8$_NfYH3W^Ie7RCakZVu(F}TpM zbF7SFbwPvQwZEAzIb7c0U*BTiFd1ChH%g9gGbR527<0#~$xxe<$wxVnDLwsqz?Mtp z6lsxGY6L?L>`}Q{TBMAPNP20_Oi>Xol-{hnPnuydz%^^Cb=EyC&B&I-56k7!<`blk z|9Z6$T!bA!miH>rFQpX6wF^QK(JyVaO&mhat1NbDo0F1ELNeU|4T^-e&D9WEef=Ou z`xVj}Cn1N=mP|(>@;Yf}1A$v_Q?|)O|D4d$xMP(x zVUHegU)$+Rq}Tnmq9o;V&lL%EjV?%vU?iDs8F@$1@Jx{m-oGHqrt@Y=XL3|Uh`umi zhL&`Qvd7}daxHJ6|ZU2dK0~>eBUu{{7 zomegHqplx2m;RC_V=vnhg+c+gE_TV{`-QTIBfc1Al*66lolDvKY{>^6GWZbZsDk)w z4`U!2Go4FmNDJ%>_Ydm5R+}VQ_uC3WMs(fKjX)i(oUum=^$D+IQG=A0XiB4*GTe4t zO?bSM!pT`Iu}<1zMwgTer%BJ~Oui|NPD(3ZQXgWAG$ELyAiNArX*ov{t6vn>cDR!g zJI)=-Sv26*4JqwnC6+8S$HM(Ig>#@$tjCZ+qd86r8B@km9$KUD1f7(AJ5}P%<|IH1 zM?=?CN#@<)qo98Oc@p&BK%meNm?W|OM>9~MkkUVcjc(of3-xwbO5Pc%$DI+tt@l$1 zTbs_xkC2l9y$FVM9nl;LeqarM~;}Wdsu)VpO40(0D5e$e- z8cHPTzKVWps%%Pn;Kn3Vr#opVQ0`;aw%4GwUts%miFCyt2ir zuEe*#EB~|8y^@VgtsGDgWf~u}yM9;>8rEh?CtKxC)1jfbL`HCk;S<#jr)5OOMIrjwfFWPo)5Y#7UKR*twe7;v>1yI}6T-7p7XlzI zOHS@bgGK2=ZvAl)&xM>W?KjG}rX$1ZQ@Qwx>2fjgxK)1X(mP9X`8e|pMAhM;ajIVQ zw?~&TtYMrr{R_h{zAn@jRFQgG*-jT0M(zBSqcLHZCks`*K(=<2HUV;Qvn`L!j0oHH zt7Z5_!7y6{J60^CDzdb4wx4$SPGr>9*CQtEa&)7Niqs^hwlOuz+H$r=|F8z?+Hjj> z`067^rqi`cjJoEAWJANsQ8~rCKdzh38)`ymoUWXV4D~NefkKVtG4oV)igi>;+La94 zXW_!|u!41EukQh_%Iml-=mee1Zq8LLh?Uf$gGO!ReRX6phr)~ojYb*bCK;;MO^+Kn z7U|KDnn>kh!jbu*#it>dCokBPcDliFQgb)U*JQ!d5C{guP1EEoSGxIOZJIhMN!46j zZX`9zrLv<~ll>sg*2t0iT#RZ+dW$MuQvbV6&Ted{t!_Ka<@q~GU*=9?X?TUw?ZG1V zg(sU=1++VtKe^F+aKj}gP6;|xF8-iGPP3t^t&Va*r(4_E@=A0Cp>FN6N~ayN^^%O} z-I!s1dv?3#WvbdGrow|YQ1{+DRJEnIM*2thvDLhJ*A*L^7HD0%{gkJw z`D5SsrTbKl^QqheOD!$ck3U(ZMUPC5-M66&C-2}}BX`H9r#*A>Zgsh0UxrGIerff% zIJ5sT0zs=#dQej*)sAY-CFJzrB(bxkGr^mG$ zC8z&;5u5h7E7?6rk>CJ=SsHh*cTaA9LL121qsIeza+dqqSWEW+*gmsE)o%Hb=~7mX zkEg3zoB^Z0Rj$WjHq6ikaI0JozC1&0|7cxI$L+CQ++TX!7daN}ai|0?zl8-0Iz4_D z$>-GwxG%@KB)5_U$tn8jG2Qul)=!oW7ycZWO&T5v(%@@|Y^NvT zgh87b-KyV{ucV78Orza^ZxHO2Gt$>zC|X_Km@j#J;30<(K^V?e9O;2#$8FC~b7j0= zC&04(Y^CJ;S0RCdD2uCcgzzGR6+NxtdsBgmi|sRrGwT(}(@F;E&c zF?!e)z3mRVA=7%_cHFd!x5?#x-VLK|T9-l@9BtFI%d?~l*G!hsVfGp>-SVZat^4K+ z)_sYLq2v_K_5~w6YAB}(n9%1kY@Q7BPeqM^(#B1coBZ{yeY9w0llVE+M=P(Y8=qJ?-vUk`sAXQiFTiG#TfA0b8PTrpZV@l^hK~P?1gi4=Zai zb_1^`n!$JFa3}49&2lySQh;mmvz;`LWeA;Yg79MD7yjp)ol6L2Lze2aaNC0E-S#<%H?|0y4ywx>jTu?q)5 z-=@2gB$D_$iH|X!KbkqiPI{k~ICC zlyIS66SXL5TjXZ$crU6Fj7`YqtdlhajX|f^p!~qG#9@I)P#+V(CHsU7WVSrTxCMfS z;0nnk|5&ND2PkY)_9tDkrhz^*Tr(^1HB(<(AlB${+lw>47pJBMZo{{G_59I`#rFU_ z2T_ei17&Br@YM*o^?t8OlVt=)Hr>P1MX%dWN-lZfB5t6bc(X-^`!@`Cx?XHl)n-ux zgfA(hVMqTCS`Xh151a`cB{;t+T&yntJ{s6b0475Dl0Fb4pOSM}(?AzQe`2@9CWuR> zk_Zj=gp47=%r_z3fyrPDH>b567KJFreK*`1?l7?ZEMGp0)QhHJTe@s}uWECLhA%TD zV0WE-1ASE~0Y2U%f)U;RXCyHRD0_EEChM`mbUIkmP(C4W=IMVM^=YKx@HuHt8Ww)| zf*s3ozvyUVt{{x)-$~$&k`7Uh`nHpl5mta6+8#jsk`^c>|0&>FABdjpAWhim<%cgR zJI(|AOaq_+@kfcIlMYuK2(*yXn;Odpk@6I-EYuLa;~GgkuYxr+7}0xk0CDgddz;ks zkKWCTrHl^=@U0xZ*_Uem*&{|U!=*PnS7k&pA*H4fbb7Z;mhvZU8cEP-XqzZKqW3L( zCry)!cq4;i?a}+9S<;&CeJv@C2-dcmlyNI9($FzYzQ}#R$YF`59G4$>HsstG_3kmGP;m8KM=m79$F$5_nXV3nk~vbmL?lcMO85DO@+%H z1aKb>pL1<0*)CBn|Vh?CMc}Bi!X$~KK-#&XhaDeO{S`HqRd%BuzXU0Vr zA7ViV=|s3`L6!Q?O_tAWHKZ_&ZE}nlG^QNi4a+Bpifrr-cY4286xc;x7A>iD6wA@l zx3)x?@bx<4v4s>dVR3F?7oR+^^$o=3jRStzJVrv7Rf}aaGcLx_bZGcsj%3?&Q(v(6 z=E)(-yJ8Fdpxj3rWh4VxqtE~f;e_pY%|~c_R4zv_GE(|PH}902*e&)A1Paw$`;gZO z88i)i`w3f}k@*8I1cC4F-yy6V4O`>o3vMiH8lcfo)=0{uD_i&8mLwC>O*(Dg*M7ipLl+R@*93DmkPqw@vV>Y`)DbfQdONH=%iOm3!JfKsx?Hm zlU}PlP(eV*BDnB`obiqS@-fB1A?kDKbV5E^a5T&oh>Uc1N52|T+GdaQl3Gp)K+rBo(!0-+Kx7LV6dIn1 z9zf~6W911ZM^x8@heNbbHgb!?R}K_nHy-1KQ)nTO)WO--TE1YxB-zgQTtSsz*kroF zF98Z+Xbd_h+w0sI)`f=QX_7-i5Ky$mXc#4HWY&f^jqCcB6mUlX2_)AI`D2inAhz}#4HG8F7sG8b4=d>{ngN<_ct_%E1rp2a63Q+``R2Vo$;A4K3`crpGiJz&&LfpfoS2MA4JWwueiEzha`auqkpT6PJl}6zi=%nA@NKO;VQ;1-e=&TizflnalTb($A z&xqXdxj^IbWI2M70oTuXC;iz>`J!Ic$*SX|KNllk`>Rjrpv(MX`7HXimcC$~Y;T^b zO{fV2v80+@P?;alL{FLVvUIaNQo7GL+v>8gL=OB>FYE?TC`OqdxkH0sR3q@BBN5Wy z;{1gdFkG~xK+w=N`g|_^)!9<^ca9+n7a+L3Fwn6jl@XnWx2ws)oJiUtC;iqIY zXf4LH^=_*4;`^l((HZW#VW+gRXD9ZRlupoMI&Tj&)pmvGyT$V1xsF;WU;Ph~r8DnJ z{`Dn&|Ag$a*BNvJ+HUMFSzM`FZf9I*Y>JVx2iHaQAWn&A2u*0>B|=|KeW|B;W#dz2pDzd}`NpJ_H+8Te*{{MThlxrw{A(5U*&!eL46DIQV-cOS995X?#dzy92ffDN@c( zFSgJR4Q*JVunhEx!j_VT^KxxAEg)#<8Yg%BHQx;N0`1#ADV!DH7mTCjpyATp^7Z(( znudO8;7IS&fvY0U?vt7*pWe7DiY-h0ZTlkGJe@xMisdW+EXi7|zsRv;W=Fwx%RaJB zHu)iERYN^zCI%pC5gmF8U>=+j&A75u2TWpVtGt>-Oj)s)`g{ZM8WpPa!KJ!T>I7< z=*A>T@mH``(n)#JlV!0+feu9Pm?w7<>jXjHmCVUsu-5jwB}DJbkXxC?t63w%_Reau zku@#^eFNRkp&ux))M%hk_dbPl!o74lxPU^<*!6y)9xK?~4AChN_}7<|i+#OW)A6rw z!o@4(2Ck}T!7R#usIW~DYTtwqT^u9x2|Um!gH3mMeU9A4Jf{1HZFuphEaqJqfq470 z^dPLjHxM*RR!MdC#QTz=Es|rz2jM8)2+HBjl8)C-F>QfH1F@r%IDFJrrvbL#XUTqU zqiePn;^aK}JSz~z<@8P&#AX)|zH%I0SJ+55Uzb0Zh+<2=5bs-}Bxl@D?FDUQ&_Yh% z8t0^6-yx<;CFqXZmz2v>Y>0%V(f5KR2}G`A15xXYWN>2$aH~sSb_Dt&16-`6eJ`vp z7n1=(3q~U-4R*`tx3$p5(U0Twjo&9%aC{97{Qwc4CI#&awUju%sMZ+Z$=uGHsL+d#jE*&O=$-A)^2)IBrh5clbO=5Mue9CtEUe-az`2~Y8v_@Gk zTRADxY`PY|}t4~3P3%b5DI>-d|72CN~B}s&lQ`0tf80X!(GHTs>lGTBC~O~IEuV2e3KZYS zkB*jIq;S)tk_VK+ObPHQsvAH9<>_j2Io-i;0OA>}PM#|*m=&Gf(fuS9L3F=?P|+rQ zHWh|vy1p-slLR{MBIH1Hy#MNHxtBRv(js2>L`noF(@*+8S0JwvY#0XI~GI!cDK8*s612Hm~nIEyWJ~aGTDe14TG$=;9pBO7&aOr!n6&)H}KkiDNZwP8SIQ#DjvYUMexLCFs zb(hI3PToP$PteI={>UJX4RGs*jF@=ISh`mK(2wI}5OlQrn9ZC|L&(Xf*+}-V?7@US zQT8o&Bb4T1AsS4SHf;WB8t9VkWH1MP-SA=kLqD*MPn3Lr3I+|L2v)?Ajds*>-K8>` zrOS>;nR{roIxc-Lv7al@0HSrF42WdVg=q2|xrP}gA>2PQI_3txI((~M<=6oKjLyk& zIh~Ha;fv1c%G~HROh(6LvXP9S8luc-%N2kx4!s*?kFLc!+iUo+4Z%83k*`@e7h+S; z5G;L;WUGfws~2L?vLPobs3e1R&;~xuag^;r8KW{}!|^wbv#oLz z#I0aqvbuGbTcqpV=Ju9v;U%awyj0Ku+F=!l5lgs{d zNtg=|73*XhRa7+uXh0mUBm4c$9oz8h2^mT9EcEcT{qJ-cNyJQ~;irv~zh+aIC^SyR z%P=Z{fbW~|pQEyiYbI4(`ZcJOHX}=nO=%1|{Tg5PE7-8n@(Rga z0oRY?^cz|%Ygrip?h8hoziE=)&+>! zS<;t`u?1FrAnvKA+Q)Y_7F%O{%hA9J_Je%ofOvAUtf;>|+yD)-HZ4CB$$Refdm&T$ zu;wqc;{^z|^*eE{4j12ZPji8cTsLiDTdKdb`f-qY==A4qQ>=~4v2yh1GeNA8wWQfj ze>SPRFY2}PA{hJrd5U6~urz=}H6*Q* zYZ2ehns5t^PaG0@kDt~od_N`0@6N*)(9>Z2eucAXK$0hLxC8Z;bOp^}|Hb%Srh-_FSM zO}{rN2!?XMuaY7Qt{Z9iPE@PQs^Ts}PN2>fng8o2nyr3x!!&uxKbyBGhm%!4A7_Q( z2_-YSpl^xfM}hhnhZAG$%-CGEm0R-^|2Xq zgAI*7@p8K#bz|k|SFDn6NH*G98kwtA>~B?BHG)py-f^X0^L?uYvnVsCxZAnOwVbYP z`e}d!OG|7&x9SIOiIpNYb^+I}M6l~}R2*qFY0!`31a8lfcOy5shC2b0XRV5fxAf7v;T9oyPLOHzBEhlbVqhJCczFP0zk}}pTHm&+=GgVSl z!YL|6-?fM~_t*^_G?y~sPNFp#N zaGsaG88^s~$?BpT zok(|(Vbjz_biy7r)`fkb-)DBAu%5&*%ETf?yJoC%c zRF%-qZvKG+#62adG3!&^PZNwT_s&oW^g@2vZYfVD$$}T6l2^!_4l8c)xb zcl?4<)*ftkymQFB=6xE^Zj%M|AhAo_lhEVNrlnlMD)r$Vk3 z$`}3f32t>8JgJ3)#z%#+*pE@90UB?TU5w8aUl$-sb7f(RP*_qLR*zF{vQ3bUZlqz| zWEJ<336SwS-G)@vxQC6Yu@IXls8-+J66M=ZSt$Pdc4Ht5vE_tnwd3DrWSa)nWrdeL z?kTgBWMm@Pp?zK!iCLR6vT1&2wY|7yY3v=RnnyZ6C_fI9SK_BeabYiKUyVIc zCj4c#ie+|SGSOWk*>P!vpVL}Ugu^aDhjV%Zn6Yq|v8KU930ik)mW|9pscS*qofs#-Qc za1>8vS~D6TN<;Tb)pC@H+|ekGqk812q_6F{DGfmMovvE?aj>=_BTps#kPw-J6v628 z1N*dM`a!TK;@$U%P%UJFrPI+6nx)!(oEfDY`wlO6jWad{1uj<~Rf!iijZ7Dc5X~2U zX%dApccsk7o`CBI6fOpTMI8w}Nw3kut0~^emi)|nQXwrrnQ4nj#N;wfCINO(l!bRmaeWQ=^5m87F zydNb8JBsB5=;~?965TvOo?{)PxmY)Rwpr%4xg<*c_vWd@`xYAYt)yGlNihe@;OhsQ zZYz``--JHVnR3uSzP1hPJE>+1tCH`b{-@QXoJ&__(BOeaStlvwL;`T1=%@RnJo9TU zhkk(gbhea#eQy{A1AR6{%84#TD2+mcdqOq1YpDq(hA0iBT`gxLQya)CN9x7-8=lqR ze*f5ZMqN13Rz$JmW!nnX_~EY&7fW>iFgf^1)M=zTTp^38Xu!YvL6?dh@-Fv*B61um zk%Q5h(lYZ3y$kZ|n*7ODjvdQX;$wj*&mF{bc7>yfqV0D_<%KgFqI&XIyHzVctA-Wq zr!(>bp&CA0hZpWXulmM6O)o4pV}IXD)zS~YEi**FKOx`QC5u)N1N|^XzO#wqG#%I; znxv9R6sL*WhSPDf?t_J4Z9(~NnrebQe72xe){%o$wMQuJC?F2)b8ld8ypRSOHA;Qe zCjW4l1{OP>tGiUx*#B{&YU*!=g960&CsZQK9z3TXX#BK7B@#%kxfr4jE7%wPh+V&- zgS^N~iyzu*pq!bZTHM(r;!3AZeK}Oyq)oJ4%3&|~5Jgj~ba5K(lr@A%Ktn$$_s={P zx7l7nwD$PtsER9z8kG(yhSyI1B#Q01bk#08g^CliM76y08ZEXJ%(;-faG?vsJvrz& ztukey|B5zTOv5LwB5rV6Y>`8p6KY9$LL|y7R&h_K=^sAP*di6nl{;;*1Ostl2bEaQ zTp{uyT9?bgNM;UbG@h+maigqCTPJar5Qbv}vs~IRkfZnMoMuTXu4~ozKBC9&Q0={m*-#VB0=j#bwQod}%p+?;j+KrY{E&S2U7C#Z`4h5j|tF zEG{rRJ(?2`9ZOUavBf@H8u+}ALiR}&zVXdBhLY^ls zC&G64Bq_SAmua98l(eHNnWTu?QD(fbjj!79n)gw#N5&Z%s&-ju=(kuU-O`L-Mmb-iImJmj!EUJ2|LcdZy{PRNnRuqCo=fD+&)2$ z6DD3-)e9foAuq6CMRi~OCs)Xe(FszV852~CuGfa;(4wDIEjU~P#W(VED`cU~DeKdi zTSrPG>r-v@v8u(7v%+o|?l?q;mG(UtW(&luB-M^PnrrA%g~ko@x~s;|F9>r1;@Pn( zzVW9~o|`vOwTxWH))1Xk0{f@hggi977AuF?5YhI)zli?Eba{(oQr&TR#yieSMJ&H= zi0We-Qe_d@#e91}gKd{X(KjHRjZ5TZ?lt-l`1Xx$s_m_PHA~+fe9p%C3z9OU%X{@z zLdvVU!)Ykw*pVoUzUdTJjs|WDzchDql<1rJ79)>yWV$$-F7r8T(=;H8U|+6O@t5_A zBFakoC41Y)=fdUlmsltwsTK@L$?UVhKa8aj8o#cSqD%YgY0Dq``}y)@10Vqtf`X&C zuYT~D*Gob5Q@ZBcFiel}v{%(%?b|8k-UzW%yExqmvyI{$kP8jh1$ zBUxVYZ$KbN$jM~OwTQSJ{W~f-gw6y1}V5z5Nb!5kR zxRXh-!S^mc8y+4A)}D#&_-Tz|mCJOx%QAc4!{?GUNtTnS1+jGlGzh<4HrAx1@=^L8 zS4kxQ_4_(RLzVL9?3Tt+5T;>liM$nAYomS+n=0LY2zwuhA$ck>zN;2nKWKYnp1j7! zT~tH5yw*Hbv(yg|S2a?NZ$2Hx<>iU;ZY1ak&z((nd)b&8tsrdgnj=g7s7~V-=dbB* z+HV^J{M*0KxbiX#Y z~TBsu1B(XlsXQaP&6SVo=mab8x}Y(xZgWtqH&8gYQ(xPj8HkOu++coGU%Z z?b-nxgQ0QA8PKqiYV_8FW{becF>q3$SM8=RP53?vhMa+%S8nZl(403IIR?rJS0CA& zf{Se!8t3MsEgjc-ZlJTIv@sKk7Twr%;EP+_b7bQyM0BJ6!29aB`COL+Ut1n7OD4Iu z5Mc(2PxOa#?*7|P8eJ@n4=Ms@NO=gjPjpjEX*;4!CV*mSWRckUA1cJa!9rxU%~jR> z%yEWBRM?WVi%o(~R%tca^zZ-Ls39y$c4EBzZhdRz$Znk{zq9RvP(H-vMJA#O zmUi4`6ZZNCcf4t^2Fjkg*gX&l)CoG-TrvB~KbwbB$jP2@(%p4yq%g0OU6AJPvIjCg z+qXu$l}8=pIip~->$pQgw7CHt`w`O0RV%s_nD z?p^2p$hkWB*0)Tmgw&D!z*b- zxsAi*_wBDl8R(CF@-5}nAZQIV=(;%fSMIG9gh%C|TjSlsWbxKL+2}IpuFdWtE<9)m z&_d3jh0EN-9Jio{wdtUXmZ(1KF4x1uYCCA|O7|dH@b-o~%%Fvn+|TKBS~+8}K?CAc zb56W94a;Tu1ounEi`Ldc42en#Oo1hnTA)eBL92Za~|^byS}&Wl{EE6S7aXxg%la$e9r5 z9%$6tIJ-=DsT?YiY~}C~#Vn1S&Ka_mlK`~UkN$kgb@!359}ZUioOjN-dl?=D;mIIp z=kfoqy|0a{s@VRgNKGx1La&O2Qdtz4g@tw_&z$Wk3>0OHg5o2FiK3z?8lrFz6*+)B z$djU^C@P8)S>D8CFS`~gX@;qkrIrtsY3aR|b+zAj*6hRKbN#>lzq@|iFPJm4)^|P3 z?AbGG));5}esvAO!M`3gPOI@p87k!NNaF}j#n+<=gtKWOz5R>}(3s@HI6@w)F)rc0 z0uVI|e50w|X?Ho|%EzV!HzkQLFv6g{N%V1dqX}c7B#Iik7V>-*-K)j&5Uf<#jQfFh zwCDq}U}vO_8Tbg+5dT83wat&$rAsLVh3K)yd7Q+M!nw&oDfrd_Ynwgo*^?P*Fz)wg zW8PU0tw9#gOV&2;s3a>)Gz3v5r+$5bJCFjlJKXqrD94_g(CZqF&&=!%++^sz85Dw9 zpJdBThW2WvkRfW8qi7&z;>#}gJ1wVaA*EH~ICc_&K|VHZsC%e!2tEpw%9KMNsxvP7 zyz3-7L>oH0S)4p|PGV3>9(ppsxQL4s9B2mCx;FG{U*ig{@~U@=Kg|7_aSAc4QtPbK zVX)kV!Bj=5C=us_>8#*RZG~kv@5R`1uB=NjxKnes&1COrhBO7T;=4Y%z)X=ri zxv}DB6)0@t*nt&+FMe=w8Sa@RrnF9HIs7G`{n;yza7>LwN;`ZCW^7&6G6M@YJQ}x$ zWnOdI<^NTqC{Z*F2$P_cDTnWk6`%Z}4j2NyO&gAt-cPX5Eo(5+;h)8bPjD2$G!$z5 z)D1Mg=K}bLu8o+vhrCs)bl`Fe(}w@zCK}5Bl7h(xO@0j$4FQ~5PZ1q4tjTEjEs=XR zTYtpDXyeCaR!bc;rV2_AQU}anFTg)oxDn;e#t?M_yMjJqS&aBj#lx5dyWFf@aBn2+ zA|Fi)>su!NdO}@!$w0lP#r(+ikuJDhCwu6v9Z5q*s6il~0Co6*xu zMX8#M&Zj3{ZQ{ss2BzQdZS+9ThNMb3y^FWujp+t{Rh}G%ypr936oQ#GLOM~qj@?`X zn;z3dJ&#;rX<6&~bJ2#kO6)9~*fbsK=i3bFA^$Leb;v<(eRYH=gng`IxwXyT=eq2o z$9ouk-)Qy4r^nqC4cOw8G!#+1flfa52jhTnHf^MPH)>;jjV~Bakv=x+6+0EZex0R- zPIN82gAcWTfb*X!#|`grklJ@y$Kx2|7-PA}9#A53&4C5XeY z3ka%?9DaYiNJe-RG$w)R+DIH$IB5n@G0|{Jpl46c=d4Zj5?HuVO3&Wt$d=<8qulF^ z+em=nfGHL9=Yq)R#gVKJHq@v=g8XsSf^;dCWmL~b;}&{0?4o>ZR6>%0%ulX(hf$;Y zi7&tQ%9tAajy=uL4IF%ytfX>4?Kx%dwj z0H6!yAEUFcicAEhfu&i1lvKxhuVSJs{^)&}<@DSuf0d7>js7`aoWq**VMSv&3f#rH z*2@eLJ(EQGBYU_8IO$r%gE`{ot_ByNnWjYy^b;$F*(DCi+NMQ}!@)lEeXx|ZP=JF< z>8_~)3Z`8ahj2s$)s?i7y@RN0_&TS?P-964aUH$VMKtfA_#8*N9pJ*UM4U^ol%ktL zK}y<)okE;ink9cVYaH_t%P{>i?E<1WLi8n8zd+P%f_9AEV$Spmcx@JFqS29 zWFfWr{4#IoZkFy4frIqFnZZw>qic~bxYHvqJSTyv8CwJJ$?{18pgph-QrFw)3=0pjBO^-PpC78#Z!Qs*vmxzwFaZeFGVkM(lRP$I=*YnSGI ztYn2l?b8+TOLT997aE$SL+^K^yzYT*4d4(RoVbaSl7mk*e$Y<_{v3eW|=XqwJFEolP7hdAJ4OIiXv)9a*xI$VVCz`|z{wiQYjiGj;3F{yD)s*F5gGGvY zjNq_@J~qLZ9zwhW{!mUg=7wD?RLf9-DAcvF{@p~LIsb@}9yAvHCC}VQRW#NX8t)*X z(0!@{p+@*kk&9H=_*Ef|>=tG%`zPOMpbEzN_*nbw`v><1MjF-8pPtitGYtp`8VgHB zGG-a4Pg(0e-t@d zXbje(5Ye5ELAu^Ko{qbEp7P3`lvgb%Geq@XEY6Q@>tqWroE|F*%p3rUXkxV}z#V1y z!}QZVUZTL9DxlD&MSboe3SLrM;HsdaleG)JUL`O*nl`ShCqnM#xM+;SA(BLRp((Ad zje8(b)GcW5(nr)EKGrsv2c-ZyYTG>cmPJ#dagIXl}U0eGXVabX$V8 z&6-fBaG-JdlE}v`BB-wXV;nYxZzHznu&V&K{$UMP*1%)a#v@7fZAAXTKa}OhFT5;H zmp<)mf*Odo{2Jm*4`S%!Kki`d@@GpbeO;SSd_&lA1G`iZYS^?1o8rYj3`4uiKo5A z;nrF3qQ9-CqW9}9U@KYc+T>t=>f8R`Tu>_q=}F7V#9{6}Mt7HLsqOSAaE zJp9GLCIhLjy7^p*KPkG-I(V*nohUAAV#&{3sDc-P#xHT=C?rjBia)uhuh@&@KhU^j zt7(%528j=_Ac6vAsL99#{taT#&_r>Wj6IxvIF?n90%%^>qOp+K^|jyFE@-A}Q^FFA z-0r2U(+jf38_+P<@I;kR@dH4ilwZluJfCa zp~s|uW4))iuM?@R-C3|K#(GcTLdiuLYU-dUF&e8q5<1hEKHg3nE$_L$v!{L-EH2=V zL->Z$`qZOt;sO>!WdWMlw5g{Siwg+O2T}Rhv^PVmt?olBWp(92u`y+q{YZF;HEnsn zkFHJI=Pv57LITm{v=#+OJ+K#HCz1v$DDHsC99$*=1EOthosa0=vRe;pKeV^RH zucQ=d%GhE3tj{1~25v1iuWQrq^%7^z2^z~X9Y@U0jqbqxw!0K=dW4%edpw*Al!D~- zS-3$+(qJyr7akR7v9t$*$_uAwxr+~Des>8seF<(J#TF+xDCk8pooAX7Ts9pL z`^@CwitVlp@nM8=DX`VFnIRKtHIg)ehRGJGDLzC3E=fb-I3mP3EJUEaGUbfB7K?K# zH@4K8uFZ&TC(i7COKMDFK|xWJ_z(#W09_T#zCWCn9yr1NQ7;uV`@uk3y2)Gi21R4$ zt#Au2`{9qG!;x3*U0uX{=_O=J%nSV32iEk_q1BJ3m41-j`B(&vpkDllI^@7a1IAk zS~O8QogZ0h9N1fDy$>}|D@>ao?QiVOye3gvaX(L(mY5Z8eDdRUT;b^^9~7)=79Sz- z034*mHZ2k7_&&nke28;TJn0|1sq_!E{DVSWTOdlsd**Q$6xg%{(~epnZddIx<$@WP ztUVDN=ca&Z+JbrR)`zjx+TA(DUpOby+IikvJO-gq*A^~}wswB~h($JG91E8;SReSa z&edPuT2Wzru&z-4suZ+%UO1Kf$CqsjAJMfWoLA4r_LIq0*B1416Q%JSX|Eh+(R+4r zAC_HAL!kze5x$F67PzJI!bLc?mua4ig-!-*$*=6ObjQX}Ti7;54k*bPe)a#tFb>fr z(c#)+iyQU+qTKqfatGAFr^a#f!wOh*Ep?zbt$8rgg7y{*7tC`=oytE{CtNbjZf)C1 zE$g$AmwXT@-bW-8Fetm&aRFO-uQ*rFwx{o=^599VAQnF*H{R+yHq<#i*EM^7H_AVg zCkIERvD8slt$wlUv@r{3Uz9|%-G=a<)+uQQI|WjC^a&n=7)R<`Rn|8W`C~*WD0N|w z^$mPBb$9k#`)58h6W{8nuDIC0tE0I~2lE^kwRCOiV*ylfpf?kBNDo>XkWPz#dfD|# zvJY^x7QK_~>gwrdgQx%@5pu50O2GAK1@hjXoOh?agmol_K%0rh;%1Bs59dMYjI6P8Q3d4$oaY*Vj)HUtne^1&~KAN_&LnKXFsO}_ZX;&81Q2K8=AGU_&bZzB#`IHlFV`ym#SzsJ|m$I zrR2;$fi(HbIM+Kn^SOAMwr)1Rl31siL;YylttTzANxf81=CDYL^_lJ*wrpDF@EV%x zH__$%naMXP8HZRQn91e%Vsf$a1angKa8Qbyz?z? z&LJoBb~{?u{4{d`7T{=ZqU4yJ44pNW`Q<@M!j~sds!=Ml(LrzA7|GJIp)&udqNHO7 z<*!a)FVWPzWNCS&)6Bp6&|A^j?1fC!(J6-#v7HYSs3s1td`c|c$4!{1W8z?1VZKcv zkFGfa1TFbAkb$X!zn`EbNYZo)G-~C?h~wFEO2qm)+N+N22#lsh$Oa6JNka|S9C`;z zJ$KU8Zyo!>DHqX$QXhCw(On&bOUl#K6v{Hk2r@k1pg>kr8Orgm2x*KScVLab2Tl=en2C9l7W z{R1il^xqO`5k4cYLpYWttIc9cUZF1h05Dz4y03;7VTA-5Qv+M(ZBJ`2FBOZ=II^ZE zQl{yaZ0xKy4w{O|glq!6f<~uyG^MVMltm%UYIB)J7V^#3O6ysc0GfX$j(tqg7#={2 z?jP^!9$ByWP@H=$D@XxL@S-Gqnxnb`mNm|u#$nyE);TlCivNbDJ6Y-ZF+n7NVf&JWV!Jb%e+o*>`(V#<+MkwrXVp&v;_D3lLgF(I1 zgR(vRDD^AVU;LS9FXYT{N301LrAnKzp9{pq5R`#&`AtOui*Bg>-x*EkOvcEITW!qgZ;OvArN{yV?s$h+v1+TP9K%QOp5(qiMg1z+PY*NOT@xv$caUo`k&PRrqcMJgx7v4Z;DPs>Z8(Xd+g7@Q31(BODZ!IZ5hN=_6-~ z8^wJyTVBRs7F7CEG7>9phL5pp=ODX!6lO{nP{97NDk6+l;VfW_PFG=LdC5h!w+7a_ zwyIqsEx|?SZcb@eJ&{fe#_Z$qG(}EGR&AL`x#kHfxD}dojG`&<(^~ktqVcq#xw9YS znIq_+#tW6S0Fx_+QrZO|mCoF*P`+2+u-+^$Wu zYVHs0K>&vuKYP%e{BmdauxYD)N~gG&3)xSV#?HpmM6V)NVT#mPZpQ|SM<|a(stRUb zqBIBxT_{;{$9YgR@}HXGqe*QyNDS-7uYNfank{&$F4PKbXuu#!c=lh`2;c%kvl* zSV1g)-jmG~g8+yY9C`Diu`;1aeW*T{pXN_%9uASvRYBgd97=g4*gsW%YS#a8$2B6PDFy-3$ z9b+l5b3iLD{VQnr(5~F04$BbD{1>Z8pYS>>2*%R2{J;Yg`rEUzK#iyQV{+)tOFLY$ z(pf~CCE?U%_!2e)8*253J`|)f zwVKdBcc-x2%T5IW*r$Os#0)!OqN`88L?duM8`@Itu=>;i8sXKJai9=fE_R?1-T5plu=o-GXX0#k(*0rL`jg<8ue~BxXDw-8aBUAF&>DZLTT@xr9 zFD+>P^53rE(#0~Ovkn6i3tVrPj1z_1{a zTFfhXCa|*0G`f1TG?aYMfSe|QQSV9!eD)AD1d|bMd_4<%z_Y1DX6%QCq+G0W5KVo7HT{dOd}8q2rjC>6wj!j z*#4K;P>i&AhlA2TZk5`m_(}}r^qwK->&leHAGV|P-{-p&RGe5&;SJkag~|*|5-AKi zmB3&*u>EHxb$^*Z0j1_qaJ!vx6gV!R27Ckshte&L2%E2UnxW+NgESV01n{d8t|UC1 z#y%e@+big6rY0I!j()-uMdniSuSVlWg;ufd$!_I<3W?5br;gS zGV1(vI_m>8Y+C7?VU%}~uNF|Ul+Ha*xyN{eTPdg%ADS_yZJF&I8Y^i^qBh9#K2Qyx zx>ovpFpcSQ(Piv4k491P1*=QzrMMu8Fd?>IDpW#!$(GnXOd#-~O@Y_d>Eib)=Qk!3@#(7K4!#+qI)S zDF-=aWdVe?K;Ql<5^sU9@twVvE7 zp?TCO2%{-iJXsbt!`cOL#uc24b^r%P2wf}dT}gSts=p{h{ZVCo4tL>wq88kt)8TGb z-ye=zd#P_ETIY$YIq^{hq^5#?#UX)$B+=Qqs<3WtBjvxwvG}S< z+0hDPS>`qVSXFF4FBK)2p_obrurn2+Bu$+q<{Im|xEnTP1HKMF1rF5sGf}*ZFkm!M z#$H!hP06iyn%85$vY`J*lBJ0>*TcAi0}mLG(2RPZZx7(ekv<36o*BR;C zSRczBu~_Ot5wf=USqn1ao6%W`-L2rNLMGuy(W*n#F z5d8M|1o3h}y(Fk~y20ll6r^*%1t(qG@Nq0N9TnBJeC9-T!VQtTDWx{T z0yz|y4cq$Bk_|rmzGdhe&NGWWIxD8QaCm?`ceMvJ!}TB*0qgsi>V`0 zFr|P79-Fq|+gR(;f#>9+oqTNC#+AX;@yI3?zh2s<;^{=nxOj|}>?R8~HdY$H;m~GV zST032sR)XrWm}h9@T*c`1@?jB5dU!s&JX*7M(iF+YojsI5ZN0l(qE!iv9|RPX5(mv z0-L*-jdNN+sf4Qt2@rEI$AvB=Yt-;-v<7;3h+fW7ssb}dJ6^yg&R$vZZLIMXVrpc8 zBKozjWwz_Qn5JzS6-h&%9l*13;25H9%4wjbFSXpxfP$%Y)DG!S?gLv!x@l$%bws8T z$e}!WQ)3VMt*rBvf0#9j>Zse!23N1#T(X!l{Be z!Qy3PCj=Lzpv}3vjrWn1S=L~-mG?B#Llghv8Jm(NCdTSzrWi&Io3^dVm7>e0e4k@LXt-mZPhu)-_Iak<} zK&xw&6O(8bf=WgREWrIf0+4po{6&E?rc<3nj~M{#H-(;xiOLmK#3SE-$uZ zCDydcW$nm*WPs$NI&$S(ku*DV8!L!aSoz`}W1%_QQW}eQP*5F$sZ>(&<}O2>^CsIItd z8{#m2nRh@si$c2X*=sbno%+y(_-OL?2J-h9!&?-9kAnUgmM3v2PXbeHpX?;|B5oXi zm<7*xh`mTs4W!D0wk`Lkl&&4+a)-$lOM7a_Ds4`GOuezs(GCmu~UZ{}b{;2Bt z7o{!Mj4Z4uv@xGqE##Q)3PtiSnG$SlB;Gniap4_59LTYHm$1R zGCgd4jk^c9prA3{*nq{*ddRX_YOLzVtJE%QJo_o*sQT$VwS(h=gJQe=H9;$T@DWu7 zYt=gE66@srQ=s@X9iN&WBs|tDQT$fNZMUA zCc))F+ryFE1ZnYFe0}?V*!GP#rN-og94jl5jJt7k7-le4cs80=PFHyq6c$vz; z_5B_DzM&NeDuFK4D79l-(;#7fN2N#)+VNXCt>_oPE~RL|aqog#I#hoN)2e%h8GEtH zBDKyns$cLH51sx-x`KQ(t$Luh7=sOepj6$X8aL86^!iJpRBWr$E9t3GhWxNnYxLE# z(#0oOM1U+xW7UbF;*vWk0d`WUCvttMgER z&;+PY`DyjHQIy$>_Xy1P`s9HRVd_7M@*z4v1a|6n9aM%W^xr#~U>W@&z#Y*tDJA9Ym=^ zT^ztTc3P6i_ovTX(~X^vWr$Lk(rlt>JAILfqH8~=_@MD}qu9FsYo~%xFe_ASMMz?b zvp4Kat2P2L=Wo&J&Qeb?!OWG#(06q#r{y>q+uixn+WBdfk%RY3i?Mfgjzf(%orco2 zo!7Ar^Y#q(G5CnC?P?!Q%PXd_!+^C-+tu|JJs!SJj>&Rzgc{EuG~{ir_*FUXuE=m> zG=NF1qYyQ=)KGuiCoXZ+GmT0?{kyA!O6+vI_D4~FEHIyPBHgvgPi#G}$pYnLyLJSL zt!6Iy5^39ApJ7RD>M_v?+fVM&F1x-uPxeob@Svn>d=Xopi^Gx^oKCmvrXMZ)^tx++ zt!aN#Y;he_(zTinXT_G*$z5yij}}{TDh((D;S#9PIY4anZ8=e9(`u|uv;vz;kW~3t z&9Fdm;@otX2L0&v-gTkg!`9cTtF zwdvwxWU-PofW@Z0*WP*4%j(`c zh{Z=O6wKIP-xZsF@DJq<@5~rn{}A5$5(ij==z>nh+1{&o2zIb6iQSB|eN-fq5-!2F z-cz0PG-bIOWsJhoFHowX@15)A#<|P;n2QDh?XLU!+1b$UD zGXHko>h8x0mQ=wX$kOcIi?4TOSzgIF8emz4>S)iy)+;L$I$2~K8@Q+%+Y&liJE)sj zrK0sgZBBn9mg0j(;((He40>|JWs#1x3#gB6JLpM#ai+gm#{ISyzy>{??k$#nX6JDX zX!Sw;4>!_MTnd3dSo}f3I4!sYq18~I;u3X#CE2Icv 0 +MAG_AUTO < 21 +FWHM_IMAGE > 0.3 / 0.187 +FWHM_IMAGE < 1.5 / 0.187 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +[MASK:flag] +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + + +[MASK:star_selection] +# Star selection using the FWHM mode +MAG_AUTO > 18. +MAG_AUTO < 22. +FWHM_IMAGE <= mode(FWHM_IMAGE{preselect}) + 0.2 +FWHM_IMAGE >= mode(FWHM_IMAGE{preselect}) - 0.2 +FLAGS == 0 +IMAFLAGS_ISO == 0 + +[MASK:fwhm_mag_cut] +FWHM_IMAGE > 0 +FWHM_IMAGE < 40 +MAG_AUTO < 35 +FLAGS == 0 +IMAFLAGS_ISO == 0 +NO_SAVE + +# Split the 'star_selection' sample into +# two random sub-samples with ratio 80/20 +[RAND_SPLIT:star_split] +RATIO = 20 +MASK = star_selection + +# The following selection is only used for plotting + +[PLOT:size_mag] +TYPE = plot +FORMAT = png +X_1 = FWHM_IMAGE{fwhm_mag_cut} +Y_1 = MAG_AUTO{fwhm_mag_cut} +X_2 = FWHM_IMAGE{star_selection} +Y_2 = MAG_AUTO{star_selection} +MARKER_1 = + +MARKER_2 = . +MARKERSIZE_1 = 3 +MARKERSIZE_2 = 3 +LABEL_1 = All +LABEL_2 = "Stars, mean FWHM: @mean(FWHM_IMAGE{star_selection})*0.187@ arcsec" +TITLE = "Stellar locus" +XLABEL = "FWHM (pix)" +YLABEL = Mag + +[PLOT:hist_mag_stars] +TYPE = hist +FORMAT = png +Y = MAG_AUTO{star_selection} +BIN = 20 +LABEL = "stars" +XLABEL = "Magnitude" +YLABEL = "Number" +TITLE = "Magnitude of stars" + +[PLOT:fwhm_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = FWHM_IMAGE{star_selection}*0.186 +MARKER = . +LABEL = "FWHM (arcsec)" +TITLE = "FWHM of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[PLOT:mag_star_field] +TYPE = scatter +FORMAT = png +X = X_IMAGE{star_selection} +Y = Y_IMAGE{star_selection} +SCATTER = MAG_AUTO{star_selection} +MARKER = . +LABEL = "Magnitude" +TITLE = "Magnitude of stars" +XLABEL = "X (pix)" +YLABEL = "Y (pix)" + +[STAT:star_stat] +"Nb objects full cat" = len(FWHM_IMAGE) +"Nb objects not masked" = len(FWHM_IMAGE{flag}) +"Nb stars" = len(FWHM_IMAGE{star_selection}) +"stars/deg^2" = len(FWHM_IMAGE{star_selection})/4612./0.187*3600.*1./2048./0.187*3600. +"Mean star fwhm selected (arcsec)" = mean(FWHM_IMAGE{star_selection})*0.187 +"Standard deviation fwhm star selected (arcsec)" = std(FWHM_IMAGE{star_selection})*0.187 +"Mode fwhm used (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187 +"Min fwhm cut (arcesec)" = mode(FWHM_IMAGE{preselect})*0.187-0.1*0.187 +"Max fwhm cut (arcsec)" = mode(FWHM_IMAGE{preselect})*0.187+0.1*0.187 From 1b22b9884d658affc2816e3b9ec38b169826a31f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:22:51 -0400 Subject: [PATCH 13/21] =?UTF-8?q?feat(orchestration):=20manifests=20become?= =?UTF-8?q?=20the=20DAG's=20currency=20=E2=80=94=20completeness=20check=20?= =?UTF-8?q?CLI=20+=20manifest-driven=20report=20(#848=20D2/D3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit completeness.py gains the 'check ' CLI every rule's shell line calls after shapepipe_run: counts products under $SP_RUN against the floor table, writes a byte-stable JSON manifest (per-runner found/expect/ floor/warn, log-scraped failure reasons, no wall-clock), exits nonzero iff a mandatory runner is below floor — manifest always written first. STAGE_DIR maps stages to the committed configs' fixed RUN_NAMEs; the ngmix entry env-expands ${SP_NGMIX_CHUNK} so chunk K checks chunk K's dir. run_report.py rewritten to read manifests instead of disk-scanning: per-tile / per-exposure tables with failure reasons, attrition aggregate (failed units excluded), tile-blocked-by-exposure join, 'not run' for missing manifests. CLI interface unchanged (--run-dir --index --status; --out/--limit added). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5KZRTAZKSBAnpdDhE7yzK --- workflow/scripts/completeness.py | 209 ++++++++++++++++++++- workflow/scripts/run_report.py | 313 ++++++++++++++++++++----------- 2 files changed, 409 insertions(+), 113 deletions(-) diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index b2246abe..dd0836c6 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -8,12 +8,18 @@ its ``floor`` files; per-CCD attrition (a sparse CCD setools rejects, ~0.2%) sits between ``floor`` and ``expect`` and is tolerated. -Two consumers share this table: - * ``sp_rule.py`` — after ``shapepipe_run``, counts products per runner and - exits nonzero if any mandatory runner is below its floor (``keep-going`` - then isolates that unit's cone; ``retries`` handle transient failures). - * ``run_report.py`` — disk-scans finished trees against ``expect`` to - enumerate shortfalls for the human, mid-run or after. +This file is also the ``check`` CLI — the second half of every rule's shell line +(PRD D2/D3):: + + shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} \\ + && completeness.py check exp_mask {output} + +It counts the unit's products under ``$SP_RUN``, writes the manifest at +``{output}``, and exits nonzero iff a mandatory runner is below its floor. The +manifest is ALWAYS written first, failure included: it is the DAG's currency and +the only thing ``run_report.py`` reads, so a failed unit must still leave a +record of *why*. Manifests carry no wall-clock — identical on-disk state must +produce a byte-identical manifest, or the mtime rerun-trigger churns the cone. Per-runner fields: expect nominal file count for a fully complete unit (report yardstick) @@ -27,6 +33,13 @@ ``ls / | wc -l`` semantics (broken symlinks excluded by the caller). """ +import argparse +import json +import os +import re +import sys +from pathlib import Path + # stage -> {runner_subdir: {expect, floor, [warn], [subpath]}} COMPLETENESS = { # --- tile prepare (phase A) --- @@ -94,3 +107,187 @@ def check_floor(stage, run_dir): if not spec.get("warn", False) and n < spec["floor"]: ok = False return ok, details + + +# --- where a stage writes ------------------------------------------------- +# +# stage -> (level, run_sp_ dir under $SP_RUN/output/). These are the +# committed configs' RUN_NAMEs (RUN_DATETIME=False makes them fixed, PRD D2), so +# the check never resolves a run-log. The ngmix entry interpolates the same env +# var its config does, so chunk K's check looks at chunk K's dir. +STAGE_DIR = { + "tile_get_images": ("tile", "run_sp_tile_Git"), + "tile_uncompress": ("tile", "run_sp_tile_Uz"), + "tile_find_exposures": ("tile", "run_sp_tile_Fe"), + "exp_get_images": ("exp", "run_sp_exp_Gie"), + "exp_split": ("exp", "run_sp_exp_Sp"), + "exp_mask": ("exp", "run_sp_exp_Ma"), + "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), + "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), + "tile_mask": ("tile", "run_sp_tile_Ma"), + "tile_detect": ("tile", "run_sp_tile_Sx"), + "tile_detect_uc": ("tile", "run_sp_tile_Uc"), + "tile_vignets": ("tile", "run_sp_tile_PiViVi"), + "tile_ngmix": ("tile", "run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u"), + "tile_merge_cats": ("tile", "run_sp_Ms"), + "tile_make_cat": ("tile", "run_sp_Mc"), +} + + +# --- failure reasons ------------------------------------------------------ + +# Lines worth showing a human who asks "why is this runner short?". Deliberately +# crude: the point is a pointer into the logs, not a taxonomy (there is no error +# whitelist in this design — the count floor is the policy). +_ERROR_RE = re.compile( + r"traceback|exception|\berror\b|\bfailed\b|no such file|not found|" + r"killed|out of memory|oom|segmentation fault|bad chi2", + re.IGNORECASE) +_TS_RE = re.compile(r"^\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2}\s*") +_NOISE_RE = re.compile(r"A total of 0 errors were recorded") + +MAX_LOG_FILES = 40 # logs are per-CCD; a handful is enough to characterise +MAX_TAIL_LINES = 120 # per file +MAX_REASONS = 3 # per runner + + +def _normalise(line: str) -> str: + """Collapse a log line to its shape, so 40 per-CCD copies dedupe to one.""" + line = _TS_RE.sub("", line.strip()) + line = re.sub(r"/\S+", "", line) # paths differ per CCD + line = re.sub(r"\d+", "N", line) + return line[:200] + + +def scrape_reasons(stage_dir, runner): + """Best-effort, bounded: distinct error-looking lines from a runner's logs. + + Two sources, in order of usefulness: the runner's per-process worker logs + (``/logs/process-*.log`` — where the module's own exception lands), + and the stage's ``logs/log_sp.log`` (where ShapePipe records its error + tally). Sorted, truncated, deduped by shape — a manifest must stay + byte-stable for a given tree. + """ + seen, reasons = {}, [] + candidates = [] + for d in (stage_dir / runner / "logs", stage_dir / "logs"): + if d.is_dir(): + candidates += sorted(p for p in d.iterdir() if p.is_file()) + for path in candidates[:MAX_LOG_FILES]: + try: + lines = path.read_text(errors="replace").splitlines()[-MAX_TAIL_LINES:] + except OSError: + continue + for raw in lines: + if not _ERROR_RE.search(raw) or _NOISE_RE.search(raw): + continue + shape = _normalise(raw) + if shape in seen: + seen[shape] += 1 + continue + seen[shape] = 1 + reasons.append([path.name, _TS_RE.sub("", raw.strip())[:300], shape]) + out = [] + for name, text, shape in reasons[:MAX_REASONS]: + n = seen[shape] + out.append(f"{name}: {text}" + (f" [x{n}]" if n > 1 else "")) + return out + + +# --- manifest ------------------------------------------------------------- + +def build_manifest(stage, run_dir, unit, stage_subdir=None): + """Count, classify and (on shortfall) scrape. Returns (manifest, ok). + + Stages absent from the table fall back to the zero-output floor: any product + anywhere under the stage dir passes, nothing at all fails. + """ + level, subdir = STAGE_DIR.get(stage, (None, None)) + subdir = stage_subdir or (os.path.expandvars(subdir) if subdir else None) + stage_dir = run_dir / "output" / subdir if subdir else run_dir + manifest = { + "stage": stage, + "level": level, + "unit": unit, + "run_dir": str(run_dir), + "stage_dir": str(stage_dir), + "runners": {}, + "failures": [], + } + + if stage not in COMPLETENESS: + produced = list(stage_dir.glob("**/output/*")) if stage_dir.is_dir() else [] + ok = bool(produced) + manifest["status"] = "complete" if ok else "failed" + manifest["n_products"] = len(produced) + if not ok: + manifest["failures"].append( + {"runner": None, "found": 0, "floor": 1, + "reasons": [f"zero output under {stage_dir}"]}) + return manifest, ok + + ok, details = check_floor(stage, stage_dir) + short = False + for runner, n, floor, expect, warn in details: + below = n < floor + if n < expect: + short = True + manifest["runners"][runner] = { + "found": n, "expect": expect, "floor": floor, "warn": warn, + "status": ("complete" if n >= expect else + "warn" if (warn or not below) else "below_floor"), + } + if below: + manifest["failures"].append({ + "runner": runner, "found": n, "floor": floor, "expect": expect, + "warn": warn, "reasons": scrape_reasons(stage_dir, runner), + }) + manifest["status"] = "failed" if not ok else ("warn" if short else "complete") + return manifest, ok + + +def _unit_from_env(): + """``SP_UNIT_NUM`` carries the rules' dashed, dash-prefixed form + (``-210-282``, ``-2605805``); the manifest records the human ID.""" + raw = os.environ.get("SP_UNIT_NUM", "") + return raw.lstrip("-") or "unknown" + + +def main(argv=None) -> int: + p = argparse.ArgumentParser(description="ShapePipe per-unit completeness check") + sub = p.add_subparsers(dest="cmd", required=True) + c = sub.add_parser("check", help="count products, write the manifest") + c.add_argument("stage") + c.add_argument("manifest", type=Path) + c.add_argument("--run-dir", type=Path, default=None, + help="the unit's $SP_RUN (default: the env var)") + c.add_argument("--unit", default=None, help="override $SP_UNIT_NUM") + c.add_argument("--stage-dir", default=None, + help="override the run_sp_* subdir (default: the stage table)") + args = p.parse_args(argv) + + run_dir = args.run_dir or Path(os.environ.get("SP_RUN", "")) + if not str(run_dir): + print("[completeness] FATAL: $SP_RUN unset and --run-dir not given", + file=sys.stderr) + return 2 + unit = args.unit or _unit_from_env() + + manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) + args.manifest.parent.mkdir(parents=True, exist_ok=True) + args.manifest.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + for runner, r in manifest["runners"].items(): + tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} + print(f"[completeness] {runner}: {r['found']}/{r['expect']} " + f"(floor {r['floor']}) {tag[r['status']]}", file=sys.stderr) + print(f"[completeness] {args.stage} {unit}: {manifest['status']} " + f"-> {args.manifest}", file=sys.stderr) + for f in manifest["failures"]: + for reason in f["reasons"]: + print(f"[completeness] {f['runner']}: {reason}", file=sys.stderr) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 683433ec..05ca9247 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -1,70 +1,169 @@ #!/usr/bin/env python3 -"""Standalone run report — NOT a DAG node. - -A report rule that declares all tiles' outputs as inputs is a descendant of every -job, so any single hard failure (under --keep-going) poisons its cone and the -report never runs — the exact scenario it exists for (finding 8). So this is a -plain script: it disk-scans the tiles/ and exp/ trees against the count-floor -table (completeness.py) and the index, and emits run_report.json. It runs -automatically via the Snakefile's onsuccess/onerror hooks, and by hand any time -(``sp report``), mid-run included. - -It distinguishes whole-exposure absence (all runners missing — a real gap or a -deletion/forest bug) from tolerated per-CCD attrition (counts between floor and -expect), so a future deletion bug cannot be silently absorbed (finding 5). - -Attrition is counted at file granularity, not unit granularity: each stage's -``products`` block aggregates found-vs-expected file counts per runner across -all passing units, with a per-unit shortfall map where files are missing. -``warn`` runners (psfex_interp) are exempt from *failing* a unit, not from -being *reported* — the P0 validation found 46/760 psfex_interp CCDs missing -(6%) behind a unit-level "zero attrition" claim, which is the failure mode -this granularity exists to prevent. +"""``sp report`` — the run's success/failure tables, read from the manifests. + +NOT a DAG node. A report rule that declared all tiles' outputs as inputs would be +a descendant of every job, so one hard failure under --keep-going would poison +its cone and the report would never run — the exact scenario it exists for. So it +is a plain script, runnable at any time including mid-run; the Snakefile's +onsuccess/onerror hooks call it so every invocation ends with one. + +It reads two things and nothing else (PRD D3): + + * the **index** (``run_index.sqlite``) — the units the run declared, and the + tile->exposure edges that let an exposure failure be blamed on the tiles it + blocks; + * the **manifests** — one per unit per stage, written by + ``completeness.py check``, carrying per-runner found/expect/floor and + log-scraped failure reasons. + +No disk scanning: counting products is the *check's* job, done once at the moment +the products were fresh. A unit with no manifest for a stage is "not run" — which +is a real and distinct answer from "ran and produced nothing". + +Manifests are discovered by glob (``tiles/**/manifests/*.json``), not by +constructed path: the unit stores are sharded (``tiles///``) and the +sharding depth is not this script's business. """ import argparse import json import sqlite3 import sys +from collections import defaultdict from pathlib import Path -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from completeness import COMPLETENESS, check_floor # noqa: E402 - -# stage -> (level, run_sp_ dir name) -STAGE_DIR = { - "tile_get_images": ("tile", "run_sp_tile_Git"), - "tile_uncompress": ("tile", "run_sp_tile_Uz"), - "tile_find_exposures": ("tile", "run_sp_tile_Fe"), - "exp_get_images": ("exp", "run_sp_exp_Gie"), - "exp_split": ("exp", "run_sp_exp_Sp"), - "exp_mask": ("exp", "run_sp_exp_Ma"), - "exp_psf": ("exp", "run_sp_exp_SxSePsfPi"), - "tile_merge_headers": ("tile", "run_sp_tile_Mh_exp"), - "tile_mask": ("tile", "run_sp_tile_Ma"), - "tile_detect": ("tile", "run_sp_tile_Sx"), - "tile_vignets": ("tile", "run_sp_tile_PiViVi"), - "tile_merge_cats": ("tile", "run_sp_Ms"), - "tile_make_cat": ("tile", "run_sp_Mc"), -} - - -def classify(stage, run_dir): - """'absent' | 'under' | 'attrition' | 'complete' for one unit's stage.""" - if not run_dir.is_dir(): - return "absent", [] - ok, details = check_floor(stage, run_dir) - if not ok: - # all mandatory runners empty -> whole-unit absence; else below floor - mand = [d for d in details if not d[4]] - if mand and all(n == 0 for _, n, *_ in mand): - return "absent", details - return "under", details - # warn runners are exempt from FAILING a unit (check_floor), not from - # attrition reporting — psfex_interp is precisely the runner that attrits. - if any(n < expect for _, n, _, expect, _ in details): - return "attrition", details - return "complete", details +# Stage order per level — the report's column order, and the definition of +# "expected" (a declared unit with no manifest for one of these is not run). +TILE_STAGES = ["tile_get_images", "tile_uncompress", "tile_find_exposures", + "tile_merge_headers", "tile_detect", "tile_vignets", + "tile_ngmix", "tile_merge_cats", "tile_make_cat"] +EXP_STAGES = ["exp_get_images", "exp_split", "exp_mask", "exp_psf"] + +STATUSES = ("complete", "warn", "failed", "not_run") + + +def load_manifests(run_dir: Path, sub: str) -> dict: + """``{unit: {stage: manifest}}`` for one store (``tiles`` or ``exp``). + + The unit key is the manifest dir's *parent directory name* — shard-depth + agnostic, and the only form that joins to the index (the manifest's own + ``unit`` field carries ``SP_UNIT_NUM``'s dashed form, ``210-282``, which is + not the index's ``210.282``). The stage comes from the manifest body, never + the filename: ngmix chunks share a stage under per-chunk filenames, and they + collapse to one worst-case entry. + """ + out: dict = defaultdict(dict) + for path in sorted((run_dir / sub).glob("**/manifests/*.json")): + try: + m = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable manifest {path}: {exc}", file=sys.stderr) + continue + unit = path.parent.parent.name + stage = m.get("stage", path.stem) + prev = out[unit].get(stage) + # Worst status wins when several manifests share a stage (ngmix chunks). + rank = lambda d: STATUSES.index(d.get("status")) if d.get("status") in STATUSES else len(STATUSES) # noqa: E731 + if prev is None or rank(m) > rank(prev): + out[unit][stage] = m + return out + + +def shortfalls(m: dict) -> dict: + """``{runner: (found, expect, floor)}`` for every runner under expect.""" + return {r: (d["found"], d["expect"], d["floor"]) + for r, d in m.get("runners", {}).items() if d["found"] < d["expect"]} + + +def reasons(m: dict) -> list: + """Flattened failure reasons, runner-tagged, for the report's why column.""" + out = [] + for f in m.get("failures", []): + head = f"{f['runner']} {f['found']}/{f.get('expect', '?')} (floor {f['floor']})" + out += [f"{head}: {r}" for r in f["reasons"]] or [head] + return out + + +def tally_level(units, stages, manifests) -> dict: + """Per-stage counts + named unit lists, for one level.""" + per_stage = {} + for stage in stages: + t = {"complete": 0, "warn": [], "failed": [], "not_run": []} + agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) + for u in units: + m = manifests.get(u, {}).get(stage) + if m is None: + t["not_run"].append(u) + continue + status = m.get("status", "failed") + status = status if status in ("complete", "warn") else "failed" + if status == "complete": + t["complete"] += 1 + else: + t[status].append(u) + if status == "failed": + # Failed units are named above, never folded into the attrition + # aggregate: a whole-unit failure is not per-CCD attrition, and + # mixing them hides real deletion bugs behind a big denominator. + continue + for runner, d in m.get("runners", {}).items(): + a = agg[runner] + a["found"] += d["found"] + a["expect"] += d["expect"] + if d["found"] < d["expect"]: + a["by_unit"][u] = d["expect"] - d["found"] + for a in agg.values(): + if not a["by_unit"]: + del a["by_unit"] + t["products"] = dict(agg) + per_stage[stage] = t + return per_stage + + +def unit_rows(units, stages, manifests) -> list: + """One row per non-clean unit: its first bad stage, shortfalls, why.""" + rows = [] + for u in units: + got = manifests.get(u, {}) + bad = [s for s in stages + if got.get(s) is None or got[s].get("status") != "complete"] + if not bad: + continue + stage = bad[0] + m = got.get(stage) + rows.append({ + "unit": u, + "stage": stage, + "status": "not_run" if m is None else m.get("status", "failed"), + "shortfalls": shortfalls(m) if m else {}, + "reasons": reasons(m) if m else [], + "n_bad_stages": len(bad), + }) + return rows + + +def print_table(title, rows, limit=25): + print(f"\n{title} ({len(rows)} affected)") + if not rows: + print(" — none") + return + print(f" {'unit':<14} {'stage':<20} {'status':<8} why") + for r in rows[:limit]: + short = ", ".join(f"{k} {v[0]}/{v[1]}" for k, v in r["shortfalls"].items()) + why = (r["reasons"][0] if r["reasons"] else short) or "-" + print(f" {r['unit']:<14} {r['stage']:<20} {r['status']:<8} {why[:90]}") + if len(rows) > limit: + print(f" … and {len(rows) - limit} more (see the JSON report)") + + +def print_stage_table(title, per_stage, n_units): + print(f"\n{title} ({n_units} units declared)") + print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} attrition") + for stage, t in per_stage.items(): + att = [f"{r} {a['found']}/{a['expect']}" + for r, a in t["products"].items() if a["found"] < a["expect"]] + print(f" {stage:<20} {t['complete']:>6} {len(t['warn']):>6} " + f"{len(t['failed']):>6} {len(t['not_run']):>8} {', '.join(att)[:60]}") def main() -> None: @@ -73,66 +172,66 @@ def main() -> None: p.add_argument("--index", required=True, type=Path) p.add_argument("--status", default="manual") p.add_argument("--out", type=Path, default=None) + p.add_argument("--limit", type=int, default=25, + help="rows per stdout table (the JSON report is complete)") args = p.parse_args() - tiles, exps = [], [] + tiles, exps, tile_exp = [], [], defaultdict(list) if args.index.exists(): con = sqlite3.connect(args.index) - tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles")] - exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures")] + tiles = [r[0] for r in con.execute("SELECT tile_id FROM tiles ORDER BY 1")] + exps = [r[0] for r in con.execute("SELECT exp_id FROM exposures ORDER BY 1")] + for tile_id, exp_id in con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + tile_exp[tile_id].append(exp_id) con.close() + else: + print(f"[run_report] no index at {args.index} — reporting manifests only", + file=sys.stderr) + + tile_m = load_manifests(args.run_dir, "tiles") + exp_m = load_manifests(args.run_dir, "exp") + tiles = tiles or sorted(tile_m) + exps = exps or sorted(exp_m) missing_json = args.index.parent / "missing.json" missing = json.loads(missing_json.read_text()) if missing_json.exists() else [] - report = {"status": args.status, "n_tiles": len(tiles), "n_exposures": len(exps), - "missing_tiles": missing, "stages": {}} - for stage, (level, prefix) in STAGE_DIR.items(): - if stage not in COMPLETENESS: - continue - units = tiles if level == "tile" else exps - sub = "tiles" if level == "tile" else "exp" - tally = {"complete": 0, "attrition": 0, "under": [], "absent": []} - # File-level aggregate per runner over PASSING units (complete/attrition); - # under/absent units are enumerated by name above, not folded in here — - # mixing them would double-count whole-unit absence as attrition. - products = {r: {"found": 0, "expect": 0, "by_unit": {}} - for r in COMPLETENESS[stage]} - for u in units: - run_dir = args.run_dir / sub / u / "output" / prefix - verdict, details = classify(stage, run_dir) - if verdict in ("complete", "attrition"): - tally[verdict] += 1 - for runner, n, _, expect, _ in details: - agg = products[runner] - agg["found"] += n - agg["expect"] += expect - if n < expect: - agg["by_unit"][u] = expect - n - else: - tally[verdict].append(u) - for agg in products.values(): - if not agg["by_unit"]: - del agg["by_unit"] - tally["products"] = products - report["stages"][stage] = tally - - finals = [t for t in tiles - if (args.run_dir / "tiles" / t / f"final_cat-{t}.fits").exists()] - report["final_cats"] = {"present": len(finals), "of": len(tiles), - "missing": [t for t in tiles if t not in finals]} + report = { + "status": args.status, + "n_tiles": len(tiles), "n_exposures": len(exps), + "missing_tiles": missing, + "tile_stages": tally_level(tiles, TILE_STAGES, tile_m), + "exp_stages": tally_level(exps, EXP_STAGES, exp_m), + "tiles": unit_rows(tiles, TILE_STAGES, tile_m), + "exposures": unit_rows(exps, EXP_STAGES, exp_m), + } + + # Blame propagation: an incomplete exposure blocks every tile that reads it. + # Without this, a tile stalled at tile_vignets looks like its own failure. + bad_exp = {r["unit"] for r in report["exposures"]} + blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} + report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} + + done = report["tile_stages"]["tile_make_cat"]["complete"] + report["final_cats"] = {"present": done, "of": len(tiles)} out = args.out or (args.index.parent / "run_report.json") - out.write_text(json.dumps(report, indent=2)) - print(f"[run_report] {report['final_cats']['present']}/{len(tiles)} final " - f"cats -> {out}", file=sys.stderr) - for stage, tally in report["stages"].items(): - for runner, agg in tally["products"].items(): - if agg["found"] < agg["expect"]: - pct = 100 * (agg["expect"] - agg["found"]) / agg["expect"] - print(f"[run_report] attrition {stage}/{runner}: " - f"{agg['found']}/{agg['expect']} files (-{pct:.1f}%) " - f"across {len(agg['by_unit'])} units", file=sys.stderr) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" + + (f" ({len(missing)} tiles missing exposure lists)" if missing else "")) + print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) + print_stage_table("TILES", report["tile_stages"], len(tiles)) + print_table("exposures not complete", report["exposures"], args.limit) + print_table("tiles not complete", report["tiles"], args.limit) + nb = report["tiles_blocked_by_exposures"] + if nb: + print(f"\ntiles waiting on incomplete exposures ({len(nb)})") + for t, e in list(nb.items())[:args.limit]: + print(f" {t:<14} {', '.join(e[:6])}" + + (f" (+{len(e) - 6})" if len(e) > 6 else "")) + print(f"\n[run_report] -> {out}") if __name__ == "__main__": From 83d26f2088447c5989060b6e5c4bac3b23c39c6d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:36:11 -0400 Subject: [PATCH 14/21] =?UTF-8?q?feat(orchestration):=20the=20rules=20rewr?= =?UTF-8?q?ite=20=E2=80=94=20manifests=20as=20DAG=20currency,=20sharded=20?= =?UTF-8?q?stores,=20parse-time=20index,=20native=20scattergather=20(#848?= =?UTF-8?q?=20D1=E2=80=93D5)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sp_rule.py is deleted; its responsibilities dissolve into the design: config normalization -> the committed S2 configs; log sync -> dead (zero 'last:'); unit furniture -> inline shell (tile_numbers.txt, pseudo-Fe exp_numbers with the ORIGINAL exposure name, star-cat symlinks); count floor -> completeness.py check writing the manifest each rule declares as its only output. SP_UNIT_NUM carries the leading-dash dashed form; the rules do the transform. Stores shard to tiles/<2ch>// and exp/<2ch>//; build_index accumulates (no DROP TABLE) and is built at parse time of the compute invocation; bin/sp reduces to run (two invocations) + report + cancel. ngmix scatters via the scattergather knob with in-job closed ID ranges (scripts/ngmix_range.py); retries with attempt-scaled mem on transient- exposed rules; protected() dropped; temp(directory()) rides as a scoped secondary output on the vignette store and chunk dirs — the one exception to no-directory-outputs, so native temp() reclaims intra-tile bulk. Deviations proven by implementation: merge_sep_cats INPUT_DIR must stay relative (MergeSep re.sub's the first '1' in the path — absolute sharded paths break chunk discovery); failed jobs need keep-incomplete in the profile or Snakemake deletes the failure manifests sp report exists to read (-> S6). Dry-runs: prepare 13 jobs, compute 141 on the 210/211 quad (19 exposures, 8 ngmix chunks); --set-scatter ngmix=1 -> 113. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5KZRTAZKSBAnpdDhE7yzK --- workflow/Snakefile | 293 ++++++++---- workflow/bin/sp | 81 ++-- workflow/config.yaml | 14 +- .../config/cfis/config_merge_sep_cats.ini | 11 +- workflow/rules/exposure.smk | 108 +++-- workflow/rules/prepare.smk | 85 ++-- workflow/rules/tile.smk | 269 ++++++----- workflow/scripts/build_forest.py | 13 +- workflow/scripts/build_index.py | 61 ++- workflow/scripts/ngmix_range.py | 55 +++ workflow/scripts/sp_rule.py | 419 ------------------ 11 files changed, 644 insertions(+), 765 deletions(-) create mode 100644 workflow/scripts/ngmix_range.py delete mode 100644 workflow/scripts/sp_rule.py diff --git a/workflow/Snakefile b/workflow/Snakefile index 5d93a181..b707fdc1 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,24 +1,31 @@ """ShapePipe real-data orchestration — Snakemake workflow. -Design / rationale: CosmoStat/shapepipe#848 (the living PRD). - -Execution is honestly THREE static invocations, chained by ``workflow/bin/sp`` -so the UX stays one command (``sp prepare`` / ``sp run``): - - snakemake --profile profiles/nibi prepare_tiles # Git_vos + Uz + Fe (keep-going) - python workflow/scripts/build_index.py ... # PLAIN SCRIPT, not a DAG node - snakemake --profile profiles/nibi prepare_exposures # Gie_vos + star cats - snakemake --profile profiles/nibi # the main compute DAG - -The run index is loaded ONCE here at parse time into plain dicts; it is never a -rule input, so appending tiles and rebuilding it changes which jobs exist without -invalidating completed work. build_index.py is a plain script *between* -prepare_tiles and prepare_exposures (the exposure job set is data-derived from -the tiles' find_exposures output, so it cannot be scheduled in the same static -DAG that produces it). +Design / rationale: CosmoStat/shapepipe#848 (the living PRD), D1-D5. + +`sp run` is TWO snakemake invocations over this one Snakefile: + + SP_PHASE=prepare snakemake prepare_all_tiles # Git -> Uz -> Fe, per tile + SP_PHASE=compute snakemake all # everything else + +They are two because the exposure job set is *data-derived*: it comes from the +tiles' find_exposures output, and a Snakemake DAG is fixed at parse time. The +join between them is the tile<->exposure index, built HERE at parse time of +invocation 2 (build_index.build(), imported — there is no `sp index` verb) and +loaded into plain dicts. The index is never a rule input, so appending tiles and +rebuilding it changes which jobs exist without invalidating completed work; it +ACCUMULATES across invocations, so a later clean_exposure (S5) sees every +consuming tile of the whole campaign, not just this tile list. + +The atom (D2): one rule == one `shapepipe_run` on one unit; its single declared +output is its MANIFEST (`/manifests/.json`), written by +`completeness.py check`. Product files are not declared — a missing CCD is often +legitimate, and at DR6 scale per-CCD declaration means millions of paths. """ +import hashlib +import os import sqlite3 +import sys from pathlib import Path # Resolved relative to THIS file, not the working directory: snakemake runs with @@ -32,16 +39,49 @@ container: config["container"] # --- paths ----------------------------------------------------------------- RUN_DIR = Path(config["run_dir"]) -CONFIG_SRC = Path(config["config_src"]) STAR_CATS = Path(config["star_cats"]) INDEX_DB = Path(config["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" -NGMIX_CHUNKS = config.get("ngmix_chunks", 8) +# The config chain is the repo's own committed dir BY CONSTRUCTION (D2): the +# configs and the rules that set the env vars they interpolate are one artefact +# and must version together. Hence no `config_src` knob. +CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis" + +sys.path.insert(0, str(SCRIPTS)) +import build_index # noqa: E402 +from completeness import STAGE_DIR # noqa: E402 + +# `prepare` suppresses the missing-tile threshold (tiles are still being +# prepared) and the report hooks. Unset == compute (a bare hand-run of `all`). +PHASE = os.environ.get("SP_PHASE", "compute") with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] -# --- parse-time index load (compute phase only) ---------------------------- +# --- ngmix scatter (D4) ---------------------------------------------------- +# Native directive: `--set-scatter ngmix=N` overrides it, N=1 degenerates to one +# ngmix job per tile. We take the count and drive our own integer `chunk` +# wildcard rather than snakemake's `{scatteritem}` ("3-of-8") token, because the +# chunk number is not ours alone: it names the run dir +# (`run_sp_tile_ngmix_Ngu`, from the template's RUN_NAME), and +# merge_sep_cats derives chunks 2..N from chunk 1's path by substituting the +# first "1" — which only works for bare integers. +scattergather: + ngmix=int(config.get("ngmix_chunks", 8)) + +NGMIX_CHUNKS = workflow._scatter["ngmix"] + +# --- parse-time index build + load (D1) ------------------------------------ +# Invocation 2's parse IS the index build. Tiles whose find_exposures output is +# absent land in index/missing.json and are dropped from the DAG; the build is +# fatal only above SP_MISSING_THRESHOLD (and never in the prepare phase, where +# absent Fe output is the normal state). +if any(build_index.exp_list_path(RUN_DIR, t).exists() for t in TILES): + build_index.build( + TILES, RUN_DIR, INDEX_DB, + missing_threshold=(None if PHASE == "prepare" + else float(os.environ.get("SP_MISSING_THRESHOLD", "0.0")))) + # EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes # verbatim into the fabricated per-unit exp_numbers list so get_images matches # .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. @@ -49,93 +89,186 @@ EXP, TILE_EXP = {}, {} if INDEX_DB.exists(): _con = sqlite3.connect(INDEX_DB) EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) - for tile, exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): - TILE_EXP.setdefault(tile, []).append(exp) + for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): + TILE_EXP.setdefault(_tile, []).append(_exp) _con.close() +# Tiles this run can actually compute: declared AND indexed. The index spans the +# campaign, so it is intersected with the declared list, not used as it. +TILES_READY = [t for t in TILES if t in TILE_EXP] + wildcard_constraints: - tile = r"\d{3}\.\d{3}", - exp = r"\d{6,7}", + tile = r"\d{3}\.\d{3}", + exp = r"\d{6,7}", + shard = r"\d{2}", chunk = r"\d+", -# --- shared wrapper invocation --------------------------------------------- -def sp_rule(stage, config_name, level, unit, *, isolate=True, exp_forest=None, - extra=""): - """Build the shell command that runs one unit through sp_rule.py. - - NUMBER_LIST is injected by default (tile-scheme stages + exp split); pass - ``isolate=False`` for get_images / exp_mask / exp_psf, whose per-CCD/download - numbering must not be constrained. ``--threads {threads}`` is appended in - each rule's ``shell:`` string (NOT here: Snakemake formats the shell string - once, so a ``{threads}`` inside params.cmd would survive literally) and - becomes SMP_BATCH_SIZE (fork width == cpus_per_task). - """ - cmd = ( - f"python {SCRIPTS}/sp_rule.py --stage {stage} --config {config_name} " - f"--unit {unit} --level {level} --run-dir {RUN_DIR} " - f"--config-src {CONFIG_SRC} --star-cats {STAR_CATS} " - f"--script-hash {SCRIPT_HASH}" - ) - if not isolate: - cmd += " --no-isolate" - if exp_forest: - cmd += f" --exp-forest {exp_forest}" - if extra: - cmd += " " + extra - return cmd +# --- the sharded stores (D2) ---------------------------------------------- +# tiles/<2-char prefix>// and exp/<2-char prefix>// — no directory +# exceeds ~1k entries at full-UNIONS scale. Rules carry the shard as its own +# wildcard because an output pattern cannot compute it; every path the DAG uses +# is built by these helpers, so a mismatched (shard, id) pair is never requested. +TILE_DIR = str(RUN_DIR / "tiles" / "{shard}" / "{tile}") +EXP_DIR = str(RUN_DIR / "exp" / "{shard}" / "{exp}") def tile_dir(tile): - return RUN_DIR / "tiles" / tile + return f"{RUN_DIR}/tiles/{tile[:2]}/{tile}" def exp_dir(exp): - return RUN_DIR / "exp" / exp - -# Content hash of the wrapper scripts, computed once at parse time. Exposed as a -# param on every rule (via sp_rule() and tile_exp_forest) so the default -# rerun-triggers' *params* trigger covers script edits: the `code` trigger only -# hashes the rule's own shell string, NOT external scripts it calls — without -# this, a wrapper/forest-script fix silently leaves stale outputs in place -# (bitten live: the forest-shard fix rebuilt nothing on rerun). -import hashlib as _hashlib -SCRIPT_HASH = _hashlib.md5(b"".join( - (SCRIPTS / n).read_bytes() - for n in ("sp_rule.py", "completeness.py", "build_forest.py"))).hexdigest()[:12] + return f"{RUN_DIR}/exp/{exp[:2]}/{exp}" + +def tile_manifest(tile, stage): + return f"{tile_dir(tile)}/manifests/{stage}.json" + +def exp_manifest(exp, stage): + return f"{exp_dir(exp)}/manifests/{stage}.json" + +def forest_dir(tile): + return f"{tile_dir(tile)}/exp_forest" + +def final_cat(tile): + return f"{tile_dir(tile)}/final_cat-{tile}.fits" + +def unit_num(unit): + """$SP_UNIT_NUM: ShapePipe's image-number convention, dot -> dash, leading + dash (tile ``210.282`` -> ``-210-282``; exposure ``2605805`` -> ``-2605805``). + The RULES do this transform; the configs just interpolate $SP_UNIT_NUM into + NUMBER_LIST (the `set_config_number_list` mechanism that replaced the retired + -e/--exclusive flag, #746).""" + return "-" + unit.replace(".", "-") + +# Content hash of completeness.py, computed once at parse time and carried as a +# param on every rule: the default rerun-triggers' `code` trigger hashes only the +# rule's own shell string, NOT external scripts it calls — without this, a fix to +# the count table silently leaves stale manifests in place (bitten live). Scoped +# to completeness.py alone, the one script every shell line runs; build_forest.py +# gets its own hash, on the forest rule only. +SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] +FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] + +# --- the shell every rule runs (D2) ---------------------------------------- + +_THREAD_CAPS = " ".join( + f"{k}={v}" for k, v in ( + ("OMP_NUM_THREADS", 1), ("OPENBLAS_NUM_THREADS", 1), + ("MKL_NUM_THREADS", 1), ("NUMEXPR_NUM_THREADS", 1), + ("MALLOC_ARENA_MAX", 2), ("MALLOC_TRIM_THRESHOLD_", 0))) + + +def unit_pre(stage, level, unit, *, exp_name=None, forest=None, env=None, + pre_run=()): + """The unit-furniture + environment prologue, as bash. + + Returned as a rule ``params`` value, NEVER inlined into the ``shell:`` + string: snakemake formats a shell string ONCE, so a ``{output}``/``{threads}`` + placeholder inside a params value would survive literally — and, conversely, + the literal ``${SP_NGMIX_CHUNK}`` braces this prologue needs would blow up + that formatting if they lived in the shell string. Params values are + substituted after formatting, so both hazards go away together. + + What it materialises (the proven v2.0 isolation-by-work-dir-content, NOT + -e/--exclusive): + * ``output/`` and ``manifests/``; + * tile: ``tile_numbers.txt`` (dot format — what get_images reads); + * exposure: a fabricated pseudo-Fe ``exp_numbers-000-000.txt`` holding the + ORIGINAL exposure name from the index (``2605805p``), so get_images + matches ``.fits.fz`` in the store — the bare base id matches + nothing. Written UNCONDITIONALLY: an exists-guard once pinned a stale + pre-fix file with the bare id; + * ``star_cat_exp`` / ``star_cat_tiles`` dir symlinks into the shared + pre-generated pool (the mask configs read them as INPUT_DIRs). + There is no per-unit ``cfis`` symlink any more: $SP_CONFIG points straight at + the committed config dir. + + Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's + FileHandler raises on an existing run dir, and it is how a rerun never sees + stale products (D2: the job clears its run dir at start). + """ + work = tile_dir(unit) if level == "tile" else exp_dir(unit) + _, subdir = STAGE_DIR[stage] + lines = [ + "set -euo pipefail", + f"export SP_RUN='{work}'", + f"export SP_UNIT_NUM='{unit_num(unit)}'", + f"export SP_CONFIG='{CONFIG_DIR}'", + # Also set via apptainer-args in the profile; kept here so a hand-run of + # this same line outside snakemake behaves identically. + f"export {_THREAD_CAPS}", + 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests"', + ] + if forest: + lines.append(f"export SP_EXP='{forest}'") + for k, v in (env or {}).items(): + lines.append(f"export {k}='{v}'") + + if level == "tile": + lines.append(f"printf '%s\\n' '{unit}' > \"$SP_RUN/tile_numbers.txt\"") + else: + fe = "$SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output" + lines += [f'mkdir -p "{fe}"', + f"printf '%s\\n' '{exp_name or unit}' > \"{fe}/exp_numbers-000-000.txt\""] + + for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): + src = STAR_CATS / sub + lines.append(f"[ -d '{src}' ] && ln -sfn '{src}' \"$SP_RUN/{name}\" || true") + + lines += list(pre_run) + lines += [f'rm -rf "$SP_RUN/output/{subdir}"', 'cd "$SP_RUN"'] + return "\n".join(lines) + + +def sp_shell(stage, config_name): + """The rule's shell string: prologue, one shapepipe_run, one completeness check. + + ``{threads}`` and ``{output}`` are placeholders HERE and nowhere else (see + unit_pre). ``-b {threads}`` makes SMP fork width and cpus_per_task one number + by construction (D4). + + The check runs even when shapepipe_run failed — the manifest is the only + thing `sp report` reads, so a failed unit must still leave a record of why. + """ + return ( + "{params.pre}\n" + "rc=0\n" + f"shapepipe_run -c \"$SP_CONFIG/{config_name}\" -b {{threads}} || rc=$?\n" + f"python {SCRIPTS}/completeness.py check {stage} {{output.manifest}} || rc=1\n" + "exit $rc\n" + ) + include: "rules/prepare.smk" include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# Only aggregation targets are localrules (no-ops under the local executor, but -# they keep the compute chain group-compatible: no mid-chain localrules). -localrules: all, prepare_tiles, prepare_exposures +# Only the aggregation targets are localrules: a mid-chain localrule would break +# `group:` fusion of the compute chains (D4). +localrules: all, prepare_all_tiles rule all: input: - expand(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits"), tile=TILES) - -# Phase A.1 — per-tile static DAG (download + find_exposures). keep-going so -# tile failures are independent; build_index.py runs next as a plain script. -rule prepare_tiles: - input: - expand(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe"), tile=TILES) + [final_cat(t) for t in TILES_READY] -# Phase A.2 — per-exposure static DAG (download + star cats), parseable now the -# index exists. Star cats are re-keyed per unit (see prepare.smk). -rule prepare_exposures: +# Invocation 1 — the static per-tile DAG, known from the tile list alone. +# keep-going makes tile failures independent; the ones that lose their exposure +# list are dropped by the index build at invocation 2's parse. +rule prepare_all_tiles: input: - expand(str(RUN_DIR / "exp/{exp}/output/run_sp_exp_Gie"), exp=sorted(EXP)) + [tile_manifest(t, "tile_find_exposures") for t in TILES] # --- report hooks ----------------------------------------------------------- # run_report is NOT a DAG node (a descendant of every job would be poisoned by # any hard failure — the exact case it exists for). It is a standalone script, -# emitted automatically at the end of every invocation, runnable any time. +# emitted automatically at the end of the COMPUTE invocation, runnable any time +# via `sp report`. def _report(status): shell(f"python {SCRIPTS}/run_report.py --run-dir {RUN_DIR} " f"--index {INDEX_DB} --status {status} || true") -onsuccess: - _report("success") +if PHASE == "compute": + + onsuccess: + _report("success") -onerror: - _report("error") + onerror: + _report("error") diff --git a/workflow/bin/sp b/workflow/bin/sp index d1f10554..1ead411a 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -1,23 +1,23 @@ #!/usr/bin/env bash -# sp — the committed launcher for the ShapePipe Snakemake workflow. +# sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). # -# Removes the shell-state dependencies the review flagged (finding 20): it loads -# the apptainer module (Snakemake resolves `apptainer` via PATH at job runtime) -# and runs the Snakemake that lives on the shared /project venv (must be on a -# shared FS — the executor re-invokes it inside jobs). One entry point, so a -# fresh tmux / a restart after a crash always launches with the right state. +# Two verbs, nothing else: # -# Usage: -# sp prepare # prepare_tiles -> build_index.py -> prepare_exposures -# sp prepare_tiles # phase A.1 only -# sp index # build_index.py only (plain script, threshold-gated) -# sp prepare_exposures # phase A.2 only -# sp run [ARGS...] # the main compute DAG -# sp rerun # forced recompute of protected/finished products -# sp cancel # scancel this workflow's jobs (name sweep) before --unlock -# sp clean-exposures [--apply] # index-driven exposure-store reclaim -# sp report # emit run_report.json now (mid-run ok) -# Extra args after the subcommand pass through to snakemake. +# sp run [ARGS...] bring the products on disk up to date with the tile list. +# Two snakemake invocations over one Snakefile: +# 1. PREPARE snakemake prepare_all_tiles +# 2. COMPUTE snakemake all <- its PARSE builds the index +# ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. +# sp report [ARGS...] emit run_report.json now (mid-run is fine). +# +# Anything else is passed straight through to snakemake with the same profile and +# state dir (the escape hatch: `sp --unlock`, `sp exp_psf ...`, `sp --dag`). +# +# It also loads the apptainer module (snakemake resolves `apptainer` via PATH at +# job runtime) and activates the snakemake venv on the shared /project FS (the +# executor re-invokes it inside jobs, so it cannot live on a node-local path). +# One entry point, so a fresh tmux or a restart after a crash always launches +# with the right state. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ @@ -34,35 +34,40 @@ source "$VENV/bin/activate" # Minimal scalar reader for workflow/config.yaml (key: value, no nesting). cfg() { sed -n "s/^$1:[[:space:]]*//p" "$CONFIG" | head -1; } RUN_DIR="$(cfg run_dir)"; INDEX_DB="$(cfg index_db)" + # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO -# THE RUN on /scratch, never on /project (hard 27/27 TiB group quota killed a -# metadata write mid-run live). --directory only moves state: all data paths are +# THE RUN on /scratch, never on /project (a hard 27/27 TiB group quota killed a +# metadata write mid-run, live). --directory only moves state: all data paths are # absolute, and the Snakefile resolves its own configfile. STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" -TILE_LIST="$(cfg tile_list)"; MISSING_THRESH="${SP_MISSING_THRESHOLD:-0.0}" + +# SP_MISSING_THRESHOLD gates the compute parse's index build: the fraction of +# declared tiles allowed to be missing their exposure list (default 0.0). +export SP_MISSING_THRESHOLD="${SP_MISSING_THRESHOLD:-0.0}" sm() { snakemake --profile "$PROFILE" --directory "$STATE_DIR" "$@"; } -build_index() { - python "$SCRIPTS/build_index.py" --tile-list "$TILE_LIST" \ - --run-dir "$RUN_DIR" --db "$INDEX_DB" --missing-threshold "$MISSING_THRESH" -} -cmd="${1:-}"; shift || true +cmd="${1:-}" case "$cmd" in - prepare) sm prepare_tiles "$@"; build_index; sm prepare_exposures "$@" ;; - prepare_tiles) sm prepare_tiles "$@" ;; - index) build_index ;; - prepare_exposures) sm prepare_exposures "$@" ;; - run) sm "$@" ;; - rerun) sm --forcerun "$@" ;; + run) + shift + SP_PHASE=prepare sm prepare_all_tiles "$@" + SP_PHASE=compute sm all "$@" + ;; + report) + shift + python "$SCRIPTS/run_report.py" --run-dir "$RUN_DIR" --index "$INDEX_DB" \ + --status manual "$@" + ;; cancel) - run="${1:?usage: sp cancel }" + # Kept only because it is two lines: scancel this workflow's jobs by name + # before an --unlock. Not part of the design surface. + run="${2:?usage: sp cancel }" squeue --me --noheader --format='%i %j' \ | awk -v r="$run" '$2 ~ r {print $1}' | xargs -r scancel - echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" ;; - clean-exposures) python "$SCRIPTS/clean_exposures.py" \ - --run-dir "$RUN_DIR" --index "$INDEX_DB" "$@" ;; - report) python "$SCRIPTS/run_report.py" \ - --run-dir "$RUN_DIR" --index "$INDEX_DB" --status manual "$@" ;; - *) echo "sp: unknown subcommand '${cmd:-}' (see the header of $0)"; exit 2 ;; + echo "cancelled jobs matching '$run'; safe to --unlock / rerun now" + ;; + *) + sm "$@" + ;; esac diff --git a/workflow/config.yaml b/workflow/config.yaml index 592ecf98..942199b1 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -13,16 +13,14 @@ tile_list: /scratch/cdaley/shapepipe-output/smk-p0/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif -# Where products land ($SP_RUN root). Per-unit work-dir forest is built under here: -# /tiles/// and /exp/// +# Where products land ($SP_RUN root). The sharded per-unit stores live under here: +# /tiles/<2-char prefix>// and /exp/// run_dir: /scratch/cdaley/shapepipe-output/smk-p0 -# The canonical config chain: the committed, wrapper-free configs (#848 D2 config -# sweep) -- RUN_DATETIME=False, fixed INPUT_DIRs, NUMBER_LIST=$SP_UNIT_NUM baked -# in. example/cfis stays untouched for the bash flow; runs/p3-batch1/cfis's real -# divergences (real data paths, external star cats, SAVE_BATCH memory tuning) -# were folded in -- see the config-sweep report for the full list. -config_src: /home/cdaley/shapepipe-smk/workflow/config/cfis +# There is no config_src knob: the config chain is workflow/config/cfis, resolved +# relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / +# $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules +# are one artefact and must version together, so the dir is fixed by construction. # Pre-staged inputs (P3 data already on /project; get_images RETRIEVE=symlink). # Star catalogues for masking, pre-generated (network step done in prepare): diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini index a033f28d..dbaf1233 100644 --- a/workflow/config/cfis/config_merge_sep_cats.ini +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -41,8 +41,15 @@ NUMBER_LIST = $SP_UNIT_NUM # NOTE: only chunk 1's ngmix output is listed here; merge_sep_cats_runner # derives the other chunks' paths itself from N_SPLIT_MAX and this pattern # (chunk dirs are run_sp_tile_ngmix_Ngu, k=1..N_SPLIT_MAX), all under the -# same fixed $SP_RUN/output. -INPUT_DIR = $SP_RUN/output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output +# same fixed output dir. +# +# RELATIVE ON PURPOSE (do not "fix" this to $SP_RUN/...): merge_sep_cats derives +# chunk k's path with re.sub("1", str(k), input_file, 1) -- it replaces the FIRST +# "1" in the path. An absolute path under the sharded store +# (.../tiles/21/210.282/...) has digits before "Ng1u", so chunk 2 would be looked +# for in a nonexistent directory. Every rule runs shapepipe_run with cwd=$SP_RUN, +# so "./output/..." resolves and "Ng1u" carries the first "1". +INPUT_DIR = ./output/run_sp_tile_ngmix_Ng1u/ngmix_runner/output # Output directory OUTPUT_DIR = $SP_RUN/output diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 82cdb4cd..66790622 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -1,74 +1,90 @@ -"""Exposure chain — per exposure, keyed by exp_id (dedup is structural). +"""Exposure chain — per exposure, keyed by exp base id (dedup is structural). -Gie -> Sp -> Ma -> psf, each in the exposure's own work dir, chained by -ShapePipe's last:/run_sp_exp_* INPUT_DIR resolver. Completeness is the count -floor (completeness.py). + exp_get_images -> exp_split -> exp_mask -> exp_psf -NO temp() on these shared exposure directories: exposures overlap tiles by -construction (~7-10 tiles per exposure), so temp() would delete an exposure the -moment its forests are built and cascade destructive reruns across spatial -neighbours whenever a tile is appended (findings 4/5/6/19). The store is -persistent; space is reclaimed by the explicit index-driven `sp clean-exposures` -verb (scripts/clean_exposures.py), which deletes an exposure's intermediates -only once every consuming tile in the index has its final_cat present. +Each in the exposure's own sharded work dir, chained by manifests; every config +reads fixed ``$SP_RUN/output/run_sp_exp_*`` INPUT_DIRs, so nothing resolves a +run log. There is no `prepare_exposures` aggregation target: these chains hang +off the compute DAG (`all` <- final_cat <- tile chain <- exposure manifests). -NUMBER_LIST is injected only for exp_split (numbering scheme == exposure id); -NEVER for get_images (download) / exp_mask / exp_psf (per-CCD schemes where the -#746 startup validation would turn tolerated per-CCD attrition into a -whole-exposure hard failure). -""" +NO temp() anywhere in this file, ever (D5). Exposures overlap tiles by +construction (~7-10 tiles each), so their consumer set closes over the CAMPAIGN, +not over one invocation — reclamation here is clean_exposure's job (S5), driven +by the accumulating index. A temp() here would delete an exposure the moment +this invocation's readers finished and cascade destructive reruns across spatial +neighbours the next time a tile is appended. -EXP_OUT = str(RUN_DIR / "exp/{exp}/output") +NUMBER_LIST is set only for exp_split (its numbering scheme IS the exposure id); +never for get_images / exp_mask / exp_psf, whose per-CCD or download numbering +would make the #746 startup validation turn tolerated per-CCD attrition into a +whole-exposure hard failure. That is now a property of the committed configs +(config_exp_Sp.ini has NUMBER_LIST = $SP_UNIT_NUM; Gie/Ma/psfex have none). +""" -# Symlink/download exposure image/weight/flag; reads the fabricated pseudo-Fe -# exp_numbers-000-000.txt the wrapper drops (via last:find_exposures_runner). rule exp_get_images: output: - directory(f"{EXP_OUT}/run_sp_exp_Gie") + manifest = f"{EXP_DIR}/manifests/exp_get_images.json" params: - cmd=lambda wc: sp_rule("exp_get_images", "config_exp_Gie_vos.ini", - "exp", wc.exp, isolate=False, - extra=f"--exp-name {EXP[wc.exp]}") + pre = lambda wc: unit_pre("exp_get_images", "exp", wc.exp, + exp_name=EXP[wc.exp]), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_get_images", "config_exp_Gie.ini") -# Split multi-HDU exposure into single-CCD files (+ headers-*.npy). Bulk output; -# persistent (see module docstring on temp()). +# Split the multi-HDU exposure into single-CCD files (+ headers-*.npy, which the +# tiles' merge_headers reads). rule exp_split: input: - rules.exp_get_images.output + rules.exp_get_images.output.manifest output: - directory(f"{EXP_OUT}/run_sp_exp_Sp") + manifest = f"{EXP_DIR}/manifests/exp_split.json" params: - cmd=lambda wc: sp_rule("exp_split", "config_exp_Sp.ini", "exp", wc.exp) + pre = lambda wc: unit_pre("exp_split", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_split", "config_exp_Sp.ini") -# Mask per-CCD. Star cats are NOT a per-unit input: the store is per-CCD -# (star_cat--.fits) and the mask config reads it as a DIR symlink -# (see prepare.smk) — a per-unit file input here has no producer and makes the -# DAG unbuildable the moment a new exposure is appended (caught live by the -# append-invariant test). A missing cat fails the mask count-floor loudly. rule exp_mask: input: - rules.exp_split.output, + rules.exp_split.output.manifest output: - directory(f"{EXP_OUT}/run_sp_exp_Ma") + manifest = f"{EXP_DIR}/manifests/exp_mask.json" params: - cmd=lambda wc: sp_rule("exp_mask", "config_exp_Ma_onthefly.ini", - "exp", wc.exp, isolate=False) + pre = lambda wc: unit_pre("exp_mask", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_mask", "config_exp_Ma.ini") # SExtractor -> setools star selection -> PSFEx model -> psfex_interp, per CCD. -# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor. +# setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's +# :warn on psfex_interp_runner. rule exp_psf: input: - rules.exp_mask.output + rules.exp_mask.output.manifest output: - directory(f"{EXP_OUT}/run_sp_exp_SxSePsfPi") + manifest = f"{EXP_DIR}/manifests/exp_psf.json" params: - cmd=lambda wc: sp_rule("exp_psf", "config_exp_psfex.ini", "exp", wc.exp, - isolate=False) + pre = lambda wc: unit_pre("exp_psf", "exp", wc.exp), + script_hash = SCRIPT_HASH + threads: 8 + retries: 2 + benchmark: + f"{EXP_DIR}/manifests/exp_psf.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 240 shell: - "{params.cmd} --threads {threads}" + sp_shell("exp_psf", "config_exp_psfex.ini") diff --git a/workflow/rules/prepare.smk b/workflow/rules/prepare.smk index d8698c82..92998e18 100644 --- a/workflow/rules/prepare.smk +++ b/workflow/rules/prepare.smk @@ -1,58 +1,67 @@ -"""Phase A — prepare: tile staging, find_exposures, per-unit star cats. +"""Invocation 1 — PREPARE: one static chain per tile. -Split into two static invocations with build_index.py (a plain script) between: - * prepare_tiles — tile_get_images -> tile_uncompress -> tile_find_exposures - * [build_index.py] — plain script, threshold-gated (not a DAG node) - * prepare_exposures — exp_get_images (exposure.smk) + per-unit star cats + tile_get_images -> tile_uncompress -> tile_find_exposures -Nibi compute nodes have internet, so downloads run in-DAG (no login-node tier). -Rules stay group-compatible: no mid-chain localrules, no run:/pipe outputs. +Known from the tile list alone, cheap, wide, idempotent. find_exposures parses +the tile FITS HISTORY header into ``exp_numbers--.txt`` — the +data-derived tile->exposure edge that invocation 2's parse aggregates into the +index. Nibi compute nodes have internet, so downloads run in-DAG (no login-node +tier). + +Rules stay group-compatible: shell only, no mid-chain localrules, no pipe outputs. + +Star catalogues for masking are pre-generated offline (create_star_cat.py on a +networked login node) and consumed as DIRECTORIES: the mask configs read +``$SP_RUN/star_cat_{exp,tiles}`` with per-CCD numbering. Every rule's prologue +materialises the two dir symlinks; there is NO per-unit star-cat DAG node — the +store is pre-run input like the image store, and a missing cat fails the mask +stage's count floor loudly. """ -# get_images reads a one-line tile_numbers.txt the wrapper drops in $SP_RUN. -# NUMBER_LIST is NEVER injected here (download stage; nothing on disk to -# validate against — #746 would hard-fail). +# NUMBER_LIST is never set for get_images (download stage; nothing on disk to +# validate against — #746 would hard-fail the unit). The committed +# config_tile_Git.ini simply has no NUMBER_LIST, so this is now a property of the +# config, not of an injection step. rule tile_get_images: output: - directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Git")) + manifest = f"{TILE_DIR}/manifests/tile_get_images.json" params: - cmd=lambda wc: sp_rule("tile_get_images", "config_tile_Git_vos.ini", - "tile", wc.tile, isolate=False) + pre = lambda wc: unit_pre("tile_get_images", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + retries: 2 + resources: + mem_mb = lambda wc, attempt: 4000 * attempt, + runtime = 60 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_get_images", "config_tile_Git.ini") rule tile_uncompress: input: - rules.tile_get_images.output + rules.tile_get_images.output.manifest output: - directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Uz")) + manifest = f"{TILE_DIR}/manifests/tile_uncompress.json" params: - cmd=lambda wc: sp_rule("tile_uncompress", "config_tile_Uz.ini", - "tile", wc.tile, isolate=True) + pre = lambda wc: unit_pre("tile_uncompress", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 60 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_uncompress", "config_tile_Uz.ini") -# find_exposures reads the tile FITS HISTORY header -> exp_numbers-.txt, the -# data-derived tile->exposure edge build_index.py aggregates (as a plain script). rule tile_find_exposures: input: - rules.tile_uncompress.output + rules.tile_uncompress.output.manifest output: - directory(str(RUN_DIR / "tiles/{tile}/output/run_sp_tile_Fe")) + manifest = f"{TILE_DIR}/manifests/tile_find_exposures.json" params: - cmd=lambda wc: sp_rule("tile_find_exposures", "config_tile_Fe.ini", - "tile", wc.tile, isolate=True) + pre = lambda wc: unit_pre("tile_find_exposures", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 1 + resources: + mem_mb = lambda wc, attempt: 2000 * attempt, + runtime = 30 shell: - "{params.cmd} --threads {threads}" - -# Star catalogues for masking are pre-generated offline (create_star_cat.py on a -# networked login node) and consumed as DIRECTORIES: the mask configs read -# INPUT_DIR $SP_RUN/star_cat_{exp,tiles} with per-CCD numbering (exp cats are -# star_cat--.fits, 40/exposure; tile cats star_cat--.fits) -# — the v2.0 mechanism. The wrapper materialises the two dir symlinks in every -# unit work dir (sp_rule.materialise_unit); there is NO per-unit star-cat DAG -# node: the store is pre-run input like the image store, and a missing cat fails -# the mask stage's count-floor loudly. (Earlier per-unit file rules linked names -# that don't exist — one file per exposure vs the store's 40 — and nothing read -# them: the configs only see the dir symlinks.) When star-cat GENERATION moves -# in-workflow (P2), it becomes real per-unit rules producing into the store. + sp_shell("tile_find_exposures", "config_tile_Fe.ini") diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 04cc4c6c..a0f048a5 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -1,151 +1,218 @@ -"""Tile post-chain — per tile: gather exposures, then detect/PSF/shape/catalogue. - -A per-tile exposure "forest" (symlink view of exactly this tile's exposures' -products) gives the tile modules a deterministic $SP_EXP to glob. But the forest -is only a runtime convenience, NEVER the DAG edge: every tile rule that reads -through $SP_EXP declares the underlying exposure directory() outputs as input via -an index dict lookup, so the last real reader is the last DAG consumer (findings -5/19). All rules are group-compatible (shell only, no mid-chain localrules). -""" +"""Tile post-chain — per tile: gather exposures, then detect / PSF / shape / catalogue. + + tile_exp_forest + tile_merge_headers -> tile_detect -> tile_vignets -> tile_ngmix x N + -> tile_merge_cats -> tile_make_cat -TILE_OUT = str(RUN_DIR / "tiles/{tile}/output") -FOREST = str(RUN_DIR / "tiles/{tile}/exp_forest") +The DAG edge to the exposures is always the exposures' MANIFESTS, looked up +through the index (TILE_EXP). The per-tile exposure "forest" — a symlink view of +exactly this tile's exposures' products — exists only so the ShapePipe configs +have one deterministic ``$SP_EXP`` to glob; it is NEVER the edge. Its 2-char +shard level is not cosmetic: ``exp_utils.get_exp_output_files`` hardwires +``///output/run_sp_*`` into its glob, so a flat forest +makes every tile gather stage fail "No split_exp_runner output found". +All rules are group-compatible (shell only, no mid-chain localrules). + +Note there is no `tile_mask` rule: the committed config chain is the +"sx_nomask" tile_detect variant (config_tile_Sx.ini reads Git + Uz + Mh, no mask +run), and no tile-mask config was committed in the S2 sweep. Adding the masked +variant is a config + one rule, at the config selector the PRD describes. +""" -def _exp_out(wc, prefix): - return [str(exp_dir(e) / "output" / prefix) for e in TILE_EXP.get(wc.tile, [])] +def tile_exp(wc): + return TILE_EXP.get(wc.tile, []) -def tile_exp_split(wc): return _exp_out(wc, "run_sp_exp_Sp") -def tile_exp_mask(wc): return _exp_out(wc, "run_sp_exp_Ma") -def tile_exp_psf(wc): return _exp_out(wc, "run_sp_exp_SxSePsfPi") -def tile_exp_products(wc): - """All of this tile's exposures' product dirs — the forest's DAG inputs.""" - return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) +def exp_manifests(wc, stage): + return [exp_manifest(e, stage) for e in tile_exp(wc)] +def tile_exp_split(wc): return exp_manifests(wc, "exp_split") +def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") +def tile_exp_psf(wc): return exp_manifests(wc, "exp_psf") +def tile_exp_all(wc): return tile_exp_split(wc) + tile_exp_mask(wc) + tile_exp_psf(wc) -# Build the per-tile symlink forest (exp_forest//output/run_sp_exp_* -> the -# real exposure products). Declaring the exposure dirs as input makes this the -# tile's wait-on-its-exposures edge; the forest itself is just the SP_EXP view. + +# Build the per-tile symlink forest. Declaring the exposure manifests as input +# makes this wait on its exposures; the forest itself is only the $SP_EXP view. +# Its output stays a directory() (it has no ShapePipe run dir and no manifest — +# it is not a shapepipe_run at all). rule tile_exp_forest: input: - tile_exp_products + tile_exp_all output: - directory(FOREST) + forest = directory(f"{TILE_DIR}/exp_forest") params: - # --forest {output} lives in the shell string: Snakemake formats shell - # ONCE — an {output} placeholder inside params.cmd survives literally - # (same trap as {threads}; all four forest jobs then race one './{output}'). - cmd=lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " - f"--run-dir {RUN_DIR} --index {INDEX_DB} " - f"--script-hash {SCRIPT_HASH}") + cmd = lambda wc: (f"python {SCRIPTS}/build_forest.py --tile {wc.tile} " + f"--run-dir {RUN_DIR} --index {INDEX_DB}"), + # build_forest.py's own content hash rides here and nowhere else. + script_hash = FOREST_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 20 shell: - # no --threads: build_forest.py is single-threaded symlinking - "{params.cmd} --forest {output}" + # --forest {output} lives in the shell string: snakemake formats shell + # ONCE, so an {output} placeholder inside params.cmd would survive + # literally and every forest job would race one './{output}'. + "{params.cmd} --forest {output.forest}" -# Merge single-exposure WCS headers into the tile-level sqlite log -# (log_exp_headers--.sqlite — the tile suffix downstream Sx/PiViVi/Ng -# consume via NUMBERING_SCHEME -000-000). Reads headers-*.npy through the forest, -# so the exposure split dirs are explicit inputs. +# Merge single-exposure WCS headers into the tile-level sqlite +# (log_exp_headers--.sqlite, which Sx / PiViVi / ngmix consume). +# Reads headers-*.npy through the forest -> the split manifests are the edge. rule tile_merge_headers: input: - forest=rules.tile_exp_forest.output, - split=tile_exp_split, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, output: - directory(f"{TILE_OUT}/run_sp_tile_Mh_exp") + manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" params: - cmd=lambda wc: sp_rule("tile_merge_headers", "config_tile_Mh_exp.ini", - "tile", wc.tile, isolate=True, - exp_forest=FOREST.format(tile=wc.tile)) + pre = lambda wc: unit_pre("tile_merge_headers", "tile", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 4 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" - -# Mask tiles (star cats: pre-run store, read via the wrapper's dir symlink). -rule tile_mask: - input: - git=rules.tile_get_images.output, - output: - directory(f"{TILE_OUT}/run_sp_tile_Ma") - params: - cmd=lambda wc: sp_rule("tile_mask", "config_tile_Ma_onthefly.ini", - "tile", wc.tile, isolate=True) - shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_merge_headers", "config_tile_Mh_exp.ini") # SExtractor object detection on the tile. rule tile_detect: input: - mask=rules.tile_mask.output, - mh=rules.tile_merge_headers.output, + uz = f"{TILE_DIR}/manifests/tile_uncompress.json", + mh = rules.tile_merge_headers.output.manifest, output: - directory(f"{TILE_OUT}/run_sp_tile_Sx") + manifest = f"{TILE_DIR}/manifests/tile_detect.json" params: - cmd=lambda wc: sp_rule("tile_detect", "config_tile_Sx.ini", - "tile", wc.tile, isolate=True) + pre = lambda wc: unit_pre("tile_detect", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 180 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_detect", "config_tile_Sx.ini") -# PSFEx interpolation to galaxies + vignet postage stamps (reads exposure PSF -# models + split through the forest — those dirs are explicit inputs). +# PSFEx interpolation to galaxies + vignet postage stamps: the last stage that +# reads exposure products, and the bulk intra-tile intermediate. +# +# The vignette store is declared as a SECOND, temp(directory()) output alongside +# the manifest. This is the ONE scoped exception to "no directory() outputs" +# (D5): the store is ~tens of GB per tile and must be reclaimed when its last +# intra-tile reader finishes, but it must not become DAG currency — so the +# manifest stays the edge, and the directory rides along purely so native temp() +# fires at the right moment. Its readers (tile_ngmix, tile_make_cat) declare +# BOTH. `--notemp` keeps it for debugging. rule tile_vignets: input: - sx=rules.tile_detect.output, - forest=rules.tile_exp_forest.output, - split=tile_exp_split, - psf=tile_exp_psf, + sx = rules.tile_detect.output.manifest, + forest = rules.tile_exp_forest.output.forest, + split = tile_exp_split, + psf = tile_exp_psf, output: - directory(f"{TILE_OUT}/run_sp_tile_PiViVi") + manifest = f"{TILE_DIR}/manifests/tile_vignets.json", + store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), params: - cmd=lambda wc: sp_rule("tile_vignets", "config_tile_PiViVi_canfar_sx.ini", - "tile", wc.tile, isolate=True, - exp_forest=FOREST.format(tile=wc.tile)) + pre = lambda wc: unit_pre("tile_vignets", "tile", wc.tile, + forest=forest_dir(wc.tile)), + script_hash = SCRIPT_HASH + threads: 16 + resources: + mem_mb = lambda wc, attempt: 32000 * attempt, + runtime = 240 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_vignets", "config_tile_PiViVi.ini") -# ngmix shape measurement — static N chunks. Each chunk computes its own closed -# object-ID range in-job from the tile's own sexcat (a declared input); chunks -# never rewrite the shared log (--no-log-sync) and read literal INPUT_DIRs. +# ngmix shape measurement — N chunks per tile (D4). Each chunk computes its own +# CLOSED object-ID range at EXECUTION time from this tile's own sexcat: a params +# function cannot, because params evaluate before the sexcat exists. Closed, not +# open-ended: `ID_OBJ_MAX = -1` on the last chunk was the 13-hour straggler's +# root cause (ngmix treats id_obj_max <= 0 as unbounded). +# +# Chunks write nothing shared: each has its own run_sp_tile_ngmix_Ngu, and +# merge_sep_cats — DAG-serialised after all chunks — is the gather. rule tile_ngmix: input: - vignets=rules.tile_vignets.output, - sx=rules.tile_detect.output, + vignets = rules.tile_vignets.output.manifest, + store = rules.tile_vignets.output.store, + sx = rules.tile_detect.output.manifest, output: - directory(f"{TILE_OUT}/run_sp_tile_ngmix_Ng{{chunk}}u") + manifest = f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.json", + # temp(directory()) for the same reason as the vignette store above. + chunkdir = temp(directory(f"{TILE_DIR}/output/run_sp_tile_ngmix_Ng{{chunk}}u")), params: - cmd=lambda wc: sp_rule( - "tile_ngmix", "config_tile_Ng_template_batch.ini", "tile", wc.tile, - extra=f"--ngmix-chunk {wc.chunk} --ngmix-nchunks {NGMIX_CHUNKS}"), + pre = lambda wc: unit_pre( + "tile_ngmix", "tile", wc.tile, + env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, + pre_run=[f'eval "$(python {SCRIPTS}/ngmix_range.py --run-dir ' + f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS})"']), + script_hash = SCRIPT_HASH + threads: 4 + retries: 2 + benchmark: + f"{TILE_DIR}/manifests/tile_ngmix_{{chunk}}.benchmark.tsv" + resources: + mem_mb = lambda wc, attempt: 14000 * attempt, + runtime = 720 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_ngmix", "config_tile_Ng_template.ini") + + +def ngmix_manifests(wc): + return [f"{tile_dir(wc.tile)}/manifests/tile_ngmix_{k}.json" + for k in range(1, NGMIX_CHUNKS + 1)] -def ngmix_chunks(wc): - return [f"{RUN_DIR}/tiles/{wc.tile}/output/run_sp_tile_ngmix_Ng{k}u" +def ngmix_chunkdirs(wc): + return [f"{tile_dir(wc.tile)}/output/run_sp_tile_ngmix_Ng{k}u" for k in range(1, NGMIX_CHUNKS + 1)] -# Merge the N chunk catalogues (DAG-serialized: the single per-tile log-sync -# point before make_cat). N_SPLIT_MAX == chunk count. +# The gather: merge the N chunk catalogues. N_SPLIT_MAX comes from the workflow's +# own chunk count via $NGMIX_N_CHUNKS (env-expanded by the module). rule tile_merge_cats: input: - ngmix_chunks + manifests = ngmix_manifests, + chunkdirs = ngmix_chunkdirs, output: - directory(f"{TILE_OUT}/run_sp_Ms") + manifest = f"{TILE_DIR}/manifests/tile_merge_cats.json" params: - cmd=lambda wc: sp_rule("tile_merge_cats", "config_merge_sep_cats_template.ini", - "tile", wc.tile, isolate=True, - extra=f"--n-split-max {NGMIX_CHUNKS}"), + pre = lambda wc: unit_pre("tile_merge_cats", "tile", wc.tile, + env={"NGMIX_N_CHUNKS": NGMIX_CHUNKS}), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + sp_shell("tile_merge_cats", "config_merge_sep_cats.ini") -# Build the final catalogue: the run's science product. protected() so a -# code/params-drift rerun cannot silently discard a finished catalogue — the -# recompute must go through `sp rerun` (--forcerun). +# The run's science product. make_cat also reads the vignette store's +# psfex_interp output, so it — not ngmix — is the store's last reader. +# +# No protected(): the full default rerun-triggers govern, and protected() only +# ever forced people through a `--forcerun` detour. rule tile_make_cat: input: - rules.tile_merge_cats.output + ms = rules.tile_merge_cats.output.manifest, + store = rules.tile_vignets.output.store, output: - protected(str(RUN_DIR / "tiles/{tile}/final_cat-{tile}.fits")) + manifest = f"{TILE_DIR}/manifests/tile_make_cat.json", + final_cat = f"{TILE_DIR}/final_cat-{{tile}}.fits", params: - cmd=lambda wc: sp_rule("tile_make_cat", "config_make_cat_psfex_nosm.ini", - "tile", wc.tile, isolate=True, - extra=f"--final-cat {RUN_DIR}/tiles/{wc.tile}/final_cat-{wc.tile}.fits"), + pre = lambda wc: unit_pre("tile_make_cat", "tile", wc.tile), + script_hash = SCRIPT_HASH + threads: 8 + resources: + mem_mb = lambda wc, attempt: 16000 * attempt, + runtime = 120 shell: - "{params.cmd} --threads {threads}" + # Publish the catalogue next to the manifest: a real file, so it is a + # real declared output (and it persists — never temp()). + "{params.pre}\n" + "rc=0\n" + 'shapepipe_run -c "$SP_CONFIG/config_tile_Mc.ini" -b {threads} || rc=$?\n' + f"python {SCRIPTS}/completeness.py check tile_make_cat {{output.manifest}} || rc=1\n" + "if [ $rc -eq 0 ]; then\n" + ' cp -f "$(ls -1 "$SP_RUN"/output/run_sp_Mc/make_cat_runner/output/final_cat*.fits' + ' | head -1)" {output.final_cat}\n' + "fi\n" + "exit $rc\n" diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index e945820e..7ccaa308 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -3,12 +3,12 @@ A plain script (not a run: block) so the tile chain stays group-compatible. Reads the tile's exposures from run_index.sqlite and symlinks each exposure's -``exp//output`` into ``///output`` by exact name (no -glob). The 2-digit ```` shard level is NOT cosmetic: ShapePipe's +``exp///output`` into ``///output`` by exact +name (no glob). The 2-char ```` shard level is NOT cosmetic: ShapePipe's ``exp_utils.get_exp_output_files`` hardwires the sharded v2.0 layout into its $SP_EXP glob (``///output/run_sp_*/...``), so a flat forest makes every tile gather stage fail "No split_exp_runner output found". -(The exposure STORE stays flat — only the module-facing view needs the shard.) +(The exposure STORE is sharded the same way, for the filesystem's sake.) The forest is a convenience view; the DAG edge to the exposures is declared in the rule's input (tile.smk), not here. """ @@ -24,11 +24,6 @@ def main() -> None: p.add_argument("--run-dir", required=True, type=Path) p.add_argument("--index", required=True, type=Path) p.add_argument("--forest", required=True, type=Path) - p.add_argument("--script-hash", default=None, - help="ignored at runtime: carries the wrapper-scripts content " - "hash into the rule's params so the params rerun-trigger " - "covers script edits (the code trigger only hashes the " - "rule's own shell string)") args = p.parse_args() con = sqlite3.connect(args.index) @@ -38,7 +33,7 @@ def main() -> None: args.forest.mkdir(parents=True, exist_ok=True) for e in exps: - src = args.run_dir / "exp" / e / "output" + src = args.run_dir / "exp" / e[:2] / e / "output" dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape dst.parent.mkdir(parents=True, exist_ok=True) if dst.is_symlink() or dst.exists(): diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 7ba865d9..31fc6178 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -5,7 +5,11 @@ at parse time into plain dicts, so extending a run's tile list changes which jobs exist without touching the mtime chain of completed work. -It records, per run: +It ACCUMULATES: tables are created if absent and rows upserted, never dropped +(D1). Successive invocations widen it, so a later ``clean_exposure`` sees every +consuming tile of the whole campaign rather than of the current tile list. + +It records: tiles(tile_id, ra_dir, n_exp, status) exposures(exp_id) -- deduplicated union over tiles @@ -14,20 +18,21 @@ The tile->exposure edges are *data-derived*: they are read from each tile's ``find_exposures`` output (``exp_numbers--.txt``), which ``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. -So this is a **plain script**, not a DAG node: it runs between the two static -invocations — after ``snakemake prepare_tiles`` (Git+Uz+Fe, keep-going) and -before ``snakemake prepare_exposures`` parses the now-existing index. It +So the build is not a DAG node: ``build()`` is imported and called at PARSE TIME +by the Snakefile of the COMPUTE invocation, after the PREPARE invocation has +produced the tiles' find_exposures output. (There is no ``sp index`` verb; the +CLI below stays for hand-inspection.) It iterates the *declared* tile list and checks each tile's Fe output at its deterministic path (no globbing — O(tile-list) existence checks, respecting the no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in -``missing.json`` and the index is built over the rest. The script exits nonzero -only if the missing *fraction* exceeds ``--missing-threshold`` (default 0.0), -so a keep-going download storm that lost a few tiles does not cost the run. -There is NO ``--allow-missing`` flag (that Snakemake flag does not exist). +``missing.json`` and the index is built over the rest. The build fails only if +the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the +check — what the PREPARE phase passes, where absent Fe output is normal), so a +keep-going download storm that lost a few tiles does not cost the run. Exposure IDs are stored with their trailing single-char suffix stripped (``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the -exposure-rule wildcard, matching the ``exp///`` forest layout. +exposure-rule wildcard, matching the sharded ``exp///`` store. """ import argparse @@ -55,8 +60,20 @@ def read_exposure_list(exp_numbers_file: Path) -> list[tuple[str, str]]: return pairs +def exp_list_path(run_dir: Path, tile_id: str) -> Path: + """This tile's find_exposures output, at its deterministic path. + + Sharded store (D2), fixed run dir (RUN_DATETIME=False) — an existence check, + never a glob (no ``ls`` at scale). + """ + idra, iddec = tile_id.split(".") + return (run_dir / "tiles" / tile_id[:2] / tile_id / "output" / + "run_sp_tile_Fe" / "find_exposures_runner" / "output" / + f"exp_numbers-{idra}-{iddec}.txt") + + def build(tile_ids: list[str], run_dir: Path, db_path: Path, - missing_threshold: float = 0.0) -> dict: + missing_threshold: float | None = 0.0) -> dict: """Build the index over ``tile_ids``; return a summary dict. For each tile, check its ``exp_numbers--.txt`` at the tile's @@ -67,14 +84,14 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, """ db_path.parent.mkdir(parents=True, exist_ok=True) con = sqlite3.connect(db_path) + # No DROP: the index accumulates across invocations (D1). con.executescript( """ - DROP TABLE IF EXISTS tiles; - DROP TABLE IF EXISTS exposures; - DROP TABLE IF EXISTS tile_exposures; - CREATE TABLE tiles(tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); - CREATE TABLE exposures(exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); - CREATE TABLE tile_exposures( + CREATE TABLE IF NOT EXISTS tiles( + tile_id TEXT PRIMARY KEY, ra_dir TEXT, n_exp INTEGER); + CREATE TABLE IF NOT EXISTS exposures( + exp_id TEXT PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS tile_exposures( tile_id TEXT, exp_id TEXT, PRIMARY KEY (tile_id, exp_id)); """ @@ -84,30 +101,26 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: ra_dir = tile_id.split(".")[0] - idra, iddec = tile_id.split(".") - # Flat forest, deterministic run dir (RUN_DATETIME=False). - exp_file = (run_dir / "tiles" / tile_id / "output" / "run_sp_tile_Fe" / - "find_exposures_runner" / "output" / - f"exp_numbers-{idra}-{iddec}.txt") + exp_file = exp_list_path(run_dir, tile_id) if not exp_file.exists(): missing.append(tile_id) continue exp_pairs = read_exposure_list(exp_file) - con.execute("INSERT INTO tiles VALUES (?,?,?)", + con.execute("INSERT OR REPLACE INTO tiles VALUES (?,?,?)", (tile_id, ra_dir, len(exp_pairs))) for exp_id, _name in exp_pairs: con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", (tile_id, exp_id)) all_exposures.update(exp_pairs) - con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", + con.executemany("INSERT OR REPLACE INTO exposures VALUES (?,?)", sorted(all_exposures)) con.commit() con.close() (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) frac = len(missing) / len(tile_ids) if tile_ids else 0.0 - if frac > missing_threshold: + if missing_threshold is not None and frac > missing_threshold: raise SystemExit( f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " diff --git a/workflow/scripts/ngmix_range.py b/workflow/scripts/ngmix_range.py new file mode 100644 index 00000000..9e305805 --- /dev/null +++ b/workflow/scripts/ngmix_range.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Print one ngmix chunk's closed object-ID range as bash exports. + +Run inside the chunk's shell, from the tile's own sexcat, because the range is +only knowable at execution time (PRD D4):: + + eval "$(ngmix_range.py --run-dir $SP_RUN --chunk 3 --n-chunks 8)" + # -> export NGMIX_ID_MIN=751; export NGMIX_ID_MAX=1125 + +SExtractor's NUMBER column (ngmix's obj_id) runs 1..N contiguous, so covering +[1, N] processes every object exactly once. The first N-1 chunks get equal +shares; the last takes the remainder, with a CLOSED upper bound at n_obj — +never ID_OBJ_MAX = -1, which ngmix reads as unbounded and would double-count. +Each tile is sized from its OWN count (the bash monolith used the campaign +average and left the last chunk open). +""" + +import argparse +from pathlib import Path + + +def id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: + base, rem = divmod(n_obj, n_chunks) + ranges, lo = [], 1 + for k in range(1, n_chunks + 1): + hi = lo + base + (rem if k == n_chunks else 0) - 1 + ranges.append((lo, hi)) + lo = hi + 1 + return ranges + + +def object_count(run_dir: Path) -> int: + """NAXIS2 of the last HDU of this tile's sexcat (get_number_objects.py).""" + from astropy.io import fits + + cats = sorted((run_dir / "output" / "run_sp_tile_Sx").glob( + "sextractor_runner/output/sexcat*.fits")) + if not cats: + raise SystemExit(f"[ngmix_range] FATAL: no sexcat under {run_dir}") + with fits.open(cats[0]) as hdul: + return int(hdul[-1].header["NAXIS2"]) + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--run-dir", required=True, type=Path) + p.add_argument("--chunk", required=True, type=int) + p.add_argument("--n-chunks", required=True, type=int) + a = p.parse_args() + lo, hi = id_ranges(object_count(a.run_dir), a.n_chunks)[a.chunk - 1] + print(f"export NGMIX_ID_MIN={lo}; export NGMIX_ID_MAX={hi}") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/sp_rule.py b/workflow/scripts/sp_rule.py deleted file mode 100644 index 8ee71c69..00000000 --- a/workflow/scripts/sp_rule.py +++ /dev/null @@ -1,419 +0,0 @@ -#!/usr/bin/env python3 -"""Thin per-unit ShapePipe rule wrapper — the single entrypoint every rule calls. - -Under the one-allocation local executor + apptainer software-deployment, -Snakemake runs each rule's shell *inside* the container on the allocation's node. -So this script already runs in-container: it calls ``shapepipe_run`` directly -(never apptainer itself), which is why the workflow never hand-invokes apptainer. - -What it owns — only what is intrinsically ShapePipe's, nothing Snakemake already -does: - -1. **Per-unit work dir + v2.0 isolation furniture.** Each tile/exposure gets its - own ``$SP_RUN`` (``tiles//`` or ``exp//``). Isolation is by work-dir - *content*, the proven v2.0 mechanism — NOT ``-e/--exclusive``: - - a ``cfis`` symlink to the canonical config dir (so ``$SP_CONFIG`` / ``.mask`` - refs resolve); - - ``star_cat_exp`` / ``star_cat_tiles`` symlinks to the shared cat pool; - - for a tile: ``tile_numbers.txt`` = this tile ID (dot format, real data — - what get_images reads); - - for an exposure: a fabricated - ``output/run_sp_tile_Fe/find_exposures_runner/output/exp_numbers-000-000.txt`` - = this exposure ID (v2.0 ``init_exp_work_dir`` pseudo-Fe), so the Gie - config's ``last:find_exposures_runner`` resolver finds exactly one unit. - -2. **Per-unit config copy.** Forces ``RUN_DATETIME = False`` (deterministic - ``output/run_sp_/`` the rule declares as its ``directory()`` output), - sets ``SMP_BATCH_SIZE = {threads}`` (fork width == cpus_per_task, so SLURM - packing density and true per-job process count are one number), and — only - when ``isolate`` (tile-scheme stages + exp split; NEVER get_images / exp_mask / - exp_psf, whose per-CCD/download numbering would make #746 turn tolerated - per-CCD attrition into a whole-exposure hard failure) — ``NUMBER_LIST = -``. - -3. **Log sync at START and end.** ``update_runs_log_file.py`` regenerates - ``log_run_sp.txt`` before ``shapepipe_run`` (after the pre-delete, so - resolution never sees a just-deleted dir) and again after. Skipped for ngmix - chunks: the log is already complete from the prior tile stages, and the N - concurrent chunk jobs share the tile work dir — a full log *rewrite* would - race, whereas ShapePipe's own single-line ``O_APPEND`` of each chunk's run is - atomic. ngmix chunk configs additionally carry *literal* INPUT_DIRs - (deterministic under RUN_DATETIME=False), so they need no log read at all. - -4. **Count-floor check** (``completeness.py`` — the ported bash ``complete_check`` - table, the single failure policy: no 3-class taxonomy, no whitelist). After - the run, count products per mandatory runner and exit nonzero below the floor. - ``keep-going`` isolates that unit's cone; per-CCD attrition between floor and - ``expect`` is tolerated. Stages absent from the table fall back to a - zero-output floor. - -Stage-specific verbs (grounded in ``runs/p3-batch1/job_p3_batch1.sh``): - -- ``--ngmix-chunk K --ngmix-nchunks N`` — read this tile's SExtractor object - count from its ``run_sp_tile_Sx`` output, split ``[1, N_obj]`` into N disjoint - closed ID ranges, and generate chunk K's config from - ``config_tile_Ng_template_batch.ini``. -- ``--n-split-max N`` — generate the merge config from - ``config_merge_sep_cats_template.ini`` with ``N_SPLIT_MAX = N``. -- ``--final-cat PATH`` — after ``make_cat``, copy the produced ``final_cat`` FITS - to PATH (the rule's declared, protected science-product output). - -Run by hand to reproduce exactly what a rule does (the escape hatch — it -pre-deletes the run dir, just like Snakemake's delete-before-rerun):: - - python sp_rule.py --stage exp_split --config config_exp_Sp.ini \\ - --unit 2243881 --level exp --run-dir $SP_RUN --config-src $SP_CONFIG \\ - --star-cats $STAR_CATS --threads 8 -""" - -import argparse -import os -import shutil -import subprocess -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) -from completeness import COMPLETENESS, check_floor # noqa: E402 - - -def unit_work_dir(run_dir: Path, level: str, unit: str) -> Path: - """Per-unit work dir in the forest: tiles/ or exp/. - - Flat (no IDra/prefix sharding): the shard existed only to keep the bash - layer's ``ls exp/`` bearable, but Snakemake never globs the forest — the - run index drives every path — so flattening keeps single-wildcard rules. - ``run`` level (aggregation stages) work directly in ``run_dir``. - """ - if level == "tile": - return run_dir / "tiles" / unit - if level == "exp": - base = unit[:-1] if unit[-1].isalpha() else unit - return run_dir / "exp" / base - return run_dir - - -def materialise_unit(work: Path, level: str, unit: str, config_src: Path, - star_cats: Path, exp_name: str | None = None) -> None: - """Create the v2.0 unit-isolation furniture in the work dir.""" - (work / "output").mkdir(parents=True, exist_ok=True) - - cfis = work / "cfis" # $SP_CONFIG -> canonical config dir (.mask refs resolve) - if not cfis.exists(): - cfis.symlink_to(config_src) - - for name, sub in (("star_cat_exp", "exp"), ("star_cat_tiles", "tiles")): - link = work / name - if not link.exists() and (star_cats / sub).exists(): - link.symlink_to(star_cats / sub) - - if level == "tile": - (work / "tile_numbers.txt").write_text(unit + "\n") # dot format, real data - elif level == "exp": - fe = work / "output" / "run_sp_tile_Fe" / "find_exposures_runner" / "output" - fe.mkdir(parents=True, exist_ok=True) - exp_numbers = fe / "exp_numbers-000-000.txt" - # Written UNCONDITIONALLY (cheap, deterministic): an exists-guard once - # pinned a stale pre-fix file with the bare base id. Content is the - # ORIGINAL exposure name (with its 'p'-style suffix, from the index) — - # the on-disk store is .fits.fz; the bare base id matches nothing. - # v2.0 copies the name verbatim from the tile's Fe output; the index - # carries it for us. - exp_numbers.write_text((exp_name or unit) + "\n") - - -def normalize_config_text(text: str, unit: str, isolate: bool, threads: int) -> str: - """Force RUN_DATETIME=False, SMP_BATCH_SIZE=threads, and (if isolating) - NUMBER_LIST=-. - - ``NUMBER_LIST`` uses ShapePipe's image-number convention (dots -> dashes), - the ``set_config_number_list`` mechanism that replaced the retired - ``-e/--exclusive`` flag (#746). - """ - lines = text.splitlines() - out, in_file, saw_datetime, saw_numberlist, saw_smp = [], False, False, False, False - number = "-" + unit.replace(".", "-") - for line in lines: - stripped = line.strip() - if stripped.startswith("[") and stripped.endswith("]"): - in_file = stripped.upper() == "[FILE]" - if stripped.upper().startswith("RUN_DATETIME"): - out.append("RUN_DATETIME = False") - saw_datetime = True - continue - if stripped.upper().startswith("SMP_BATCH_SIZE"): - out.append(f"SMP_BATCH_SIZE = {threads}") - saw_smp = True - continue - if isolate and stripped.upper().startswith("NUMBER_LIST"): - out.append(f"NUMBER_LIST = {number}") - saw_numberlist = True - continue - out.append(line) - if in_file and stripped.upper() == "[FILE]" and isolate and not saw_numberlist: - out.append(f"NUMBER_LIST = {number}") - saw_numberlist = True - if not saw_datetime: - # ShapePipe reads RUN_DATETIME from [DEFAULT]; inject after its header. - patched, done = [], False - for line in out: - patched.append(line) - if not done and line.strip().upper() == "[DEFAULT]": - patched.append("RUN_DATETIME = False") - done = True - out = patched if done else ["[DEFAULT]", "RUN_DATETIME = False"] + out - if not saw_smp: - # SMP_BATCH_SIZE lives in [JOB]; inject after its header. - patched, done = [], False - for line in out: - patched.append(line) - if not done and line.strip().upper() == "[JOB]": - patched.append(f"SMP_BATCH_SIZE = {threads}") - done = True - out = patched if done else out + ["[JOB]", f"SMP_BATCH_SIZE = {threads}"] - return "\n".join(out) + "\n" - - -def run_prefix_from_text(text: str, config_label: str) -> str: - """The RUN_NAME (== run_sp_) the module will write under.""" - for line in text.splitlines(): - s = line.strip() - if s.upper().startswith("RUN_NAME"): - return s.split("=", 1)[1].strip() - raise SystemExit(f"no RUN_NAME in {config_label}") - - -# --- ngmix chunking -------------------------------------------------------- - -# Literal, deterministic INPUT_DIRs for the ngmix chunk (RUN_DATETIME=False), so -# concurrent chunks need no log read at all (finding 18). -NGMIX_LITERAL_INPUT = ( - "./output/run_sp_tile_Sx/sextractor_runner/output, " - "./output/run_sp_tile_PiViVi/psfex_interp_runner/output, " - "./output/run_sp_tile_PiViVi/vignetmaker_runner_run_2/output, " - "./output/run_sp_tile_Mh_exp/merge_headers_runner/output") - - -def ngmix_id_ranges(n_obj: int, n_chunks: int) -> list[tuple[int, int]]: - """Split ``[1, n_obj]`` into ``n_chunks`` disjoint *closed* ID ranges. - - SExtractor's ``NUMBER`` column (what ngmix reads as ``obj_id``) runs 1..N - contiguous, so covering ``[1, N]`` processes every object exactly once. The - first n_chunks-1 chunks get ``n_obj // n_chunks`` each; the last gets the - remainder — a *closed* upper bound at ``n_obj`` (never ``ID_OBJ_MAX = -1``: - ngmix treats ``id_obj_max <= 0`` as unbounded, which would double-count). - - Deviation from the nibi monolith (deliberate): the monolith sizes every - tile's chunks from the *average* object count across all tiles and leaves the - last chunk open (``-1``); here each tile is an independent job, so we size - from *this* tile's own count and close the last chunk. The merged catalogue - is identical either way (merge_sep_cats recombines all chunks, ngmix fits - each object independently) — this just removes the open-ended overflow. - """ - base, rem = divmod(n_obj, n_chunks) - ranges, lo = [], 1 - for k in range(1, n_chunks + 1): - size = base + (rem if k == n_chunks else 0) - hi = lo + size - 1 - ranges.append((lo, hi)) - lo = hi + 1 - return ranges - - -def tile_object_count(work: Path) -> int: - """Object count for this tile from its run_sp_tile_Sx SExtractor catalogue. - - Matches ``get_number_objects.py``: the ``NAXIS2`` of the catalogue's last - HDU. Per-unit isolation means exactly one ``sexcat*.fits`` lives here. - """ - from astropy.io import fits - - sexcats = sorted((work / "output" / "run_sp_tile_Sx").glob( - "sextractor_runner/output/sexcat*.fits")) - if not sexcats: - raise SystemExit( - f"[sp_rule] FATAL ngmix: no sexcat under {work}/output/run_sp_tile_Sx") - with fits.open(sexcats[0]) as hdul: - return int(hdul[-1].header["NAXIS2"]) - - -def apply_ngmix_subs(text: str, chunk: int, lo: int, hi: int) -> str: - """Fill the ngmix batch template for one chunk (the monolith's sed recipe) - plus literal INPUT_DIRs so the chunk needs no shared-log read.""" - text = text.replace("NgXu", f"Ng{chunk}u").replace("X_interp", "psfex_interp") - out = [] - for line in text.splitlines(): - s = line.strip() - if s.startswith("ID_OBJ_MIN"): - out.append(f"ID_OBJ_MIN = {lo}") - elif s.startswith("ID_OBJ_MAX"): - out.append(f"ID_OBJ_MAX = {hi}") - elif s.startswith("INPUT_DIR") and "sextractor_runner" in s: - out.append(f"INPUT_DIR = {NGMIX_LITERAL_INPUT}") - else: - out.append(line) - return "\n".join(out) - - -def apply_merge_subs(text: str, n_split_max: int) -> str: - """Fill N_SPLIT_MAX in the merge template (the monolith's sed recipe).""" - out = [] - for line in text.splitlines(): - if line.strip().startswith("N_SPLIT_MAX"): - out.append(f"N_SPLIT_MAX = {n_split_max}") - else: - out.append(line) - return "\n".join(out) - - -# --- in-job count floor ---------------------------------------------------- - -def count_floor(run_out: Path, stage: str) -> None: - """Fail loudly if any mandatory runner is below its ``floor`` (completeness.py). - - Stages absent from the table skip this floor (only the zero-output floor - applies to them). - """ - if stage not in COMPLETENESS: - return - ok, details = check_floor(stage, run_out) - for runner, n, floor, expect, warn in details: - tag = "warn" if warn else ("OK" if n >= floor else " <-- BELOW floor") - print(f"[sp_rule] {runner}: {n}/{expect} (floor {floor}) {tag}", - file=sys.stderr) - if not ok: - print(f"[sp_rule] FATAL {stage}: count floor breached in {run_out}", - file=sys.stderr) - sys.exit(1) - - -def main() -> None: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--stage", required=True) - p.add_argument("--config", required=True, help="config filename in --config-src") - p.add_argument("--unit", required=True) - p.add_argument("--level", required=True, choices=["tile", "exp", "run"]) - p.add_argument("--run-dir", required=True, type=Path, help="$SP_RUN root") - p.add_argument("--config-src", required=True, type=Path, - help="dir holding the canonical configs") - p.add_argument("--star-cats", type=Path, default=None, - help="shared star-cat pool (tiles/ + exp/ subdirs)") - p.add_argument("--threads", type=int, default=1, - help="SMP_BATCH_SIZE == cpus_per_task (fork width)") - p.add_argument("--exp-forest", type=Path, default=None, - help="$SP_EXP: per-tile exposure forest (tile post-stages)") - p.add_argument("--exp-name", default=None, - help="original exposure name incl. suffix (e.g. 2605805p); " - "written into the fabricated exp_numbers list so " - "get_images finds .fits.fz in the store") - p.add_argument("--no-isolate", action="store_true", - help="skip NUMBER_LIST (get_images / exp_mask / exp_psf)") - p.add_argument("--ngmix-chunk", type=int, default=None, - help="1-indexed ngmix chunk K (with --ngmix-nchunks)") - p.add_argument("--ngmix-nchunks", type=int, default=None, - help="total ngmix chunks N (with --ngmix-chunk)") - p.add_argument("--n-split-max", type=int, default=None, - help="N_SPLIT_MAX for the merge_sep_cats template") - p.add_argument("--final-cat", type=Path, default=None, - help="copy the produced final_cat FITS here (make_cat)") - p.add_argument("--script-hash", default=None, - help="ignored at runtime: carries the wrapper-scripts content " - "hash into the rule's params so the params rerun-trigger " - "covers script edits (the code trigger only hashes the " - "rule's own shell string)") - args = p.parse_args() - - ngmix_mode = args.ngmix_chunk is not None - if ngmix_mode and args.ngmix_nchunks is None: - raise SystemExit("--ngmix-chunk requires --ngmix-nchunks") - - work = unit_work_dir(args.run_dir, args.level, args.unit) - materialise_unit(work, args.level, args.unit, args.config_src, - args.star_cats or args.run_dir, exp_name=args.exp_name) - - # Build the config copy. ngmix chunks and merge fill template placeholders - # first; every copy is then normalized (RUN_DATETIME=False, SMP_BATCH_SIZE, - # [+ NUMBER_LIST]). - src_text = (args.config_src / args.config).read_text() - isolate = not args.no_isolate - if ngmix_mode: - n_obj = tile_object_count(work) - lo, hi = ngmix_id_ranges(n_obj, args.ngmix_nchunks)[args.ngmix_chunk - 1] - print(f"[sp_rule] ngmix chunk {args.ngmix_chunk}/{args.ngmix_nchunks}: " - f"n_obj={n_obj} -> ID_OBJ [{lo}, {hi}]", file=sys.stderr) - src_text = apply_ngmix_subs(src_text, args.ngmix_chunk, lo, hi) - cfg_name = f"config_tile_Ng{args.ngmix_chunk}u.ini" # distinct per chunk - elif args.n_split_max is not None: - src_text = apply_merge_subs(src_text, args.n_split_max) - cfg_name = args.config - else: - cfg_name = args.config - - cfg_mod_dir = work / "cfis_mod" - cfg_mod = cfg_mod_dir / cfg_name - cfg_text = normalize_config_text(src_text, args.unit, isolate, args.threads) - cfg_mod_dir.mkdir(parents=True, exist_ok=True) - cfg_mod.write_text(cfg_text) - - prefix = run_prefix_from_text(cfg_text, str(cfg_mod)) - run_out = work / "output" / prefix - - # Pre-delete the deterministic run dir (delete-before-rerun for the hand-run - # escape hatch; ShapePipe's FileHandler.mkdir raises on an existing run dir). - if run_out.exists(): - shutil.rmtree(run_out) - - env = dict(os.environ) - env["SP_RUN"] = str(work) - env["SP_CONFIG"] = str(work / "cfis") # canonical dir (.mask refs), not cfis_mod - if args.exp_forest: - env["SP_EXP"] = str(args.exp_forest) - # Thread caps also come via apptainer-args; keep here for the hand-run path. - env.setdefault("OMP_NUM_THREADS", "1") - env.setdefault("OPENBLAS_NUM_THREADS", "1") - env.setdefault("MKL_NUM_THREADS", "1") - env.setdefault("NUMEXPR_NUM_THREADS", "1") - env.setdefault("MALLOC_ARENA_MAX", "2") - env.setdefault("MALLOC_TRIM_THRESHOLD_", "0") - - # Log sync before the run (after pre-delete). Skipped for ngmix chunks: N - # concurrent chunks would race a full rewrite (literal INPUT_DIRs make it moot). - if not ngmix_mode: - subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) - - print(f"[sp_rule] stage={args.stage} unit={args.unit} work={work}", - file=sys.stderr) - rc = subprocess.call(["shapepipe_run", "-c", str(cfg_mod)], - cwd=str(work), env=env) - if rc != 0: - print(f"[sp_rule] WARN shapepipe_run exit {rc} — checking count floor", - file=sys.stderr) - - if not ngmix_mode: - subprocess.call(["update_runs_log_file.py"], cwd=str(work), env=env) - - # Zero-output floor for stages without a count table; count floor otherwise. - if args.stage in COMPLETENESS: - count_floor(run_out, args.stage) - else: - produced = list(run_out.glob("**/output/*")) if run_out.exists() else [] - if not produced: - print(f"[sp_rule] FATAL {args.stage} {args.unit}: zero output in {run_out}", - file=sys.stderr) - sys.exit(1) - - # make_cat: publish the science product to its declared, protected path. - if args.final_cat is not None: - finals = sorted(run_out.glob("**/output/final_cat*.fits")) - if not finals: - print(f"[sp_rule] FATAL {args.stage} {args.unit}: no final_cat under {run_out}", - file=sys.stderr) - sys.exit(1) - args.final_cat.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(finals[0], args.final_cat) - print(f"[sp_rule] final_cat -> {args.final_cat}", file=sys.stderr) - - print(f"[sp_rule] OK {args.stage} {args.unit}", file=sys.stderr) - - -if __name__ == "__main__": - main() From b149e187618b4c086fc6637b4768a8ec4a2346a8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 22:55:48 -0400 Subject: [PATCH 15/21] fix(orchestration): harden the rewrite against its adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All eleven findings from the fresh-eyes review, repro-verified: keep-incomplete:true (failure manifests are the post-mortem record — Snakemake otherwise deletes exactly what sp report reads); sp run survives a partial prepare (the missing-threshold is the gate, not set -e); the parse-time index build runs only under SP_PHASE=compute, always evaluates the threshold there, and refuses a silent empty run (WorkflowError on zero ready tiles) while passthrough invocations (--unlock, --dag, targets) never build; profile set-threads/set-resources blocks deleted — they silently replaced the rules' attempt-scaled lambdas and fork widths; manifests write only on content change (a byte-identical rewrite churned the whole cone via mtime); per-tile edge refresh in build_index (stale tile→exposure edges are unrepresentable-shrink no more); blame join blocks on failed/missing only, never warn (else all-tiles-blocked-by-all- exposures at production attrition); ngmix range eval no longer swallows script failure; compute DAG gets a regeneration edge to tile_find_exposures; threshold check precedes any durable index write. Manifest 'unit' is the human ID (basename of SP_RUN); build_forest handles a real-dir dst; stale comments (N_SPLIT_MAX expansion, && check form, sp_rule/protected refs) corrected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01B5KZRTAZKSBAnpdDhE7yzK --- profiles/nibi/config.yaml | 71 ++++------------- workflow/Snakefile | 37 ++++++--- workflow/bin/sp | 26 +++++- .../config/cfis/config_merge_sep_cats.ini | 12 ++- workflow/rules/tile.smk | 19 ++++- workflow/scripts/build_forest.py | 9 ++- workflow/scripts/build_index.py | 79 +++++++++++++------ workflow/scripts/completeness.py | 48 +++++++---- workflow/scripts/run_report.py | 19 ++++- 9 files changed, 200 insertions(+), 120 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 07f68833..029329aa 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -37,67 +37,26 @@ rerun-incomplete: true # re-do jobs left incomplete by an unclean deat show-failed-logs: true printshellcmds: true +# D3 DEPENDS ON THIS. Snakemake's default is to DELETE the declared outputs of a +# failed job — and this workflow's declared output IS the manifest, the only +# record of *why* a unit failed and the only thing `sp report` reads. Without +# keep-incomplete, every failed unit reads back as "not_run" and the post-mortem +# debris is gone. Safe here because each rule `rm -rf`s its own run dir at start, +# so a rerun never sees stale products. +keep-incomplete: true + # rerun-triggers left at the full Snakemake v9 default # {mtime,params,input,code,software-env}: params/code fixes SHOULD propagate. -# No override — the wrapper writes outputs only on change so mtimes move only -# when reality moves; protected() guards finished science products. +# No override — completeness.py writes the manifest only on change, so mtimes +# move only when reality moves. default-resources: mem_mb: 4000 runtime: 120 # minutes cpus_per_task: 1 -# Per-stage resources — measured footprints from the nibi P3 runs (#808). -# threads == cpus_per_task == ShapePipe SMP fork width (sp_rule sets -# SMP_BATCH_SIZE={threads}); mem_mb == threads x measured per-worker footprint. -# -# set-threads is REQUIRED for {threads} to resolve above 1: the local scheduler -# derives cpus_per_task from threads, NOT the reverse (verified snakemake 9.23 — -# set-resources cpus_per_task alone leaves {threads}==1, i.e. SMP_BATCH_SIZE=1, -# serial). Keep the two blocks in lockstep. -set-threads: - exp_split: 4 - exp_mask: 4 - exp_psf: 8 - tile_mask: 16 - tile_detect: 16 - tile_merge_headers: 8 - tile_vignets: 8 - tile_ngmix: 8 - tile_merge_cats: 4 - tile_make_cat: 4 - -set-resources: - exp_split: - cpus_per_task: 4 - mem_mb: 8000 - exp_mask: - cpus_per_task: 4 - mem_mb: 8000 - exp_psf: - cpus_per_task: 8 - mem_mb: 16000 - runtime: 240 - tile_mask: - cpus_per_task: 16 - mem_mb: 16000 - tile_detect: - cpus_per_task: 16 - mem_mb: 16000 - tile_merge_headers: - cpus_per_task: 8 - mem_mb: 8000 - tile_vignets: - cpus_per_task: 8 - mem_mb: 24000 - runtime: 240 - tile_ngmix: - cpus_per_task: 8 - mem_mb: 24000 - runtime: 360 - tile_merge_cats: - cpus_per_task: 4 - mem_mb: 8000 - tile_make_cat: - cpus_per_task: 4 - mem_mb: 8000 +# NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a +# rule's own values (verified snakemake 9.23), which would kill the +# attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +# ngmix thread count. The RULES own threads and resources; this profile only +# sizes the allocation and sets the defaults for rules that state nothing. diff --git a/workflow/Snakefile b/workflow/Snakefile index b707fdc1..7e9300c5 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -28,6 +28,8 @@ import sqlite3 import sys from pathlib import Path +from snakemake.exceptions import WorkflowError + # Resolved relative to THIS file, not the working directory: snakemake runs with # --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project # (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). @@ -51,9 +53,12 @@ sys.path.insert(0, str(SCRIPTS)) import build_index # noqa: E402 from completeness import STAGE_DIR # noqa: E402 -# `prepare` suppresses the missing-tile threshold (tiles are still being -# prepared) and the report hooks. Unset == compute (a bare hand-run of `all`). -PHASE = os.environ.get("SP_PHASE", "compute") +# SP_PHASE is set by bin/sp on the two invocations of `sp run` and NOWHERE else. +# It gates the two parse-time side effects — the index build and the report +# hooks — so that a passthrough invocation (`sp --unlock`, `sp --dag`, `sp +# exp_psf ...`) never mutates durable state or dies on a threshold it was not +# asked about. Unset means "read the index, build nothing". +PHASE = os.environ.get("SP_PHASE", "") with open(config["tile_list"]) as f: TILES = [ln.strip() for ln in f if ln.strip()] @@ -72,15 +77,18 @@ scattergather: NGMIX_CHUNKS = workflow._scatter["ngmix"] # --- parse-time index build + load (D1) ------------------------------------ -# Invocation 2's parse IS the index build. Tiles whose find_exposures output is -# absent land in index/missing.json and are dropped from the DAG; the build is -# fatal only above SP_MISSING_THRESHOLD (and never in the prepare phase, where -# absent Fe output is the normal state). -if any(build_index.exp_list_path(RUN_DIR, t).exists() for t in TILES): +# The COMPUTE invocation's parse IS the index build, and it runs UNCONDITIONALLY +# there — no "some Fe output exists" guard. That guard used to make a +# totally-failed prepare produce an empty index, an empty DAG and a green exit 0; +# with it gone, zero Fe outputs means a missing fraction of 1.0, which trips the +# SP_MISSING_THRESHOLD gate and fails loudly, as it should. +# +# In every other phase (prepare, or unset for a passthrough invocation) the parse +# builds NOTHING and only loads whatever index is already on disk. +if PHASE == "compute": build_index.build( TILES, RUN_DIR, INDEX_DB, - missing_threshold=(None if PHASE == "prepare" - else float(os.environ.get("SP_MISSING_THRESHOLD", "0.0")))) + missing_threshold=float(os.environ.get("SP_MISSING_THRESHOLD", "0.0"))) # EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes # verbatim into the fabricated per-unit exp_numbers list so get_images matches @@ -97,6 +105,15 @@ if INDEX_DB.exists(): # campaign, so it is intersected with the declared list, not used as it. TILES_READY = [t for t in TILES if t in TILE_EXP] +# A compute invocation with nothing to compute is never a success. Without this, +# an empty intersection yields `rule all` with no inputs, an empty DAG and exit +# 0 — the silent green run the threshold gate exists to prevent. +if PHASE == "compute" and not TILES_READY: + raise WorkflowError( + f"No declared tile has an indexed exposure list: 0 of {len(TILES)} tiles " + f"are ready to compute (index {INDEX_DB}). Run the prepare phase first " + f"(`sp run`), or check {INDEX_DB.parent / 'missing.json'}.") + wildcard_constraints: tile = r"\d{3}\.\d{3}", exp = r"\d{6,7}", diff --git a/workflow/bin/sp b/workflow/bin/sp index 1ead411a..3424584d 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -51,8 +51,30 @@ cmd="${1:-}" case "$cmd" in run) shift - SP_PHASE=prepare sm prepare_all_tiles "$@" - SP_PHASE=compute sm all "$@" + # PREPARE failing is NOT fatal to the run: keep-going means a failed tile + # poisons only its own cone, and the tiles that lost their exposure list are + # dropped at the compute parse. The real gate is SP_MISSING_THRESHOLD, which + # the compute parse's index build enforces over the WHOLE tile list — so we + # record the failure and go on rather than letting `set -e` abort here. + prep_rc=0 + SP_PHASE=prepare sm prepare_all_tiles "$@" || prep_rc=$? + if [ "$prep_rc" -ne 0 ]; then + echo "" >&2 + echo "############################################################" >&2 + echo "## WARNING: the PREPARE phase exited $prep_rc." >&2 + echo "## Some tiles may be missing their exposure list and will be" >&2 + echo "## dropped from the compute DAG. Continuing to COMPUTE; the" >&2 + echo "## SP_MISSING_THRESHOLD gate (now $SP_MISSING_THRESHOLD) decides" >&2 + echo "## whether that is tolerable." >&2 + echo "############################################################" >&2 + echo "" >&2 + fi + + comp_rc=0 + SP_PHASE=compute sm all "$@" || comp_rc=$? + if [ "$prep_rc" -ne 0 ] || [ "$comp_rc" -ne 0 ]; then + exit "$([ "$comp_rc" -ne 0 ] && echo "$comp_rc" || echo "$prep_rc")" + fi ;; report) shift diff --git a/workflow/config/cfis/config_merge_sep_cats.ini b/workflow/config/cfis/config_merge_sep_cats.ini index dbaf1233..750fe112 100644 --- a/workflow/config/cfis/config_merge_sep_cats.ini +++ b/workflow/config/cfis/config_merge_sep_cats.ini @@ -84,11 +84,9 @@ NUMBERING_SCHEME = -000-000 WARNING = always # Maximum number of separated catalogues per input. -# BLOCKER: merge_sep_cats_runner.py reads this with plain config.getint (not -# getexpanded) -- $NGMIX_N_CHUNKS would NOT expand here, unlike ID_OBJ_MIN/MAX -# in the ngmix module. Committed as a constant matching workflow/config.yaml's -# ngmix_chunks: 8; if that value changes, this line must be edited by hand -# (or the module gets the same getexpanded treatment ngmix_runner got). -# Chunk count comes from the workflow (single source of truth: the -# scattergather count); env-expanded by the module. +# merge_sep_cats_runner.py reads this with getexpanded (runner .py line ~31), so +# $NGMIX_N_CHUNKS DOES expand here, exactly like ID_OBJ_MIN/MAX in the ngmix +# module. The tile_merge_cats rule exports it from the workflow's scattergather +# chunk count, which is the single source of truth; this value must equal that +# count, so do not replace it with a literal. N_SPLIT_MAX = $NGMIX_N_CHUNKS diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index a0f048a5..e894309b 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -63,6 +63,12 @@ rule tile_merge_headers: input: forest = rules.tile_exp_forest.output.forest, split = tile_exp_split, + # config_tile_Mh_exp.ini reads run_sp_tile_Fe output, which the PREPARE + # phase produced. Declaring the Fe manifest gives the COMPUTE DAG a + # regeneration path for it instead of a silent dependency on a + # previous invocation (prepare.smk is included in every parse, so the + # rule exists here too). Normally a satisfied no-op. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: manifest = f"{TILE_DIR}/manifests/tile_merge_headers.json" params: @@ -109,6 +115,9 @@ rule tile_vignets: forest = rules.tile_exp_forest.output.forest, split = tile_exp_split, psf = tile_exp_psf, + # config_tile_PiViVi.ini reads run_sp_tile_Fe output — same reason as + # tile_merge_headers above. + fe = f"{TILE_DIR}/manifests/tile_find_exposures.json", output: manifest = f"{TILE_DIR}/manifests/tile_vignets.json", store = temp(directory(f"{TILE_DIR}/output/run_sp_tile_PiViVi")), @@ -144,8 +153,14 @@ rule tile_ngmix: pre = lambda wc: unit_pre( "tile_ngmix", "tile", wc.tile, env={"SP_NGMIX_CHUNK": wc.chunk, "NGMIX_N_CHUNKS": NGMIX_CHUNKS}, - pre_run=[f'eval "$(python {SCRIPTS}/ngmix_range.py --run-dir ' - f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS})"']), + # Two steps, not `eval "$(...)"`: a command substitution inside eval + # discards the script's exit status, so a missing sexcat would fall + # through to shapepipe_run with an unset range and fail as something + # else. Capture, check, then eval — the range script fails as itself. + pre_run=[f'ngmix_range_out=$(python {SCRIPTS}/ngmix_range.py --run-dir ' + f'"$SP_RUN" --chunk {wc.chunk} --n-chunks {NGMIX_CHUNKS}) ' + f'|| exit 1', + 'eval "$ngmix_range_out"']), script_hash = SCRIPT_HASH threads: 4 retries: 2 diff --git a/workflow/scripts/build_forest.py b/workflow/scripts/build_forest.py index 7ccaa308..427e9c64 100644 --- a/workflow/scripts/build_forest.py +++ b/workflow/scripts/build_forest.py @@ -14,6 +14,7 @@ """ import argparse +import shutil import sqlite3 from pathlib import Path @@ -36,8 +37,14 @@ def main() -> None: src = args.run_dir / "exp" / e[:2] / e / "output" dst = args.forest / e[:2] / e / "output" # sharded: the module glob's shape dst.parent.mkdir(parents=True, exist_ok=True) + # A symlink (the normal case) is unlinked; a REAL directory left behind + # by a hand-run or an older layout must be removed as a tree — unlink() + # raises IsADirectoryError on it and would kill the job. if dst.is_symlink() or dst.exists(): - dst.unlink() + if dst.is_dir() and not dst.is_symlink(): + shutil.rmtree(dst) + else: + dst.unlink() dst.symlink_to(src) print(f"[build_forest] {args.tile}: {len(exps)} exposures -> {args.forest}") diff --git a/workflow/scripts/build_index.py b/workflow/scripts/build_index.py index 31fc6178..73d08609 100644 --- a/workflow/scripts/build_index.py +++ b/workflow/scripts/build_index.py @@ -5,9 +5,19 @@ at parse time into plain dicts, so extending a run's tile list changes which jobs exist without touching the mtime chain of completed work. -It ACCUMULATES: tables are created if absent and rows upserted, never dropped -(D1). Successive invocations widen it, so a later ``clean_exposure`` sees every -consuming tile of the whole campaign rather than of the current tile list. +It ACCUMULATES ACROSS INVOCATIONS, but is AUTHORITATIVE FOR THE CURRENT TILE +LIST (D1). Precisely: + + * a tile in the current list that has find_exposures output has its ``tiles`` + row replaced and its ``tile_exposures`` edges DELETED AND REBUILT — the Fe + output is the truth, so a tile whose exposure list shrank or changed must + not keep stale edges to exposures it no longer reads; + * a tile NOT in the current list is left completely untouched, which is what + makes the index span the campaign and lets a later ``clean_exposure`` see + every consuming tile; + * ``exposures`` rows only ever accumulate (``INSERT OR IGNORE``). An exposure + no tile references any more is a harmless orphan: nothing reads that table + except by joining through ``tile_exposures``. It records: @@ -19,16 +29,20 @@ ``find_exposures`` output (``exp_numbers--.txt``), which ``find_exposures_runner`` produces by parsing the tile FITS ``HISTORY`` header. So the build is not a DAG node: ``build()`` is imported and called at PARSE TIME -by the Snakefile of the COMPUTE invocation, after the PREPARE invocation has -produced the tiles' find_exposures output. (There is no ``sp index`` verb; the -CLI below stays for hand-inspection.) It +by the Snakefile of the COMPUTE invocation ONLY — ``SP_PHASE == "compute"``, +which ``bin/sp`` sets — after the PREPARE invocation has produced the tiles' +find_exposures output. No other parse builds anything: a prepare parse or a +passthrough invocation (``sp --unlock``, ``sp --dag``) just loads whatever is +already on disk. (There is no ``sp index`` verb; the CLI below stays for +hand-inspection.) It iterates the *declared* tile list and checks each tile's Fe output at its deterministic path (no globbing — O(tile-list) existence checks, respecting the no-``ls``-at-scale ban). A bad tile costs only that tile: it is recorded in ``missing.json`` and the index is built over the rest. The build fails only if -the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the -check — what the PREPARE phase passes, where absent Fe output is normal), so a -keep-going download storm that lost a few tiles does not cost the run. +the missing *fraction* exceeds ``missing_threshold`` (``None`` disables the check +entirely), so a keep-going download storm that lost a few tiles does not cost the +run. The threshold is checked BEFORE anything is written, so a failed build +leaves the previous index and ``missing.json`` intact. Exposure IDs are stored with their trailing single-char suffix stripped (``2243881p`` -> ``2243881``); that ``exp_base`` is the dedup key and the @@ -81,7 +95,24 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, tile whose exposure list is missing is recorded in ``missing.json`` and the index is built over the rest (a bad tile costs that tile, not the run). The build is fatal only if the missing fraction exceeds ``missing_threshold``. + + ORDER MATTERS: the threshold is evaluated FIRST, from the missing set, and + the database + ``missing.json`` are written only if it passes. A build that + aborts must leave no trace — an aborted parse that had already mutated + durable state was the bug this ordering fixes. + + The write is idempotent, which is what makes it acceptable that the compute + parse runs it even under ``-n``: re-running over an unchanged tree produces + an identical database. """ + missing = [t for t in tile_ids if not exp_list_path(run_dir, t).exists()] + frac = len(missing) / len(tile_ids) if tile_ids else 0.0 + if missing_threshold is not None and frac > missing_threshold: + raise SystemExit( + f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " + f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " + f"Re-run prepare_tiles for them, or raise --missing-threshold.") + db_path.parent.mkdir(parents=True, exist_ok=True) con = sqlite3.connect(db_path) # No DROP: the index accumulates across invocations (D1). @@ -97,34 +128,32 @@ def build(tile_ids: list[str], run_dir: Path, db_path: Path, """ ) - missing = [] + missing_set = set(missing) all_exposures: set[tuple[str, str]] = set() for tile_id in tile_ids: - ra_dir = tile_id.split(".")[0] - exp_file = exp_list_path(run_dir, tile_id) - if not exp_file.exists(): - missing.append(tile_id) + if tile_id in missing_set: continue - exp_pairs = read_exposure_list(exp_file) + ra_dir = tile_id.split(".")[0] + exp_pairs = read_exposure_list(exp_list_path(run_dir, tile_id)) con.execute("INSERT OR REPLACE INTO tiles VALUES (?,?,?)", (tile_id, ra_dir, len(exp_pairs))) - for exp_id, _name in exp_pairs: - con.execute("INSERT OR IGNORE INTO tile_exposures VALUES (?,?)", - (tile_id, exp_id)) + # Replace this tile's edge set wholesale. INSERT OR IGNORE alone only + # ever added, so a tile whose exposure list shrank kept edges to + # exposures it no longer reads — and those stale edges would block it in + # the report and pin those exposures against cleanup. + con.execute("DELETE FROM tile_exposures WHERE tile_id = ?", (tile_id,)) + con.executemany("INSERT INTO tile_exposures VALUES (?,?)", + [(tile_id, exp_id) for exp_id, _ in exp_pairs]) all_exposures.update(exp_pairs) - con.executemany("INSERT OR REPLACE INTO exposures VALUES (?,?)", + # Exposures accumulate: OR IGNORE, never REPLACE (the name never changes, + # and orphans left by a shrunken tile are harmless). + con.executemany("INSERT OR IGNORE INTO exposures VALUES (?,?)", sorted(all_exposures)) con.commit() con.close() (db_path.parent / "missing.json").write_text(json.dumps(missing, indent=2)) - frac = len(missing) / len(tile_ids) if tile_ids else 0.0 - if missing_threshold is not None and frac > missing_threshold: - raise SystemExit( - f"Missing exposure lists for {len(missing)}/{len(tile_ids)} tile(s) " - f"(fraction {frac:.3f} > threshold {missing_threshold}): {missing}. " - f"Re-run prepare_tiles for them, or raise --missing-threshold.") return {"n_tiles": len(tile_ids) - len(missing), "n_exposures": len(all_exposures), "n_missing": len(missing)} diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index dd0836c6..1a8820c1 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -9,17 +9,24 @@ sits between ``floor`` and ``expect`` and is tolerated. This file is also the ``check`` CLI — the second half of every rule's shell line -(PRD D2/D3):: +(PRD D2/D3). The rules capture ShapePipe's return code rather than ``&&``-ing +onto it, so the check runs — and the manifest is written — even when +``shapepipe_run`` failed:: - shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} \\ - && completeness.py check exp_mask {output} + rc=0 + shapepipe_run -c $SP_CONFIG/config_exp_Ma.ini -b {threads} || rc=$? + completeness.py check exp_mask {output} || rc=1 + exit $rc It counts the unit's products under ``$SP_RUN``, writes the manifest at ``{output}``, and exits nonzero iff a mandatory runner is below its floor. The -manifest is ALWAYS written first, failure included: it is the DAG's currency and -the only thing ``run_report.py`` reads, so a failed unit must still leave a -record of *why*. Manifests carry no wall-clock — identical on-disk state must -produce a byte-identical manifest, or the mtime rerun-trigger churns the cone. +manifest is ALWAYS written, failure included: it is the DAG's currency and the +only thing ``run_report.py`` reads, so a failed unit must still leave a record of +*why*. (The profile sets ``keep-incomplete`` so snakemake does not delete that +record on the way out.) Manifests carry no wall-clock, and are rewritten ONLY +when their content changes — identical on-disk state must leave a byte-identical +manifest with an UNMOVED mtime, or the mtime rerun-trigger churns the cone on +every unrelated ``--forcerun``. Per-runner fields: expect nominal file count for a fully complete unit (report yardstick) @@ -246,11 +253,15 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): return manifest, ok -def _unit_from_env(): - """``SP_UNIT_NUM`` carries the rules' dashed, dash-prefixed form - (``-210-282``, ``-2605805``); the manifest records the human ID.""" - raw = os.environ.get("SP_UNIT_NUM", "") - return raw.lstrip("-") or "unknown" +def _unit_from_run_dir(run_dir): + """The human unit ID: the basename of ``$SP_RUN`` (``210.282``, ``2605805``). + + NOT ``SP_UNIT_NUM``, which carries ShapePipe's dashed numbering form + (``-210-282``) and would put ``210-282`` in the manifest — a key that joins + to nothing. ``run_report`` keys units on the store directory name, which is + exactly this basename, so the two now agree. + """ + return Path(str(run_dir)).name or "unknown" def main(argv=None) -> int: @@ -261,7 +272,8 @@ def main(argv=None) -> int: c.add_argument("manifest", type=Path) c.add_argument("--run-dir", type=Path, default=None, help="the unit's $SP_RUN (default: the env var)") - c.add_argument("--unit", default=None, help="override $SP_UNIT_NUM") + c.add_argument("--unit", default=None, + help="override the unit ID (default: basename of $SP_RUN)") c.add_argument("--stage-dir", default=None, help="override the run_sp_* subdir (default: the stage table)") args = p.parse_args(argv) @@ -271,11 +283,17 @@ def main(argv=None) -> int: print("[completeness] FATAL: $SP_RUN unset and --run-dir not given", file=sys.stderr) return 2 - unit = args.unit or _unit_from_env() + unit = args.unit or _unit_from_run_dir(run_dir) manifest, ok = build_manifest(args.stage, Path(run_dir), unit, args.stage_dir) args.manifest.parent.mkdir(parents=True, exist_ok=True) - args.manifest.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + # Write ONLY on change. An unconditional write moves the mtime on every run, + # and mtime is a rerun-trigger: a `--forcerun` of one upstream stage would + # then rewrite this manifest byte-identically and drag the whole downstream + # cone along with it. + text = json.dumps(manifest, indent=2, sort_keys=True) + "\n" + if not args.manifest.exists() or args.manifest.read_text() != text: + args.manifest.write_text(text) for runner, r in manifest["runners"].items(): tag = {"complete": "OK", "warn": "warn", "below_floor": "<-- BELOW floor"} diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 05ca9247..1a8d128f 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -206,9 +206,24 @@ def main() -> None: "exposures": unit_rows(exps, EXP_STAGES, exp_m), } - # Blame propagation: an incomplete exposure blocks every tile that reads it. + # Blame propagation: a BLOCKING exposure blocks every tile that reads it. # Without this, a tile stalled at tile_vignets looks like its own failure. - bad_exp = {r["unit"] for r in report["exposures"]} + # + # Blocking means "failed" or "never ran" — NOT "warn". Warn is the expected + # per-CCD attrition (setools rejecting a sparse CCD, psfex_interp short an + # epoch); it is present in essentially every exposure at production scale, so + # counting it here made every exposure block every tile and the table said + # nothing. + # Judged over ALL the exposure's stages, not just the first bad one, so an + # exposure that warns early and fails late still blocks. + def _blocks(unit): + for stage in EXP_STAGES: + m = exp_m.get(unit, {}).get(stage) + if m is None or m.get("status", "failed") == "failed": + return True + return False + + bad_exp = {e for e in exps if _blocks(e)} blocked = {t: sorted(set(tile_exp.get(t, [])) & bad_exp) for t in tiles} report["tiles_blocked_by_exposures"] = {t: e for t, e in blocked.items() if e} From 12ecfc2b1c1cc4b9c88e60c38798420d392e5a89 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 23:13:06 -0400 Subject: [PATCH 16/21] feat(orchestration): clean_exposure becomes an in-DAG rule with tombstones (#848 D5, S5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposure-store reclamation moves from a standalone script you remember to run into the DAG itself. One `clean_exposure` job per exposure takes every consuming tile's `tile_vignets` manifest as `input:` — vignets is the last stage that reads exposure products — deletes the store, and leaves a `cleaned.json` tombstone as its output. Writer, then readers, then cleaner: DAG-ordered, race-free, rolling mid-run while ngmix still grinds. The consumer set comes from the accumulating index (new EXP_TILES inverse edge), so it spans the campaign, not one invocation. The tombstone / late-append interaction, which is the whole design problem here: * The job deletes the exposure's `manifests/` along with its `output/`. That is load-bearing. The manifests are the exposure rules' DECLARED outputs; had they survived, a tile appended after the clean would find the exposure chain "up to date" and run its vignets against a store that is no longer on disk. With them gone the DAG sees an unbuilt chain and regenerates it — the accepted cost of a late append, expressed as ordinary Snakemake bookkeeping rather than a special case. The tombstone deliberately does NOT stand in for those manifests: it is input to nothing, so it can never mask their absence. * Tiles already finished are not disturbed: Snakemake demands a missing intermediate only when something downstream of it must run. * `params.consumers` carries the consumer set, so growing it makes the tombstone stale under the default `params` rerun-trigger; the clean job then reruns after the new tile's vignets, against the enlarged set. Cleaned once per consumer set, not once per campaign. * Nothing is lost to the report: each manifest's content is copied verbatim into the tombstone before deletion. Scope guard: an exposure is only eligible when every consuming tile is either in this run's scope or already has its vignets on disk. Without that parse-time test, requesting a tombstone for an exposure shared with a later batch would drag that batch's whole tile chain into this DAG through the clean rule's input — scope expansion by cleanup. Ineligible exposures are deferred, never lost: the invocation that finishes their last consumer picks them up. Gated by `clean:` in workflow/config.yaml (default false for the 4-tile P0 run, where the exposure store is exactly what you want to inspect) and by SP_PHASE=compute, so prepare and passthrough parses schedule no deletions. Flipping it on later reclaims retroactively — the missing tombstones schedule exactly the outstanding clean jobs. Invariants held: no temp() on exposure-level outputs; the loud WorkflowError on zero ready tiles; keep-incomplete untouched. Dry-runs on the 210/211 quad (19 exposures): prepare 13; compute 145 with clean off (identical to b149e187 under the same disk state); compute 164 with clean on = 145 + 19 clean_exposure. Prepare and passthrough parses show no clean rules and leave the index untouched. Deviation from the gap-map wording: one wildcarded rule with an index-driven input function, not literally one generated rule object per exposure. The DAG is identical, and at DR6 scale ~20k rule objects is a parse cost with nothing bought. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- workflow/README.md | 173 ++++++++++++++++++---------- workflow/Snakefile | 54 ++++++++- workflow/config.yaml | 10 ++ workflow/rules/exposure.smk | 40 +++++++ workflow/scripts/clean_exposure.py | 82 +++++++++++++ workflow/scripts/clean_exposures.py | 64 ---------- 6 files changed, 293 insertions(+), 130 deletions(-) create mode 100644 workflow/scripts/clean_exposure.py delete mode 100644 workflow/scripts/clean_exposures.py diff --git a/workflow/README.md b/workflow/README.md index 4d8abd1f..f42a7630 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -1,56 +1,88 @@ # ShapePipe Snakemake orchestration -Snakemake workflow that orchestrates real-data ShapePipe runs, replacing the +Snakemake workflow that orchestrates real-data ShapePipe runs. It replaces the `curl_canfar_local.sh → run_job_sp_canfar_v2.0.bash → job_sp_canfar_v2.0.bash` -bash layers (and per-site sbatch reimplementations). **Module code is untouched**: -rules call `shapepipe_run -c ` on the existing config chains. Design and -rationale: [CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) +bash layers and the per-site sbatch reimplementations. **Module code is +untouched**: rules call `shapepipe_run -c ` on the existing config +chains. Design and rationale: +[CosmoStat/shapepipe#848](https://github.com/CosmoStat/shapepipe/issues/848) (the living PRD). -## Quick start (nibi) +Use `workflow/bin/sp` for everything. Bare `snakemake all` outside `sp` is +unsupported: `sp` sets the state directory, the SLURM profile, and +`SP_PHASE`, which the Snakefile needs to build the tile/exposure index at +parse time. Running snakemake directly skips all of that. -**One-allocation mode.** `sbatch` one node, then run Snakemake with the *local* -scheduler inside it: it bin-packs the DAG into the allocation (no per-job -sbatch). Each job's shell runs `shapepipe_run` **inside the container** (the -profile's apptainer software-deployment — you never type `apptainer`). +## Quick start (nibi) ```bash -# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which is -# node-local; the executor re-invokes this python inside jobs). +# One-time: a snakemake env on a SHARED filesystem (/project — NOT /tmp, which +# is node-local; the SLURM executor re-invokes this python inside every job). uv venv /project/def-mjhudson/cdaley/snakemake-env --python 3.12 source /project/def-mjhudson/cdaley/snakemake-env/bin/activate -uv pip install 'snakemake>=9,<10' +uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' -# Edit workflow/config.yaml: tile_list, run_dir, config_src, container, star_cats. +# Edit workflow/config.yaml: tile_list, run_dir, container, star_cats. -# The committed launcher loads apptainer/1.4.5 + the /project venv, so a fresh -# shell always has the right state. Inside your allocation: -workflow/bin/sp prepare # prepare_tiles -> build_index.py -> prepare_exposures -workflow/bin/sp run # the main compute DAG +# The committed launcher loads apptainer/1.4.5 + the /project venv, so a +# fresh shell always has the right state. +workflow/bin/sp run # bring products on disk up to date with the tile list +workflow/bin/sp report # emit run_report.json now (mid-run is fine) +workflow/bin/sp cancel # scancel this workflow's jobs ``` -`sp` subcommands: `prepare` (chains all three prepare steps), `prepare_tiles`, -`index`, `prepare_exposures`, `run`, `rerun ` (forced recompute of -protected products), `cancel ` (scancel sweep before `--unlock`), -`clean-exposures [--apply]`, `report`. +Installed and pinned versions on nibi (`/project/def-mjhudson/cdaley/snakemake-env`, +queried 2026-07-30): `snakemake==9.23.1`, `snakemake-executor-plugin-slurm==2.7.1`. +Pin range: `snakemake>=9,<10`, `snakemake-executor-plugin-slurm>=2.7,<3`. The +v8→v9 breaks matter here: `--use-singularity` became `--sdm`, executors became +plugins, and full `rerun-triggers` became the default. + +Anything other than `run`, `report`, `cancel` passes straight through to +snakemake with the workflow's profile and state dir — the escape hatch for +`sp --unlock`, `sp --dag`, `sp exp_psf ...`. + +## Execution: two static invocations + +The exposure job set is data-derived from the tiles' `find_exposures` output, +so it cannot live in the same static DAG that produces it. `sp run` is +therefore two snakemake invocations over one Snakefile: + +1. **PREPARE** — `snakemake prepare_all_tiles`: per-tile static DAG + (`Git_vos → Uz → Fe`), `keep-going` so tile failures are independent. A + nonzero exit here is not fatal to the run — tiles that lost their + exposure list are dropped at the compute parse — but it is a warning: + `SP_MISSING_THRESHOLD` (default 0.0) is the real gate. +2. **COMPUTE** — `snakemake all`: this invocation's *parse* builds the + tile↔exposure index (`build_index.py`, imported at parse time, not a DAG + node) and runs the full tile/exposure compute chain. The index + accumulates across invocations, so appending tiles later changes which + jobs exist without invalidating completed work. + +`sp run` chains both so the UX is one command; both exit codes are checked +and the run fails if either phase failed. -## Execution is three static invocations +## Execution mode: one SLURM job per rule -The exposure job set is data-derived from the tiles' `find_exposures` output, so -it cannot be scheduled in the same static DAG that produces it. Hence: +The profile (`profiles/nibi/config.yaml`) sets `executor: slurm`. Every rule +instance becomes its own SLURM job carrying that rule's own attempt-scaled +resources (`cpus_per_task = threads`, `mem_mb`, `runtime`) and runs inside +`apptainer exec` via the profile's software-deployment method — the workflow +never calls apptainer directly. Snakemake feeds the queue as jobs finish, so +the full campaign's job count never needs to queue at once, and multi-node +scaling is inherent. `jobs:` in the profile caps concurrent submissions at a +fraction of the cluster's per-user submit limit (queried, not invented — see +the comment in the profile file for the query and date). -1. `snakemake prepare_tiles` — per-tile static DAG (Git_vos → Uz → Fe), - `keep-going` so tile failures are independent. -2. `build_index.py` — a **plain script**, not a DAG node. It iterates the - declared tile list, checks each tile's Fe output at its deterministic path (no - glob), writes `run_index.sqlite` + `missing.json`, and exits nonzero only if - the missing *fraction* exceeds a threshold. (There is no `--allow-missing` - Snakemake flag; that mechanism does not exist.) -3. `snakemake prepare_exposures` — per-exposure static DAG (Gie_vos + per-unit - star cats), parseable now the index exists. -4. `snakemake` (main DAG) — the tile/exposure compute chain. +The profile intentionally sets no `set-resources` / `set-threads` overrides: +those replace a rule's own values wholesale, which would kill the +attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned +ngmix thread count. Rules own their own resources; the profile only supplies +defaults for rules that state nothing. -`sp prepare` chains 1–3 so the UX stays one command. +`group:` labels that fuse short rules (uncompress, merges) into their chunky +neighbours (queue-latency amortization, per the PRD) are **not yet wired**: +they require labels in `workflow/rules/*.smk`, out of scope for this +profile-only pass. ## Layout @@ -58,7 +90,7 @@ it cannot be scheduled in the same static DAG that produces it. Hence: workflow/ Snakefile parse-time index load; global container:; onsuccess/onerror report hooks config.yaml the run: tile list, paths, container, chunk count - bin/sp committed launcher (module load + /project venv + subcommands) + bin/sp committed launcher (module load + /project venv + run/report/cancel) rules/ prepare.smk tile get_images/uncompress/find_exposures + per-unit star cats exposure.smk per-exposure: get_images, split, mask, psf (no temp()) @@ -69,37 +101,50 @@ workflow/ build_forest.py per-tile exposure symlink forest (group-compatible shell) completeness.py the ported count-floor table (shared by sp_rule + run_report) run_report.py standalone report (NOT a DAG node; run_report hooks call it) - clean_exposures.py index-driven exposure-store reclaim (replaces temp()) -profiles/nibi/config.yaml local scheduler; apptainer SDM; resources; keep-going + clean_exposure.py ONE exposure's store + manifests -> tombstone (the clean_exposure rule) +profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; keep-going ``` ## How it works -- **Completeness = the count floor.** Each stage declares its deterministic - `output/run_sp_/` directory as `directory()` output (the per-unit - config copy forces `RUN_DATETIME = False`, so the path is known at DAG time). - After the run, `sp_rule.py` counts products per mandatory runner against - `completeness.py`'s floor and exits nonzero below it; per-CCD attrition between - floor and `expect` is tolerated (the directory simply lacks that product). - This *is* the failure policy — no 3-class taxonomy, no error-signature - whitelist. `--keep-going` isolates a failure to its own cone. -- **Per-unit isolation is by work-dir content, not `-e/--exclusive`.** Every - tile/exposure runs its own `shapepipe_run` in `tiles//` or `exp//`, - with a `cfis` config symlink, `star_cat_{exp,tiles}` symlinks, a - `tile_numbers.txt` (tiles) or fabricated `exp_numbers-000-000.txt` pseudo-Fe - (exposures), and a config copy setting `RUN_DATETIME=False` and - `SMP_BATCH_SIZE={threads}`. `NUMBER_LIST=-` is injected **only** for - stages whose numbering scheme is the unit ID (tile-scheme stages + exp split), - never for get_images / exp_mask / exp_psf. -- **No `temp()` on shared exposure dirs.** Exposures overlap tiles, so temp() - would cascade destructive reruns when a tile is appended. The store is - persistent; `sp clean-exposures` reclaims space only for exposures all of - whose consuming tiles have a final_cat. -- **The index is parse-time data, never a rule input.** Appending tiles changes - which jobs exist without invalidating completed work. Star cats are re-keyed - per unit for the same reason. `final_cat` is `protected()`. +- **The atom is one rule == one `shapepipe_run` on one unit.** Its single + declared output is that unit's manifest + (`/manifests/.json`), not its product files — a missing + CCD is often legitimate, and at DR6 scale per-CCD declaration means + millions of paths. +- **Manifests are the DAG's currency.** `completeness.py check` writes the + manifest, and it is the only record of *why* a unit failed. That is why + the profile sets `keep-incomplete: true` — Snakemake's default deletes a + failed job's declared outputs, which would erase the manifest the report + needs. +- **Completeness is a count floor, not a taxonomy.** After a run, + `sp_rule.py` counts products per mandatory runner against + `completeness.py`'s floor and exits nonzero below it. Per-CCD attrition + between floor and `expect` is tolerated. No 3-class taxonomy, no + error-signature whitelist. `--keep-going` isolates a failure to its own + DAG cone. +- **Stores are sharded.** Every tile/exposure runs its own `shapepipe_run` + in `tiles/<2-char prefix>//` or `exp///`, with a `cfis` + config symlink, `star_cat_{exp,tiles}` symlinks, and a config copy setting + `RUN_DATETIME=False`. Configs are committed under `workflow/config/cfis/` + and version with the rules that set the env vars they interpolate — there + is no `config_src` knob, by construction. +- **The index is parse-time data, never a rule input.** Appending tiles + changes which jobs exist without invalidating completed work. Star cats + are re-keyed per unit for the same reason. `final_cat` is `protected()`. +- **Exposure products are not `temp()`.** Exposures overlap tiles, so + `temp()` would cascade destructive reruns when a tile is appended later. + Reclamation is the in-DAG `clean_exposure` rule instead: one job per + exposure, taking every consuming tile's `tile_vignets` manifest as input + (the campaign-wide consumer set comes from the accumulating index), which + deletes the store *and* the exposure's manifests and leaves a `cleaned.json` + tombstone. Deleting the manifests is what makes a late append correct: the + appended tile finds an unbuilt chain and regenerates it. The `clean:` flag + in `config.yaml` gates it; flipping it on later reclaims retroactively, + since the missing tombstones schedule exactly the outstanding clean jobs. - **Failure is a report, not a gate.** `run_report.py` disk-scans the trees against the count table and enumerates shortfalls (whole-unit absence vs - per-CCD attrition). It is a standalone script — a DAG report node would be - poisoned by the very failures it must enumerate — emitted automatically by the - `onsuccess`/`onerror` hooks and runnable any time via `sp report`. + per-CCD attrition). It runs standalone — a DAG report node would itself be + poisoned by the failures it must enumerate — and fires automatically from + the COMPUTE invocation's `onsuccess`/`onerror` hooks, or on demand via + `sp report`. diff --git a/workflow/Snakefile b/workflow/Snakefile index 7e9300c5..50dc7577 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -93,12 +93,15 @@ if PHASE == "compute": # EXP: exposure base-id -> original name (2605805 -> 2605805p; the name goes # verbatim into the fabricated per-unit exp_numbers list so get_images matches # .fits.fz in the store). TILE_EXP: tile -> [exp_ids]. -EXP, TILE_EXP = {}, {} +# EXP_TILES is the inverse edge — the CAMPAIGN-WIDE consumer set clean_exposure +# is keyed on (D5). +EXP, TILE_EXP, EXP_TILES = {}, {}, {} if INDEX_DB.exists(): _con = sqlite3.connect(INDEX_DB) EXP = dict(_con.execute("SELECT exp_id, name FROM exposures")) for _tile, _exp in _con.execute("SELECT tile_id, exp_id FROM tile_exposures"): TILE_EXP.setdefault(_tile, []).append(_exp) + EXP_TILES.setdefault(_exp, []).append(_tile) _con.close() # Tiles this run can actually compute: declared AND indexed. The index spans the @@ -162,6 +165,52 @@ def unit_num(unit): # gets its own hash, on the forest rule only. SCRIPT_HASH = hashlib.md5((SCRIPTS / "completeness.py").read_bytes()).hexdigest()[:12] FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest()[:12] +CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] + +# --- exposure reclamation (D5, S5) ----------------------------------------- +# `clean:` in config.yaml gates the whole mechanism. Off => the rule generates no +# jobs at all (nothing requests a tombstone); flipping it on later reclaims +# RETROACTIVELY, because the exposures already cleaned are exactly the ones with +# a tombstone, so the missing tombstones schedule exactly the clean jobs. +# Only ever active under SP_PHASE=compute: the prepare and passthrough parses +# read no index of their own and must schedule no deletions. +CLEAN = bool(config.get("clean", False)) and PHASE == "compute" + + +def tombstone(exp): + """The clean_exposure output. Lives BESIDE manifests/, not inside it: the + clean job deletes manifests/ wholesale, and `sp report` scans it.""" + return f"{exp_dir(exp)}/cleaned.json" + + +def clean_consumers(exp): + """Every tile in the campaign that reads this exposure — the set whose + vignets must all exist before the store may go.""" + return sorted(EXP_TILES.get(exp, [])) + + +def clean_targets(): + """Which exposures this invocation may clean. + + An exposure is eligible only when every consuming tile is either in this + run's scope or has already produced its vignets on disk. Without that test, + requesting a tombstone for an exposure shared with a LATER batch would drag + that batch's whole tile chain into this DAG through the clean rule's input — + scope expansion by cleanup, which is not a trade anyone asked for. Ineligible + exposures are simply skipped; the invocation that finishes their last + consumer picks them up. Deferral, never loss. + """ + if not CLEAN: + return [] + ready = set(TILES_READY) + out = [] + for exp, tiles in EXP_TILES.items(): + if not tiles: + continue + if all(t in ready or Path(tile_manifest(t, "tile_vignets")).exists() + for t in tiles): + out.append(tombstone(exp)) + return sorted(out) # --- the shell every rule runs (D2) ---------------------------------------- @@ -264,7 +313,8 @@ localrules: all, prepare_all_tiles rule all: input: - [final_cat(t) for t in TILES_READY] + [final_cat(t) for t in TILES_READY], + clean_targets(), # Invocation 1 — the static per-tile DAG, known from the tile list alone. # keep-going makes tile failures independent; the ones that lose their exposure diff --git a/workflow/config.yaml b/workflow/config.yaml index 942199b1..ab06a459 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -32,6 +32,16 @@ star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats # 60-day purge for anything that must survive. index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite +# Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one +# `clean_exposure` job per exposure. It fires once every campaign tile that reads +# that exposure has its vignets, deletes the exposure's store AND its manifests, +# and leaves `cleaned.json` (which absorbs the manifests, so the report keeps +# them). Off here because P0 is a 4-tile debugging run where the exposure store +# is exactly what you want to inspect; on for any run big enough to care about +# disk. Flipping it on later reclaims retroactively — the missing tombstones +# schedule exactly the outstanding clean jobs. +clean: false + # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). ngmix_chunks: 8 diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 66790622..b752bbee 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -88,3 +88,43 @@ rule exp_psf: runtime = 240 shell: sp_shell("exp_psf", "config_exp_psfex.ini") + + +# --- reclamation (D5) ------------------------------------------------------- +# The one exception to "no reclamation in this file": clean_exposure OWNS +# exposure-level deletion, and it is a real job, not temp() bookkeeping, because +# an exposure's consumer set closes over the CAMPAIGN. The index supplies that +# set (EXP_TILES, accumulated across invocations); the input is every consuming +# tile's tile_vignets manifest — vignets is the last stage that reads exposure +# products, everything after it reads tile-level files. +# +# Three properties make the late append behave (see clean_exposure.py): +# * the job deletes the exposure's manifests too, so a tile appended after the +# clean sees an unbuilt chain and regenerates it instead of running against +# an empty store. The tombstone deliberately does NOT stand in for those +# manifests — it is not an input to anything but itself. +# * a finished tile is not disturbed: Snakemake demands a missing intermediate +# only when something downstream of it must run. +# * params.consumers carries the consumer set, so growing it makes the +# tombstone stale under the default `params` rerun-trigger; the clean job +# reruns after the new tile's vignets, against the enlarged set. +# +# Not a localrule: at DR6 scale these are ~20k rmtrees and the local-cores cap +# would serialise them behind the aggregation targets. +rule clean_exposure: + input: + lambda wc: [tile_manifest(t, "tile_vignets") + for t in clean_consumers(wc.exp)] + output: + tombstone = f"{EXP_DIR}/cleaned.json" + params: + consumers = lambda wc: ",".join(clean_consumers(wc.exp)), + script_hash = CLEAN_HASH + threads: 1 + resources: + mem_mb = 2000, + runtime = 30 + shell: + f"python {SCRIPTS}/clean_exposure.py" + " --exp-dir $(dirname {output.tombstone}) --exp {wildcards.exp}" + " --tombstone {output.tombstone} --consumers '{params.consumers}'" diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py new file mode 100644 index 00000000..2d3d40cc --- /dev/null +++ b/workflow/scripts/clean_exposure.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Reclaim ONE exposure's store and leave a tombstone (PRD #848 D5, S5). + +Run as the shell of the in-DAG ``clean_exposure`` rule, never by hand: the rule's +``input:`` is every consuming tile's ``tile_vignets`` manifest, so by the time +this executes, every campaign tile that reads this exposure has already extracted +its postage stamps. Writer, then readers, then cleaner — DAG-ordered, race-free. + +What it deletes: the exposure's whole ``output/`` tree (the bulk store — +run_sp_exp_Gie/Sp/Ma/SxSePsfPi) AND its ``manifests/``. Deleting the manifests is +deliberate and load-bearing, not tidiness: + + * the manifests are the exposure rules' DECLARED outputs. If they survived, a + tile appended later would find the exposure chain "up to date" and run + tile_vignets against products that are no longer on disk. With them gone the + DAG sees the chain as unbuilt and regenerates it — the accepted cost of a + late append (D5), expressed as ordinary Snakemake bookkeeping rather than as + a special case. + * Snakemake only demands a missing intermediate when something downstream of it + needs to run, so tiles already finished are NOT rerun by their exposures' + manifests vanishing. + +Nothing is lost to the report: each manifest's content is copied verbatim into +the tombstone under ``manifests`` before deletion, so `sp report` can still say +what this exposure produced and why any runner was short. + +The tombstone records the consumer set it was cleaned against. The rule carries +that same set as a ``params`` value, so when the index grows a new consumer the +tombstone goes stale under the default ``params`` rerun-trigger and the clean job +is rescheduled after the new tile's vignets — the exposure is cleaned once per +consumer set, not once per campaign. +""" + +import argparse +import json +import shutil +import time +from pathlib import Path + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--exp-dir", required=True, type=Path) + p.add_argument("--exp", required=True) + p.add_argument("--tombstone", required=True, type=Path) + p.add_argument("--consumers", default="", + help="comma-separated tile ids this exposure was cleaned against") + args = p.parse_args() + + consumers = [t for t in args.consumers.split(",") if t] + + # Absorb the manifests before they go: the tombstone becomes the exposure's + # surviving record. + manifests = {} + mdir = args.exp_dir / "manifests" + if mdir.is_dir(): + for f in sorted(mdir.glob("*.json")): + try: + manifests[f.stem] = json.loads(f.read_text()) + except (OSError, json.JSONDecodeError) as exc: + manifests[f.stem] = {"unreadable": str(exc)} + + removed = [] + for target in (args.exp_dir / "output", mdir): + if target.exists(): + shutil.rmtree(target) + removed.append(str(target)) + + args.tombstone.parent.mkdir(parents=True, exist_ok=True) + args.tombstone.write_text(json.dumps({ + "exp": args.exp, + "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), + "consumers": consumers, + "removed": removed, + "manifests": manifests, + }, indent=2) + "\n") + print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " + f"{len(consumers)} consuming tile(s)") + + +if __name__ == "__main__": + main() diff --git a/workflow/scripts/clean_exposures.py b/workflow/scripts/clean_exposures.py deleted file mode 100644 index 4f46bf19..00000000 --- a/workflow/scripts/clean_exposures.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 -"""Index-driven exposure-store cleanup (`sp clean-exposures`). - -temp() is deliberately NOT used on the shared exposure directories (findings -4/5/6/19): they overlap tiles by construction, so temp() deletion would cascade -destructive reruns across spatial neighbours whenever a tile is appended. Instead -the store is persistent and space is reclaimed explicitly here. - -For each exposure in run_index.sqlite, delete its bulky intermediates -(run_sp_exp_Sp / run_sp_exp_Ma / run_sp_exp_SxSePsfPi) ONLY when every tile that -consumes it (tile_exposures) has its final_cat-.fits present across the whole -output root. An exposure still feeding an unfinished tile is left untouched. - -Caveat: if a later batch appends a tile that shares a deleted exposure, -Snakemake will regenerate that exposure — cleanup trades peak storage for -recompute. Run it between campaign phases, not mid-DAG. --dry-run by default. -""" - -import argparse -import shutil -import sqlite3 -import sys -from pathlib import Path - -PREFIXES = ("run_sp_exp_Sp", "run_sp_exp_Ma", "run_sp_exp_SxSePsfPi") - - -def main() -> None: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--run-dir", required=True, type=Path) - p.add_argument("--index", required=True, type=Path) - p.add_argument("--apply", action="store_true", - help="actually delete (default: dry-run, list only)") - args = p.parse_args() - - con = sqlite3.connect(args.index) - edges = con.execute("SELECT exp_id, tile_id FROM tile_exposures").fetchall() - con.close() - - consumers: dict[str, list[str]] = {} - for exp_id, tile_id in edges: - consumers.setdefault(exp_id, []).append(tile_id) - - def tile_done(tile): - return (args.run_dir / "tiles" / tile / f"final_cat-{tile}.fits").exists() - - freed, held = 0, 0 - for exp_id, tiles in consumers.items(): - if not all(tile_done(t) for t in tiles): - held += 1 - continue - for prefix in PREFIXES: - d = args.run_dir / "exp" / exp_id / "output" / prefix - if d.is_dir(): - freed += 1 - print(f"{'DELETE' if args.apply else 'would delete'} {d}") - if args.apply: - shutil.rmtree(d) - print(f"[clean_exposures] {freed} dirs {'freed' if args.apply else 'reclaimable'}, " - f"{held} exposures held (consuming tile unfinished)", file=sys.stderr) - - -if __name__ == "__main__": - main() From e677f679f41b398d7630ce1aeee9494abbef9726 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 30 Jul 2026 23:13:49 -0400 Subject: [PATCH 17/21] feat(orchestration): nibi profile goes slurm-executor (#848 D4, S6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The profile flips from the P0 one-allocation/local-scheduler mode to the PRD's executor: slurm. jobs: 800 = 80% of the queried MaxSubmitPU=1000 on def-mjhudson (sacctmgr, 2026-07-30). default-resources carry mem_mb, runtime, slurm_account: def-mjhudson; keep-incomplete stays (a failed job's manifest is its only post-mortem record). No set-resources/set-threads — rules own their attempt-scaled lambdas. Env pins recorded in the README: snakemake>=9,<10, snakemake-executor-plugin-slurm>=2.7,<3 (installed: 9.23.1 / 2.7.1). group: fusion deferred — it needs rule edits, tracked in the README. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 41 ++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 029329aa..7819a0fd 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -1,20 +1,29 @@ # Snakemake profile for the Nibi cluster (Digital Research Alliance). # -# ONE-ALLOCATION mode: sbatch one node up front, then run snakemake with the -# LOCAL scheduler inside it (no per-job sbatch; the local scheduler bin-packs the -# DAG into the allocation against `cores` + `resources: mem_mb`). This is the -# proven nibi path (job_p3_batch1) and the single honest first-pass mode; -# multi-node scaling = multiple tile-disjoint invocations, deferred. +# SLURM-EXECUTOR mode (PRD #848 D-profile): one SLURM job per rule instance, +# each carrying its rule's own attempt-scaled resources (cpus_per_task, +# mem_mb, runtime). Snakemake feeds the queue as jobs finish, so DR6's ~170k +# total jobs never queue at once, and multi-node scaling is inherent — this +# supersedes the earlier one-allocation/local-scheduler mode. # # Launch via `workflow/bin/sp` (loads apptainer/1.4.5, uses the /project venv). # software-deployment-method wraps every job's shell in `apptainer exec` — the # user never types apptainer; the container is set in the Snakefile (local .sif). -# Local scheduler (no `executor:` -> built-in local). Size to the allocation. -cores: 192 # node cores available to the DAG -local-cores: 4 # cap localrule/aggregation concurrency -resources: - mem_mb: 720000 # node RAM cap the local scheduler bin-packs under +executor: slurm + +# Per-user submit cap, queried 2026-07-30 on nibi: +# sacctmgr show assoc user=cdaley format=Account,MaxSubmitPU -P +# -> def-mjhudson_cpu 1000, def-mjhudson_gpu 1000 (MaxSubmitPU; no site-wide +# MaxSubmitJobs in `scontrol show config`, so the association limit governs) +# Set to ~80% of that (1000) so this workflow never starves other submissions +# under the same account. Re-query if the association limits change. +jobs: 800 + +default-resources: + mem_mb: 2000 + runtime: 120 # minutes + slurm_account: def-mjhudson software-deployment-method: [apptainer] # Explicit container environment (finding 2/3): the apptainer SDM otherwise drops @@ -50,13 +59,13 @@ keep-incomplete: true # No override — completeness.py writes the manifest only on change, so mtimes # move only when reality moves. -default-resources: - mem_mb: 4000 - runtime: 120 # minutes - cpus_per_task: 1 - # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a # rule's own values (verified snakemake 9.23), which would kill the # attempt-scaled `mem_mb = lambda wc, attempt: ...` OOM retries and the tuned # ngmix thread count. The RULES own threads and resources; this profile only -# sizes the allocation and sets the defaults for rules that state nothing. +# sets defaults for rules that state nothing (default-resources above). +# +# `group:` fusion of the short rules (uncompress, merges) into their chunky +# neighbours (PRD D-profile) is NOT set here — it requires labels in the rule +# files themselves, which is out of scope for this profile-only pass. Deferred +# to whichever slice next touches workflow/rules/*.smk. From 96c78cdb1fbb2f48c46b96f1f9c351738718ec60 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:08:07 -0400 Subject: [PATCH 18/21] fix(orchestration): stop the clean_exposure cascade; report reads tombstones (#848 D5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An adversarial review of the in-DAG clean_exposure design found eight problems. This fixes all eight. The central one is a rerun cascade. THE CASCADE, AND WHAT ACTUALLY CUT IT clean_exposure deletes an exposure's manifests on purpose: that is what makes a tile appended later rebuild the chain instead of running against an empty store. But those manifests are the tile side's inputs, and an exposure is read by ~7-10 tiles. So one rebuilt exposure made every other tile that reads it rerun, and that rerun rebuilt ITS exposures, spreading across the whole exposure-overlap connected component. On fixture t4 (four tiles chained by shared exposures, all complete, all exposures cleaned, one tile's catalogue deleted), asking for the one damaged tile scheduled all four tiles' chains. ancient() alone does NOT fix it, and the measurement says why. Snakemake suppresses the pending-producer propagation only when every shared file is ancient AND EXISTS (dag.py); reclamation has deleted these. Job counts with ancient() alone were identical to before. Three changes cut it, and t4 proves each: * a FINISHED tile (final_cat on disk) drops the exposure manifests that are gone from its input list, and holds the rest through ancient(). It has already extracted everything it will ever read from them, so the propagation has nowhere to go. Only missing manifests are dropped, so a campaign that has cleaned nothing declares exactly the edges it always did. * clean_exposure declares only IN-SCOPE consumers' vignets manifests. An out-of-scope consumer is checked for existence at parse time instead; declaring it pulled that finished tile's whole chain into the DAG. * the profile drops the `input` rerun-trigger. With it on, the conditional edge reads as "set of input files has changed" and reruns the finished tiles anyway — against a store that is gone. Nothing is lost: a changed exposure list still arrives through the find_exposures manifest's mtime, and the ngmix chunk count rides in params. t4, one damaged tile of four: 82 jobs before, 70 with the edge cut but the trigger on, 28 with all three — exactly the damaged tile's chain, its own two exposures, and the clean jobs. Same 28 whether the tile list is the one tile or all four. Appending a tile that shares a cleaned exposure still rebuilds that exposure and reschedules its clean, and touches no other tile (25 jobs). A complete campaign schedules nothing, clean on or off. The cost to know: --forcerun on a tile whose final_cat exists will not rebuild its reclaimed exposures. Delete the final_cat first. THE OTHER SEVEN * sp report read no tombstones, so every cleaned exposure came back "not_run" and blocked the complete tiles whose completion had authorised the deletion. It now reads the absorbed manifests out of cleaned.json, reports the unit as `cleaned` with its warn counts intact, and never blames it for a tile. Fixture t3: 3 exposures "not run" + 2 tiles blocked, before; 3 cleaned + 0 blocked, after. * clean_ignore_tiles: in config.yaml. A permanently-failed tile pinned its ~80 exposures forever; listed there, it leaves the consumer sets. Retrying it later rebuilds those chains from scratch, and the config says so. * the exp_psf benchmark tsv moved out of manifests/, which reclamation deletes. It is the measured-memory feed for mem_mb sizing (D4). * clean_exposure is a localrule. At DR6 scale it was ~20k sbatch submissions to run rmtree. * `--config clean=false` was truthy, so switching reclamation off switched it on. Config booleans now parse strings, and reject what they cannot read. * the tombstone is written, complete and atomically, BEFORE anything is deleted. The crash window now costs disk, never the record. * the false comments in clean_exposure.py and config.yaml are true again. Dry-runs only; no jobs were run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 22 +++++++-- workflow/README.md | 22 +++++++++ workflow/Snakefile | 71 +++++++++++++++++++++++++----- workflow/config.yaml | 25 ++++++++--- workflow/rules/exposure.smk | 26 +++++++++-- workflow/rules/tile.smk | 63 +++++++++++++++++++++++++- workflow/scripts/clean_exposure.py | 36 +++++++++++---- workflow/scripts/run_report.py | 69 ++++++++++++++++++++++++++--- 8 files changed, 294 insertions(+), 40 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 7819a0fd..b79985e6 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -54,10 +54,24 @@ printshellcmds: true # so a rerun never sees stale products. keep-incomplete: true -# rerun-triggers left at the full Snakemake v9 default -# {mtime,params,input,code,software-env}: params/code fixes SHOULD propagate. -# No override — completeness.py writes the manifest only on change, so mtimes -# move only when reality moves. +# rerun-triggers: the v9 default MINUS `input`. params/code/mtime fixes still +# propagate — completeness.py writes the manifest only on change, so mtimes move +# only when reality moves. +# +# `input` is dropped because reclamation needs the tile->exposure edge to be +# CONDITIONAL: a tile whose final_cat is on disk declares no exposure inputs, so +# a neighbour rebuilding a shared exposure cannot drag it along (see tile.smk). +# With the `input` trigger on, that same conditional reads as "set of input files +# has changed" and reruns every finished tile — against an exposure store that +# reclamation has already deleted. Measured on fixture t4: 70 jobs with the +# trigger, 28 without, for one damaged tile in a four-tile chain. +# +# Nothing this workflow relied on is lost. The two genuinely data-derived input +# sets are covered another way: a changed exposure list arrives through the +# (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides +# in params. And the Snakefile refuses to schedule reclamation while the trigger +# is enabled, so a bare snakemake invocation cannot quietly undo this. +rerun-triggers: [mtime, params, code, software-env] # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a # rule's own values (verified snakemake 9.23), which would kill the diff --git a/workflow/README.md b/workflow/README.md index f42a7630..c64047e7 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -142,6 +142,28 @@ profiles/nibi/config.yaml SLURM executor; apptainer SDM; per-user jobs cap; kee appended tile finds an unbuilt chain and regenerates it. The `clean:` flag in `config.yaml` gates it; flipping it on later reclaims retroactively, since the missing tombstones schedule exactly the outstanding clean jobs. + The tombstone is written *before* anything is deleted, so a crash can cost + disk but never the record. +- **A finished tile declares no reclaimed exposures.** Deleting an exposure's + manifests would otherwise rerun every other tile that reads it, and those + reruns spread across the exposure-overlap component. So a tile whose + `final_cat` exists drops the exposure manifests that are gone from its input + list, and holds the rest through `ancient()`. This is why the profile runs + with `rerun-triggers: [mtime, params, code, software-env]`: the `input` + trigger reads that cut as a reason to rerun the very tiles it protects. + Know the consequence — `--forcerun` on a tile whose `final_cat` exists will + not rebuild its reclaimed exposures. Delete the `final_cat` first. +- **A dead tile can be told to stop pinning exposures.** An exposure is + cleanable only once every consuming tile has its vignets, so one + permanently-failed tile holds its ~80 exposures for the life of the + campaign. List it under `clean_ignore_tiles:` in `config.yaml` and it leaves + the consumer sets. Retrying an ignored tile later is legal and expensive: + its exposure chains are gone and rebuild from scratch. +- **A reclaimed exposure reports as `cleaned`.** `run_report.py` reads the + absorbed manifests out of `cleaned.json`, so a reclaimed exposure keeps its + per-runner counts and blocks no tile. The `exp_psf` benchmark tsv lives + beside `manifests/`, not inside it, so reclamation does not eat the + memory-sizing data. - **Failure is a report, not a gate.** `run_report.py` disk-scans the trees against the count table and enumerates shortfalls (whole-unit absence vs per-CCD attrition). It runs standalone — a DAG report node would itself be diff --git a/workflow/Snakefile b/workflow/Snakefile index 50dc7577..419e56cb 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -107,6 +107,7 @@ if INDEX_DB.exists(): # Tiles this run can actually compute: declared AND indexed. The index spans the # campaign, so it is intersected with the declared list, not used as it. TILES_READY = [t for t in TILES if t in TILE_EXP] +READY_SET = set(TILES_READY) # A compute invocation with nothing to compute is never a success. Without this, # an empty intersection yields `rule all` with no inputs, an empty DAG and exit @@ -168,13 +169,50 @@ FOREST_HASH = hashlib.md5((SCRIPTS / "build_forest.py").read_bytes()).hexdigest( CLEAN_HASH = hashlib.md5((SCRIPTS / "clean_exposure.py").read_bytes()).hexdigest()[:12] # --- exposure reclamation (D5, S5) ----------------------------------------- + + +def flag(value, default=False): + """Truthiness for a config value that may arrive as a STRING. + + `--config clean=false` delivers the string "false", and every non-empty + string is truthy in Python — a plain bool() read that as ON and scheduled + the deletions the user had just switched off. YAML booleans pass through + unchanged; only strings are parsed, and an unparseable one is an error, not + a guess. + """ + if value is None: + return default + if isinstance(value, str): + v = value.strip().lower() + if v in ("1", "true", "yes", "on"): + return True + if v in ("0", "false", "no", "off", ""): + return False + raise WorkflowError(f"Cannot read {value!r} as a boolean (use true/false).") + return bool(value) + + # `clean:` in config.yaml gates the whole mechanism. Off => the rule generates no # jobs at all (nothing requests a tombstone); flipping it on later reclaims # RETROACTIVELY, because the exposures already cleaned are exactly the ones with # a tombstone, so the missing tombstones schedule exactly the clean jobs. # Only ever active under SP_PHASE=compute: the prepare and passthrough parses # read no index of their own and must schedule no deletions. -CLEAN = bool(config.get("clean", False)) and PHASE == "compute" +CLEAN = flag(config.get("clean", False)) and PHASE == "compute" + +# Reclamation REQUIRES the `input` rerun-trigger to be off (profiles/nibi sets +# the list; tile.smk explains why). The two are already tied together: CLEAN is +# gated on SP_PHASE=compute, which only workflow/bin/sp sets, and bin/sp always +# launches with that profile. A bare snakemake invocation without SP_PHASE +# schedules no clean job at all, so it cannot meet the incompatible combination. +# There is no runtime assertion because the trigger set is not readable at parse +# time (workflow.dag_settings is None until the DAG is built). + +# Tiles that must not pin an exposure's store. See config.yaml: a permanently +# failed tile otherwise holds every exposure it touches (~80) forever, because +# its vignets manifest will never exist and its exposures are therefore never +# eligible. Listing it here drops it from the consumer sets. +CLEAN_IGNORE_TILES = set(config.get("clean_ignore_tiles") or []) def tombstone(exp): @@ -185,8 +223,9 @@ def tombstone(exp): def clean_consumers(exp): """Every tile in the campaign that reads this exposure — the set whose - vignets must all exist before the store may go.""" - return sorted(EXP_TILES.get(exp, [])) + vignets must all exist before the store may go — minus the ignored tiles.""" + return sorted(t for t in EXP_TILES.get(exp, []) + if t not in CLEAN_IGNORE_TILES) def clean_targets(): @@ -199,15 +238,23 @@ def clean_targets(): scope expansion by cleanup, which is not a trade anyone asked for. Ineligible exposures are simply skipped; the invocation that finishes their last consumer picks them up. Deferral, never loss. + + Consumer sets are the IGNORE-FILTERED ones (clean_consumers), so a tile in + `clean_ignore_tiles` neither gates eligibility nor appears in the job's + input — which is the whole point of that list. """ if not CLEAN: return [] - ready = set(TILES_READY) out = [] - for exp, tiles in EXP_TILES.items(): - if not tiles: + for exp, raw in EXP_TILES.items(): + if not raw: continue - if all(t in ready or Path(tile_manifest(t, "tile_vignets")).exists() + # An empty set AFTER filtering means every consumer is ignored: nothing + # is left that could ever read this exposure, so it is eligible now. + # That is the whole point of clean_ignore_tiles — all() of an empty set + # is True, and it is true here in the intended sense. + tiles = clean_consumers(exp) + if all(t in READY_SET or Path(tile_manifest(t, "tile_vignets")).exists() for t in tiles): out.append(tombstone(exp)) return sorted(out) @@ -307,9 +354,13 @@ include: "rules/exposure.smk" include: "rules/tile.smk" # --- top-level targets ------------------------------------------------------ -# Only the aggregation targets are localrules: a mid-chain localrule would break -# `group:` fusion of the compute chains (D4). -localrules: all, prepare_all_tiles +# Only the aggregation targets and clean_exposure are localrules: a mid-chain +# localrule would break `group:` fusion of the compute chains (D4). +# clean_exposure is safe there because it is not in any chain — it hangs off +# `all` — and it is seconds of rmtree. As an sbatch job it would be ~20k +# scheduler submissions at DR6 scale to delete directories; the local-cores cap +# serialises them instead, which costs nothing at rmtree speed. +localrules: all, prepare_all_tiles, clean_exposure rule all: input: diff --git a/workflow/config.yaml b/workflow/config.yaml index ab06a459..e52aa445 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -35,13 +35,28 @@ index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, -# and leaves `cleaned.json` (which absorbs the manifests, so the report keeps -# them). Off here because P0 is a 4-tile debugging run where the exposure store -# is exactly what you want to inspect; on for any run big enough to care about -# disk. Flipping it on later reclaims retroactively — the missing tombstones -# schedule exactly the outstanding clean jobs. +# and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them +# back out of the tombstone and reports the exposure as `cleaned`. Off here +# because P0 is a 4-tile debugging run where the exposure store is exactly what +# you want to inspect; on for any run big enough to care about disk. Flipping it +# on later reclaims retroactively — the missing tombstones schedule exactly the +# outstanding clean jobs. clean: false +# Tiles that may NOT pin an exposure store (default: empty). +# +# An exposure is eligible for cleaning only once EVERY consuming tile has its +# vignets. One permanently-failed tile therefore holds all ~80 exposures it +# touches for the life of the campaign. A tile listed here is dropped from the +# consumer sets, and its exposures become eligible. +# +# READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its +# exposures' stores. If you later retry that tile, those exposure chains are +# gone and will be REBUILT from scratch — get_images, split, mask, psf, per +# exposure. That is correct, and expensive. Ignore a tile when you have decided +# it is dead, not while you are still debugging it. +clean_ignore_tiles: [] + # ngmix within-tile chunking: static N chunks (closed ID ranges computed # per-tile, in-job, from the tile's own sexcat). ngmix_chunks: 8 diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index b752bbee..e04dd76f 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -82,7 +82,11 @@ rule exp_psf: threads: 8 retries: 2 benchmark: - f"{EXP_DIR}/manifests/exp_psf.benchmark.tsv" + # BESIDE manifests/, not inside it: clean_exposure deletes manifests/ + # wholesale, and this tsv is the measured-memory feed for mem_mb sizing + # (D4). Inside manifests/ it died with the first reclamation and took + # the campaign's only record of exp_psf's real footprint with it. + f"{EXP_DIR}/exp_psf.benchmark.tsv" resources: mem_mb = lambda wc, attempt: 16000 * attempt, runtime = 240 @@ -109,12 +113,26 @@ rule exp_psf: # tombstone stale under the default `params` rerun-trigger; the clean job # reruns after the new tile's vignets, against the enlarged set. # -# Not a localrule: at DR6 scale these are ~20k rmtrees and the local-cores cap -# would serialise them behind the aggregation targets. +# The tile side reads the exposure manifests through ancient() (see tile.smk), +# which is what keeps this deletion from rebuilding every neighbouring tile. +# This rule's OWN inputs are deliberately not ancient: a tile that really did +# rebuild its vignets must reschedule the cleans of the exposures it read. +# +# A localrule (declared in the Snakefile): it is an rmtree, not science, and one +# sbatch per exposure would be ~20k scheduler submissions at DR6 scale. Local +# execution serialises them under local-cores, which costs nothing at rmtree +# speed and never blocks the compute chains (this rule is in none of them). rule clean_exposure: input: + # ONLY the consumers this invocation may actually build. A consumer that + # is out of scope had its vignets manifest checked for existence at parse + # time (clean_targets' eligibility test) — declaring it here as well would + # pull that finished tile's whole chain into the DAG, where a rebuilt + # shared exposure then reruns it. That is how one damaged tile reached its + # spatial neighbours. In-scope consumers keep their edge: they may run in + # this DAG, so the clean must be ordered after them. lambda wc: [tile_manifest(t, "tile_vignets") - for t in clean_consumers(wc.exp)] + for t in clean_consumers(wc.exp) if t in READY_SET] output: tombstone = f"{EXP_DIR}/cleaned.json" params: diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index e894309b..8b665d1c 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -23,8 +23,69 @@ variant is a config + one rule, at the config selector the PRD describes. def tile_exp(wc): return TILE_EXP.get(wc.tile, []) +# --- the tile->exposure edge, and why it is cut for finished tiles ---------- +# +# THE cascade fix. clean_exposure deletes the exposure's manifests on purpose: +# that is what makes a tile appended later rebuild the chain instead of running +# against an empty store. But those manifests are the tile side's inputs, and an +# exposure is read by ~7-10 tiles. So the moment ONE tile's chain rebuilt an +# exposure, every other tile reading it saw "input files updated by another job" +# and reran — and that rerun rebuilt ITS exposures, which reran THEIR other +# consumers, propagating across the whole exposure-overlap connected component. +# On fixture t4, asking for one damaged tile scheduled all four tiles' chains. +# +# Two mechanisms, and only the second one actually cuts it: +# +# 1. ancient() on every exposure-manifest edge. Correct on its own terms — a +# tile has no business rerunning because an exposure manifest is NEWER — and +# it is what keeps a pure-mtime disturbance (a re-touched manifest, a +# restored backup) from waking finished tiles. But ancient() governs +# TIMESTAMPS only. Snakemake propagates "my input is produced by a job that +# will run in this DAG" separately, and ancient does not suppress it +# (measured: t4 counts were identical with ancient alone). +# +# 2. Cutting the RECLAIMED edges of a FINISHED tile — the mechanism that works. +# A tile whose final_cat is on disk needs nothing further from its +# exposures: it has already extracted everything it will ever read. So for +# such a tile the input list drops the manifests that are GONE, and the +# propagation has nowhere to go. An UNfinished tile keeps its full edge set +# and therefore still drags in — and rebuilds — every exposure it needs, +# which is the accepted price of a late append, unchanged. +# +# Only the missing ones are dropped, never a manifest that still exists: a +# campaign that has cleaned nothing then declares exactly the edges it always +# did, and the cut cannot perturb it. +# +# The marker is final_cat, not the tile's own vignets manifest: on a tile whose +# catalogue was lost, the vignets manifest still exists while the vignette store +# (temp()) does not, so keying on vignets would cut the edge on exactly the tile +# that has to rerun, and run it against a deleted exposure store. +# +# THE CUT REQUIRES THE `input` RERUN-TRIGGER TO BE OFF (profiles/nibi sets the +# trigger list). Dropping an input is itself a change in the set of input files, +# which that trigger reads as a reason to rerun — reinstating the very cascade, +# now as "Set of input files has changed", and running finished tiles against a +# store that is gone. Measured on fixture t4, one damaged tile of four: 82 jobs +# with neither fix, 70 with the cut but the trigger on, 28 with both (= exactly +# the damaged tile's own chain, its two exposures, and the clean jobs). +# +# The cost, stated plainly: `--forcerun` on a tile whose final_cat exists will +# NOT rebuild its reclaimed exposures, because those edges are not in the DAG. +# Delete that tile's final_cat first and the whole chain comes back. +# +# What none of this weakens: clean_exposure's own inputs are neither ancient nor +# cut, so a tile that really did rebuild its vignets still reschedules the cleans +# of the exposures it read, and a grown consumer set still travels through +# params.consumers. +def tile_finished(tile): + return Path(final_cat(tile)).exists() + + def exp_manifests(wc, stage): - return [exp_manifest(e, stage) for e in tile_exp(wc)] + paths = [exp_manifest(e, stage) for e in tile_exp(wc)] + if tile_finished(wc.tile): + paths = [p for p in paths if Path(p).exists()] + return [ancient(p) for p in paths] def tile_exp_split(wc): return exp_manifests(wc, "exp_split") def tile_exp_mask(wc): return exp_manifests(wc, "exp_mask") diff --git a/workflow/scripts/clean_exposure.py b/workflow/scripts/clean_exposure.py index 2d3d40cc..e441fc3b 100644 --- a/workflow/scripts/clean_exposure.py +++ b/workflow/scripts/clean_exposure.py @@ -21,8 +21,20 @@ manifests vanishing. Nothing is lost to the report: each manifest's content is copied verbatim into -the tombstone under ``manifests`` before deletion, so `sp report` can still say -what this exposure produced and why any runner was short. +the tombstone under ``manifests``, and ``run_report.py`` reads a cleaned +exposure's record out of the tombstone — it reports the unit as ``cleaned``, +warn counts and shortfalls intact, instead of "not run". + +Order matters, and it is the reverse of the obvious one: the tombstone is +written FIRST, complete, and only then is anything deleted. A crash between the +two leaves a tombstone beside a store that is still there — the next invocation +treats the exposure as cleaned and only the disk is lost. Deleting first would +put the crash window where the manifests are already gone and the record that +replaces them was never written, and the report would be blind to that exposure +forever. + +The exp_psf benchmark tsv lives beside ``manifests/``, not inside it, so it +survives this job — it is the measured-memory feed for resource sizing (D4). The tombstone records the consumer set it was cleaned against. The rule carries that same set as a ``params`` value, so when the index grows a new consumer the @@ -60,20 +72,26 @@ def main() -> None: except (OSError, json.JSONDecodeError) as exc: manifests[f.stem] = {"unreadable": str(exc)} - removed = [] - for target in (args.exp_dir / "output", mdir): - if target.exists(): - shutil.rmtree(target) - removed.append(str(target)) + targets = [t for t in (args.exp_dir / "output", mdir) if t.exists()] + # Tombstone first, complete, fsync'd — then delete. See the module docstring: + # the crash window has to sit where the data still exists, not where the + # record does not. args.tombstone.parent.mkdir(parents=True, exist_ok=True) - args.tombstone.write_text(json.dumps({ + tmp = args.tombstone.with_suffix(".json.tmp") + tmp.write_text(json.dumps({ "exp": args.exp, "cleaned_at": time.strftime("%Y-%m-%dT%H:%M:%S"), "consumers": consumers, - "removed": removed, + "removed": [str(t) for t in targets], "manifests": manifests, }, indent=2) + "\n") + tmp.replace(args.tombstone) # atomic: no half-written tombstone, ever + + removed = [] + for target in targets: + shutil.rmtree(target) + removed.append(str(target)) print(f"[clean_exposure] {args.exp}: removed {len(removed)} tree(s) after " f"{len(consumers)} consuming tile(s)") diff --git a/workflow/scripts/run_report.py b/workflow/scripts/run_report.py index 1a8d128f..43ce8dc6 100644 --- a/workflow/scripts/run_report.py +++ b/workflow/scripts/run_report.py @@ -16,6 +16,11 @@ ``completeness.py check``, carrying per-runner found/expect/floor and log-scraped failure reasons. +A reclaimed exposure has no manifests: ``clean_exposure`` deleted them after +copying them into ``/cleaned.json``. That tombstone is read as the +unit's record and the unit is reported as **cleaned** — not "not run", and it +blocks no tile. + No disk scanning: counting products is the *check's* job, done once at the moment the products were fresh. A unit with no manifest for a stage is "not run" — which is a real and distinct answer from "ran and produced nothing". @@ -69,6 +74,36 @@ def load_manifests(run_dir: Path, sub: str) -> dict: return out +def absorb_tombstones(run_dir: Path, sub: str, manifests: dict) -> set: + """Fill in reclaimed units from their ``cleaned.json``; return their ids. + + A cleaned exposure has NO ``manifests/`` — ``clean_exposure`` deleted it, + after copying every manifest verbatim into the tombstone. Read them back, or + the report inverts the truth exactly when reclamation works: the exposure + shows as "not run" and blocks the very tiles whose completion authorised the + deletion. + + Manifests on disk win if both exist — that is a re-built chain, and the + tombstone is then a stale record of the previous generation. + """ + cleaned = set() + for path in sorted((run_dir / sub).glob("**/cleaned.json")): + unit = path.parent.name + try: + tomb = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"[run_report] unreadable tombstone {path}: {exc}", file=sys.stderr) + continue + if manifests.get(unit): + continue + for key, m in (tomb.get("manifests") or {}).items(): + if not isinstance(m, dict): + continue + manifests[unit][m.get("stage", key)] = m + cleaned.add(unit) + return cleaned + + def shortfalls(m: dict) -> dict: """``{runner: (found, expect, floor)}`` for every runner under expect.""" return {r: (d["found"], d["expect"], d["floor"]) @@ -84,17 +119,25 @@ def reasons(m: dict) -> list: return out -def tally_level(units, stages, manifests) -> dict: - """Per-stage counts + named unit lists, for one level.""" +def tally_level(units, stages, manifests, cleaned=frozenset()) -> dict: + """Per-stage counts + named unit lists, for one level. + + ``cleaned`` units are counted by the status their absorbed manifests carry + (complete or warn — attrition is preserved) and additionally listed under + ``cleaned``, so a reclaimed campaign reads as reclaimed rather than as a + campaign that never ran. + """ per_stage = {} for stage in stages: - t = {"complete": 0, "warn": [], "failed": [], "not_run": []} + t = {"complete": 0, "warn": [], "failed": [], "not_run": [], "cleaned": []} agg = defaultdict(lambda: {"found": 0, "expect": 0, "by_unit": {}}) for u in units: m = manifests.get(u, {}).get(stage) if m is None: t["not_run"].append(u) continue + if u in cleaned: + t["cleaned"].append(u) status = m.get("status", "failed") status = status if status in ("complete", "warn") else "failed" if status == "complete": @@ -158,12 +201,14 @@ def print_table(title, rows, limit=25): def print_stage_table(title, per_stage, n_units): print(f"\n{title} ({n_units} units declared)") - print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} attrition") + print(f" {'stage':<20} {'ok':>6} {'warn':>6} {'fail':>6} {'not run':>8} " + f"{'cleaned':>8} attrition") for stage, t in per_stage.items(): att = [f"{r} {a['found']}/{a['expect']}" for r, a in t["products"].items() if a["found"] < a["expect"]] print(f" {stage:<20} {t['complete']:>6} {len(t['warn']):>6} " - f"{len(t['failed']):>6} {len(t['not_run']):>8} {', '.join(att)[:60]}") + f"{len(t['failed']):>6} {len(t['not_run']):>8} " + f"{len(t.get('cleaned', [])):>8} {', '.join(att)[:60]}") def main() -> None: @@ -190,6 +235,8 @@ def main() -> None: tile_m = load_manifests(args.run_dir, "tiles") exp_m = load_manifests(args.run_dir, "exp") + # Reclaimed exposures speak through their tombstones (D5, S5). + cleaned_exp = absorb_tombstones(args.run_dir, "exp", exp_m) tiles = tiles or sorted(tile_m) exps = exps or sorted(exp_m) @@ -201,7 +248,8 @@ def main() -> None: "n_tiles": len(tiles), "n_exposures": len(exps), "missing_tiles": missing, "tile_stages": tally_level(tiles, TILE_STAGES, tile_m), - "exp_stages": tally_level(exps, EXP_STAGES, exp_m), + "exp_stages": tally_level(exps, EXP_STAGES, exp_m, cleaned_exp), + "cleaned_exposures": sorted(cleaned_exp), "tiles": unit_rows(tiles, TILE_STAGES, tile_m), "exposures": unit_rows(exps, EXP_STAGES, exp_m), } @@ -216,7 +264,13 @@ def main() -> None: # nothing. # Judged over ALL the exposure's stages, not just the first bad one, so an # exposure that warns early and fails late still blocks. + # A CLEANED exposure never blocks: its store is gone precisely because every + # consuming tile already had its vignets. Its absorbed manifests are read + # above, so a cleaned exposure that genuinely failed still shows in the + # tables — it just does not get to hold complete tiles hostage. def _blocks(unit): + if unit in cleaned_exp: + return False for stage in EXP_STAGES: m = exp_m.get(unit, {}).get(stage) if m is None or m.get("status", "failed") == "failed": @@ -235,7 +289,8 @@ def _blocks(unit): out.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") print(f"[run_report] status={args.status} {done}/{len(tiles)} final cats" - + (f" ({len(missing)} tiles missing exposure lists)" if missing else "")) + + (f" ({len(missing)} tiles missing exposure lists)" if missing else "") + + (f" ({len(cleaned_exp)} exposures reclaimed)" if cleaned_exp else "")) print_stage_table("EXPOSURES", report["exp_stages"], len(exps)) print_stage_table("TILES", report["tile_stages"], len(tiles)) print_table("exposures not complete", report["exposures"], args.limit) From 4d28897550cc566b80ce5ea3b171ca5f75fb294a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:09:29 -0400 Subject: [PATCH 19/21] docs(orchestration): profile comment tells the truth about the clean/trigger gate Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index b79985e6..63997cb0 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -69,8 +69,10 @@ keep-incomplete: true # Nothing this workflow relied on is lost. The two genuinely data-derived input # sets are covered another way: a changed exposure list arrives through the # (non-ancient) find_exposures manifest's mtime, and the ngmix chunk count rides -# in params. And the Snakefile refuses to schedule reclamation while the trigger -# is enabled, so a bare snakemake invocation cannot quietly undo this. +# in params. And clean scheduling is structurally gated to bin/sp (SP_PHASE=compute +# + this profile), so a bare snakemake invocation cannot quietly recombine +# reclamation with the `input` trigger; a runtime assertion is impossible +# (the trigger set is unreadable at parse time) — the gate is the launcher. rerun-triggers: [mtime, params, code, software-env] # NO set-threads / set-resources here, deliberately. Profile overrides REPLACE a From 831a1273b369762c859622625bb2bbb557e5cbd9 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:10:40 -0400 Subject: [PATCH 20/21] run: point the quad at a fresh smk-g3 run dir (old smk-p0 tree is unsharded, pre-manifest) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- workflow/config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index e52aa445..e06235f1 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -8,14 +8,14 @@ # 210/211 quad: 19 unique exposures, 15 reused across tiles — exercises the # structural exposure dedup (the 196 quad had zero overlap; kept for the # append-invariant test). -tile_list: /scratch/cdaley/shapepipe-output/smk-p0/tiles4.txt +tile_list: /scratch/cdaley/shapepipe-output/smk-g3/tiles4.txt # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # Where products land ($SP_RUN root). The sharded per-unit stores live under here: # /tiles/<2-char prefix>// and /exp/// -run_dir: /scratch/cdaley/shapepipe-output/smk-p0 +run_dir: /scratch/cdaley/shapepipe-output/smk-g3 # There is no config_src knob: the config chain is workflow/config/cfis, resolved # relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / @@ -30,7 +30,7 @@ star_cats: /home/cdaley/projects/def-mjhudson/cdaley/runs/p3-batch1/star_cats # BLOCKED for now: def-mjhudson /project is hard-full (27/27 TiB), so index and # report live with the run on /scratch until space is reclaimed. Mind the # 60-day purge for anything that must survive. -index_db: /scratch/cdaley/shapepipe-output/smk-p0/index/run_index.sqlite +index_db: /scratch/cdaley/shapepipe-output/smk-g3/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads From f82e7f46e93f95fb348bab05162fedfc509bb7ea Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 31 Jul 2026 00:11:55 -0400 Subject: [PATCH 21/21] fix(orchestration): slurm_account is the association name (def-mjhudson_cpu) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The executor plugin validates against sacctmgr associations, which on nibi carry the _cpu/_gpu suffix — the bare account that sbatch scripts accept is rejected at submission ('appears to be invalid'). First-contact finding from the first real slurm-executor run. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01UxXRKzfu6RUGNLv7ytNsHw --- profiles/nibi/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiles/nibi/config.yaml b/profiles/nibi/config.yaml index 63997cb0..e4ad8e2c 100644 --- a/profiles/nibi/config.yaml +++ b/profiles/nibi/config.yaml @@ -23,7 +23,7 @@ jobs: 800 default-resources: mem_mb: 2000 runtime: 120 # minutes - slurm_account: def-mjhudson + slurm_account: def-mjhudson_cpu software-deployment-method: [apptainer] # Explicit container environment (finding 2/3): the apptainer SDM otherwise drops