From d117dd6640db0c19f1e708457dd91a73a2185f57 Mon Sep 17 00:00:00 2001 From: Nuoya Wang Date: Tue, 14 Jul 2026 13:40:30 -0400 Subject: [PATCH 01/18] feat: edge-file dropdown to flip between multiple edges.parquet (#46) Lets users compare different edge computations (e.g. raw vs. normalized scoring) on the same tissue without duplicating cell/transcript/boundary files. Additional edge sets live in a dedicated edges/ subfolder of the dataset; a single global dropdown (top of the Edge Data panel) switches between them and applies to all viewer panels. Backend: - GET /edges/{dataset}/files now returns {files:[{id,label}], default}, scoped to the legacy top-level edges.parquet + the edges/ subfolder so it never lists cells/transcripts/boundary parquet. - _reader() resolves edge_file under the dataset dir with a path-traversal guard (escape -> 400, missing -> 404). Every endpoint already accepted edge_file; the reader cache is keyed by (dataset, edge_file). Frontend: - New global store.edgeFile; setEdgeFile/setDataset reset edge-scoped state (lrmCatalogue, hiddenLrms, selectedEdge, edge color range/clamp). - edge_file threaded through all six edge fetch sites (useEdges x2, useEdgeColors, EdgeSection schema+catalogue, EdgeCategoricalLegend, EdgeInfoPanel). Picker shown only when >1 edge file exists. Tooling/docs: - make_edges.py gains --out for writing into edges/. - Two demo edge sets added under sample_data/mouse_ileum_tiny/edges/. - CLAUDE.md + docs/data_format.md document the convention. - Version bump to v0.3.1. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 61 +++++++++++++++++- backend/app/main.py | 2 +- backend/app/routers/edges.py | 41 ++++++++++-- docs/data_format.md | 27 ++++++++ frontend/package.json | 2 +- frontend/src/components/EdgeInfoPanel.jsx | 7 +- frontend/src/components/LayerPanel.jsx | 55 ++++++++++++++-- frontend/src/components/Viewer.jsx | 7 +- frontend/src/hooks/useEdgeColors.js | 9 ++- frontend/src/hooks/useEdges.js | 13 ++-- frontend/src/store.js | 24 ++++++- sample_data/make_edges.py | 7 +- .../edges/edge.normalized.product.parquet | Bin 0 -> 38756 bytes .../edges/edge.raw.minimum.parquet | Bin 0 -> 20231 bytes 14 files changed, 225 insertions(+), 30 deletions(-) create mode 100644 sample_data/mouse_ileum_tiny/edges/edge.normalized.product.parquet create mode 100644 sample_data/mouse_ileum_tiny/edges/edge.raw.minimum.parquet diff --git a/CLAUDE.md b/CLAUDE.md index af18f1e..b4c4884 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -79,7 +79,9 @@ backend/ tiles.py DZI descriptor + tile serving; auto-builds pyramid on first request spatial.py Platform-agnostic router: all /spatial/... endpoints xenium.py DEPRECATED — kept for reference; not registered in main.py - edges.py edge query, LRM catalogue, edge color values, edge detail + edges.py edge query, LRM catalogue, edge color values, edge detail; + all endpoints take an edge_file param (multi-file support); + /files lists edge sources (top-level + edges/ folder) layers.py generic parquet layer router readers/ base_reader.py Abstract base class — SpatialDatasetReader interface @@ -203,12 +205,65 @@ of how many color-by requests arrive. --- +## Multiple Edge Files (edges/ folder) + +A dataset can carry more than one edge set so users can flip between different +computational approaches (e.g. raw-count vs. normalized scoring) on the **same** +tissue without duplicating the cell / transcript / boundary parquet files. This is +issue #46. + +``` +dataset_dir/ + experiment.xenium + cells.parquet ← untouched by this feature + transcripts.parquet ← untouched + cell_boundaries.parquet ← untouched + edges.parquet ← optional legacy top-level file (still the default) + edges/ ← dedicated folder for additional edge sets + edge.raw.minimum.parquet + edge.normalized.product.parquet +``` + +Every file (top-level and in `edges/`) follows the same `edges.parquet` schema +documented below. Generate extra sets with +`sample_data/make_edges.py --out edges/.parquet …`. + +**Discovery** — `GET /edges/{dataset}/files` returns +`{ files: [{id, label}], default }`. It looks in exactly two places so it never +sweeps up cells/transcripts/boundary parquet: the legacy top-level `edges.parquet` +(listed first, kept as the default for backward compatibility) and every `*.parquet` +in the `edges/` subfolder. `id` is the value passed back as the `edge_file` query +param (e.g. `"edges/edge.raw.minimum.parquet"`); `label` is the display name +(folder + `.parquet` stripped). + +**Backend** — every `/edges/*` endpoint already accepted an `edge_file` query param +(default `edges.parquet`); the reader cache in `edges.py` is keyed by +`(dataset, edge_file)`. `_reader()` resolves `edge_file` under the dataset directory +and rejects anything that escapes it (path-traversal guard → 400; missing file → 404). + +**Frontend** — a single global `edgeFile` in the store applies to **all** open viewer +panels (see the Split-Screen note; a per-panel edge file was deliberately deferred +because LRM catalogue / color ranges are edge-file-specific and the sidebar is shared). +The picker is a ` setEdgeFile(e.target.value)} + style={SELECT_STYLE} + title="Which edge-source parquet to render" + > + {edgeFiles.map((f) => ( + + ))} + + + )} )} {mode === "metadata" && field && isCategorical && ( - + )} {/* ── LRM Mechanisms checklist ─────────────────────────────── */} @@ -1281,12 +1321,13 @@ function EdgeSection() { ); } -function EdgeCategoricalLegend({ field, apiBase, dataset }) { +function EdgeCategoricalLegend({ field, apiBase, dataset, edgeFile = "edges.parquet" }) { const [categories, setCategories] = useState([]); useEffect(() => { if (!field) return; - fetch(`${apiBase}/edges/${dataset}/edge-color-values`, { + const efParam = `?edge_file=${encodeURIComponent(edgeFile)}`; + fetch(`${apiBase}/edges/${dataset}/edge-color-values${efParam}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ mode: "metadata", field }), @@ -1294,7 +1335,7 @@ function EdgeCategoricalLegend({ field, apiBase, dataset }) { .then((r) => r.json()) .then((d) => { if (d.type === "categorical") setCategories(d.categories); }) .catch(() => {}); - }, [apiBase, dataset, field]); + }, [apiBase, dataset, field, edgeFile]); if (!categories.length) return null; return ( diff --git a/frontend/src/components/Viewer.jsx b/frontend/src/components/Viewer.jsx index e01cb3d..f18937d 100644 --- a/frontend/src/components/Viewer.jsx +++ b/frontend/src/components/Viewer.jsx @@ -125,7 +125,7 @@ function ViewerPanel({ panelIndex }) { cellColorEnabled, colorBy, cellColorPalette, categoryColorOverrides, allGenes, selectedGenes, transcriptColorOverrides, selectedCell, setSelectedCell, - edgeMinStrength, edgeDensity, + edgeMinStrength, edgeDensity, edgeFile, edgeColorBy, edgeColorPalette, edgeDirectional, showAutocrine, edgeWidth, showArrowheads, arrowStyle, arrowheadScale, edgeOffset, @@ -541,7 +541,7 @@ function ViewerPanel({ panelIndex }) { const { edges, loading: edgesLoading } = useEdges( apiBase, dataset, viewport, imageSize, edgesVisible || tissueGraphVisible, - edgeMinStrength, hiddenLrms, lrmCatalogue, edgeDensity + edgeMinStrength, hiddenLrms, lrmCatalogue, edgeDensity, edgeFile ); const { colorValues, vmin: cellVmin, vmax: cellVmax, loading: cellColorsLoading } = useCellColors( @@ -554,7 +554,7 @@ function ViewerPanel({ panelIndex }) { const edgeColorEnabled = edgeColorBy.mode !== "default"; const { colorValues: edgeColorValues, vmin: edgeVmin, vmax: edgeVmax, p95: edgeP95, loading: edgeColorsLoading } = useEdgeColors( - apiBase, dataset, edgeColorBy, hiddenLrms, lrmCatalogue, edgeColorPalette, edgeColorEnabled, edgeColorClamp, edges + apiBase, dataset, edgeColorBy, hiddenLrms, lrmCatalogue, edgeColorPalette, edgeColorEnabled, edgeColorClamp, edges, edgeFile ); useEffect(() => { if (panelIndex === 0) setEdgeColorRange(edgeVmin, edgeVmax); @@ -1011,6 +1011,7 @@ function ViewerPanel({ panelIndex }) { apiBase={apiBase} dataset={dataset} edgeId={selectedEdge} + edgeFile={edgeFile} onClose={() => setSelectedEdge(null)} /> )} diff --git a/frontend/src/hooks/useEdgeColors.js b/frontend/src/hooks/useEdgeColors.js index 0af42f6..6e1a849 100644 --- a/frontend/src/hooks/useEdgeColors.js +++ b/frontend/src/hooks/useEdgeColors.js @@ -27,8 +27,11 @@ import { valueToColor, QUAL_PALETTE } from "../utils/colormap"; export function useEdgeColors( apiBase, dataset, edgeColorBy, hiddenLrms, lrmCatalogue, palette, enabled, clamp, - edges // array from useEdges — used for client-side lrm_set coloring + edges, // array from useEdges — used for client-side lrm_set coloring + edgeFile = "edges.parquet" // which edge-source parquet the metadata fetch reads ) { + // Appended to the metadata edge-color-values request (lrm_set is client-side only). + const efParam = `?edge_file=${encodeURIComponent(edgeFile)}`; const [result, setResult] = useState({ colorValues: null, type: "continuous", vmin: 0, vmax: 0, p95: null, categories: [], categoryColors: new Map(), @@ -93,7 +96,7 @@ export function useEdgeColors( setLoading(true); try { - const res = await fetch(`${apiBase}/edges/${dataset}/edge-color-values`, { + const res = await fetch(`${apiBase}/edges/${dataset}/edge-color-values${efParam}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ mode: "metadata", field }), @@ -128,7 +131,7 @@ export function useEdgeColors( } }, 400); return () => clearTimeout(timerRef.current); - }, [apiBase, dataset, edgeColorBy?.mode, edgeColorBy?.field, enabled]); // eslint-disable-line + }, [apiBase, dataset, edgeColorBy?.mode, edgeColorBy?.field, enabled, efParam]); // eslint-disable-line // ── Effect 2: apply clamp + palette to continuous metadata (no fetch, no debounce) ── useEffect(() => { diff --git a/frontend/src/hooks/useEdges.js b/frontend/src/hooks/useEdges.js index 80c6d12..32a9618 100644 --- a/frontend/src/hooks/useEdges.js +++ b/frontend/src/hooks/useEdges.js @@ -29,8 +29,11 @@ const DEBOUNCE_MS = 400; export function useEdges( apiBase, dataset, viewport, imageSize, enabled, - minStrength, hiddenLrms, lrmCatalogue, density = 1.0 + minStrength, hiddenLrms, lrmCatalogue, density = 1.0, + edgeFile = "edges.parquet" ) { + // Which edge-source parquet to query; appended to every /edges request. + const efParam = `?edge_file=${encodeURIComponent(edgeFile)}`; // ── Structural state ────────────────────────────────────────────────────── const [structuralEdges, setStructuralEdges] = useState([]); const [loadingStructural, setLoadingStructural] = useState(false); @@ -68,7 +71,7 @@ export function useEdges( if (minStrength != null && minStrength > 0) body.min_strength = minStrength; try { - const res = await fetch(`${apiBase}/edges/${dataset}/query-grouped`, { + const res = await fetch(`${apiBase}/edges/${dataset}/query-grouped${efParam}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), @@ -83,7 +86,7 @@ export function useEdges( }, DEBOUNCE_MS); return () => clearTimeout(structTimerRef.current); - }, [apiBase, dataset, viewport, imageSize, enabled, minStrength, density]); // eslint-disable-line + }, [apiBase, dataset, viewport, imageSize, enabled, minStrength, density, efParam]); // eslint-disable-line // ── Effect 2: score fetch ────────────────────────────────────────────────── // Runs when viewport OR hiddenLrms changes. @@ -139,7 +142,7 @@ export function useEdges( } try { - const res = await fetch(`${apiBase}/edges/${dataset}/query-scores`, { + const res = await fetch(`${apiBase}/edges/${dataset}/query-scores${efParam}`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body), @@ -163,7 +166,7 @@ export function useEdges( }, DEBOUNCE_MS); return () => clearTimeout(scoreTimerRef.current); - }, [apiBase, dataset, viewport, imageSize, enabled, hiddenLrms, lrmCatalogue]); // eslint-disable-line + }, [apiBase, dataset, viewport, imageSize, enabled, hiddenLrms, lrmCatalogue, efParam]); // eslint-disable-line // ── Merge: overlay scores onto structural edges ─────────────────────────── // edgeScores === null → no filter; use score_sum from structural as visible_score_sum diff --git a/frontend/src/store.js b/frontend/src/store.js index 4eff5f8..b4bde44 100644 --- a/frontend/src/store.js +++ b/frontend/src/store.js @@ -8,9 +8,31 @@ export const useStore = create((set, get) => ({ dataset: null, // initialized from /spatial/datasets on first load activeImage: "morphology", // which OME-TIFF is loaded as the background - setDataset: (dataset) => set({ dataset, selectedGenes: null, allGenes: [], genesLoaded: false, platformCapabilities: null, categoryColorOverrides: {}, transcriptColorOverrides: {} }), + // edgeFile: which edge-source parquet the app renders. "edges.parquet" is the + // legacy top-level default; multiple sets live under the dataset's edges/ folder + // and are identified as "edges/.parquet" (see issue #46). Applies to all + // open viewer panels — the single sidebar drives every panel equally. + edgeFile: "edges.parquet", + + // Switching datasets resets all edge-file-scoped state so a stale LRM catalogue, + // filter, selection, or color range from the previous dataset never leaks through. + setDataset: (dataset) => set({ + dataset, selectedGenes: null, allGenes: [], genesLoaded: false, + platformCapabilities: null, categoryColorOverrides: {}, transcriptColorOverrides: {}, + edgeFile: "edges.parquet", lrmCatalogue: [], hiddenLrms: new Set(), + selectedEdge: null, edgeColorRange: { vmin: null, vmax: null }, + edgeColorClamp: { low: null, high: null }, + }), setActiveImage: (activeImage) => set({ activeImage }), + // Switching the edge file resets the same edge-scoped state: the LRM catalogue, + // hidden-LRM filter, current selection, and auto-computed color range/clamp are + // all specific to a given edges.parquet and must be re-derived for the new file. + setEdgeFile: (edgeFile) => set({ + edgeFile, lrmCatalogue: [], hiddenLrms: new Set(), selectedEdge: null, + edgeColorRange: { vmin: null, vmax: null }, edgeColorClamp: { low: null, high: null }, + }), + // ── Platform capabilities (fetched from /spatial/{dataset}/info) ────────── // null = not yet loaded; object = { has_morphology, has_transcripts, has_boundaries, unit_label } platformCapabilities: null, diff --git a/sample_data/make_edges.py b/sample_data/make_edges.py index de082f7..6928192 100644 --- a/sample_data/make_edges.py +++ b/sample_data/make_edges.py @@ -167,6 +167,10 @@ def main(): ap.add_argument("--autocrine-fraction", type=float, default=0.15, help="Fraction of cells with autocrine self-loops") ap.add_argument("--seed", type=int, default=42) + ap.add_argument("--out", default="edges.parquet", + help="Output path relative to the dataset dir. Use e.g. " + "'edges/edge.raw.minimum.parquet' to add a selectable " + "edge set to the dataset's edges/ folder (issue #46).") args = ap.parse_args() here = Path(__file__).parent @@ -175,7 +179,8 @@ def main(): df = make_edges(dataset_dir, k_neighbors=args.k, lrms_per_pair=args.lrms, autocrine_fraction=args.autocrine_fraction, seed=args.seed) - out = dataset_dir / "edges.parquet" + out = dataset_dir / args.out + out.parent.mkdir(parents=True, exist_ok=True) df.to_parquet(out, index=False) print(f"Wrote {out}") diff --git a/sample_data/mouse_ileum_tiny/edges/edge.normalized.product.parquet b/sample_data/mouse_ileum_tiny/edges/edge.normalized.product.parquet new file mode 100644 index 0000000000000000000000000000000000000000..66d836d621152dd492066052cf106d5b50596274 GIT binary patch literal 38756 zcmeFZcU%+O*Ec!|lVm21P!f{}3K$d=6c7=y>P!!WbU+G~B+THm#^J^t>2j#5(k zK2e%$Hd1(`i_dE_R)KHj0>-%7xWXI(2vwH3DYBor3#MAKDF3CG{?;@)GCVRgF+LDlfP~Wh`ei8wTTW|#pm2iu+a+O?p}*5# zq^sQ)aNw>x$T{E{EMo0GveqqAwmmv1}61O4}G~T<2gwgaDW@?u$dX4Z5n9=oDd16d-bWGTc z!mEmjZd=-$K)d;jBBn7~?iXV@-yFsUB)K#b=1y=?^Z*(9S-K%MGYo>&fh4dvMkYoG za|I|GUV&w$OjD3Fht*+6Si*nqKoF5i%kegh>M>Y z6SE5}9~T)hGhznQ$ZTdzM09Lo#7qFXJRv$hCNw-wsTC*~paTS@w}|9+xjVEDg8h33 z`j#ie%#4qS30FQ5Dj0C|%NSC~=uf|BP?y3@f9Y6S`*K%9J99<9F4l#hgz=zUW&#FC z|8>!f6ENUr)cpGj-Srj^Y6ri5G)C8z1df~$8yy`R35uB!9u_?#JSsjG-pC`uXGX-t z#m6vzm?nlsM23dNtIZW=u@Pb6kuyRfnMKD&MnujGkMbS?w^32?F>x`&oj{8TGvmTy zl%&Kw+%#cEL{!wwE~JFfO1lW{VEXS@b4BPD!Ym;!bVh9Aj4)f1-Y`u3jQH@FczZ9! zCBOl*76x6QlYZyi=Ie0+>iO4)+SRk4NFH!HFqU>w zWOg*uTOROjq4yvQ184~#Pl9L%qjcCC%o@X>T6*EXnAPe$?MCq!fqq}j6?aZ>wkX0~ zC?53fxU@F_SNtt@`x2l5vq(kezo7vBZhr-_!;sJ1>SLAFTZ*-%qjh_ie76w{IKXiM zOg%ab&~663&>hPAJ%c`Q!tDKa>jA1CgXzgcw*T#-0PnS$NxN+&83F#{GS`lP(|-HE zJXpZdhmJEPZkXGlSx_%b&0Nj|SWkoaaRKZN6YApuyy*xWaHkJR9n$J00S&)qHBT(I zN&fBD_Ks3QT4^fXZ)zvaG?q%FTq)Pgg|Lw(v_mi!T;n%1?H%RP|1o7kD&5cjc!zBO zH<`vPdKvsCBo;V@V3U-@A-FV!1-~u)^KdGgE0jAVHkGCtzyq9P;hskFO>O*(g3Ho4 z@FrN3=f0dPHgipEYbsmLW5$qJdZbgvDxVyOkSpmlw9WI+_sHPO?1G9Co3CUDpi^m% z<zQTT zePph~td?t8CNMBwZ{@YhwCC6Y|KiZ{RZ^HCSi9G2wXFA)lEil{<*Pd}OEK!^oh^3? zugq}_yPmCpIr9Scd9P8r#N95M_4xXl&Mw@Ti0bs z^4GR^k1N)71#wu+k}P0pM$A~2AeUv}exLPLFt}sVlgjno;H~=xmG1`C@J%)WC&Q~Y zSc9Y?y8XTzyN}xGI_v$Ds*ODujT&1+1E#A}MD-?HkUc-}!0^pI#~%$UPI_9sxfd*= zY=iZPEq0SmM{7?;)NHYb)imo4jM&Xl8lu-NU;;B*YE z2-MqhqY%X2J!s+kkF6k^`-MCyg9`>R{TKx>rW$t`24ZwH_a(GuWZT>mp0Q+YbI;7U zd3YRHAQXNb8Gn4r`J_)TZyuSzz+Bo3OvBi-U+k@;6B)Uo#j#0eQI)(JGV0@@Q zg9P`h#01t?PX;sd-`+nzo3Yi_-cv8kv9y<*oB80vT-HGetm=<>tggvl-ahzaKI7`0 z`T!UiO!rTKff?JudM_*1xvfRL+O`%J%_{2k13F5%JTB|QKtk3kiGF8%rA z+lMcIJ^FJQV1|^ZE^=mb_@;~(rUG_UL6#{(u1M+Zp5eS4AmqtdJDm^4w45i&04RmC z{*tT^o4d_nMNe7D#2UcRwM@pT00h^v81Mw8T3EVDzzF55Sr$ZYsyJH-7yvz7UjsOM z0?Mvf+hy>gk04ORIwqpgO#FkQqm3v1%o!>5(bpVb4V?M#_&))Z7Mr%$9JKh~f&<%`g%N zb#N$qM@G~C$zYJ;_OqmiS^qu7@&D66&i{{Dpl4uPKu0R$Vgj=;8yjE}b1)b4FokK% z#{w+GA}q#+Sb~kPF*d=bSc+x16P9BIR^ras44dOFxGT26me>k+qs-dz@biDi!|%eA z_5Tcm-$m#DjdlOEDOmFNg5dUdjmhAsqa*|qwc{uhhU&nCQBTwMLQmAwEUKf%p?k_Z zDh9CDZy*%>4?x)dZ=pbY|4)`P`)xT<(|L4WDC0+rhD?;QIv&^V83wI;ADdcQ;DwR_VlpF|pgekOJwbzF3OZ(7rXhhLc z+J|{qbJ9-Q(+oNbuR2uBV#z7L24`m~iCZGJAB=(MZ&opD zYUAA9&lf)Wvc!T)gNJU$ghhya#a^3L(5SaPL0V8EIrZ3eEj_}`g%JUI^$sRzIXyx; zmkyRLrdQDmKsZJfwiMk)ntCa2_m&f6gYy8&JBCGWi5TEd`w%_)srtpx3K#%nq~FqI zz1PDaZd{hz9YXq!nG#k3IsxAx*#mh(J2{zB{@sGnmQ#9 z4~cSQ4Ky1X76(tmk~rIhFtQV!!0JhONzY33sfCVk^^Tk4$b-RrU^>5uIV?rEM|`4l zGF+0J{v#bDV-iFBO9j$u!T;g`?JH*1Cg=aBGmP29`S|EJ#pf)u$aMF1*-p{4w^#P^k|byKEqHmAjTxlQwi+@~swk#RkD8}VYpmA7xDm%AtJ zcGxSD?DR}14P|$GX1l*ME##m}UfhX{Z`=X8odW_VW_Ui8SUk%t=EnTxc_n>Y%uyFC zG`UDT7R=12Z)ZiWKc_RX*}l+Xp)LQ3#!gok`ONom?rEPSj=lGWNjc=gE9txepEfwK z6+t&biv~O5*`v}l#&UaUWy^XtsoLXf>333>B|M*w1suQp+os)|$8CtUQ(3s12+JF; zPnn%_t~58XM6$_3G~B3SAg9>%?)s5wI4E$q+~3oofehH0GrQoEZmd=J()*h}5WERy zB46$vJCos-Q8axt%D=P4Aj=yguAFvSZzia8H}&-Li5fp}D0xm}Qhn?ev3ObZAg3{A zX?A&B(o^Z;x!v*fQ*VQ3sw>?CBQLyD^@^ZWPkikLSt$qgF?;G`bV%x0KGcCdXOi$) zv2%AT-_6|BL2Sn>nF0BdPWPu&Nj|f8KJj!&v+b2PKhu{bx4A>q*1CI+t*%Y;nDyAa z|GG&@gC7q!Gb}hobn`g7F(oL_=0kNj$B7-`9LjZKO<^CTdh{jAXOWX*3NoMA9psF= zJ;m6)c)w+EPgx@->YKM)vi8fiM(3aJJ86^W_Agf5d`-LEn&s`#mbSBI(3ibK{qkM& zc8=clf*r@%o46v3mUc7WnPN%znlDWX%#o>rovs{~OlV>X54*IhH) zb|uw7VwSYFai>w%v2G9DQZJdP+4gSzwhu41J?Ss7+R2kD95S|XL*jEY)2#NJ`|e-Z zZE)z`&OPF#mRE-NbjS+oTX^Y1bsWFyY*1Qxm1?r&FK3g3L-VXBgepInhD)+mq|Eb9 zY{`9}JxO=QHps99@yr=DF39iL+y#XX|>b*NkGrur&?v6Rv`gadJ z$ts&>JWT3l9Yig3KR4_0Nq6J0Y2LFoIj@=G5Miu16*{=5SEutVbJDocc&At1t0b?z% zwV386rR#XrPr^&LnomC7W7M49qfB|W<2p;M(vkv4jO{{qSw2Bkzd6>jY@$`=c|p%* zQl~lQPPCl&_<-dw^Brb_i2lPfyv$mLn+CSbQ#zfDGF~xAbylScZ1g{CZoSC8-^$a& zHUzr4Ww>80+caUH@vtzFk(a{SWiD&(an>Z$Zfk-(rPnO4hv)7N9BIyurrla%c^i5M zdF-F#_I#>`Be^xHao8k(^X}U_-G1tMCUyU&X+2u|;euTOiKZ9(vCi!}SsYjV=CQTT zt;g66*RA=+qb6R_O<8?&)~K{wX6NQeWk)BBtsjP0Xu8M+OLdBix)FH+=e?2whMIX# zPCrmQr+Vm>VE;~|ENiR}#aPAUhsM+QvWpj7a-G=Oxcf?z`R<+eb-pQeJ@xFd^(?2G zW<7SOj2FsZ9+-SC)Z=u}<-+;bmR`0Do$NlPsNzPa@Ui0>`W4B_LcIcW>qm_8BLjNK z=df74R$qHDcWv1wm;6Ba$qOFiw6n|mlNC4h3F#VJx|=;x+FcL-y{u>kzzqD8LK$>2 z+>~($Ju77yQNY|dl#rpLtX`&irsu@!W~%3*B+HIsIH-}Go`EtD2lC013xRYxv*Z#z z-Gm{4Y;qjQp<%QKD}tjCGNh6bYZ#ok{}Sv8-XmlqAWy(wu}`;Er#+SStr|99{2b+5O8Mb!0$xW>Ue^4P)8e zOS7}z6CY$|BCpR=?B07BMbNYz!yR2MHvxCF3NRB~bH6dv%QKAOj;C@=s*m0xW-p6& zwK8G2<8=cSaK{Y}rz7;-Q5ERDz1fQ4jxD}+u2y#k_BM-PxT9spP($_{rSN*OvkJK5 z=T*PCqe(V!$7feIKEVukv|zYnZYSW5mG0!&>dG_^$E)Vu)=ioiFKu*@J6UHo3+6tEw+i#&AcQ%)JbE%$;I1xOlH6?#XS$O;sTbcjN+h+}2l_ ztx`U>&~ryeZ-*w2O*O8M^W6O`U8ijtWgdMc_Gy0NiV#|AQnwMfV=!<>$)4rCa=Qm_ z{Fgh@!}Q#->Y8xQe^<&)lCC9n=fr^{Dd8h#Iyk=39^n${ieH+|ZbwZr9YVDKvUHN!DDjK;rCE`DF+FUD^tc?@|mk%L#b z-YSoGrty4x%dQ_^bFu9k(Pf8iJZ!K)ptAYN#jJ;HdR12;o?_wtF07V=-SrDJz`a(= z?$mz~;!ZyPKjgE8`1<`b>B;^Sc3~E3WQdbPoW?oyR`*O)r{I*s4#9%Of8J0C}b_-CX%{Qb?V(gFKuj7>tWHXPyOt{JZa zO1e;(S^n5E5?}K;7V~hH0si>UjRma-DZJyE`Joq;99->mt+@A0kQ1bxI)aBU_zKy* zRucHdj9#C&PKEXOrHe-Qp)s#s5pcJ&A+}mOcUI>$TP3U^jFpjpvQ-BAekyr7)FP9@9NInL+eC#L3APmvhU@n$Rwdbk1#Fr=~Lwx06MW7+uOMy4Z$ zXF8{x!)Y80jR7ER46E-Ip&&4Zzr;Ivc;2keqNZb1M?W5(Zd)U%aHFu`%)x~fwjFx# zq1yXxF==E=YP?&*Zz88(x8!mD=a+$`7(?J7T5uTnqQkq>2`qg`bphGSSSJ*O$ z9-tNUT{^t1XxOF&ORH*8=Io*_xlO3E&&GIM&o7o81VcIOg}W5FB`HnQ>J<$XpTTxEdG7{C;9@yva! ziyY7!qvmJeox6;fnMiEpGU!sEjNf5@9)9nxj=UMg!Hib-KjPtI->1ZvFXiB}0o{f# zUe3nP0S-RL4Y0~)Y4-V8A!dB|$_g&lc;0UNSj}tq`m2I=zr0W3o`ojxir2ob1|d$Y z&(u8(Ho%N`q@1JhkQaB3e!nlk;Dvv=meY8PB>-m!yIoJeDBR=9ibE5IbMa8KS1m4p zW4#}&r|>iefJb=v>y_}s!DD$Cg22zA-zf}W`mlBkjrHaY72s5m?$887Jm7TYyvut8 zSUmc~rU*)m88Mnxi|}hef5s~@*4!Nby3asiyPh`i+vAOl!m#K*BOywS);kOz>*3%A z>+b!_@}JKJ_{s9!b%)gyE&~*tolWEOKURLc*GOXqZ7mf795ObrM_!h=BOVFx-J^^` z`R#Ub;Nf(p`#1`(8v_Al84rgY@TG+KZHAAH01sk9+fp7TIIL1GW(3a+6JuE3991VF z9>3?vvD=R*tRt9E(&5p)1i17yxcpeIpN|cRS00)Gj?2S!2OKC0yB&~cvT@s?>m&Np z1~?;Gb86yT60fmgLKB5&Frg%zgYV=ws?LzuDvohcZeeiSVKEPTfpv#ZB=I;Vj+Sh55C&E)0f4|Rb zfB%Y$IZc^aJe&o#6>Q|U`}QdcZ!Am;@HOXRy7TU7wl!RQ;H%AZ9)>`D=jb91E}u_m zMJ1~D91PHZKH~^gm6<5x;{I9g!A~LbPA>ivGz!{d9N+{GB28InCLcq%u+OpJ;bk)} z&enSb=PelG}GX+fz3c>Pis5z@a`g3luZtdF%WH15MhvkVfuAGE2k<#qTJ z)!Js0PU3hVA7?>35ebq90(?#z5(2|JGWdJK#j~uN!Mj3IV^rG&9U+ABE3%=mr9Ph1 zcvi?4VWI#p6jxD_KPdb&B7_a!fSug+7dQe(pGfthti^&ycQJ5Qo zRQEafIup}M`D-Xjf^99okOkr@EPNuuLN9H!0P~ofAmrmXhGFy-wWrPpLTnq-NN(fX zOH^X4Pws^P<7MG)91QenbzBq|>q9Ng!sK|d{mEShvP8=6FopSGsRiFTIA2WBo@dw{ z8IrIs;>mcHd#DXg(elDhV2Jn#MQX z*Li;BVY>)LOVCyxX4130B^3`lhQ7JfaYJmpl27S`*z25R0~f--WeBh;_jSU|K!`3; z5+IIJ^N$wvKQIx%_1;3qGeBmf<1O(r>k@o5Jl!wC^bbl4$4NQ)CBlNALrW#`F zt#(ObJdcT6mkh99xi4&wmWBh<1LAO@gpId?GrIILz;`<{DflPK|7kkZj0c1&xz1CldlwyJq ztVvWE;^3zmt9#%0IO7ZXh4O79Y>bKknM_8V@`e-zks@qd!WX{dW5W=wY8byG$%trd zGoQu`K@$ao@*7pa8{a{AUs1w2mJ0xVH9OzkPJpxTK|Es7koxnrFswcUD``Jhg0hu( z-lVa)Ym{fqUj`VgUpa3GhnA2D+^kIEh7qOWU<*BQ5#g#PQh?q2n;zZC#}hlH2Eqav zrek_sa4eDJ;#9SecZrX)nkXLbmod=>hzY|d4)JlIv4w!33T=evxVWLLku2of`j7-g z+c@-~@uW}Z@2*UHDns4%%dQoj`p$wxo3D~nUT1z7Jf!5eZ(UnPC_XHM485rVLzRp#Kj)& z3UabACWSo8v;PVhff*-EcI4trhSY{p7=k1OZ7yzpNJ)lq{0ayI+XBg7JiMZ+td*eS z6A8kP5)yb{H_M*X4`grojXX=V=6DVf;Sn!MXQQ)_t(fwsj4Ml_B(G@PxsUjznBe$k zQjqP$$Ayx@cRXBFw&%s0pXg691Z8Iqt^vaGVF`hu3<-NXiPK&gbmQPWWuFJC6%-~R zFuxIT+bf671~{w4lZ}C_0^|9G7BhVHy8&jc6uJx|v2m24>MV`ppZ^WHiI({ol3nOR z2hxOlED;MltW)J ze%eQvM`I_JmNcO19PIe*_0$J0{$`kZhv3xV6fR~M_L+2pcB;(hV5aa{CFWqr>v>;` zDN!E5X$`h$Bk&EV6k2Y_OMGleJa?m<9KV)}2SZi$%p6=V7r!TQArzM;&FuCn=)}9* zgg~hg6I@xFfIKMhtt2D4N*|gaZNC!kkk=@ZhdC4M4ndrE2L3&mgP&XeDZI?Vz9EGq zm-1TxE|c1WOlT{108tl^zfgIKB#?_oOceK&07gR^sQ_04SrA1^ShhCgat>WWBx!>o zs+5S+_>`Q7#nWi&m!-k6OoHp@Pq1y9NEY*b4qw!!;$b5QPit1t_`n46CpV^Pqz~0m z)pFXasXzI6p79-u;8jJF&&ezYs>%R|Oec%~bo3_(S*Fn8G1*>LDhd3dYicXOi+6xT zQJ*QE1s>~H1nW)_2u%!dfq12ma%5w#7dBE7v$s>7cRvIj@u+Eqyg~`tn@~FNZCLm| zRI1*lEKG>2XwNoQ2c;I^lf5C$4-&p9q6GL}>Q-Smg$F#Xf|~z`8p!$yAfdp(=LXYw zpyfLoH3we`gyP{73mYRNl>@{#W`jV(DfFlM3$b`DEpbcb;lbi6K4lMOfT6_Z22JBk z$Y-e^`1EaZHDCFHPG+?i+!`(($W$uRSorUnw^dg?1o(9+R6u_R_!Hc^T!@Bx)|bv+ zQz?X+v?;{Vhu2Q^X*M?1*Fpx6KvQW87XAVo4kI`7WtFsWssUbdfVKc4#t@Rbggqt= zKomF_I^_r2=*DVgjJt$7bC6w^NRfC-{xD+X8w;|6XWs_NB4Zr|g*|I4MULmZp_o{O zXi2C3T>Mx(--ML#WR(;-nAb{><0u)$r=WOboujgfxO5`Xy-TK;U}xo!yJ=aFz`=`$ zx6g+imH~X+MV9}Ppz7KPek&GU{)JTH0b+77rPR>CZQI#4orP6r1Q@n2ddj7g64Ma* z#u}%X0E$DFQI`T~We^`sG}3sRhEEbZU_)t6tC5qA<%p}!1d{sQt2FPq_EX)rsy?vltPk?s*%p#k2$jl}}e>i|(3&Wbz*38g9%5;MC?JJ%c7 z=8$7(NU%nBX9tHx}>73R8 z=RROIPsqYt>Ld%dHW6f2B3Z;y%7u^-U)UTc+c#$xgRO>9!p&Sf*?53}L4cv^?+YC} zKH0@N1f|U36C4>FA?n9brig`XT(1ys5D=t&8^DHR>`oCv*#a>+P;|_O5ZY?USps`3 zgqkkiJ&oYWB!(1+jm7ms$&%h&oSq*=je~@srpe<5tzXGpTIoXpTMaWVFebP$6q&=% zvLG*!qFh24j{zXjV$zmohOg&?_*Wnh^Mxdi%oNiSkPI_@mx?C}s8$ZZ!H7zQUx*?e z22>J!8BoRvu*EY>EP>Lq(OEpnh!Bc$7|(kvw0VVT+~5b~_dHKoK*Ith6e;H07N`OY zaP2amKbP#{+dF`-pJFx!Uy+5S)D^~MMUn*q*yc1Yqeu>hObPRg#i=GFB~VflLKss? zSwkXpF{Hlb@s%Zpkm*(cM|ivm5ECr`wxJSF;FI_GG6!SwqR>9YLfElo0!0%=V}}Fh zl>mMWNml^li=%|pYZ6lUzPDH>fWT)FCf;re?j{$KGev&U z1dq-myBf$SAxRo2^Y|cuv8y5V4~YH(vSDih`GjL%L~vppgmW;?o1(BK>!~!qVVD1W;a4CYZDT+U5Y?! zvPvUK6)=X&;sFpud|!~Sc(U74@(72{qcS1OJU0?%SA#htJ5#G))5>5GLCG9U$x|F< zrr6E^o5P;SBiQV&KP)z)A$>10ZWR%v?5og%GQg{Wojz#>!O2Z*Wu^&08S}|h=qUwy zVEju;4h608VF@4&J1dCPv21S>;Y!Mse9KW*QcB89b(M{c^&5)deFE|lPxh4}*|bb! zLb!L)7y>iPHH6S*e8QMKEGWc6E)QIIn-tt4+LYi|1(RO8~n9Shd z&s+1efCuDA$w#C;R3eZN00M*&qzHRa*g$+oM9yR>gH0iFh%3w}^)v9MI|zgE1u~$5OIN6G{Q9eq=(On!)jtL+e}$yfQ~hy=G&X1LPPP zHYKOGHCHws!^$t)q`11o7!u(HWqsODjJvDTaz2xAFDNW55_N|E2W zvKJ;oaGFlybz&+EGFcx!(a;6Z3i)P2CsiKQm)-QcybvzQo1~IrvV{1#km9znl}%z$ z=N~0v+b$4=Gsz{C5>l}c_bm${yYZAkazMMMWeDX9fznW<`XM*y*w|n1L$UE&@d-n+ znytiuQDt5iP)drF2rsU*;C5^eQ&j|M%6*edQ1+m1Wr@J3gy6>K5rQ{11XHJhXRZ{H zIg~O*2qdk9(t@7n%PEbOK;%P+-Yg)5z7)|5I0M;RaLg49P8pN`u#e>eg)q@?D3iWl z9rAyn_Aez^_~ru3?}R~>obXY8H6>L-->*bIWCn6PTb3dsN3)eShOqY3X~H#ILE}w^ zgoUzAMz-7yBzx$um@=~5`fsh=| zrZKscmDNT~hG@D@Mn&%x$Zqo`6Akd$bg@xq27cg*(|NXuf+M0bMF z;%`jG8x#`cTT&*cNH`!tTJ*mj?K6*mC~q#Dk?<3fHmmQ}H1MHC=!qwGxaRKWWID!ztwgCX3mKRp&^Y z!))H{p_CSUPYU}m^?HS{W6xTI;fzSOLb^rqU(Set&+C6ASWc~CSPsNv<~Y~?{uG{E z7k>2O8o&+w8!Evvf=VH{IE5{5A}F5}wyZRT>)OPUWTueaaVkHki7l^8;kuNjiWf8) zSWu~yH%>FoZ6a-aQu*Uc(_|%09EZ$QVI*FzyxqigtxOfqD_!2@YZIJcPm`>`=~nV) z%Eu?ocuQ$|ch_dxKQqnrAkOF+)XWd6Op~20&FHfr7+uP~#SB^uPWp;>UpZvNj*J(q3c7($|Mg0P<^Ipp}G5-6Mw@SD!C-c|D zC+_R~@GIB#$~%Ku*~`tII#dkMtQXHcy4?ItV#UBFBg2KY%ez<=tQ`C{bzSm{u&zH| ztVBOqIVn`Q1!-I9C`^*1=U7_uCsIR|EjcT(%F1{f<<&EE{pw&9OQEgwcGVfJOD0v8 zMpZsOfyNtmlGe5XRl|dH##=9v-5u9gjf@U7*>OKy=vtLEYTlKNyT3l^0ecAkD?B#s zl|Qxd@m&?T<;tdmu1{_KvsR5g=&|{5(9@n1N{5d>eNuXSPI<5CtPw%ioMb0AmD@$R zkC-$hZ_AmY1NJjxswOuXcRF92(R*%j&D6FHoi4r3=(FTq&2+*pmd(symrma99T zxjrAD$yz(-q@(%Eu@!EOWuxYuK4t!PPQ}1StkDau_3iS0Q^lZV&jaX1x_!Gp6&)J< zcA;T%OSe7WG>3+KK5w}6{k}c_v>ie}zZs^mRe3C<0?c)jq#5te<2V+$ix)}KtyKAR zK!Jzsf@Fo${(RA_f}vf$OENuFdyUF7JguFyS^k@>q{lM7?31*sgH5|B>N36ioz$*L z+|X#ynGNTfeN~<4)=}7j3e(kIx!0 z^}TLuscCn|HHED3o*TE-ryq1KjPZ+`v2n*k@{ns)%&0ksH|~0yerQl@q4naHF?)V| z?14GM{g+#f%@@wK@#ORfST)IXUyIIW*kFF(`VBYs+l;dvF_k}No9@Oz*PFJZm+;5t z8{aJO8P{{nnI~fl(uN)$d#303Gu7iP7fO$YS|6FHJv{#GjH+Xan|n=tvueVn_cu?Z zNbN#ItAom&ZWZTjwu`V|J+Ut7)~P(HeT?7gNjFd4I#aaSK0ZFy{r<}>=Pq-OCuYUk zKepPsyCtyq>|KJXt$~dfZr|)Z|B_(ZyA6#OUySRs_<>;hH(lf9uQ&VPZ?C2kg4|NB zcTt++bucv|_e$IuhZTywA>zgIYgXNhvjUEW%2wYjci7^%CO$jN{KM@EEOT1FCOh1^ z@0}|DEl!({Wk=Z0zEcw{>$|NkJF?%YJ9UX$`j)+jiyHK5n|j*2zIhzK=%L*vX|v}! z?{~M1aSXhB-T#d9Y@du6zu9*i@?`xA{npGFck1rVqAmT7&5DSf@@jkIW$xLctoXR_ zUOVp8uQ+?UFdqIf`i^@Kx#vo%;%Ck|vg5(q73VIt#wR4d+VSWox1^MwnV8<|eywo6 z>ox09vz#WIJyt$1sc`R|v|)$Y)1Fc1YeIU@-hR)_+O=_jdWm>WzR80ZJ`>y;c8TW} zj(PAhsL`$Ql6YS6#s{yXCk(vzKs^8a#*DY~D*kx*?d|*?rLR2N()$l;k*XI|44w9V z)44&<{@J^*esPzNdwX1J4LGrAZPCL|MOz2IiC?$)!H0)mN;?gCwQ`R= zSg01f3Y-4-{VzklmXBWgVc+zBUe8BAo{nDj_v1&;zd!%;7t0}qZDzp|M3)*^Ii%k9 zZyNvl6y~~UQVr9C$@!`>I59V&r=6Cf*yyXBTO3xp-M2Jc{Nid?wqvHfQHx|UTc29s7_(*_OL})5 z9Gm>!eir4$u#eBX{Z;X1>*(=Iuq+LDNIv`@YNmt7EG^0CsXMA7k1PkKi^x;FM4Xx1=9(e zmrT&2b!M%??2DBsDrEG)AN}gkxV^9KpC;>2n#+J>U6u}c=)X#=Lr3QM`NRtAk(upe z)f;auDy_tI@-Wf4uRStwWPf6Y)nT)}iCZcH*d| zT4d+D`_>!n$5XUb?9lEZx7>Ko2d7vfExiimOJmX7^MOhwIV2ehaH-T&P8DiYyJ~ zqC@S+7BuMF$J<+2Q5xiP;I`KFh8Fc*?H|YSt3qGx0n{Vty!*uk9a_91 zvG!z-8uVsoWa_+?8g#ELa|zoS;<=iw`SKMD>-lL+5sAL*6&?OfvgVY1zJ}>46H?`w#FYi zX4HU=KCK_=uv3k`{&4VEa9WGL3k&-1dscQ864@ik~?;Iv-( zf7B!EjbjF0y`e$t6b0GK_NdY7_&{yY-)bZe?z-#3`#MBF`n&UK!0R`Y&7#eeSVH1&iASq{4E8fvdWajy>jI{HnGY)XOxmdDhh!uy`~b;qjF z>NN|^XI1Kuh*`!FEowg|q($xD)~Jzq*M8jXQFPP zM~4n;u$=~(70dRpgX_`NkC#?7m1xn6i`RPxTvDTX1qW5H66;a-argY%{B-C_*~4o& zJ1dd?3*vA!YWAA$zP(tB`agE|l>XGBoe#_Zp7nPX(suJ!+t1OWaVDM*cou5(=duqi z*Fdk~mTu%)QiJ5DtdCgTuSa|N%U1M{tw(ikkN<87tV7H0nW)bG0db&q)2&Cw8q}vC zVOv9l4jrNr7j^op4y~$Q(QmVt2K`8|9_Yf>pdMwNbp@pw6uDx}$)$a@XwuCMYm*!_ zsBjd6#5&aY?Oe^$2WrF|YN>Y8qN9O}4sMm}(D<7*e>rcfLUr@oOn4eKnl60m`3K-c z7Sm$hb(4AXcy_-rfbkO)s)I^ z5FfjITb}y-VI7KoITu2K8XdbYsO5XrBU5T%`C)e*THT59Tn%dfmbVU#NuHXmI0~W(&)Dv}(QeY`ed}A7NKy-y99fbNadDdWITZPUnmJWY(fs zfqFza;Noen-CG3f>X~>_bU(EgIq3mghuV*gfn1|&45xs9d^l$KYRx1y`gQ7A#q>$_ zsJ){GRkgnQw*9FFZU3vOhd8eay={HDw%kR75^l}~%j?js%HyZA!C!uD-rW5_o(2he z#Qe;?Rf&3xl-E^%R-@>m!Vd*sb?D|a&wGyG_k;Ew8*cS?J?gxt>gPgbJ<9FGI2PoM z9r6tuvuaS$0G>L%y9Uh~FfH|kuMRz3Vpg|)vl{I^w_t%f75p#Ya$W|hLn}5^748aw z{F6HWeQR+In)xMZj~S;PB?abwac!taYfiONO_5r({7_D>Pk`S|dp9pmoe1$cG{w`z zMuVOW3Y&iN9psHJ`v!eTsz?9)F=Y0wL$&DYp5{8`MK#hNBf6{_ z_Q+AA%{S)^*1|eQ_I>bb|K)lVQTS=;K69}DUvcXmCTh^fZ#La}6IwJ8yj?y7aFSJ5 zaVEbOy}mPh+jfw9=Ki?#r#shl;1TksWZkuz1`W!Vbcr`B)}gFmvng)i*Ml2IWJi40 zpeK!n#bsA&&|go7KuW1WJF|OUE$Q5V?9Q($PvEOj`**);^x1t2$7ES8I?b35?9qPk zL5(b?dvI2E(jw(G-n<{3G-&;>#BLF*wa9X?m)NL3_{)*%%e~Sd&Si^zu3QE=qBI&* zR)ZEx73Mh*clzmp1CPd3ZeAqQps@Gc+;3vw0R<0Eup!S3u{!8!e5oF_%x|zCcn#LG zyB2*1`$hiwcFuFK=P57U%#V&b^yP3f(DCQ$(Opu$1wE=oc?Y)~`vm^V z6K)uCY;PUvx?Fl_l0btDcU0DLdTWsS-tE^upCJC6dhHtwdD(mpz4Bp89r7B%Ah#Yx z-u(&4Q={U8Cv9Hu)u6}OA)_?iYmiqo*}3sg4Ps6$-k`Ope2(S#W$9W}(yCs%KU0h5 zEPA+xlTn3y^@Ko!TF+-3_spwBGo=6IZ|k8!3=7x={Nh>HB^wXAYY;0AriBZf;yu6% zc7-3Q0p9TE?h_MxcZT@4VpjjmTlMJZocX&%-yj}u7HnQ-uSVX~I+wLUpy$NrSA6=^ zqpIu6?dO5tWK4hg+z<5iWn#p^(?4rbXNl$UotLy|=&XtzKV({@Kg0%cXTCIWY2X_* z3aYAj%eJmV!9ylGPBE`X=;QZ|U+>cmh&T00{<#IU zNPke|Xf>)0`C@)GOpDHpnzD9+g%(AhT(+AltVZy^KH}{Erw%o%>bf@E)ge=>CduI& z^+-{9s^;8~YV`5JF+1%^f|Acoo_)aRe*omj(^j_1lCs{Hu3-GX+-}CTNHEP&b2p#}@YFGEZ569P{9fuc(&xSbi z{QAdq>n>#-JT=Qi`6kI^6}c26_+o*IP9%2z2z zYSGQ9cR0&|2L$9_IREutJ=$^P*#N6YbsceEhg8}cn=O+8AJ3(u2kQV=c5iM>0XcVF ztR#OZHE7m_rKOy60RC_2(tT}&04hK`S&4as+(w^n|bA;88v8~eDFC9@CEXCugG*5zm(9Ihm)(&T&uFNePcoYD)$mLzYZmYGnr6>-k*uuLKbK{{1) z2YfU8UC8H?pjRUEk3~`)^6B^b%QZJO+AUnWVZ!rTl<)IpHvfGUI%2`S@R_4Ut#+rV za(?9lTEZZ;tOG1?|I>iqG*In+swie@nxL5t33P1-&g^6j}TRf&W4)S+3GTwq8# z)M0I|Abk()gzThx7#H@b?Ev(!wwET z^t+C!MIkNg_SWC8L$zCuu1efk*Rd`QT8L);nS8PyU1DGc^_648=MzVrwdl*BWp@ik z)ggP@e@qR;waZzS1GbfD(BA2j(LYfQXwOmkadAp5${3pPu5EV>8d=cPmX@jM$Pd-% zJDkd%vk~fdf`$ru=hTRfGCBPuq6x zr3S5<#{@hL;(pHciE&YP)PdmtHcKp@Kt7-8a3dsRaxG%2%%fRaG$ZfZ&OfHsp~HO9 zwUgVl=wgjma*s&R)3&qAA`a-#miadgo>{8V{;c^aJ?!hzsGiE$Wl`!5zNSS>)IRCn zMPOfsrUHL{kh{C+nn69<&bezID6B;;MdQ-SAt(XE5b7Sulf3k8I%M_ezzyriwP>PZP4cvNwMaOdsem=;H0yQ>HA{m^ zr>vIc#cNSQx|i3^Q93lx?YQ^60~KiaRHwD;Uu%##Co@ekrW&o782#K?phbJVEvt?` z)S&R=!o#9bT6B1cTbI86TI9J`bMkFhHQK|;zqoE+4SL`GPp=*$)F|z+_6Tcu4YICZ zRgg6Vcmnipo(KHq)7y1l4r|cRH69x#0^W|T>|^5Pqe0#;2XAhGh?mlq?>VB28ucAB zY1s&(9xcmS04ZOMqPBV2oCSVt*(Gc1q1mu65OIx@7f_GJRuTT?YoJcK6O^@ZJn%S( z4xc{OAHCi_fiqGEFX1o({V>|GlJm3L-pVbX1kcZPhEfaqLK6-oyQ?O|f``v}5 zWizW$+sufpdMA+U(D><7!T-Zb+=!Orz|WL^y|0(nA}C_Nza0;L+c5HrTOs&O&fRNO z|J0%tU3ZQ)KT?mH@@+Id-hLORCjqfcCgY^=J)hFlCMU3+miEFO!C2s0U9iD0c81Rf7^2 z9#tqH|END2hizM`LBR{Gt7b2$LF;cG+r1I&b9DXG7zN0+>&vfMSJLay#BUJ}-YP9J z@Oaf+;|Y3L+Uaa$FAeI+s3Wi*>0bl?+f%u5e{6RxTEIUas7}+N!{J0h%k?^RA@Y5l zOMeYws%fM0T68d&VZ_zwY-8HP&O>!bF=*u9&wgsq#rZdKC196I-OAnt300{5XsibP z+#PhN`ilXwy9LoSi0iQMqG;NUK`NauEVyT!{| zG-6O*T(wk-Dn>BJz11j2e*hi$oX{fKMq7^-PBalD1App) zHiIHY5Cla<(6~baf}qqPVGSw>$|@=Z!kPfGgBx{0Kt!P}0$N2yR8(9_sVfCkWJy8- zYH_C^OV!qWr~2J_35tMS{Ixvi%a0!jLb*SP^Q7tko%>6CKMS@P- z_#6uNsg?0V1H!qvyG7_%KgAyhGek)A`{mZM%ObSSpR9f&2IP*IhiZ`POXD-1YeDY! z4mEybAx6_s#5WVD0#uFNdMe$xtPVZFY<~C>I%1vwdO)Ta(RVPd-YLL1xQ}g{a2e*K zWt-~#0auwU&)3j8UX8BivN}vaKit@%d+g>HVkF%@B|!rc_w&z9l%Qll)vpKCqdRBb zZB_@nprAFx-yQ5r%8|v_B0L0$*p852hh9&>4iX{S=tKsRmyz3ph*{I8iqJBPo#bZ$ zBII{}-PYzyFu%j5t2cELWE5LDC*p2Bx*IRBV1lU;_7F5rI zR3Z9x=RWe<0+6qT^PQhf0e{8tiIa|keU`E1ON-NhPio9;kK65UM5U((be3(eLBqzz z%z9JXfCx8TBiL6qgT$qI5)`*#nEbdr5i)L(6HiYr z0>I%Z&ukCVSBep_n`wnPMr+!mev3T|`0S^h=Q|ulNNwm_8*xq}N`EqA7nR)Ye-)x36X@Sw z;fvAQ>)6o=^?HOSEbs@d^DWOhC`Q!UH)eVIfSWaDgfUPfI~ zvO+Q%>^4y=J?BRj;D^Wht1gTXAUtXThK;Cr@|$AHTOlI0Po>nO9QBf#p8@x9yWXS^ z8X!Q}`r8QeNg<%o!P*iuXdBs}Ivebx)%iC{AHaO$+)(jiALUcR4q&4}r1gZOYyj=ly$=>T~C!19Oe-JxhpnY0#YlA=@)?LL8 z0H`P0r;DKebV86|{Gb0iPOqyT-B`2tRMJ)$Z>Olodg`E8YSlm_8WH_ziX;mB17!sa zFFRMzOJJ3i4Xr_K6D@}ca=@=!WMj=pkjVTp1o(9Zwj?E}plV&t9MH4J%f77Hq}PBBW3gETcKt%rLkFgcQP?YcheYrT8ZE;X zgb3}AytW|H5$ye8(aYI>VuYpA$GskjZfWyf)#{P6np*J4)nKnPb8Wu@f6I5i^SI7U z_2>X&Yr&a_@|G~x|z^o-hkNLG;_+#TI4$Xq0u>* ze?3~r+y%v8S6(xoBL{YNmkSD??9zY;H_((ibolwv;RBl*k>RVIT{mZeKjGragE1|& zsMa}YVh#iBB+`ZIEOQaMg{Q45kP|>RUpx?@!T5AF3h-V4@22)78>4+eLJ-mt$aK!FcmY`mH!L<`+#gGdUHd4slAg6AcKFg(kK zb&{%Ux32<@e8k;NIdC2D>Vzko%fR0{;F}W}%PvTe_q*cy?a%7b@c{oVrzv8@P#*r6 z5+*oueNRL4Mm>T>gjv`3h|$T4 z`5QFCz%L9a_!QvXx~Fr_FX{js{H$^Hl`awTKJp8b3I2L4-G%prD7zCIsv;CU*m(L5 z3iu`PThq${H`*#XmVzBUXsc_JN<=*}oHw-jseu^Dt-_Xe0~%=%>Es?)i*8~!GvE>$ zp)e$9Px1FJ4#2vb+qakHmd|ZO#P;UObBJ&coCN#l*9m?ravIP(j}+A}_3P38R)+jH zkAx_ma6n5CwPuUoJ#Fy2Uw2%j4}QauW$R;FZvq~C=_p79{4pS}{^a#%0;JqxvArDp z(zBw5z>*30O9(|OKr2+I+ErVM(4Nh&k}LzoXvfMnW@1_ulD{k5{|fvbw`Y_+7i3*R z)?vvbAHurH>yos$VgalV3}!5O0_#_}7iU0!O6_AYdiZ9Bq2r1=^o+XcH(kIXF?H~k zA;|5H@fZkzp9@d_FwTQG`JqcK>k;FJVlx^1PxytlwdD=ysLojV1n_^L z%n9tD0QbMpyOwd_v;ak>`){5Pet}zcDXzA&p$%Xk-mPnJ6t+pwTWjvQyjlqY;sXQb)}b>s;gT9yM{zvi-hTL06-t>B z_#nL${L{D6&MTNSBKNHeJHQ2~ zZr-Ja8(?QeUbx_VGF5;S%qRL5{@RGXQ>IT{U3L+DG5PI-6u@c3^ls3AUK|Q7(dmTw zNk*;;TqS6OGu5K(Yz-pZIbgSRf!C%Da9&ObcC1Q} zve~-HCd*;|w^^ArizhxrVe@`_s_6RkGzQzW-!%y>0Tf*z?p zELdL${wIEg%i?k|5=F?*M(MROKRW20(eq8)2EzK*$RBccjuasPYaU&Nur9oPXN%cG zcs`Ojj^Yk@@ssMgTTcLZ+MG+b@#J6kuZ?JCoYUq!g!NB)~A3=bAqFc45g8WJn zHZ9%_e7$Mgh@jaW;9npVc>}u1ygzr>lWu=g4f^S(#pSUxY7jBqysJfBe5WlNtV zW`SGdvql+SY(T_kAz-~?k3cEc|BeKi6TrAmh8yaT%l17P8SoxOY9qV!)OHC1ED2gNgm&TL1QFVrndsD30`}~QHvLJz)SxYDr%QhTJ$UklK1+n|j;}KM9q@|OKLGRF^i@B9GY;0nllND`QkcyDEkdxJ z8Mem*e#7ns7o4hdHMp@=?fd2%uWODKcld3_Lehs=*HO_6YR+FK7;=u z(|1R}?=?QMLL2yL4_=1bRgDDqQmjn3RHJv}X2-38b%aZoN8zP*lzXzg?N~q~a=EPF z{DLe%jW=UV@4$M>9zDkiSHaImz=Te)AIY!OCYv=NT6j|K1JI{$;oWXuSZAAqm-(j( z(Wb#S?+*d|yG{+S5D8^=5a7jwS=FbP7m{l<|B6C`GI_)>iyi$=7d!lvJ!;}pc?aa-IO|7F} zR&SX7ykhgJm#t$+$|;sgtSxJ_?&wn3DJ;#(Ey-qg#!^#LtZA&wG>*cg9U*r#iS*W@VqaGoGHBIyZpzRbJB_+TVHq)T1WNt+eu+;`p{H zKFS+Cs#w{_bK0f`u{U~Msmv}t(Kan2b)$DXYwM||HiP(@jZ2oQXhmX8+G|%N!nYA;vq^A4Rta58T+KqB*(gRG-<%;6ljq{Z^1=(8VHRQBU-_PC@ z;&v|Y%87Q9qp6#C0an|tHMN_bsM!=2dv4pUm+dpkl`|ret+u!Hed)B&>f5^>9W$G1 zGGa^5efwK{$E;@M&GA)MJ09nBSah;CCtf+XFCM0DUeRu~^L10l?A0%8Hm`bq zZs*&V9f&k=OOldxzMOWaCBgVzMqKT*HLK0ch&{h&^s7!g%Yj*2lC2B$ zw12U;cgV`hRv1PzsbM>?!pfj|=vF@#uils@LlIa^v?ihZfCGo;hr{+TQ&( zj3VcZ#9;+`;}5W%i(Il#4J$O-dtlzOq6Ir%58KZgU*w!ww6JidMv=?jBG+R@uEmKT z-ZVX>Q5?DV;NnMVn*F?K>Kx=L2+96#hvMNNcBt2-qU`UB*B|Buhc5c?rm6Rx$y-l~ z))z+~2wi+XWb5f$ibvvWLYF)(+Isf4^+#6x61ueO?$#gQDjrQz=J_a0$*E9HKDura z&sQZh=iF$eV<~JNXV}4<3kJ!@(t~+^I`?ue&Q?0UC6(ttZc1*AWAgE@5Ae8CLv!nv zD1Dbx!wZ;xFjp9s{N46nq_3B*QYy(;<_B3%$&+LxmlRCm2hR=7Yuusq{eCt-WZ}WQ z%f-px9}4FGKfhD@#`R~}JEa@_yjyi%m{r?2Mt-%&yKQQmxq#~!wXM~=!{UXxc)erv z9{t6?%#E9QX|H38w~hJzZs3n^el}eF`udo6eP1LU=&dV1va@^ui@ru}yRPDt zpo&2p`q~jUbO+4ptQfL}zD`ePtn!?oO7-3J^^?8F4qDXt2XBwEbo7R93_7o`$4GVX z)>Gfsd42+mk>;{pPvc?`CWY@zx)Cy|ZdY zB_ktpyZ)$qK^JGWFg7RN&>!M z!*0(#Hi}eA(ic}?Gv`BfxGj$zqe!7Nr<%bvXc)|ERmv@95FM9z8QSWT*a`9ZFTR4SuE`dh zBlDD^871jl1dfe5jB`7vLT;bB2eD+>LcCEMK1ZqW`(B7h8lHwTgtFnPyUGc=J8G9A zouV5`RZODjI*=e&k~ZXmGZs=;DSA+vlYDWm>^8VI-)>jbRcY2&k0MW`Bu&nT%rAiq z3_%23Nt+7cYB?`d(Sxq*>l=?7l&=~d|H*+MBBh&ipn}~4(JK@R`4I8#N`G1@`#f`!u%ncc*K#m!JNo&4$n6#8Z;jqcbL}e z?4iLQ6{j`G=0kSKZ9iAG3snC7@k4T!3W`cp`6MzqiL9jqe=Oya;8cRqK4eQ0oURAy zm_{VK2mG|gw zj3Q~vxU8?oQAtCxO8D2a$tsjA1rk|BL5ZZOMN(8&vDPBXLkLw~9uiXJlu5d5jcGbm zxdGR+$BoM+o2yXehF@1Sr;MA#Cfkz~$Tm~qjx9%mJZx4Lc_CenDz6S%*?MdsFj!eZ zfuy`(7^I<+254xhP)Srd@*V}Yp5h=ac_@-9uM7uF;wlW84h7TL*Of@@<9x{NqC%oj zIRW z<3#W+?UQ^eS)Ked={R|@GEZBDTtbp3SCZt$j)G%v+%!~_7i!p(eMs5lTr(&W$UvF6 z9eQjH1?{e(CL}GTD8C>s-&aGIq@gliPD_F7AIKd~nyUl+K?JBoal`zB!vdH3ae2I+ z5vq|~KW=br?q|+csA#?&M9T=KtqH^}`RLe(KILhqx%5Ah??p#i=wnsQE2^}z7+O*k zF0|J-^ZORNzgMvXWt}1wK53Z`cat@<)5x?dkKq58Wat-QS@`{h?+^4L_;J5qmLo*JosedOiFDS$h#FF5X zXG*l3oAR{B*Z(Oo&8JTkU>c!OG_SSD!;oqzMZhHJ8>Mx4$a1~lFn@0RQonFs3_mPN zZ>H{|#Zz_lq>oF9IBu^{;&HiNm@l6T$@TCwt_N8Hg1J2ZrM*L?gqGrV;lwdv!G7WX zMEbB8eni6ih;U8_*Do5E!{SB80v)d&hoW+8=|EzSR51 ze_r}MkxpOgy<>aP=7fjyaHfwty*u=ex*R6IzSLxay@>q7!Ik2)eu)8o!{NQ?1&2kO zPVd{Q_rZVIECL7iL+pKcFJk|&aor%ZFU6iQpO+?qTl*4xAKZ&t059CP-wy2&@kd?y z0VaQfCIO*w+-D{n&*)1~8upn9CGUebo5N&IXR-pBwoGOq(<+b%W3fJpkHdjB@8g}E?WNCfB}yUU*-T~}6Y{}dT!@t! z3xeSpD9|B2YX+Ooi}A8^o9htGquU1a%97#t6liHEU^iJzS`KgY;n znU}jGU0Oba=Wk~VWw-?4x^cqXqB!eBDjLpq)I9U0m>7E2syb2lp-k zJ?h8 z3+e}D&^>Yg*g>1@;?3t;;r{>7FMg2T%bjQR7wVY+?dSP=c)_^%2Y4Dmo27LAtNx7e zW0(i|d$`1U8aqH8TtC&H{if@$KRx;8v9dPw)0Ymh93$zNaopYLp6)I|{&qH)Pa$o5 zQ~>ZQH^Cvs)14vf`}hQol)w1C?ru?Dw&wlC{a1Yz<;Q0P__{lPs*gU3@2_uwALx!D z()_^R(SG*4p7IjG33E1b{2P46vEzkdx%-rSeiR?Z3If>=g7N;q+hJ}ozV42(ut$Pu zQ$KZz_2ftJJdI^_{wre^=D-6!j`O!OPxw^52>I|}(K*I!f^L6(^{>b!=tjR!^LQfY zfrLNMXMBzw$cnY6OkefYXI}jHSWl0?H->&ho!mfonSYws!yH36#tuBty-?MJ-(2fY^YyGF@}-9N4;q0b`xF}~{7ofr@OTl&;ZipfAn!rTZt z6fg+PiBZ1p4hfzf4kiR931gYSeFDeWkB~oqzFUHy5igb#j4>NKVcvr2adShM!FWFH zQBNZ;4~Ilwck>vcj1YFryoBkA>@Ye;k#_jft%u>3K%Qn8ZqA9b0(?0g!%i3c%)-ys zfjC7phOKO8fT1jB{A`b(=i`#B;fBfN%l`{*&TOqU2zl-|T&UX(gfR@asVz+Q?*?%!BsMjRic9 ziwm?4Hw-W|#1YJRxH6fp_(HS?-vA)T7m#}<)0P97aQoOyoB1=O`JLwxM8F7uA8||E z5m4Sf6iQ@G!KMQQS~0Cd;GXE8KdRq!Tw=ug`mq8H;U4mL*Ut|V=utngLtuP+&NYyq z@jkzm^Pl7=sxHm%itQO_s3-m_{RLVxyYsVLh@jr|ai=nS%=<*5Xxu+j61&?g}N;yV96x z;b)U58!s6@_Th(lkcb7?j?Q$N4@r%!yVKkC&%Xr!^~TGX64MvwH+`S~FY`ZP7eIbK z&Tsx_`Frud)J`JkC*b_RZ7`V9@rz^P#$o0sx&m%lpBO))sf4{JO-W>t_V*WGkhF9o qd?S~|aHFUBMMO*^3~F7O!K^DarC~J{mX`n4KR6=<4x(ndF#ZQl+{$WFAdrN}kRdAIwqg+%&f?a# z?rPPiwMZSP9oA77l7QA~Yg=lqTDA54-J9S*?bGLd`tRrcy#IW-C+D8=+uz?gC->?o zX`-KiDL6hvu+~0Ku%98IInZbudQBA2Xf(N8uFF&8YP|i*<#MGqSDvTx_6rr!oE6#f z+`K%!B2q*X8q}GZoV+|4G%Is6<$6_)!CqvaMk^QPu~h=mFcnQC3RTerqJdDuQ0pmD z_|t@$x!IXn8Xd2-r@wPnPOdgLCnuNPBzg1CvPI#lETuuC%*qB?RXJH2jUkiwh#Zlt%rfNavSt1> z;NR1ql!TPcjie1^MgEH6H0Lb2GFPussCml-8$p1cB0(~1Du_K5s{USdQQy#QAP^7$ zih^YNRG12M@;h`BfvhZ*NHoxko_0MHF(?fO+E@($Jq?=dY>hIUZ6Y~L7*2zVpl_%bFb!d`nneDhL&HQgATO{R1~QX^ zgN%1rDk$gDMd9EMrACn}2j#OBhCI1i<2?bGXcS7VAyZr326oHMljmu5dX48*xY29W z*|`R_o+mOJgHqPwD9T&HY@IeYORbG;0!EIx*-Et`lOzKc5m7e8!${!-Ls_lVc+efD zlM;jcK`c552GUYy!kIa_+1a_7AX|<>q0Uq&y-5*r)k;-njy%(|5oQQ8)tP!#mJZ^{ zF*i@IRcd~?EzD6VvvX8gc7X(V9tJsPW$83p4G(BL=H;lfvh=-r5M;Y9N2h{F+UcC5 zR_WCmt&ZL#$aCr1AuWhZ${plFD+gnOn?yq)hAB-3Mu2Z=)97_-jjDT+E%cD~hmawG zKt-QFWoFX-qC=#(qGVE+mxB>*LCLwSsY?q&KVp#5%_QlQaXIiKM%r~yf~HM)LPZh; zE`+1M*h*wa>P9L^ITWPNqfhO(f-AD6izJnt&{8&8F5pP|u!mhIT#wu4KzV_nPdzv| z^bu`EmtozRYaR=1>BB>aVO{A#ghzRYfxP_$G(if#>9zoxk8~-s#I!qW@c6vWC;zVz+A0Xr_<~6@(hMalk)Qm3JMF0iY8Ak zE-oo4EiEgXGNrtH>eOk|rcbY^sI082nlWSM%vrN$&z>`9?%a9v=FflYtpy9Js~0YO z`|X;VMT=@{7cXA2Wa-jn%a$*%t6Q;R=zJJkPEp%|l~HmWOIb(&gDXT5@i%P-V`zJm=@Bx}USUBT?HuOZ>slvM$@XBOUUf z(XtlsoI6q{@af2Jz{i~4c)`MKbBf{_gxTi&1VUO89xusRW63(~NM}nP(Llc?KL8nX zCIT{+JVe&aNr=}^m}|~7_Rf}M=0d>Wt5XYj4~Og@Wm&9~tb zd^VrMx8-yBJiZ+tqMvWici?yBJMsm5Cw@1+kT2qQ=l9?{^Lz4p@q6>dd>6hezmLtQ z6NSWjkJD|zV`j{> zSvUnhi=jZbju@gc=S?et$?C6$$S-u<5hC$4KC{Y~%d}@%w-52-M#zOsI!)@!h2|5L z;+XXwP@c|rCK`QD*SbOd(;Bz=Rb+YMh_AH4clK9-F;(J)N?}YRof{_|<;Xo37;(fG z8osoWCb-3%W74hLQ@;#L;Jd|qIeH9~=k5)G_GLxmh0D|zH)*aOpESOB{L1~hwGXCV zoFyOsV5)GlhHP1;9zRDeZkn{Vcs#Ux*>!fU+kBX~K6v4mT_d1OaGYHm$g*~Gn-6nV z0+lVSk*QlWSFNk6K)3`_5FjJmuMQ7f#q=J%SoUotFtoY<&oM#Kl>is zNEpxJiANg@e&u1Rz(Bf(TZ+Cg2>Q!Zeu7Dk{^haqEI)cra-csum~Q7G(-{KGp~&~= z()YQQD7AhdUx_lGHH|$faJVYp23SOcoFmkQbPvZQkjk%|Zq3a1rz>d@f_*f9`WSWv zGck`n#>?DEmigpbTxTVy?;KHPVG{H5{e_(8oPot>^!C#h3$o(HbEWg`ZQRFHq{Rh( zQ#VD?5Itd$=S-=?iqCno#U zo}{SUDsk9dKf8Ov==pz4m38MX2-T+ z(z@^n8W%1CE-Scf;bOvN4;KqAK3tA)b%l!!7Y8mLTnxBuc*GEEFc|rvo4?5L>}+fy z;b`d;5*@=JTwl`)!~Q=seg#8t1B~B`!d?Y^E)&!wu}8)jK#sJa8F4JAQD9@yyK_mD z(fbg9ExL$L0gFDE05Gxo@u|oa425{_K>&DI1ec6odUr>#Y}^)QMP9n)JXPgo9XURH+z+$WrNheM$S$OuGKH~V;JF~M|>s^-3C4>1WLr~9%6G$j~uBSVj?Fm_c;F1}P zPNb5Vi!`e_XOWgLrz^HEm8!@IWRjxW^T{Ly-&`s|5d@?r7N3B;#B$+NiHP3Mi_AcQ zq-`Jr(QODa`KY<1W!WxtHEV7lnL$Wqf0k^lsZM_(iy{hxUmxZn!+kF#7bu2AkDEv4 zm__eb&+273CH?se(V6NE;i3K3yCleG2pR0kyA6H|tD=@!E%h9EbnnpwlQXl6Imfte zE}=0@J(G?~3I0*YF%8RRi~Aktn`+LklZ{k;=Kbq>uZ_{4o}Kpt;U*$;%*x7>?kB6l zHu}l&j2*af{$jQJmc%j-H@9w{*N1EqGUnTRBu#POw@H}PM%(Juz zn0vOxWJ};mN#pD-7uk5Q0a_+-@Ne~5d%r8-;KT#HW_|w^fgco^R@=K<;r#gvZ`|!| zi;p+&n^``JWu|w6jWg=v1Pe~t;*~4N4}o+0-(P;ME5R$dWsP^`XD|BU2?xgwnkSv8 z=HUKwoGyRyfQ98riGAzY99$9o$<&?)xH$M>qv~A~8_!jaZ;qTn;AEHh_P6pl_^>Z% zE3m@v{(N?Xg3rOLkBWh$HTENoki@~h{kC;IbCiR%Y>+yDgDDZua&Y^slSxOmf{vnu zD?Ogr;!n@@SBn%Jynkl-nZ1_@9JiSh>HaH$<99Am&sB0T>AqDLIN0P}Q~%J<28V0% zfCOQlUv7g>x}5O!;ow_2q;BBXx%6#rOniB0Y3kN}Hkf{b2+m@Awx3yPi+{a(v9chN zi;w-WXUg4O1YTLZdVQaRJZwXXvWkiCG=%hteayz={|I~Z!}m-qVoIe1foIb{BsN-O zR)dF!4L+NfVe`cbE~fAFD5|o?*W9ctmxkGR3gectaoGva(Awb~40?xOGFamPI=x-9 zmyQR1K6vC;_8goV0^v5y7LVQi$(^+yad71RUw%LT&<3ynU=Y}ti+5bgm>j>Bg;P)Y zAKeYnl00?Ce#KKB4y`>sJsR2Ic-&R~>1bPgbBFtl8NF<9x!iw@Z$F5_)BF>M4lrVx zR22+7;Hk^+_kXp;kH_giPd3&=WMT%}a~E-o!1OMLsy1sJ%VqSRv&FE)^ zh=m8!N6h5K^KfgKw3jVc~&8K2BV=j*VTRt4B2xpV^!C>%nv@ z%)kBNPOB_i4ETqDXX1ImTmttCbvdG9<5gqcd2f5FHBP#|Y1L6641u_K&MXe5#}h6K zxz!Z}8$Xq9i5q2u0YYFzHty?3yziXO!YL6UVm4-c#By*%)}C9#m)K(N;%!w4Fs22N0OClIXFvg@fIjm|xoP9x@56p{Y3d^UvTDXN~u)`Dx4X(>C~aG=$w$7T#3s zTg$}TA3^L*Be31!%1MBqI+woTSJ)UFGU69IYkVjd62iCK?ZHegR@ZxNWnd6>gW+Rq zT*ulP$G``x!4pj!JPZIF^f?n}*vNmT*^H=T^00KR@AnP1xNrTuU9Gp-xInss8_&Ua zD~-%cHZc>qKUrgc`6Rv-7k^(M^<)ANgJCq4M#i&o$k)dpOG7NUEq_`wjg2Q9iUAm~ zF$uEiX>2^=mxZGX-B>!B8-ZsuoLU#LA29FZ@z@$KnKLW>+z?xQEQ4sa#g%KhPv)?2 zWqfFFE{vH4T*F+1G8<@oSHYy$6Yy}fkL0yf^7LZX<7_mSA{2Pr@NQv1DATnvte zENzPybtxP8-eHz`Jz7WL(`pmJ#G}+_+-%I=YPG^gy2O1m$`;q@9Y7!!h9J9?ai4=* z`*M4pWMld4QDu%cxNMVGm~*Zzwuc;0ZEcNZ>Vr%crqAXN(A!~EYo*&9Hik@FF5hEY zT@lW~UuItTHs?KC>~((S&7NDBSUrcB#=(^tjw=Z}d=p6z!kBj<0@r}8hJVKlwBHW` z(RPbz)|#!%A8ohS5w`e=Jj<)i76YOF_q%a$34I-NHSsydZ0tzs$-_q=l}?#Kuv-7y+To!B^-1u702* zuy@44XjngIeV=e(Eff3xt8W+EeO!F(pJt?_nE%cS{skm9PB>m97@u)|h@fQ;v| z!*N}N9d21HmC|h4^|TwG*AombzCD*_7Zc5W%EdYK%gmW}BX-evP6mRX0&vKa%Xqs8 z5f9HYaIXLbyORX*JA}oDae3`LygC4qdkGtpxp+!G7sEFA3`=n3#b0EFt@X9 zB9_e+O4rYv*VQ9&YwD5pv)=0Ft7xi|ZkS!&BRK!or6U{WyxkjD5hB^fxr_Ql*LtKi zZk)Hc@2K_BD`cDIFYTYYC;M{arni>+Ol+Jh3fsJ3ML_o1t!bZbu3j}rXF9qlV9v;% zgNxg5e6@E@d~YAIkT``q+E&AF{(_D<%F9jLo{py-$Tym7&kt~V_WD7G6PTJ-&;eFYEfCJZV4<<^D0 z#a%LnO?~|E*IyoVnTRW`rtrO@&zPh$*@6M8OJ`cc=GwcD+?e`RTf_oUkp9M{udYYc zxI`>;OFMfbW{KyRUa2SdxKlsbnk0hnByr@==j7wLU0v_(WRGL{o!RHpiei3Uld+`! z8oFXg?fx*K87)3>?yl9vR`kJb!j$=Q3zEF``0cnkCUgL{b|>ntAS#{jZ9|PCBA3dW zucF?U#|3?9--fETJ9yb;T}Eiz{o5sR*HLxfs^BhtTTsZI!^)C7CbV07v*vuG39Xd` zv?Q0cpz$#4w*e+ZQsJ#PBBdn3=V71;h3p-_uFEnb`t{@4rUMIFk<-k(!xr9bMw73N zT~sG-MO`DJuYGa&8d|utF3Wwp5rymvh|_tRP*CLeTXU*RXnBF>j((3@(CFn0lx|b5 zpp*|nzR)$cp^bl(&0;-hMms8GL#NJcMI}E^Yo7L-3EB5#L@{_RC}@0q;rJ30S{fnm z7SY9s)WYH!+1@~x6mydi#ooEnqK~?R&w5N_QR8zcZ!_5AqyHxw-w2oe@pA%$kw=qY))^ojU9kVL}Yg-MLf# zWkl0+`^L&#O(@{WmL4AME$Cai*WK^IUlIKuF!sv8&N(xuZEi6k^UkIfQM<5Dni2Nf zqgwb)3!3=t=FfwxO=#P!+LPPX7*YR^2lnXiXF^^F=O4@CTthh*VZ9t?1fUPxHOguOWwS6_K|STT#QebL<;87?GGe;B;8ptx)3Nd8RjjkH#@edHITntyIa+b-2zw4c+8ep^3u zLbOdA>T+miSl-W9(f5Vo@3)OFZ@Iyy5&NzfKvMl}t+r~NVI zGCI(^y@4jXj)u(q`P_j)CN%efbLzqxBMSCCn4sEjLN)JHj~lcJ;%Uf$cO$}0C~eM$ z_X>-Rh(4?A5%C1<|8Zm0W4Z}#j}MW&HN}KBPpsa1n=qofha0vA6r0c)y8}Dw?2YJF z?y}R7-y-aqpA-K*%A8qeNU{n}9cHy>=dvAPANR=dAnz1xJ| z_2T@?qs)ljyIr>adzlGs`Sx1sDB%_4#a(pn;xBFJ$kXIK%05QayW+fLq~3%QveO>C zJ*yQ>@EKumy4Q@x_|uXmU2j9ltDdIx47q}8s@x4#wTRWjjBZ9=ZU--pwv5hZQA zY_%PmQ0n%$hnuDuk>`?GJ9qxtigsMQS6}bgj6UT(eBvuJqF%qvsQh+<2^H4)_%sYN zA>o(x>!w#3k<&bO&O;|7DjNLw)QmzCQg{zzH2|K6Y?|@wbii{r?v8^Sp0=QSzj^c- zP+>xS@1Gf#+-^d9x=yGX-Ma-Hf*r2p=S!6z;fqP{U^}sHJG&jdxa<5SOL3 z>2tpYmHi%5{^Lv&I>Gy7l|IdcCf%rPavcHrZEN@Yd$C;GN?`L7tDkZOET#MC;a^^E&fmGxG6Uc6<5UR%AEhxBjXffZyM^TL*1u zMu7vIiIa>rRJY;8Pr64}(fzUqC#H;PLtQ!c24c)r^he+64;Dh+v)V7$dqnWz`>dPM8qM zMPg_{?|PmLbsi7-D0fxcqy!Tp&wo{IMs$-}flMgfXLkPj@z>DkTOS=CkkE>H<_9!a zd}%~KN8Vh7gpd#6?5T%*WIh6z&{xd)sYiD;qvgLJI=5y^D~fjVtsMIR{C@78-=poD zQDsJ-7(VFx6d%+XLEq*z`>qZMg#6~`UVBU3iWXe19b^AvE4meTCFxq9X0&lG_q~!0 zt?0uaZ*H?Y*o=%vYgWgNgLs_Way&EY8X6$}px5EgTG6IYmN(AGY({q;vHm3oeLXn$ z<_!UT7w#Oo*9~7oAs+J0KV$&?wPSC^GEL~)#%Q0M2S%j*it)kr%vR(%{?YP{{@2ja zVRXkn!%e8T>i3HWHGm7x63)TIHuTH(F5g$DUqyJ{mun_Iy@nc1Om+ zqxKN@dFMIjAb-WXqSwOlKi!1x+nmxp?qx)*C*5PJ3?{VljN9&w&EW3|``7G`Y(dorXo0cI zjp)#Zr{AQ*x?i3#?%lJ?OvpwW^~T8=iN%e zuA#6PLZ6uE-uf3gYIyMW&^SFM8eZp);+X`fz2c3x3UxK|t6vo?Kd z(S=sDx-pg|7}Jd65(ZRfl(eGk@f*IuU9KV7_>H5SU|*5()%J_`V4q=vQ)>X|Oa5AO z6ZC}xJ{-Y}h&s=WFrf>}H?3KDx&=7}dhhm!xS!r_e`bFl$j|pS#^q$2(8+v<*bKmt zuKU$o=}*n5k5|98!dfHx@%DEMj|~KzOvt?C@x2i}J@eV(Tn^-G`a8WN=bF&lPmVMc z{M3R>10EM`8Qg+uvKbLxZ6@^EEqBLeOGtEVzsN23tF8Z!ZMh9?O)J8-EL=0V3b==6em|kyti!o*aP{uuN-@C`Ob`nDt3>^_v>~mn`%AB9)Ewu-mEk0 zV^&3OU%4;$(jIuGxP8@!d8WpBJ)(B3K3H(;?6$EdcC7hVaeLdzRZ%1B z?0n~w=`<^jbM&tIBQuDu{l+!zTKDN3exKOY(Yx0ldrKIQ)7-Rs!-==WVe_41_G~=4 z*kkm&<4*3`bb7h3^3>{>y%K86-M|zy!9zmxTN3q*U~qwHaJLy4zG-ETtLX5H!iEU= zeY|4m*K9tsy(?53Cii9$glS$dlof8*Sz@C85`%27>qEBRn*rs-%w#CVzpfx#T69p> zdmmsC9uwy%%ShVdlPq|IQSaqhO6PM;Q8!C~(M01YmXO=PU9^DCCuo~VGC2DNlno!; zViU~vbx@s%v}D842~e7%hmkF&aCS8FUK&(s>vpiR2^>fXbEO$XGF7f+`T8~_crPHC znof^hK%NOZzm7ni34zH1I1`e?Kpbp26LxBa-hb@%gif9zpV*i}izj3FrtP>2>f)rs zP&Np6gC%L7i;L-EZo`m4lqJh9fbtW|Zb5(jgT+vagKEebiw?1ZiHc7RWN*QCa(1j8 z%$|7qr^)naF6}K*K8s-fz6m-iy3HpiXF9b4k$=)NhdW`0#0sehfMo`R@(XAIG=cv2WyxBp19fKsw>ml21Fe0*(KL0kVKDSO_l%aHwCoBJ0Tq z$uT-zkzP%{PZOz8O@dwQydHiq1r-{uk+R=S!Y{Em#6w}34VkW$MNAKs>OqU9?Q*}m-@RcOtzp0WS@Mrg!;G=A+8&ru!K69hT89;Gv=wd+ zF_PMFNyQSk3bz;sS`Bo?xW-^tdIdAOBD{8<8$GeMCPo&UD2t7$jgG0U!HgOjjr=>g zKL%b5<(G&Y1n$WsHnXT5z1!9_j>&_4|)v7GDJ1x!`>`gUr3)Fg*MxT|TRBN>_ zcG%^sm1@o8&RsMh!qpUH$cqZ|l=&LH+Dyhuo3HDb&4eM=aG#-pcbiG3)>=&#{1b(a zezJ)msT&IO@;e$i(8wxK=H&xHvd*FE8T!0@owGXb}$Qi6c8fS>-YfVB1lbOJ0 zgwr235|fXltv@B>8(x@u)e^OSFI$3XajzvV&D0-SJ3V4a4h2a}0nY!y8)C0;d%YHx zs5i6vfaO%rlC%t#TnQl+dFj=sSLiwIae7($j^KQT8Z*&T)=oRDC5NPvN5e>>o!;p8pgVPIX@Hwvf8`yej;kW6Ej4BRVV`62j@DBXn@6(AogYp1eS~y~`9=jb5cL$x!BLi*)(|*HH1; z@m^wA^KAyzr|#5B-A-}U%XMlP?n*A)t}rE2qt>c2I=7k`Wsqu-gNyVUWuA%}uP@RW zN}qM)DRR`xLXwV7T_{({3*|6eU!>Je^%6hl_2Qo3Ri2%D|6blNZ#!P){%Zf<%m2l_ z&8yry_jcl~$jj4`lU_D@N9&)N9G1LSxmg-JG5If@Yu=f?mKp5W$o5YBH2T7Ufv;-y ztocvXqIQ0-VfL)O6SMzPxsGk+s~lhK`FnX%JJnYiK5Op8EmNB(f2~1Z==d{}KHnI> zfhVx!2&lU3gQgRJ6l*;q-^NL9i zPgds_X!7)AtH0j>-vPd^u-tXPyJ-}Hs3=@QK9-{q|#z3%!6-|NLa88 zn&BQe_`-PEpctLDC_O4UFx7wLWJPpxp+Y}G6R(jeQb!kO#po<;+7N?6m#9@oC6~qI z4m*QUel$lu_pS3jf$i z=_w<9&Gh}Ws;Eeyk(f=&rqIK`FNg;DqF<7yEFJjBQ-buMr&bYFtceQ)Idod6CuY7N z_iM)~qawjZk!3H+ph^icr1^*BTJ$JVrT7)AlS6#fsWL4oM~p5+BiAM8sKO{c3)4Y9 zpy8WF+9wKhiYf_-3nT6S+%8HOpPr)i{|j=Kg8o{0YC8Bum6_%bI-B|YQ}#3{{X(); zsfm-*0%AdiF>kWxYsbCLo@u&}$rc@6W6Rjd3V*ZD6e-ERX(@@>s;CGuPGM|GK_3^L&W! zLS?k}MZ6dk`UL-x|3RE7qO^JvcW=VyOZ|FTHef#+{QW$(^~pu@l#!Fu0goh(UL(`w zG@U`47GRP2ulP(KtA#i&Rz-!BzDX_$AE`25ML-P2_jR`VAHgN$M&+AhybSU{>7V2? zogxab5}szsSFh%?bY02hwABC1he}GOWXN41Z;o~S$Q(sLtQK-F$nX0`n|0#<8vLaf zDEuRz$KYS`XI9CZ{HP9(_=jc5eX}e{|FkSwZ;Wqs_?zSUf5u0-Z}$+}4WjlWdRY+EOBDf13V$kHa;ef^J6WM2_iQnxBSr@n$K^;hlMums?_yogUL?kr`sF62RNfqQ! z*{l8m`YEFVe0+lb-F%Xt?D_uByx*iKYsmS*&*qzj1kvI10)Nm^DhnC``o>L${X9u7 zD?HC9)5nMGkd{Cxm5w0`N*}TU8*;J$+)JgA3YbLd7bA@rHOM?aVFbkl4gv6?s#J~I z%17q{MOlCH=m3pj((oLpQ}+2Y`2tCbhG+80vV5Qp^E>2Ik_=vu56&Ur-xt>!nD6&& zeptf4nNNvso&w!$7o9WYEGV zqRiqiOFX_34zBBsIls@1a9!La{-f qKdDGv=&dvuyr~1V*m7VNn~&1){1hH0|3`ncHND^;#>0Ps_kREgTO&gN literal 0 HcmV?d00001 From 9fde74764de72dee50f6aac9fa484f2f569d6161 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:53:33 -0400 Subject: [PATCH 02/18] feat: discover morphology images in subdirectories Xenium writes its multi-channel stack to morphology_focus/, but list_images only scanned the dataset root, so those channels were unreachable from the image picker even though the files were sitting right there. Both sample datasets ship four of them. list_images now scans the root and one level of subdirectories, returning bare filename stems. _find_source resolves a stem back to a path by searching the same two locations in the same order, so the picker can never list an image the tile builder cannot open. Root-level files are added first and stems are de-duplicated, so a root file wins any name collision with a subdirectory file in both functions. Hidden directories are skipped so .dzi_cache is never scanned. Salvaged from a GitHub Desktop stash (28616a9) whose edge-file changes were superseded by the merged #46 implementation; this part had never landed. Co-Authored-By: Claude Fable 5 --- backend/app/routers/spatial.py | 54 +++++++++++++++++++++++++++------- backend/app/tiling/pyramid.py | 13 ++++++++ 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/backend/app/routers/spatial.py b/backend/app/routers/spatial.py index 821f9ad..57438ba 100644 --- a/backend/app/routers/spatial.py +++ b/backend/app/routers/spatial.py @@ -66,22 +66,56 @@ def dataset_info(dataset: str): return {**r.info(), "capabilities": r.capabilities()} +_TIFF_EXTS = (".ome.tiff", ".ome.tif", ".tiff", ".tif") + + +def _strip_tiff_ext(name: str) -> Optional[str]: + """Return the filename stem if it is a TIFF variant, else None.""" + for ext in _TIFF_EXTS: + if name.lower().endswith(ext): + return name[: -len(ext)] + return None + + @router.get("/{dataset}/images") def list_images(dataset: str): - """Base names of available morphology images (OME-TIFF / TIFF) in a dataset folder.""" + """Base names of available morphology images (OME-TIFF / TIFF) in a dataset folder. + + Searches the dataset root and one level of subdirectories, so multi-channel + sets such as Xenium's ``morphology_focus/`` are selectable alongside the + top-level ``morphology.ome.tif``. + + Returns bare filename stems with no extension and no directory prefix; + ``pyramid._find_source`` resolves a stem back to a path by searching the + same two locations in the same order. + """ path = DATA_ROOT / dataset if not path.exists(): raise HTTPException(404, f"Dataset '{dataset}' not found") - names = [] + + names: list[str] = [] + seen: set[str] = set() + + def _add(f: Path) -> None: + stem = _strip_tiff_ext(f.name) + if stem and stem not in seen: + seen.add(stem) + names.append(stem) + + # Root-level images first, so a top-level stem always wins a name collision + # with a subdirectory file — matching _find_source's resolution order. for f in sorted(path.iterdir()): - if not f.is_file(): - continue - name = f.name - for ext in (".ome.tiff", ".ome.tif", ".tiff", ".tif"): - if name.lower().endswith(ext): - names.append(name[: -len(ext)]) - break - # Morphology variants first + if f.is_file(): + _add(f) + + # One subdirectory level; skip hidden dirs so .dzi_cache is never scanned. + for sub in sorted(path.iterdir()): + if sub.is_dir() and not sub.name.startswith("."): + for f in sorted(sub.iterdir()): + if f.is_file(): + _add(f) + + # Morphology variants first, then alphabetical names.sort(key=lambda n: (not n.startswith("morphology"), n)) return names diff --git a/backend/app/tiling/pyramid.py b/backend/app/tiling/pyramid.py index bda6a38..e285ea1 100644 --- a/backend/app/tiling/pyramid.py +++ b/backend/app/tiling/pyramid.py @@ -321,8 +321,21 @@ def _pyramid_root(dataset_path: Path, image_name: str) -> Path: def _find_source(dataset_path: Path, image_name: str) -> Optional[Path]: + """Resolve an image stem from /spatial/{dataset}/images back to a file path. + + Searches the dataset root first, then one level of subdirectories, so + multi-channel sets such as Xenium's ``morphology_focus/`` resolve. The + root-first order matches ``spatial.list_images`` so a stem that exists in + both places always resolves to the same file the picker listed. + """ for ext in (".ome.tif", ".ome.tiff", ".tif", ".tiff"): candidate = dataset_path / f"{image_name}{ext}" if candidate.exists(): return candidate + for ext in (".ome.tif", ".ome.tiff", ".tif", ".tiff"): + for subdir in sorted(dataset_path.iterdir()): + if subdir.is_dir() and not subdir.name.startswith("."): + candidate = subdir / f"{image_name}{ext}" + if candidate.exists(): + return candidate return None From 700ba3800bc113e2294230cf6e238ed7503c5577 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:53:48 -0400 Subject: [PATCH 03/18] docs: align CLAUDE.md and README with shipped code; archive superseded plans; v0.4.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md had drifted far enough to mislead. Corrections: - Visium HD was entirely absent despite being a registered reader (second in detection order). Added it, plus a note that VisiumHDReader.transcripts and .cell_boundaries still carry the pre-refactor limit= signature and would raise TypeError if the capability flags did not stop the frontend calling them. - Per-panel rotation (#31) and RenderingStatus.jsx were undocumented. Added a Rotation section covering the two places the angle must be applied, why the fetch bbox is padded, and why viewportActual exists separately. - The cloud deployment story was missing entirely: docs/cloud-deploy.md, Caddyfile, deploy.sh, docker-compose.prod.yml, upload-data.sh. Corrected the claim that there is no auth at all — Caddy basicauth exists but is opt-in and off by default. - Fixed the r/ listing, which named a file deleted in b30bc82, and the App.jsx path. Documented the actual three-script pipeline. - Replaced the stale fracW zoom-skip thresholds with the sampling fractions that actually govern fetch volume now. - Reordered What's Not Built Yet to lead with the real bottleneck: spatial reads pull whole parquet files into pandas on every viewport change instead of pushing the bbox down to DuckDB the way the edge path does. Added the CellInfoPanel supplemental-metadata gap and open issues #35 and #45. README: replaced your-lab placeholder URLs with RaredonLab, added Visium HD to the platform table with an honest note on per-platform completeness, documented multi-channel morphology, multiple edge sets, split-screen and rotation, linked the cloud-deploy runbook, corrected export_for_TissuePlex to export_to_TissuePlex, and added a roadmap pointing at the open issues. vite.config.js defaulted the dev server to port 3000 — the same port docker compose binds — which is why launch.json had to override it. Set it to 5173 so the config matches the documented intent and npm run dev stops colliding with a running container. PLAN.md, PLAN_v2_2026-05-02.md and EDGE_UI_PLAN.md move to OBS/ with a README explaining why each is obsolete. EDGE_UI_PLAN in particular specifies the opposite of what shipped on both edge aggregation and lrm_set coloring. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 282 +++++++++++++++--- EDGE_UI_PLAN.md => OBS/EDGE_UI_PLAN.md | 0 PLAN.md => OBS/PLAN.md | 0 .../PLAN_v2_2026-05-02.md | 0 OBS/README.md | 17 ++ README.md | 64 +++- backend/app/main.py | 2 +- frontend/package.json | 2 +- frontend/vite.config.js | 4 +- 9 files changed, 311 insertions(+), 60 deletions(-) rename EDGE_UI_PLAN.md => OBS/EDGE_UI_PLAN.md (100%) rename PLAN.md => OBS/PLAN.md (100%) rename PLAN_v2_2026-05-02.md => OBS/PLAN_v2_2026-05-02.md (100%) create mode 100644 OBS/README.md diff --git a/CLAUDE.md b/CLAUDE.md index b4c4884..337cc8b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -50,22 +50,43 @@ The backend uses an abstract reader pattern. All platform readers inherit from `SpatialDatasetReader` (base_reader.py) and implement the same interface. `ReaderFactory` auto-detects the platform from directory contents. -**Detection order:** +**Detection order** (first match wins — see `reader_factory.py::_DETECTORS`): | Platform | Sentinel file | |---|---| | Xenium (10x Genomics) | `experiment.xenium` | +| Visium HD (10x Genomics) | a `square_???um/` subdirectory | | MERSCOPE (Vizgen) | `cell_by_gene.csv` or `cell_metadata.csv` | | CosMx (Nanostring) | `*_tx_file.csv` | **Coordinate contract**: Every reader converts native coordinates to image pixel space before returning data. The frontend always receives pixel coordinates. +**Capability flags**: `capabilities()` on the base class returns +`{has_morphology, has_transcripts, has_boundaries, unit_label}`. The frontend reads these +from `/spatial/{dataset}/info` and hides layers a platform cannot serve. Readers override +it to declare what they lack — this is how spot-based platforms suppress the transcript +and boundary layers rather than returning empty arrays for them. + **Implementation status:** -- Xenium: fully implemented +- Xenium: fully implemented; the only reader with supplemental `cell-metadata/` support +- Visium HD: bins as points (`cells`, `cells_schema`, `cell_detail` from + `tissue_positions.parquet`); declares `has_transcripts: False`, `has_boundaries: False`, + `unit_label: "bin"`. `gene_list`, `cell_expression`, and gene-set color-values are stubs + pending `filtered_feature_bc_matrix.h5` parsing. - MERSCOPE: cells, transcripts, genes, color-values (metadata + gene-set) implemented; - cell boundaries stub (MERSCOPE uses HDF5 boundary format, not yet parsed) + `cell_boundaries()` returns empty and `has_boundaries: False` (HDF5 polygon format + not yet parsed) - CosMx: cells, transcripts, genes, metadata color-values implemented; - gene-set color-values stub (requires transcript aggregation per cell) + gene-set color-values stub (requires transcript aggregation per cell); + `has_boundaries: False` (boundaries are per-FOV label TIFFs) + +**Interface caveat**: `VisiumHDReader.transcripts()` and `.cell_boundaries()` still carry +the pre-refactor signature (`limit=` instead of `fraction=`, returning `[]` instead of the +`{"transcripts"/"boundaries": [...], "total": N}` dict every other reader returns). The +router calls them with `fraction=`, so a direct call would raise `TypeError`. It is +unreachable today only because the capability flags stop the frontend from asking. Fix the +signatures before relying on those flags. Note also that `base_reader.py`'s docstrings +still say `cell_boundaries -> list[dict]` while every implementation returns the dict form. --- @@ -87,9 +108,10 @@ backend/ base_reader.py Abstract base class — SpatialDatasetReader interface reader_factory.py ReaderFactory: auto-detect platform, instantiate reader xenium_reader.py Xenium implementation (inherits SpatialDatasetReader) + visium_hd_reader.py Visium HD implementation — bins as points; partial (see status above) merscope_reader.py MERSCOPE implementation (inherits SpatialDatasetReader) cosmx_reader.py CosMx implementation (inherits SpatialDatasetReader) - edge_reader.py reads edges.parquet; query_grouped(), lrm_catalogue(), edge_color_values(), edge_detail() + edge_reader.py reads edges.parquet; query_grouped(), query_scores(), lrm_catalogue(), edge_color_values(), edge_detail() layer_reader.py generic parquet reader tiling/ pyramid.py OME-TIFF → DZI; pyvips streaming primary, tifffile+Pillow fallback @@ -98,15 +120,18 @@ backend/ frontend/ src/ + App.jsx Root component; React ErrorBoundary + top-level layout. + NOTE: sibling of components/, not inside it. store.js Zustand store — ALL shared state lives here components/ - App.jsx Root component; wraps everything in a React ErrorBoundary Viewer.jsx Split-screen wrapper (Viewer) + per-panel logic (ViewerPanel) LayerPanel.jsx Right-side panel: toggles, opacity, color-by, legends, dataset/image picker, transcript species filter CellInfoPanel.jsx Floating panel on cell click; shows color-by value highlight EdgeInfoPanel.jsx Floating panel on edge/autocrine click - AnnotationToolbar.jsx Region drawing + measurement tools; ⊞ Split / □ Single toggle; ⇔ Match zoom + AnnotationToolbar.jsx Region drawing + measurement tools; ⊞ Split / □ Single toggle; + ⇔ Match zoom; per-panel rotation (⟲ / angle / ⟳) + RenderingStatus.jsx Per-panel loading badge, driven by the store's loadingKeys set hooks/ useTranscripts.js Viewport-bounded transcript fetch (bbox always sent; skip at low zoom) useCellBoundaries.js Viewport-bounded cell boundary fetch (skip when fracW >= 0.5) @@ -121,14 +146,27 @@ frontend/ Dockerfile Multi-stage: node build → nginx serve docker-compose.yml Repo root; mounts DATA_PATH (or sample_data/) as /data:ro +docker-compose.prod.yml Production stack used by the cloud deployment docker/docker-compose.yml Legacy path (kept for compatibility) -sample_data/ GITIGNORED — default data mount for local dev/demo -r/ - export_NICHESObject_for_viewer.R draft R function for NICHESv2 → edges.parquet export +Caddyfile Reverse proxy + TLS for the cloud deployment; optional basicauth +deploy.sh One-shot droplet bootstrap (see docs/cloud-deploy.md) +upload-data.sh rsync datasets to a deployed server +sample_data/ Partially gitignored — default data mount for local dev/demo. + mouse_ileum_tiny is tracked; larger datasets are ignored. +r/ Personal analysis scripts with hardcoded paths — a pipeline, + not reusable functions. Run in this order: + ExportMetaDataforTissuePlex.R dump a Seurat @meta.data to CSV + run_NICHESv2_Xenium_PPLR.R run NICHESv2 (rad=25, method="product") → .rds + export_NICHES_for_TissuePlex_PPLR.R call export_to_TissuePlex(), validate the parquet docs/ data_format.md edges.parquet column spec for NICHESv2 R export - setup.md Docker deployment guide + setup.md Docker deployment guide (lab-facing) + cloud-deploy.md DigitalOcean deployment runbook (~$106–116/mo) public_datasets.md Links to public Xenium datasets used for development + index.html, demo.gif Landing page + README demo animation +OBS/ Archived, superseded planning docs. Provenance only — + NOT a specification. See OBS/README.md. +NICHESv2_package_design.md Design doc for the separate NICHESv2 R package (not this repo) ``` --- @@ -281,11 +319,26 @@ All shared state lives in a single Zustand store. Key sections: - **LRM filter**: `hiddenLrms` (Set of "ligand|receptor" strings), `lrmCatalogue` - **Selection**: `selectedCell`, `selectedEdge` - **Annotations**: `regions`, `measurements`, `activeRegion`, `annotationMode` +- **Sampling**: `transcriptFraction` (default 0.1) and `cellBoundaryFraction` + (`null` = auto) control how much of the viewport each hook requests; + `transcriptStats` / `cellBoundaryStats` hold live `{shown, total}` counts that the + LayerPanel displays. Both stats are written by panel 0 only. +- **Color overrides**: `categoryColorOverrides` (keyed `${field}::${category}`) and + `transcriptColorOverrides` (keyed by gene name) hold user-picked swatch colors. + Both reset on dataset change. `merge*` actions exist for bulk CSV import. +- **Loading**: `loadingKeys` — a Set of in-flight keys, one per panel + (`panel-0`, `panel-1`). `RenderingStatus.jsx` shows a badge whenever it is non-empty. + Each ViewerPanel ORs together every hook's `loading` flag into its own key. - **Split-screen**: `panelCount` (1 or 2), `viewports` (array of two viewport objects, one per panel — `{xmin,ymin,xmax,ymax}` in image pixels), `pendingZoomMatch` (`null` or `{ fromPanel }` — consumed by the target panel to match zoom while keeping its own center). `requestZoomMatch(fromPanel)` / `clearZoomMatch()` are the corresponding actions. +- **Rotation**: `panelRotations` — `[deg, deg]`, one per panel, normalized to 0–359 by + `setPanelRotation`. See the Rotation section below. +- **`viewportActual`**: the *un-expanded* OSD bounds per panel. Distinct from `viewports`, + which is padded when a panel is rotated. Only ⇔ Match zoom reads it, so that matching + uses the true visible width rather than the rotation-padded fetch bbox. --- @@ -340,7 +393,10 @@ Layers rendered in order (bottom to top): 5. `edges-directed` — LineLayer, directed edges (LRM-filtered, colored) 6. `edges-arrowheads` — SolidPolygonLayer, filled arrowhead triangles (full or harpoon style) 7. `edges-autocrine` — ScatterplotLayer (stroked only), autocrine rings -8. Annotation layers (region fills, outlines, measurement lines) +8. Annotation layers (region fills, outlines, active region + vertices, measurement + lines, endpoints, first-point marker) + +Every layer receives `modelMatrix: rotModelMatrix` so rotation applies uniformly. **Tissue graph vs Edge data**: Tissue graph = binary structural layer (which cells are connected at all, regardless of LRM). Edge data = quantitative/categorical overlay on top. Analogous to @@ -377,7 +433,8 @@ Results set `selectedCell` or `selectedEdge` in the store. - OSD viewer instance (`viewerRef`) - deck.gl ref (`deckRef`) - deck.gl view state (`deckViewState`) -- Per-panel viewport in store (`viewports[panelIndex]`) +- Per-panel viewport in store (`viewports[panelIndex]`, `viewportActual[panelIndex]`) +- Rotation angle (`panelRotations[panelIndex]`) and the derived `rotModelMatrix` - `osdOpenCount` — local counter incremented on each OSD `open` event; used as dep for the morphology opacity effect to ensure it fires regardless of whether `imageSize.w` changed (fixes the bug where morphology stayed visible after @@ -398,6 +455,67 @@ Results set `selectedCell` or `selectedEdge` in the store. - `setCellColorRange`, `setEdgeColorRange`, `setEdgeColorClamp` updates - EdgeInfoPanel rendering +--- + +## Per-Panel Rotation (issue #31) + +Each panel can be rotated independently, via ⟲ / angle input / ⟳ in `AnnotationToolbar`. +`setPanelRotation(panelIndex, angle)` normalizes to 0–359. + +Rotation has to be applied in **two** places that must stay consistent: + +1. **OSD tiles** — `viewer.viewport.setRotation(panelRotation)` rotates the morphology + image. +2. **deck.gl layers** — a column-major 4×4 `modelMatrix` from `makeRotMatrix(angle, cx, cy)`, + pivoting around the *current viewport center*, passed to every layer. + +Because the pivot is the viewport center, the matrix must be recomputed whenever the +viewport moves — which is why `syncDeckFromOSD` rebuilds it on every viewport-change event +rather than only when the angle changes. + +Three consequences worth knowing before touching this: + +- **Fetch bboxes are padded.** A rotated viewport rectangle covers more of the image than + its axis-aligned bounds suggest, so `rotatedBbox()` grows the box outward (no-op at 0° + and 180°). That padded box goes to `viewports`; the true bounds go to `viewportActual`. + ⇔ Match zoom reads `viewportActual` so padding never inflates the matched zoom. +- **Picking and annotation clicks must inverse-rotate.** `screenToData()` projects screen → + rotated view space, then calls `inverseRotate()` to get back to original image + coordinates. Skip that and annotations land in the wrong place at any non-zero angle. +- **Measurement labels forward-rotate.** `forwardRotate()` maps an image-space midpoint + into rotated view space before projecting it to a screen position for the HTML label. + +The rotation effect depends on `[panelRotation, osdOpenCount]` so it re-applies after an +OSD reinitialization, not just on an angle change. + +--- + +## Morphology Image Discovery + +`GET /spatial/{dataset}/images` returns bare filename **stems** (no extension, no directory +prefix) for every `.ome.tiff` / `.ome.tif` / `.tiff` / `.tif` in the dataset root **and one +level of subdirectories**. This is what makes Xenium's multi-channel `morphology_focus/` +set selectable alongside the top-level `morphology.ome.tif`: + +``` +dataset_dir/ + morphology.ome.tif → "morphology" + morphology_focus/ + morphology_focus_0000.ome.tif → "morphology_focus_0000" + morphology_focus_0001.ome.tif → "morphology_focus_0001" +``` + +`pyramid.py::_find_source()` resolves a stem back to a path by searching the **same two +locations in the same order** — root first, then subdirectories. These two functions are a +matched pair: if you change the search order or depth in one, change it in the other, or +the picker will list images the tile builder cannot open. + +Hidden directories are skipped so `.dzi_cache` is never scanned. Stems are de-duplicated, +and root-level files are added first, so a root file always wins a name collision with a +subdirectory file. Names sort morphology-first, then alphabetically. + +--- + **⇔ Match zoom flow:** `requestZoomMatch(fromPanel)` → both panels' effects fire → source panel early-returns (`fromPanel === panelIndex`) → target panel reads `viewports[fromPanel]` via @@ -409,20 +527,30 @@ center, calls `viewport.fitBounds(newBounds, false)` (animated), then `clearZoom ## Viewport-Bounded Data Fetching -All data hooks (transcripts, cell boundaries, edges) are debounced and skip fetches -that would be wasted at the current zoom level: +All data hooks (transcripts, cell boundaries, edges) are debounced (400 ms) and abort +in-flight requests when superseded. Rendering is no longer gated on a zoom threshold — +the old `fracW >= 0.7` / `fracW >= 0.5` skip conditions were removed so layers draw at +every zoom level including whole-tissue ("bird's-eye view", PR #28). Volume is instead +controlled by user-adjustable sampling fractions: -| Hook | Skip condition | Bbox filter | Limit | +| Hook | Volume control | Bbox filter | Backend cap | |---|---|---|---| -| `useTranscripts` | `fracW >= 0.7` | Always sent when viewport available | 50K (random sample) | -| `useCellBoundaries` | `fracW >= 0.5` | Always sent | 20K cells | -| `useEdges` | no viewport | Always sent | 10K–50K edges (grouped) | +| `useTranscripts` | `transcriptFraction` (default 0.1) | Always sent | 200K rows | +| `useCellBoundaries` | `cellBoundaryFraction` (`null` = auto, targets ~5K cells) | Always sent | — | +| `useEdges` | `edgeDensity` (default 0.1) | Always sent | 500K grouped rows | + +Each hook reports live `{shown, total}` counts into the store so the LayerPanel can show +what fraction of the data is actually on screen. -`fracW = (xmax - xmin) / imageSize.w` — fraction of image width visible. +**Transcript sampling**: the backend uses `df.sample(n=...)` (random, not `head`) so the +returned transcripts are spatially uniform across the viewport rather than biased toward +whatever region appears first in the parquet row order. Cell boundaries sample *unique +cell IDs* before filtering rows, so a sampled cell keeps all of its vertices and never +renders as a partial polygon. -**Transcript sampling**: the backend uses `df.sample(n=limit)` (random, not `head`) -so the 50K returned transcripts are spatially uniform across the viewport rather than -biased toward whatever region appears first in the parquet row order. +**Edge sampling** uses DuckDB `USING SAMPLE ... (bernoulli)` on the grouped result, so +each edge is included independently at probability `density` — spatially uniform, and +no sampling clause is emitted at all when `density = 1.0`. **Edge aggregation**: `useEdges` POSTs to `/edges/{dataset}/query-grouped` which returns one row per directed edge (GROUP BY edge, ORDER BY RANDOM()). For a 168M-row parquet @@ -487,8 +615,10 @@ npm install npm run dev # → http://localhost:5173, proxies /api → :8000 ``` -Note: dev server runs on port **5173** (not 3000) to avoid conflicting with Docker, -which binds port 3000. This is configured in `.claude/launch.json`. +Note: the dev server runs on port **5173**, set in `frontend/vite.config.js`. It must not +be 3000 — `docker compose` binds 3000 for the production frontend, so a dev server on 3000 +collides with any running container. `.claude/launch.json` passes `--port 5173` explicitly +as well, so both entry points agree. **Docker — demo data (sample_data/):** ```bash @@ -504,6 +634,22 @@ DATA_PATH="/absolute/path/to/datasets" docker compose up --build `DATA_PATH` must be an absolute host path with no colons. Drop any supported platform output folder under `DATA_PATH` — TissuePlex auto-detects the platform on first access. +**Cloud deployment:** `docs/cloud-deploy.md` is a complete DigitalOcean runbook +(~$106–116/month: 16 GB / 4 vCPU droplet + 200 GB block storage). The moving parts are +`deploy.sh` (droplet bootstrap), `docker-compose.prod.yml` (production stack), +`Caddyfile` (reverse proxy + automatic TLS), and `upload-data.sh` (rsync datasets up). +Tuning knobs live in a `.env.prod` file that is gitignored and must be created by hand; +`DUCKDB_MEMORY_LIMIT` is the one to reach for if the backend OOMs on large edge files. + +**Access control is opt-in and off by default.** The Caddyfile supports `basicauth`, but +unless it is enabled anyone with the URL can view the data. There is no application-level +auth, no user accounts, and no per-dataset permissions. + +**No tests, no CI, no linter.** There is no test suite, no `.github/workflows`, and no +ESLint or Python lint configuration in this repo. Changes are verified by running the app. +Be correspondingly careful with refactors that touch the reader interface or the +OSD ↔ deck.gl coordinate bridge, since nothing will catch a regression automatically. + --- ## Known Issues / Gotchas @@ -544,27 +690,75 @@ output folder under `DATA_PATH` — TissuePlex auto-detects the platform on firs - **`Math.min/max` spread on large arrays** (fixed in `useEdgeColors.js`): spreading 100K+ element arrays causes `RangeError: Maximum call stack size exceeded`. Use a `for` loop to find min/max instead of `Math.min(...arr)`. +- **`list_images` and `_find_source` are a matched pair.** They must search the same + locations in the same order (root, then one subdirectory level). Changing the depth or + order in one without the other makes the picker list images the tile builder can't open. +- **Edge color clamp has two different defaults, by design.** `Viewer.jsx` auto-sets + `edgeColorClamp.high` from the p95 of `visible_score_sum` so the initial view isn't + washed out by outliers. But `useEdgeColors` computes its own fallback `hi` as `max`, not + p95, so that "reset range" lands on a value matching the legend endpoints. They disagree + intentionally — don't "fix" one in isolation. +- **`edges.py` validates path traversal; the other routers don't.** `edges.py::_reader` + resolves `edge_file` and rejects anything escaping the dataset directory. + `spatial.py::_reader`, `tiles.py`, and `layers.py` do a bare `DATA_ROOT / dataset` with + no equivalent check. Harmless for a local single-user tool; worth closing before any + deployment where the URL is reachable by someone untrusted. +- `zarr==2.18.2` is still pinned in requirements.txt although the readers use parquet and + HDF5, not zarr. Likely stale; verify before removing. --- ## What's Not Built Yet -1. **R export function** — `export_for_TissuePlex()` is implemented in the NICHESv2 R - package (separate repo). The draft in `r/export_NICHESObject_for_viewer.R` is - superseded. See `docs/data_format.md` for the column spec. - -2. **Cell expression bar chart** — click panel currently shows cell metadata but not a sorted - gene expression readout. The `/spatial/{dataset}/expression/{cell_id}` endpoint exists - but the UI component is not built. - -3. **MERSCOPE cell boundaries** — MERSCOPE stores boundaries as HDF5 polygon data; - `MerscopeReader.cell_boundaries()` is a stub returning `[]`. - -4. **CosMx gene-set coloring** — requires per-cell expression aggregation from the - transcript file; `CosMxReader._color_values_gene_set()` is a stub returning empty. - -5. **Performance at scale** — edge rendering is now fast (query-grouped returns ~300K - edges as 300K rows instead of 168M rows; colors computed client-side). Remaining - bottlenecks: LOD for arrowheads at low zoom, transcript rendering at very high density. - -6. **Authentication** — no auth. Fine for local/lab use, needs work for any public deployment. +1. **Spatial reads have no streaming path** — this is the biggest remaining performance + gap. `XeniumReader._read_parquet()` does an uncached `pq.read_table(...).to_pandas()` + and then filters by bbox *in pandas*, so every viewport change re-reads the entire + `transcripts.parquet` / `cell_boundaries.parquet` before discarding everything outside + the box. On a full Xenium run that read dominates the cost of a pan. The edge path + already solved exactly this with DuckDB predicate pushdown — porting `transcripts()` + and `cell_boundaries()` to the `EdgeReader` pattern is the highest-leverage change + available, and the pattern to copy is already in the repo. (`_cells_full()` and + `_load_supplemental_metadata()` *are* cached, but those are the small tables.) + +2. **Supplemental metadata is not shown in the cell info panel** — `CellInfoPanel.jsx` + renders a hardcoded field list (`cell_id`, x, y, `transcript_counts`, `total_counts`, + `cell_area`, `nucleus_area`) plus expression. Supplemental columns merged by + `_cells_full()` reach the color-by dropdown but only appear in the panel if one happens + to be the active color-by field. Compounding this, **no sample dataset has a + `cell-metadata/` folder**, so the feature cannot be exercised locally as shipped. + +3. **Cell expression bar chart** — click panel shows cell metadata but not a sorted gene + expression readout. `/spatial/{dataset}/expression/{cell_id}` exists; the UI does not. + +4. **Reader interface drift** — `VisiumHDReader.transcripts()` / `.cell_boundaries()` use + the old `limit=` signature and return `[]`. See the caveat under Platform Support. + +5. **MERSCOPE cell boundaries** — HDF5 polygon data; `MerscopeReader.cell_boundaries()` + returns empty and the reader declares `has_boundaries: False`. + +6. **CosMx gene-set coloring and boundaries** — gene-set coloring requires per-cell + expression aggregation from the transcript file; boundaries are per-FOV label TIFFs. + Both are stubs. + +7. **Visium HD expression** — `gene_list()`, `cell_expression()`, and gene-set color-values + need `filtered_feature_bc_matrix.h5` parsing. + +8. **Rendering performance** — edge rendering is fast now (query-grouped returns ~300K + rows instead of 168M; colors computed client-side). Remaining: LOD for arrowheads at + low zoom, transcript rendering at very high density. + +9. **Authentication** — no application-level auth. Caddy `basicauth` is available for + cloud deployments (`docs/cloud-deploy.md`) but is **opt-in and off by default**. There + are no user accounts and no per-dataset permissions. + +### Open GitHub issues + +- **#45 — Select cells/edges by metadata.** Let the user restrict the view to a subset + (a sample, or 2–3 cell types) rather than all data at once. +- **#35 — Force-categorical toggle for numeric metadata columns.** Integer-coded + categoricals (Seurat cluster IDs, `*_snn_res.*`, phenotype codes) currently route to a + continuous viridis gradient. Partially mitigated already: `_color_values_meta()` treats + an integer column with ≤ 30 unique values as categorical, and `edge_color_values()` does + the same. Above that threshold users still fall back to renaming values to strings. + The ask is an explicit per-column "treat as categorical" toggle in the color panel, + with numeric sort order preserved in the legend. diff --git a/EDGE_UI_PLAN.md b/OBS/EDGE_UI_PLAN.md similarity index 100% rename from EDGE_UI_PLAN.md rename to OBS/EDGE_UI_PLAN.md diff --git a/PLAN.md b/OBS/PLAN.md similarity index 100% rename from PLAN.md rename to OBS/PLAN.md diff --git a/PLAN_v2_2026-05-02.md b/OBS/PLAN_v2_2026-05-02.md similarity index 100% rename from PLAN_v2_2026-05-02.md rename to OBS/PLAN_v2_2026-05-02.md diff --git a/OBS/README.md b/OBS/README.md new file mode 100644 index 0000000..f2b8d0c --- /dev/null +++ b/OBS/README.md @@ -0,0 +1,17 @@ +# OBS — Obsolete planning documents + +These documents describe earlier designs that have since been superseded by the shipped +implementation. They are kept for provenance and to explain why certain decisions were +made, **but they are not a specification and should not be used to guide new work.** + +For current architecture, read `CLAUDE.md` at the repo root. For current data contracts +and deployment, read `docs/`. + +| File | Written | Why it is obsolete | +|---|---|---| +| `PLAN.md` | 2026-05-01 | The original v1 plan, under the project's former name *ConnectivityExplorer*. Its edge schema (integer `lrm_id` 1–488, `strength`, `cell_id_source`/`cell_id_target`, Xenium pixel coordinates) was fully replaced by the NICHESv2 schema documented in `docs/data_format.md`. Still useful for the rationale behind storing connectivity as vector edges rather than 488 rasterized PNGs. | +| `PLAN_v2_2026-05-02.md` | 2026-05-02 | Status snapshot declaring v1 feature-complete. Its API reference lists the `/xenium/...` routes, which were replaced by the platform-agnostic `/spatial/...` router. Its state schema predates the `lrm_set` color mode and string-keyed `hiddenLrms`. Its P3 performance backlog is still partly relevant and has been carried into `CLAUDE.md`. | +| `EDGE_UI_PLAN.md` | 2026-05-02 | Design doc for the NICHESv2 edge-UI migration. Overtaken during implementation: it specifies **client-side** edge aggregation and a **server** round-trip for `lrm_set` coloring, and the shipped code does the opposite of both (server-side `/query-grouped`, client-side `lrm_set`). It also predates the tissue-graph layer, `edgeDensity`, `arrowStyle`, and `edgeColorClamp`. | + +`NICHESv2_package_design.md` remains at the repo root: it documents the separate NICHESv2 +R package rather than this codebase, and is still current for that package. diff --git a/README.md b/README.md index f1deeb9..8d4a857 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TissuePlex -An interactive spatial transcriptomics viewer for exploring cell-cell communication from [NICHESv2](https://github.com/your-lab/NICHESv2) directly on the tissue image. +An interactive spatial transcriptomics viewer for exploring cell-cell communication from [NICHESv2](https://github.com/RaredonLab/NICHESv2) directly on the tissue image. ![TissuePlex demo](docs/demo.gif) @@ -8,7 +8,7 @@ An interactive spatial transcriptomics viewer for exploring cell-cell communicat ## What it does -Spatial transcriptomics platforms (Xenium, MERSCOPE, CosMx) produce high-resolution images with hundreds of genes measured per cell. NICHESv2 infers which cells are communicating and through which ligand-receptor mechanisms (LRMs). TissuePlex bridges those two outputs: it overlays the NICHESv2 communication graph on the tissue image and lets you explore it interactively. +Spatial transcriptomics platforms (Xenium, Visium HD, MERSCOPE, CosMx) produce high-resolution images with hundreds of genes measured per cell. NICHESv2 infers which cells are communicating and through which ligand-receptor mechanisms (LRMs). TissuePlex bridges those two outputs: it overlays the NICHESv2 communication graph on the tissue image and lets you explore it interactively. **Key capabilities:** @@ -16,10 +16,14 @@ Spatial transcriptomics platforms (Xenium, MERSCOPE, CosMx) produce high-resolut - **Color edges by communication score or metadata** — visualize LRM set strength, cell type, or any custom column from your analysis as a continuous or categorical color scale - **Click any edge for full detail** — inspect every active LRM for a given cell pair with their individual scores - **Directed edges with arrowheads** — A→B and B→A are visually distinct; autocrine communication renders as rings +- **Multiple edge sets per dataset** — drop several `.parquet` files into an `edges/` folder and flip between scoring approaches on the same tissue without duplicating the image or cell data - **Pan and zoom on high-resolution morphology images** — OME-TIFF tile pyramid with smooth zoom from whole-tissue to single-cell scale -- **Transcript dot overlay** — per-gene colored dots, filterable by gene species -- **Cell/spot segmentation** — polygon boundaries with color-by-gene-set or color-by-metadata -- **Region drawing and measurement tools** — annotate areas, export cell selections +- **Multi-channel morphology** — Xenium `morphology_focus/` channels are selectable alongside the top-level morphology image +- **Split-screen comparison** — two independently navigable panels sharing one set of layer controls, with a match-zoom button +- **Per-panel rotation** — rotate either panel to any angle to align tissue orientation +- **Transcript dot overlay** — per-gene colored dots, filterable by gene species, with hover tooltips +- **Cell/spot segmentation** — polygon boundaries with color-by-gene-set or color-by-metadata, and editable per-category colors +- **Region drawing and measurement tools** — annotate areas, export cell selections, save PNG screenshots - **Supplemental metadata** — drop any CSV or parquet into a `cell-metadata/` folder to add custom color-by columns (clusters, pseudotime, etc.) without touching the original data - **Multi-dataset support** — switch between datasets without restarting; each is auto-detected by platform @@ -30,9 +34,12 @@ Spatial transcriptomics platforms (Xenium, MERSCOPE, CosMx) produce high-resolut | Platform | Vendor | Morphology | Transcripts | Cell segments | Edges | |---|---|:---:|:---:|:---:|:---:| | **Xenium** | 10x Genomics | ✓ | ✓ | ✓ | ✓ | +| **Visium HD** | 10x Genomics | ✓ | — | — | ✓ | | **MERSCOPE** | Vizgen | — | ✓ | — | ✓ | | **CosMx** | Nanostring | — | ✓ | — | ✓ | +Xenium is the most complete implementation. The other readers cover cells, transcripts, and metadata coloring; boundary parsing is platform-specific and not yet implemented for them (MERSCOPE stores polygons in HDF5, CosMx in per-FOV label TIFFs). Visium HD renders bins as points rather than polygons and has no per-molecule transcript coordinates. Each reader declares what it supports via a capability flag, and the UI hides layers the platform cannot serve. + The edge connectivity layer (NICHESv2 output) works with any platform — it is platform-agnostic as long as cell barcodes match. --- @@ -44,7 +51,7 @@ The edge connectivity layer (NICHESv2 output) works with any platform — it is ### Demo with sample data ```bash -git clone https://github.com/your-lab/TissuePlex.git +git clone https://github.com/RaredonLab/TissuePlex.git cd TissuePlex docker compose up --build ``` @@ -64,40 +71,55 @@ DATA_PATH=/absolute/path/to/your/datasets docker compose up --build xenium_run_A/ experiment.xenium ← Xenium sentinel morphology.ome.tif + morphology_focus/ ← optional; extra channels appear in the image picker cells.parquet transcripts.parquet cell_boundaries.parquet edges.parquet ← NICHESv2 output (optional) + edges/ ← optional; additional edge sets to flip between + raw_minimum.parquet + normalized_product.parquet + + visium_hd_run_B/ + square_008um/ ← Visium HD sentinel + edges.parquet - merscope_run_B/ + merscope_run_C/ cell_by_gene.csv ← MERSCOPE sentinel cell_metadata.csv detected_transcripts.csv edges.parquet - cosmx_run_C/ + cosmx_run_D/ my_experiment_tx_file.csv ← CosMx sentinel edges.parquet ``` -If `edges.parquet` is absent the edge layers are hidden — all other layers work normally. +If `edges.parquet` is absent the edge layers are hidden — all other layers work normally. When a dataset has more than one edge source, a dropdown appears at the top of the Edge Data section; the selection applies to every open panel. The first launch builds DZI tile pyramids from OME-TIFF morphology images. This takes ~30 seconds per dataset and is cached across restarts. +### Deploying to a server + +[docs/cloud-deploy.md](docs/cloud-deploy.md) is a step-by-step DigitalOcean runbook (~$106–116/month) covering droplet setup, block storage for data, DNS, automatic TLS via Caddy, and data upload. Note that **access control is opt-in**: unless you enable Caddy's `basicauth`, anyone with the URL can view the data. + --- ## NICHESv2 workflow -TissuePlex is designed as a downstream visualization step for [NICHESv2](https://github.com/your-lab/NICHESv2). After running NICHESv2 on your spatial dataset, export the connectivity object: +TissuePlex is designed as a downstream visualization step for [NICHESv2](https://github.com/RaredonLab/NICHESv2). After running NICHESv2 on your spatial dataset, export the connectivity object: ```r # In R, after running NICHESv2: -export_for_TissuePlex( +export_to_TissuePlex( niches_object, - output_path = "/your/datasets/xenium_run_A/edges.parquet" + output.path = "/your/datasets/xenium_run_A/edges.parquet", + celltype.col = "Type.6" ) ``` +Working examples of the full pipeline — exporting Seurat metadata, running NICHESv2, and exporting the parquet — are in [`r/`](r/). Those scripts have hardcoded paths and are meant to be read and adapted, not run as-is. + Then launch TissuePlex — the edge layer will appear automatically. The `edges.parquet` format is one row per **(directed edge) × (LRM)**. A→B and B→A are separate rows. Any additional columns in the file (cell types, scores, custom metadata) are automatically available as color-by options in the UI. See [docs/data_format.md](docs/data_format.md) for the full column specification. @@ -138,9 +160,13 @@ DATA_ROOT=../sample_data uvicorn app.main:app --reload # Frontend (React + Vite) — in a separate terminal cd frontend npm install -npm run dev # → http://localhost:3000, proxies /api → :8000 +npm run dev # → http://localhost:5173, proxies /api → :8000 ``` +The dev server uses port 5173 so it does not collide with `docker compose`, which binds 3000 for the production frontend. + +There is currently no automated test suite, CI, or linter — changes are verified by running the app. + --- ## Architecture @@ -154,6 +180,7 @@ FastAPI backend /tiles — OME-TIFF → DZI tile pyramid (pyvips / tifffile fallback) /spatial — transcripts, cell boundaries, cell metadata, gene expression /edges — edge query, LRM catalogue, per-edge color values, edge detail + /layers — generic parquet layer serving ``` All data is served directly from parquet files via DuckDB — no database setup or import step. Tile pyramids are built on first access and cached. @@ -169,6 +196,17 @@ The frontend automatically adapts its layer controls to the capabilities your re --- +## Roadmap + +Tracked in [GitHub issues](https://github.com/RaredonLab/TissuePlex/issues). Currently open: + +- **[#45](https://github.com/RaredonLab/TissuePlex/issues/45)** — select cells and edges by metadata, so you can focus on a sample or a few cell types instead of the whole dataset +- **[#35](https://github.com/RaredonLab/TissuePlex/issues/35)** — a "treat as categorical" toggle for numeric metadata columns, so integer-coded cluster IDs get a discrete editable palette instead of a continuous gradient + +Also known and not yet addressed: transcript and cell-boundary queries read their full parquet file on every viewport change rather than pushing the bbox filter down to DuckDB the way the edge queries do. This is the main performance limit on very large datasets. + +--- + ## Citation If you use TissuePlex in published work, please cite: *(preprint / paper link — coming soon)* diff --git a/backend/app/main.py b/backend/app/main.py index cb37663..a414261 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,7 @@ from app.routers import tiles, spatial, edges, layers -APP_VERSION = "0.3.1" +APP_VERSION = "0.4.0" app = FastAPI(title="TissuePlex API", version=APP_VERSION) diff --git a/frontend/package.json b/frontend/package.json index 8b7f56b..1ad5d42 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "tissueplex", - "version": "0.3.1", + "version": "0.4.0", "private": true, "scripts": { "dev": "vite", diff --git a/frontend/vite.config.js b/frontend/vite.config.js index c7c731e..bae85cb 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -11,7 +11,9 @@ export default defineConfig({ }, plugins: [react()], server: { - port: 3000, + // 5173, not 3000: docker compose binds 3000 for the production frontend, so a + // default of 3000 made `npm run dev` collide with a running container. + port: 5173, proxy: { "/api": { target: "http://localhost:8000", From 5900569e3d58c513d40e7fd2663e721623d99377 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:09:08 -0400 Subject: [PATCH 04/18] perf: stream transcript and boundary queries through DuckDB Both methods loaded their entire parquet with pq.read_table(...).to_pandas() and then masked in pandas, so every viewport change materialized the whole file to keep a viewport-sized slice of it. Measured on a synthetic 40M-row / 0.78 GB transcripts.parquet, one zoomed-in viewport query: pandas full read + mask 2903 MB peak RSS 912 ms DuckDB streaming 233 MB peak RSS 1369 ms 12x less memory. That is the binding constraint: production runs on a 16 GB droplet, where a multi-GB transcripts.parquet under the old path exhausts RAM long before it is merely slow. DuckDB is somewhat slower here, and the reason is worth recording. Xenium writes transcripts.parquet in row order, not spatial order, with very large row groups -- the bundled breast dataset is 1.1M rows in 2 row groups, the first spanning the entire x-range. Row-group statistics are therefore useless and DuckDB scans everything anyway, paying predicate-evaluation cost with none of the pruning payoff. Sorting the file spatially and rewriting it with small row groups takes the same query from 1010 ms to 29 ms; that is a follow-up, noted in CLAUDE.md. Also fixes polygon clipping. Boundaries previously filtered individual vertices by bbox, so cells straddling the viewport edge came back missing part of their outline and rendered as torn shapes -- 97 such cells on the bundled breast dataset. Selection is now per cell: a cell qualifies if any vertex is in the bbox, and all of its vertices are returned. Sampling likewise draws whole cells. Shared query helpers live in readers/duck.py so the other platform readers can adopt the same path. Verified against pandas ground truth on both bundled datasets: identical totals for full-extent, bbox-quadrant and gene-filtered queries, identical pixel-space conversion, and sampling that is deterministic across repeated calls. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 69 +++++++++-- backend/app/readers/duck.py | 111 +++++++++++++++++ backend/app/readers/xenium_reader.py | 179 ++++++++++++++++++++------- 3 files changed, 303 insertions(+), 56 deletions(-) create mode 100644 backend/app/readers/duck.py diff --git a/CLAUDE.md b/CLAUDE.md index 337cc8b..bc4d7a0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -560,6 +560,52 @@ pre-computed server-side. --- +## Spatial Query Path (readers/duck.py) + +`transcripts()` and `cell_boundaries()` query parquet through DuckDB rather than loading +it into pandas. `readers/duck.py` holds the shared pieces — `connect()`, `scan()`, +`columns()`, `bbox_predicate()`, `in_predicate()`, `reservoir_sample()`, `to_records()` — +so every reader builds queries the same way. `EdgeReader` predates it and has its own +equivalent helpers; the two should converge. + +**The reason is memory, not raw speed.** The old path did +`pq.read_table(...).to_pandas()` and masked in pandas, which materializes the whole file +on every viewport change. Measured on a synthetic 40M-row / 0.78 GB transcripts file, +one zoomed-in viewport query: + +| | peak RSS | wall time | +|---|---|---| +| pandas full read + mask | 2903 MB | 912 ms | +| DuckDB streaming | 233 MB | 1369 ms | + +12× less memory. Production runs on a 16 GB droplet, so a multi-GB `transcripts.parquet` +under the old path would OOM well before it was slow. DuckDB is somewhat *slower* here +because the file is not spatially sorted (see What's Not Built Yet #1) — pruning cannot +skip anything, so it pays predicate-evaluation cost without the row-group savings. Fixing +the layout closes that gap and then some. + +Things to preserve when editing these methods: + +- **`total` is a pre-sample count.** Both endpoints return `{rows, total}` where `total` is + the count *after* bbox/gene filtering but *before* sampling. `useCellBoundaries` divides + its ~5K target by `total` to pick the next fraction, so returning a post-sample count + makes the auto-fraction oscillate. +- **Boundaries select whole cells, never loose vertices.** A cell qualifies if *any* vertex + falls in the bbox, and then all of its vertices are returned. Filtering vertices directly + clips cells at the viewport edge into torn polygons — measured at 97 clipped cells on the + bundled breast dataset before this changed. +- **Sampling is seeded** (`duck.SAMPLE_SEED`). Re-fetching an unchanged viewport must return + the same rows or the layer visibly flickers. +- **`USING SAMPLE` goes on a subquery** wrapping the filtered SELECT. Applied alongside a + WHERE clause, DuckDB may sample before filtering. +- **DuckDB cannot bind numpy scalars.** `bbox_predicate()` casts to builtin `float` for + this reason. +- A fresh `connect()` per call is deliberate — DuckDB's global connection is not + thread-safe and returns corrupt results under FastAPI's threadpool rather than raising. + It costs ~5 ms, which is noise next to the scan. + +--- + ## Color System `valueToColor(value, vmin, vmax, palette)` in `colormap.js` maps a scalar to RGBA. @@ -710,15 +756,20 @@ OSD ↔ deck.gl coordinate bridge, since nothing will catch a regression automat ## What's Not Built Yet -1. **Spatial reads have no streaming path** — this is the biggest remaining performance - gap. `XeniumReader._read_parquet()` does an uncached `pq.read_table(...).to_pandas()` - and then filters by bbox *in pandas*, so every viewport change re-reads the entire - `transcripts.parquet` / `cell_boundaries.parquet` before discarding everything outside - the box. On a full Xenium run that read dominates the cost of a pan. The edge path - already solved exactly this with DuckDB predicate pushdown — porting `transcripts()` - and `cell_boundaries()` to the `EdgeReader` pattern is the highest-leverage change - available, and the pattern to copy is already in the repo. (`_cells_full()` and - `_load_supplemental_metadata()` *are* cached, but those are the small tables.) +1. **Spatial queries are not yet spatially indexed.** `transcripts()` and + `cell_boundaries()` now stream through DuckDB (see the Spatial Query Path section), + which fixed the memory problem, but **row-group pruning does not currently help**: + Xenium writes `transcripts.parquet` in row order, not spatial order, with very large + row groups. Measured on the bundled breast dataset — 1.1M rows in **2** row groups, + the first spanning the entire x-range. DuckDB therefore still scans every row to + evaluate the bbox predicate. + + Sorting the file spatially and rewriting it with small row groups makes the statistics + selective and is dramatically faster. Measured on a synthetic 40M-row / 0.78 GB file, + zoomed-in viewport query: **COUNT 206 ms → 9 ms, SELECT 1010 ms → 29 ms**, for a + one-time 4.1 s sort. That is the natural next step, and it fits the existing + "build a derived artifact on first access and cache it" pattern that `ensure_pyramid` + already uses for tiles. 2. **Supplemental metadata is not shown in the cell info panel** — `CellInfoPanel.jsx` renders a hardcoded field list (`cell_id`, x, y, `transcript_counts`, `total_counts`, diff --git a/backend/app/readers/duck.py b/backend/app/readers/duck.py new file mode 100644 index 0000000..a7536e4 --- /dev/null +++ b/backend/app/readers/duck.py @@ -0,0 +1,111 @@ +""" +Shared DuckDB helpers for parquet-backed readers. + +Every reader that queries a large parquet file should go through here rather +than loading the file into pandas. The win is predicate pushdown: DuckDB reads +only the row groups whose statistics can satisfy the WHERE clause, so a viewport +query touches a fraction of the file instead of all of it. + +The alternative — ``pq.read_table(path).to_pandas()`` followed by a pandas mask — +reads and materializes every row on every request, which is what made transcript +and boundary panning slow on full-size datasets. +""" +import math +import os +from pathlib import Path + +import duckdb +import pyarrow.parquet as pq + +_MEMORY_LIMIT = os.getenv("DUCKDB_MEMORY_LIMIT", "8GB") +_THREADS = os.getenv("DUCKDB_THREADS", "4") + + +def connect() -> duckdb.DuckDBPyConnection: + """Return a fresh, isolated DuckDB connection. + + A new connection per call is deliberate: DuckDB's default global connection + is not thread-safe, and sharing it under FastAPI's threadpool produces empty + or corrupt result sets rather than an error. + """ + conn = duckdb.connect() + conn.execute(f"SET memory_limit='{_MEMORY_LIMIT}'") + conn.execute(f"SET threads={_THREADS}") + return conn + + +def scan(path: Path) -> str: + """SQL FROM-clause fragment that reads a parquet file. + + Single quotes in the path are escaped so a path like ``/data/o'brien/x.parquet`` + cannot terminate the string literal. + """ + return "read_parquet('{}')".format(str(path).replace("'", "''")) + + +def columns(path: Path) -> set[str]: + """Column names in a parquet file, read from its footer (no data scan).""" + return set(pq.read_schema(path).names) + + +def bbox_predicate(x_col: str, y_col: str, bbox: tuple) -> tuple[str, list]: + """Build a bounding-box WHERE fragment and its bind parameters. + + ``bbox`` must already be in the file's native coordinate space. Returns + ``("", [])`` when the bbox is absent or has any None component, so callers + can splice the result unconditionally. + """ + if not bbox: + return "", [] + xmin, ymin, xmax, ymax = bbox + if None in (xmin, ymin, xmax, ymax): + return "", [] + # Cast to builtin float: DuckDB cannot bind numpy scalars, which is what you + # get whenever a bound comes from a pandas/numpy computation rather than the + # router's float query params. + return ( + f'("{x_col}" >= ? AND "{x_col}" <= ? AND "{y_col}" >= ? AND "{y_col}" <= ?)', + [float(xmin), float(xmax), float(ymin), float(ymax)], + ) + + +def where_clause(conditions: list[str]) -> str: + """Join conditions into a WHERE clause, or return '' when there are none.""" + conditions = [c for c in conditions if c] + return f"WHERE {' AND '.join(conditions)}" if conditions else "" + + +def in_predicate(col: str, values: list) -> tuple[str, list]: + """Build an IN (...) fragment. Empty values yield a never-true predicate.""" + if not values: + return "FALSE", [] + placeholders = ", ".join("?" for _ in values) + return f'"{col}" IN ({placeholders})', list(values) + + +# Sampling is seeded so that re-fetching an unchanged viewport returns the same +# rows. Without this, every refetch reshuffles which transcripts are drawn and +# the layer visibly flickers. +SAMPLE_SEED = 42 + + +def reservoir_sample(n: int) -> str: + """``USING SAMPLE`` clause drawing exactly n rows, or '' to keep all rows. + + Reservoir sampling gives an exact row count (unlike bernoulli, which gives an + expected count), matching the pre-DuckDB ``df.sample(n=...)`` behaviour. + Always attach this to a subquery wrapping the filtered SELECT — applied + directly alongside a WHERE clause, DuckDB may sample before filtering. + """ + if n <= 0: + return "" + return f"USING SAMPLE reservoir({int(n)} ROWS) REPEATABLE ({SAMPLE_SEED})" + + +def to_records(df) -> list[dict]: + """DataFrame to JSON-safe records (NaN/Inf → None).""" + return [ + {k: (None if isinstance(v, float) and not math.isfinite(v) else v) + for k, v in row.items()} + for row in df.to_dict(orient="records") + ] diff --git a/backend/app/readers/xenium_reader.py b/backend/app/readers/xenium_reader.py index fd2793a..a3a42e4 100644 --- a/backend/app/readers/xenium_reader.py +++ b/backend/app/readers/xenium_reader.py @@ -10,11 +10,17 @@ import pandas as pd import pyarrow.parquet as pq +from app.readers import duck from app.readers.base_reader import SpatialDatasetReader _UNSET = object() # sentinel: "not yet loaded" vs "loaded, no data" +# Hard ceiling on transcripts returned in one response, independent of `fraction`. +# Guards against a request for fraction=1.0 over a whole-tissue viewport trying to +# serialize tens of millions of rows. +_MAX_TRANSCRIPTS = 200_000 + class XeniumReader(SpatialDatasetReader): @@ -89,29 +95,69 @@ def transcripts( genes: Optional[list[str]] = None, fraction: float = 1.0, ) -> dict: - df = self._read_parquet( - "transcripts.parquet", - columns=["x_location", "y_location", "feature_name", "qv"], - ) - if df is None: + """Transcript detections in pixel space, bbox- and gene-filtered. + + Queried through DuckDB so the bbox and gene predicates push down into the + parquet scan. Only matching row groups are read; a zoomed-in viewport on a + multi-GB transcripts.parquet touches a small fraction of the file. + + ``total`` is the count *after* filtering but *before* sampling, because + the frontend uses it to report "showing N of M" and to calibrate density. + """ + path = self.path / "transcripts.parquet" + if not path.exists(): return {"transcripts": [], "total": 0} - if bbox: - xmin, ymin, xmax, ymax = self._bbox_to_native(bbox) - if None not in (xmin, ymin, xmax, ymax): - df = df[ - (df["x_location"] >= xmin) & (df["x_location"] <= xmax) & - (df["y_location"] >= ymin) & (df["y_location"] <= ymax) - ] - if genes: - df = df[df["feature_name"].isin(genes)] - total = len(df) - fraction = max(0.0001, min(1.0, fraction)) - sample_n = min(round(fraction * total), 200_000) - df = (df.sample(n=sample_n, random_state=42).copy() - if sample_n < total else df.copy()) - df["x_location"] = df["x_location"] / self.pixel_size - df["y_location"] = df["y_location"] / self.pixel_size - return {"transcripts": self._to_records(df), "total": total} + + cols = duck.columns(path) + if not {"x_location", "y_location"} <= cols: + return {"transcripts": [], "total": 0} + # qv is absent from some exports — select only what the file actually has. + select_cols = [c for c in ("x_location", "y_location", "feature_name", "qv") + if c in cols] + select = ", ".join(f'"{c}"' for c in select_cols) + + conditions: list[str] = [] + params: list = [] + + bbox_sql, bbox_params = duck.bbox_predicate( + "x_location", "y_location", + self._bbox_to_native(bbox) if bbox else None, + ) + if bbox_sql: + conditions.append(bbox_sql) + params.extend(bbox_params) + + if genes and "feature_name" in cols: + gene_sql, gene_params = duck.in_predicate("feature_name", genes) + conditions.append(gene_sql) + params.extend(gene_params) + + where = duck.where_clause(conditions) + src = duck.scan(path) + + with duck.connect() as conn: + total = conn.execute( + f"SELECT COUNT(*) FROM {src} {where}", params + ).fetchone()[0] + total = int(total or 0) + if total == 0: + return {"transcripts": [], "total": 0} + + fraction = max(0.0001, min(1.0, fraction)) + sample_n = min(round(fraction * total), _MAX_TRANSCRIPTS) + if sample_n <= 0: + return {"transcripts": [], "total": total} + + sample = duck.reservoir_sample(sample_n) if sample_n < total else "" + df = conn.execute( + f"SELECT * FROM (SELECT {select} FROM {src} {where}) {sample}", + params, + ).df() + + ps = self.pixel_size + df["x_location"] = df["x_location"] / ps + df["y_location"] = df["y_location"] / ps + return {"transcripts": duck.to_records(df), "total": total} # ── Cells ───────────────────────────────────────────────────────────────── @@ -157,33 +203,72 @@ def cells_schema(self) -> dict: # ── Cell boundaries ─────────────────────────────────────────────────────── def cell_boundaries(self, bbox: Optional[tuple] = None, fraction: float = 1.0) -> dict: - df = self._read_parquet("cell_boundaries.parquet") - if df is None: + """Cell polygon vertices in pixel space for cells visible in the bbox. + + Selection is per *cell*, not per vertex. A cell qualifies if any one of its + vertices falls in the bbox, and then **all** of its vertices are returned. + That matters at the viewport edge: filtering vertices directly (the previous + behaviour) clipped boundary cells into partial polygons that rendered as + torn shapes. Sampling likewise draws whole cells, so a sampled cell is never + missing part of its outline. + + ``total`` is the number of distinct cells touching the bbox before sampling — + ``useCellBoundaries`` divides its ~5K target by this to pick the next fraction, + so it has to stay a pre-sample count. + """ + path = self.path / "cell_boundaries.parquet" + if not path.exists(): return {"boundaries": [], "total": 0} - x_col = next((c for c in df.columns if "vertex_x" in c), None) - y_col = next((c for c in df.columns if "vertex_y" in c), None) - if bbox and x_col and y_col: - xmin, ymin, xmax, ymax = self._bbox_to_native(bbox) - if None not in (xmin, ymin, xmax, ymax): - df = df[ - (df[x_col] >= xmin) & (df[x_col] <= xmax) & - (df[y_col] >= ymin) & (df[y_col] <= ymax) - ] - total_cells = 0 - if "cell_id" in df.columns: - unique_ids = df["cell_id"].drop_duplicates() - total_cells = len(unique_ids) + + cols = duck.columns(path) + x_col = next((c for c in cols if "vertex_x" in c), None) + y_col = next((c for c in cols if "vertex_y" in c), None) + if not x_col or not y_col or "cell_id" not in cols: + return {"boundaries": [], "total": 0} + + bbox_sql, bbox_params = duck.bbox_predicate( + x_col, y_col, self._bbox_to_native(bbox) if bbox else None + ) + where = duck.where_clause([bbox_sql]) + src = duck.scan(path) + select = f'"cell_id", "{x_col}", "{y_col}"' + + with duck.connect() as conn: + total = conn.execute( + f"SELECT COUNT(DISTINCT cell_id) FROM {src} {where}", bbox_params + ).fetchone()[0] + total = int(total or 0) + if total == 0: + return {"boundaries": [], "total": 0} + fraction = max(0.0001, min(1.0, fraction)) - sample_n = round(fraction * total_cells) - if sample_n < total_cells: - keep = unique_ids.sample(n=sample_n, random_state=42) - df = df[df["cell_id"].isin(keep)] - df = df.copy() - if x_col: - df[x_col] = df[x_col] / self.pixel_size - if y_col: - df[y_col] = df[y_col] / self.pixel_size - return {"boundaries": self._to_records(df), "total": total_cells} + sample_n = round(fraction * total) + if sample_n <= 0: + return {"boundaries": [], "total": total} + + # Resolve the visible cell ids first, sample among them, then fetch + # every vertex belonging to a surviving id. The second scan re-reads + # the parquet, but both scans are predicate-pushed and together still + # read far less than materializing the whole file. + sample = duck.reservoir_sample(sample_n) if sample_n < total else "" + df = conn.execute( + f""" + WITH visible AS ( + SELECT DISTINCT cell_id FROM {src} {where} + ), + keep AS ( + SELECT cell_id FROM visible {sample} + ) + SELECT {select} FROM {src} + WHERE cell_id IN (SELECT cell_id FROM keep) + """, + bbox_params, + ).df() + + ps = self.pixel_size + df[x_col] = df[x_col] / ps + df[y_col] = df[y_col] / ps + return {"boundaries": duck.to_records(df), "total": total} # ── Expression ──────────────────────────────────────────────────────────── From aa7ad5414c78a84f5a27f03e51726df763657aef Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:38:57 -0400 Subject: [PATCH 05/18] docs: propose seqFISH (Spatial Genomics GenePS) support Design proposal only -- no reader implemented yet. Records the format research and the decisions taken: target Spatial Genomics GenePS rather than academic seqFISH+ (which has no standard output), one ROI per folder, bare integer cell labels as cell_id, and an edge-metadata/ folder mirroring the existing cell-metadata/ convention. The format claims are verified against the real scverse CI fixture rather than taken from documentation. Two findings changed the plan: A single seqFISH dataset mixes coordinate systems. Measured on the 1000x1000 DAPI at 0.107161 um/px: CellCoordinates center_x spans 1.82-105.66 and TranscriptList x spans 0-107.05, both microns, while Boundaries.geojson vertices span 0-999, pixels. Cells and transcripts need dividing by pixel_size and boundaries must pass through untouched. One global transform in either direction would put cells and their own outlines in different places, which presents as a rendering bug rather than a unit bug. The px-vs-um heuristic (ratio of max coordinate to image width) separates all three cases here with two orders of magnitude of headroom. GeoJSON features carry the cell label in their `id` field -- strings "1".."62" matching CellCoordinates.label exactly. spatialdata-io does not read this and maps polygons to cells positionally, with an open issue about the fragility (scverse/spatialdata-io#249). Joining on id is correct by construction; a silent off-by-one there would draw every outline on the wrong cell. sample_data/.gitignore excludes seqfish*/ (and visium-*/). The test dataset is public for CI by written permission from Spatial Genomics, not under an open licence, so it must not be redistributed from this repo. Co-Authored-By: Claude Fable 5 --- docs/seqfish_plan.md | 259 +++++++++++++++++++++++++++++++++++++++++ sample_data/.gitignore | 8 ++ 2 files changed, 267 insertions(+) create mode 100644 docs/seqfish_plan.md diff --git a/docs/seqfish_plan.md b/docs/seqfish_plan.md new file mode 100644 index 0000000..6e8e4dc --- /dev/null +++ b/docs/seqfish_plan.md @@ -0,0 +1,259 @@ +# Plan: seqFISH (Spatial Genomics GenePS) support + +**Status: PROPOSAL — not yet approved, nothing implemented.** + +## Context + +TissuePlex needs to read seqFISH data alongside Xenium without regressing Xenium, and to +make cell metadata, edge data (NICHESv2), and edge metadata behave the same way on every +platform. Edge metadata does not exist for any platform today, so this is where that +contract gets defined. + +**Decisions already made** (from the design questions): + +| Decision | Choice | +|---|---| +| Target format | Spatial Genomics GenePS (the commercial platform) | +| Multi-ROI | One ROI per folder — no ROI-as-dataset machinery | +| Cell IDs | Bare integer label, `"42"` — matches the raw file, no R-side change | +| Edge metadata | `edge-metadata/` folder mirroring `cell-metadata/` | + +Those choices remove the two most invasive parts of the original sketch. What remains is +a self-contained reader plus one genuinely shared piece of infrastructure. + +--- + +## What we're building against + +"seqFISH" names two unrelated things. **Academic seqFISH/seqFISH+** (Cai lab) is a method +with no standard output — published data is ad-hoc `.txt`/`.mat` tables, and the HuBMAP +schema covers *raw acquisition*, not analysis output. **Spatial Genomics Inc. GenePS** is +the commercial platform with a real, stable format. We target GenePS, confirmed. + +I read the `spatialdata-io` reader source rather than relying on documentation prose, so +the following is what a working implementation actually expects. + +### Layout (current "v2" format) + +Flat directory, files prefixed by an ROI name, no subdirectories: + +``` +seqfish_dataset/ + Roi1_CellCoordinates.csv label, area, center_x, center_y + Roi1_CellxGene.csv unnamed first col = label; remaining cols = genes (dense) + Roi1_TranscriptList.csv name, x, y, [z], [refid] + Roi1_DAPI.tiff OME-TIFF (OME-XML despite the .tiff extension), often pyramidal + Roi1_Segmentation.tiff integer label mask + Roi1_Boundaries.geojson cell polygons +``` + +**Sentinel:** `glob("*_CellCoordinates*.csv")`. There is no manifest or version file, so +detection has to be a glob. Registered last in `ReaderFactory` so it cannot shadow the +existing exact-filename sentinels. + +### Legacy "v1" layout — also supported, per your decision + +Spatial Genomics renamed things at some point. Both are handled; the reader picks a +variant by which filenames are present and records the choice in `info()`. + +| v1 (legacy) | v2 (current) | +|---|---| +| `{prefix}_CellCoordinates_section{N}.csv` | `{roi}_CellCoordinates.csv` | +| `{prefix}_CxG_section{N}.csv` | `{roi}_CellxGene.csv` | +| `{prefix}_TranscriptCoordinates_section{N}.csv` | `{roi}_TranscriptList.csv` | +| `{prefix}_DAPI_section{N}.ome.tiff` | `{roi}_DAPI.tiff` | +| `{prefix}_CellMask_section{N}.tiff` | `{roi}_Segmentation.tiff` | +| *(no boundaries file)* | `{roi}_Boundaries.geojson` | +| transcripts have `cell`, no `z` | transcripts have `z`, no `cell` | + +Two consequences: v1 has **no GeoJSON**, so boundaries must come from polygonising +`CellMask_section{N}.tiff`, which is more work than reading vertices and is the main cost +of v1 support — I would implement v1 cells/transcripts first and treat v1 boundaries as a +follow-on, declaring `has_boundaries: False` until it lands. And v1 *does* carry the +transcript→cell assignment that v2 dropped, so if it is present we should keep it. + +The public SGI Mouse Kidney release is v1, which is the practical reason this matters. + +### Four things that will bite, and the plan for each + +**1. A single dataset mixes microns and pixels — now measured, not guessed.** +This was the biggest risk in the plan. It is real, and it is confirmed against the +downloaded dataset: + +| Source | Extent | Verdict | +|---|---|---| +| `Roi1_DAPI.tiff` | 1000 × 1000 px, `PhysicalSizeX` = 0.107161 µm/px → 107.16 µm | reference | +| `CellCoordinates.csv` `center_x` | 1.82 → 105.66 | **microns** | +| `TranscriptList.csv` `x` | 0.0 → 107.05 | **microns** | +| `Boundaries.geojson` vertices | 0 → 999 | **pixels** | + +So cells and transcripts need dividing by `pixel_size`; boundaries must be passed through +untouched. Applying one global transform — in either direction — puts cells and their +own outlines in different places, which reads as a rendering bug rather than a unit bug. + +The auto-detection heuristic is validated on this data: compute +`ratio = max(coord) / image_width_px`. A ratio near `pixel_size` (0.106, 0.107 above) +means microns; a ratio near 1.0 (0.999 above) means pixels. It cleanly separates all +three cases here with two orders of magnitude to spare. Read `PhysicalSizeX`/`PhysicalSizeY` +from the DAPI OME-XML for `pixel_size`, fall back to 0.107, apply the heuristic per source +table, and log the verdict on load. + +**2. GeoJSON features carry the cell label in `id`** — verified: `id` values are strings +`"1"`, `"2"`, … matching `label` in `CellCoordinates.csv` exactly, all 62 unique. + +This matters because `spatialdata-io` does *not* read it — it maps polygons to cells +**positionally** and has an open issue about the fragility (scverse/spatialdata-io#249). +We can join on `id` and be correct by construction, falling back to positional order only +if `id` is missing. Worth doing better than the reference implementation here, since a +silent off-by-one in this mapping would draw every outline on the wrong cell. + +**3. v2 dropped the transcript→cell assignment.** Older exports had a `cell` column in +`TranscriptList.csv`; the current format removed it, so transcripts arrive unassigned. +Nothing in TissuePlex needs per-transcript cell assignment today (the transcript layer is +positional dots), so this costs us nothing now — but it means we cannot derive expression +from transcripts, and `CellxGene.csv` is the only expression source. + +**4. There is no QV column.** Xenium's per-transcript quality filter has no seqFISH +equivalent. Not a blocker; just means that control has nothing to bind to. + +### Column mapping + +| Concept | Xenium | seqFISH | +|---|---|---| +| cell id | `cell_id` (string barcode) | `label` (int → `str(label)`) | +| centroid | `x_centroid`, `y_centroid` | `center_x`, `center_y` | +| cell area | `cell_area` | `area` | +| transcript position | `x_location`, `y_location` | `x`, `y` | +| transcript gene | `feature_name` | `name` | +| transcript quality | `qv` | *(none)* | +| boundaries | long parquet, `vertex_x`/`vertex_y` | GeoJSON polygons | +| counts | `cell_feature_matrix.h5` | `CellxGene.csv` (dense) | +| pixel size | `experiment.xenium` | DAPI OME-XML | + +--- + +## Test data — downloaded and verified + +`seqfish-2-test-dataset.zip` (1.7 MB, `s3.embl.de/spatialdata/raw_data/`) is now at +`sample_data/seqfish_instrument2/`. Real Spatial Genomics v2 output: + +``` +Roi1_CellCoordinates.csv 62 cells label, area, center_x, center_y +Roi1_TranscriptList.csv 9,051 rows name, x, y, z (12 genes, z all = 1) +Roi1_CellxGene.csv 62 × 12 unnamed first col + gene columns +Roi1_DAPI.tiff 1000×1000 uint16, OME, 3 pyramid levels +Roi1_Segmentation.tiff 1000×1000 uint32, 62 labels +Roi1_Boundaries.geojson 62 Polygon features, ~20 vertices each +``` + +Confirmed v2: **`TranscriptList.csv` has no `cell` column**, so transcripts are +unassigned, and there is no `qv`. + +Licence: public for CI by written permission from Spatial Genomics, not an open licence. +`sample_data/.gitignore` now excludes `seqfish*/` so it cannot be committed or +redistributed from this repo. + +Because the fixture cannot be committed, I still want a **synthetic seqFISH generator** in +the shape of `sample_data/make_edges.py`, emitting a tiny valid ROI. That is what makes +the format testable in CI without licence questions. + +Everything else found is unsuitable and worth recording so nobody re-searches it: the +squidpy `seqfish.h5ad` (32 MB) and the Bioconductor/Giotto fixtures are academic-format +AnnData or count tables with no transcripts, boundaries, or image; the SGI Mouse Kidney +release is email-gated and tens of GB; **GEO has essentially nothing** in GenePS format. + +--- + +## Design + +### 1. `SeqfishReader` (new) + +Subclasses `SpatialDatasetReader` like every other platform, implements the same +interface, and returns pixel coordinates per the existing contract. One ROI per folder: +glob for `*_CellCoordinates.csv`, take the single match, and warn if there are several +rather than silently picking one. + +- `cells()` — `CellCoordinates.csv`; `label`→`cell_id` (as string), `center_x/y`→ + `x_centroid`/`y_centroid`, `area`→`cell_area` +- `transcripts()` — `TranscriptList.csv`; `name`→`feature_name`, `x`/`y`→ + `x_location`/`y_location` +- `cell_boundaries()` — `Boundaries.geojson` flattened to the same long-format + `{cell_id, vertex_x, vertex_y}` rows Xenium already produces, so `useCellBoundaries` + needs no change at all +- `color_values()` / `cell_expression()` — from `CellxGene.csv` +- `capabilities()` — all three layers true, `unit_label: "cell"` +- Morphology — `Roi1_DAPI.tiff` is already OME and often pyramidal, so the existing + `tiles.py` path should work unmodified. The subdirectory-aware `list_images` from + v0.4.0 already handles finding it. + +### 2. Normalising ingest cache (shared) + +seqFISH ships CSV. CSV cannot be range-scanned, has no column statistics, and re-parses on +every request — strictly worse than the parquet path just optimised. `CellxGene.csv` for a +real run (670k cells × 1092 genes) is multi-GB. + +Propose `ensure_normalized(dataset)`, cached on disk exactly like the DZI pyramid: + +``` +dataset_dir/ + .tissueplex_cache/ + transcripts.parquet canonical columns, spatially sorted + cells.parquet + cell_boundaries.parquet GeoJSON flattened to vertex rows +``` + +Three jobs at once: makes seqFISH queryable at Xenium speed; normalises column names so +the readers become thin adapters over one query implementation; and delivers the spatial +sort measured during the perf work — **COUNT 206 ms → 9 ms, SELECT 1010 ms → 29 ms** on a +40M-row file for a one-time 4.1 s build. + +Xenium adopts it incrementally: if the cache is absent, query the original parquet as +today. Nothing breaks while it rolls out. + +### 3. Edge metadata (new, cross-platform) + +Mirror `cell-metadata/` exactly: + +``` +dataset_dir/ + edge-metadata/ + annotations.csv key column `edge` = "SendingCell|ReceivingCell" +``` + +Outer-joined on `edge`, columns surfacing automatically in the edge color-by dropdown — +exactly how supplemental cell metadata reaches the cell color-by dropdown now. Same loader +shape, same caching, same column-resolution rules. Platform-agnostic by construction, +since it lives in `EdgeReader`. + +### 4. Lift supplemental metadata into the base class + +`_load_supplemental_metadata()` and `_read_csv_with_barcodes()` are Xenium-only today but +contain nothing Xenium-specific. Move them to `base_reader.py` so seqFISH gets +`cell-metadata/` for free and the other three readers can adopt it. Mostly a move, not a +rewrite — and it is the concrete form of the "similar across platforms" ask. + +--- + +## Sequencing + +1. **Golden-output snapshot of both Xenium datasets** — the regression guard (below). +2. Supplemental metadata moves to `base_reader` — no behaviour change. +3. Synthetic seqFISH fixture generator. +4. `SeqfishReader` + factory detection, against the fixture, then the real test dataset. +5. Boundaries (GeoJSON → long vertices) and the px/µm detection. +6. Normalising + spatial-sort cache; Xenium opted in second. +7. Edge metadata. + +Steps 1–5 deliver a working seqFISH viewer. 6 and 7 are independent and can be deferred +or reordered. + +--- + +## Not breaking Xenium + +There is no test suite, so "without breaking current functionality" needs teeth. Before +any of this lands I will capture a **golden-output snapshot** — recorded responses from +every spatial and edge endpoint for both bundled Xenium datasets — and re-check it after +each step. Cheap to build, and it turns "I think Xenium still works" into something +actually verified. It is also the seed of a real test suite, which the repo currently +lacks entirely. diff --git a/sample_data/.gitignore b/sample_data/.gitignore index b50f322..628277f 100644 --- a/sample_data/.gitignore +++ b/sample_data/.gitignore @@ -3,6 +3,14 @@ xenium_human_breast_2fov/ merscope-*/ cosmx-*/ +visium-*/ + +# seqFISH / Spatial Genomics GenePS datasets. +# The scverse test dataset (s3.embl.de/spatialdata/raw_data/seqfish-2-test-dataset.zip) +# is public for CI by written permission from Spatial Genomics, not under an open +# licence — usable for local development, but must not be redistributed from this repo. +seqfish*/ +seqfish-*/ # ── Xenium files present in the repo but not needed by TissuePlex ──────────── # Xenium Explorer analysis outputs From d4406a91394ea38796b7419e355511d63314f2e5 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Wed, 5 Aug 2026 08:10:25 -0400 Subject: [PATCH 06/18] test: golden-output guard; lift supplemental metadata to the base reader Groundwork for seqFISH support. Two changes, neither altering Xenium behaviour. backend/tests/golden_snapshot.py exercises every reader method against the bundled datasets, digests the results, and diffs them against a recorded baseline -- 62 probes across the two Xenium datasets, covering info, capabilities, gene lists, cells, schema, transcripts (full/bbox/gene-filtered/ sampled), boundaries (full/sampled), per-cell detail and expression, colour values, and every edge endpoint for each edge file. It calls readers directly rather than over HTTP, so no server is needed and failures point at the reader instead of the transport. This repo has no test suite, so cross-platform reader work had nothing to catch a regression. Verified the guard actually bites: flipping duck.SAMPLE_SEED from 42 to 43 fails exactly the six sampled probes and passes again on revert. Two sources of false failure had to be removed first. Record-list digests are now order-independent, because query_grouped uses ORDER BY RANDOM() and SQL GROUP BY promises no ordering -- hashing in returned order failed on every run. And the edge_detail probe picked grouped[0], a different edge each time; it now takes the lexicographic minimum. _load_supplemental_metadata and _read_csv_with_barcodes move from XeniumReader to SpatialDatasetReader, so cell-metadata/ becomes available to every platform rather than Xenium alone. Nothing platform-specific was in them; all Xenium contributes now is the list of its own root CSVs to ignore. That list is the trap this opens, so it is closed here too: MERSCOPE and CosMx both write CSVs at the dataset root that the shared loader would otherwise ingest as user metadata. MERSCOPE gets an exact-name skip set; CosMx prefixes every output with the experiment name, so it needs suffix matching, which _is_platform_csv now supports. Co-Authored-By: Claude Fable 5 --- backend/app/readers/base_reader.py | 128 ++++++ backend/app/readers/cosmx_reader.py | 8 + backend/app/readers/merscope_reader.py | 6 + backend/app/readers/xenium_reader.py | 105 +---- backend/tests/golden_baseline.json | 572 +++++++++++++++++++++++++ backend/tests/golden_snapshot.py | 313 ++++++++++++++ 6 files changed, 1035 insertions(+), 97 deletions(-) create mode 100644 backend/tests/golden_baseline.json create mode 100644 backend/tests/golden_snapshot.py diff --git a/backend/app/readers/base_reader.py b/backend/app/readers/base_reader.py index 7fa934b..9ae7b66 100644 --- a/backend/app/readers/base_reader.py +++ b/backend/app/readers/base_reader.py @@ -13,6 +13,9 @@ import pandas as pd +_UNSET = object() # sentinel: "not yet loaded" vs "loaded, no data" + + class SpatialDatasetReader(ABC): """ Interface every platform reader must satisfy. @@ -25,8 +28,20 @@ class SpatialDatasetReader(ABC): always receives pixel coordinates and never needs to know the native unit. """ + # Filenames in the dataset root that belong to the platform itself rather than + # to the user, and must never be picked up as supplemental metadata. Subclasses + # override this with their own output filenames. `.csv.gz` at the root is always + # skipped, since every platform uses that only for its own data files. + _ROOT_CSV_SKIP: frozenset = frozenset() + + # Suffix patterns for platforms whose output filenames carry a run-specific + # prefix (CosMx writes `_tx_file.csv`), where an exact-name set + # cannot work. + _ROOT_CSV_SKIP_SUFFIXES: tuple = () + def __init__(self, dataset_path: Path): self.path = dataset_path + self._supp_meta_cache = _UNSET # ── Identity ────────────────────────────────────────────────────────────── @@ -143,6 +158,119 @@ def capabilities(self) -> dict: "unit_label": "cell", } + # ── Supplemental cell metadata (platform-agnostic) ──────────────────────── + # + # Users add their own per-cell columns (clusters, pseudotime, phenotype calls) + # by dropping CSV/parquet into a `cell-metadata/` subdirectory, without touching + # the platform's own output. These live on the base class so every reader gets + # the feature; only `_ROOT_CSV_SKIP` is platform-specific. + + def _load_supplemental_metadata(self) -> Optional[pd.DataFrame]: + """ + Merge user-defined cell metadata from: + 1. {dataset}/cell-metadata/ — all CSV / parquet + 2. {dataset}/ — plain .csv only, skipping platform filenames + Multiple files are outer-joined on cell_id. Cached per reader instance. + """ + if self._supp_meta_cache is not _UNSET: + return self._supp_meta_cache # type: ignore[return-value] + + candidate_files: list[Path] = [] + meta_dir = self.path / "cell-metadata" + if meta_dir.is_dir(): + candidate_files.extend(sorted(meta_dir.iterdir())) + for f in sorted(self.path.iterdir()): + if not f.is_file(): + continue + nl = f.name.lower() + if not nl.endswith(".csv"): + continue + if self._is_platform_csv(nl): + continue + candidate_files.append(f) + + frames: list[pd.DataFrame] = [] + for f in candidate_files: + try: + nl = f.name.lower() + if nl.endswith(".parquet"): + df = pd.read_parquet(f) + if "cell_id" not in df.columns: + print(f"[{self.platform}] skip {f.name}: no 'cell_id' column") + continue + elif nl.endswith(".csv.gz") or nl.endswith(".csv"): + df = self._read_csv_with_barcodes(f) + if df is None: + continue + else: + continue + df["cell_id"] = df["cell_id"].astype(str) + frames.append(df) + print(f"[{self.platform}] loaded supplemental metadata: {f.name} " + f"({len(df)} rows, {len(df.columns)-1} extra columns)") + except Exception as exc: + print(f"[{self.platform}] warning: could not load {f.name}: {exc}") + + if not frames: + self._supp_meta_cache = None + return None + + merged = frames[0] + for frame in frames[1:]: + new_cols = ["cell_id"] + [c for c in frame.columns if c not in merged.columns] + merged = merged.merge(frame[new_cols], on="cell_id", how="outer") + self._supp_meta_cache = merged + return merged + + def _is_platform_csv(self, lowercase_name: str) -> bool: + """True if a root-level CSV is the platform's own output, not user metadata.""" + if lowercase_name in self._ROOT_CSV_SKIP: + return True + return any(lowercase_name.endswith(s) for s in self._ROOT_CSV_SKIP_SUFFIXES) + + def _read_csv_with_barcodes(self, path: Path) -> Optional[pd.DataFrame]: + """Read a CSV and promote the barcode column to 'cell_id'. + + Resolution order: an explicit `cell_id` column; then `Unnamed: 0`, which is + what pandas calls R's unnamed rowname column from `write.csv(row.names=TRUE)`; + then the first column if it holds unique strings. + """ + try: + df = pd.read_csv(path, index_col=0) + df.index.name = "cell_id" + return df.reset_index() + except Exception: + pass + df = pd.read_csv(path) + if "cell_id" in df.columns: + return df + if "Unnamed: 0" in df.columns: + return df.rename(columns={"Unnamed: 0": "cell_id"}) + first = df.columns[0] + if df[first].dtype == object and df[first].is_unique: + return df.rename(columns={first: "cell_id"}) + print(f"[{self.platform}] skip {path.name}: cannot identify barcode column") + return None + + def _merge_supplemental(self, cells: Optional[pd.DataFrame]) -> Optional[pd.DataFrame]: + """Left-join supplemental metadata onto a platform cells table. + + Either side may be absent: with no supplemental files this returns `cells` + unchanged, and with no cells table it returns the supplemental frame alone + (so metadata-only datasets still expose their columns). + """ + supp = self._load_supplemental_metadata() + if cells is None and supp is None: + return None + if supp is None: + return cells + if cells is None: + return supp + new_cols = [c for c in supp.columns if c not in cells.columns] + if not new_cols: + return cells + return cells.merge(supp[["cell_id"] + new_cols], on="cell_id", how="left") + # ── Shared utilities ────────────────────────────────────────────────────── def _to_px(self, val: float) -> float: diff --git a/backend/app/readers/cosmx_reader.py b/backend/app/readers/cosmx_reader.py index f02e3ae..a96ec8c 100644 --- a/backend/app/readers/cosmx_reader.py +++ b/backend/app/readers/cosmx_reader.py @@ -30,6 +30,14 @@ class CosMxReader(SpatialDatasetReader): + # CosMx prefixes every output with the experiment name (`_tx_file.csv`), + # so the shared supplemental-metadata loader has to match by suffix rather than + # by exact filename to avoid ingesting the platform's own tables. + _ROOT_CSV_SKIP_SUFFIXES = ( + "_tx_file.csv", "_metadata_file.csv", "_fov_positions_file.csv", + "_exprmat_file.csv", + ) + def __init__(self, dataset_path: Path): super().__init__(dataset_path) self._cells_cache: Optional[pd.DataFrame] = None diff --git a/backend/app/readers/merscope_reader.py b/backend/app/readers/merscope_reader.py index 24eaf19..dface95 100644 --- a/backend/app/readers/merscope_reader.py +++ b/backend/app/readers/merscope_reader.py @@ -31,6 +31,12 @@ class MerscopeReader(SpatialDatasetReader): + # Root CSVs that are MERSCOPE's own output, so the shared supplemental-metadata + # loader never mistakes them for user-supplied columns. + _ROOT_CSV_SKIP = frozenset({ + "cell_by_gene.csv", "cell_metadata.csv", "detected_transcripts.csv", + }) + def __init__(self, dataset_path: Path): super().__init__(dataset_path) self._pixel_size: Optional[float] = None diff --git a/backend/app/readers/xenium_reader.py b/backend/app/readers/xenium_reader.py index a3a42e4..03c4725 100644 --- a/backend/app/readers/xenium_reader.py +++ b/backend/app/readers/xenium_reader.py @@ -11,10 +11,7 @@ import pyarrow.parquet as pq from app.readers import duck -from app.readers.base_reader import SpatialDatasetReader - - -_UNSET = object() # sentinel: "not yet loaded" vs "loaded, no data" +from app.readers.base_reader import _UNSET, SpatialDatasetReader # Hard ceiling on transcripts returned in one response, independent of `fraction`. # Guards against a request for fraction=1.0 over a whole-tissue viewport trying to @@ -27,7 +24,6 @@ class XeniumReader(SpatialDatasetReader): def __init__(self, dataset_path: Path): super().__init__(dataset_path) self._pixel_size: Optional[float] = None - self._supp_meta = _UNSET self._cells_full_cache = _UNSET # ── Identity ────────────────────────────────────────────────────────────── @@ -380,109 +376,24 @@ def _color_values_meta(self, field: str) -> dict: "min": float(valid.min()), "max": float(valid.max())} # ── Supplemental metadata ───────────────────────────────────────────────── + # The loader itself lives on SpatialDatasetReader so every platform gets it. + # All Xenium contributes is the list of its own root CSVs to ignore. # Plain-CSV filenames in the dataset root that are standard Xenium outputs. # .csv.gz files are always skipped at root (exclusively Xenium data files). - _XENIUM_ROOT_SKIP = frozenset({ + _ROOT_CSV_SKIP = frozenset({ "cells.csv", "transcripts.csv", "metrics_summary.csv", "analysis_summary.csv", "gene_panel.csv", }) - def _load_supplemental_metadata(self) -> Optional[pd.DataFrame]: - """ - Merge user-defined cell metadata from: - 1. {dataset}/cell-metadata/ — all CSV / parquet - 2. {dataset}/ — plain .csv only, skipping known Xenium filenames - Multiple files are outer-joined on cell_id. Cached per reader instance. - """ - if self._supp_meta is not _UNSET: - return self._supp_meta # type: ignore[return-value] - - candidate_files: list[Path] = [] - meta_dir = self.path / "cell-metadata" - if meta_dir.is_dir(): - candidate_files.extend(sorted(meta_dir.iterdir())) - for f in sorted(self.path.iterdir()): - if not f.is_file(): - continue - nl = f.name.lower() - if not nl.endswith(".csv"): - continue - if nl in self._XENIUM_ROOT_SKIP: - continue - candidate_files.append(f) - - frames: list[pd.DataFrame] = [] - for f in candidate_files: - try: - nl = f.name.lower() - if nl.endswith(".parquet"): - df = pd.read_parquet(f) - if "cell_id" not in df.columns: - print(f"[xenium_reader] skip {f.name}: no 'cell_id' column") - continue - elif nl.endswith(".csv.gz") or nl.endswith(".csv"): - df = self._read_csv_with_barcodes(f) - if df is None: - continue - else: - continue - df["cell_id"] = df["cell_id"].astype(str) - frames.append(df) - print(f"[xenium_reader] loaded supplemental metadata: {f.name} " - f"({len(df)} rows, {len(df.columns)-1} extra columns)") - except Exception as exc: - print(f"[xenium_reader] warning: could not load {f.name}: {exc}") - - if not frames: - self._supp_meta = None - return None - - merged = frames[0] - for frame in frames[1:]: - new_cols = ["cell_id"] + [c for c in frame.columns if c not in merged.columns] - merged = merged.merge(frame[new_cols], on="cell_id", how="outer") - self._supp_meta = merged - return merged - - def _read_csv_with_barcodes(self, path: Path) -> Optional[pd.DataFrame]: - """Read a CSV and promote the barcode column to 'cell_id'.""" - try: - df = pd.read_csv(path, index_col=0) - df.index.name = "cell_id" - return df.reset_index() - except Exception: - pass - df = pd.read_csv(path) - if "cell_id" in df.columns: - return df - if "Unnamed: 0" in df.columns: - return df.rename(columns={"Unnamed: 0": "cell_id"}) - first = df.columns[0] - if df[first].dtype == object and df[first].is_unique: - return df.rename(columns={first: "cell_id"}) - print(f"[xenium_reader] skip {path.name}: cannot identify barcode column") - return None - def _cells_full(self) -> Optional[pd.DataFrame]: """cells.parquet merged with supplemental metadata. Cached.""" if self._cells_full_cache is not _UNSET: return self._cells_full_cache # type: ignore[return-value] - cells = self._read_parquet("cells.parquet") - supp = self._load_supplemental_metadata() - if cells is None and supp is None: - self._cells_full_cache = None - return None - if supp is None: - self._cells_full_cache = cells - return cells - if cells is None: - self._cells_full_cache = supp - return supp - new_cols = [c for c in supp.columns if c not in cells.columns] - merged = cells.merge(supp[["cell_id"] + new_cols], on="cell_id", how="left") if new_cols else cells - self._cells_full_cache = merged - return merged + self._cells_full_cache = self._merge_supplemental( + self._read_parquet("cells.parquet") + ) + return self._cells_full_cache # type: ignore[return-value] # ── Internal helpers ────────────────────────────────────────────────────── diff --git a/backend/tests/golden_baseline.json b/backend/tests/golden_baseline.json new file mode 100644 index 0000000..7d6e6ef --- /dev/null +++ b/backend/tests/golden_baseline.json @@ -0,0 +1,572 @@ +{ + "mouse_ileum_tiny": { + "bounds_full": { + "digest": "751d82c098ea8c71", + "keys": [ + "cell_id", + "vertex_x", + "vertex_y" + ], + "n": 900, + "n_cells": 36, + "total": 36 + }, + "bounds_half": { + "digest": "12202c48c61571a9", + "n": 450, + "n_cells": 18, + "total": 36 + }, + "capabilities": { + "has_boundaries": true, + "has_morphology": true, + "has_transcripts": true, + "unit_label": "cell" + }, + "cell_detail_0": "1ecb75740676415d", + "cell_expression_0": { + "digest": "e4e6708c3bf53c95", + "n": 3 + }, + "cells_all": { + "digest": "61dc45819d71ad01", + "keys": [ + "cell_area", + "cell_id", + "control_codeword_counts", + "control_probe_counts", + "deprecated_codeword_counts", + "genomic_control_counts", + "nucleus_area", + "nucleus_count", + "segmentation_method", + "total_counts", + "transcript_counts", + "unassigned_codeword_counts", + "x_centroid", + "y_centroid" + ], + "n": 36 + }, + "cells_schema": { + "columns": { + "cell_area": "float64", + "control_codeword_counts": "int64", + "control_probe_counts": "int64", + "deprecated_codeword_counts": "int64", + "genomic_control_counts": "int64", + "nucleus_area": "float64", + "nucleus_count": "int64", + "segmentation_method": "object", + "total_counts": "int64", + "transcript_counts": "int64", + "unassigned_codeword_counts": "int64", + "x_centroid": "float64", + "y_centroid": "float64" + } + }, + "color_gene_set": { + "digest": "dc12ce5b5034d05e", + "max": 1.0, + "min": 0.0, + "n": 36, + "type": "continuous" + }, + "color_meta__transcript_counts": { + "digest": "5d8ffb43c4670f47", + "n": 36, + "type": "categorical" + }, + "color_meta__x_centroid": { + "digest": "a8cef95f5f41c61d", + "n": 36, + "type": "continuous" + }, + "color_meta__y_centroid": { + "digest": "d359977cedf42c6a", + "n": 36, + "type": "continuous" + }, + "edge__edges.parquet__catalogue": { + "digest": "556c70f23429ddd8", + "n": 20 + }, + "edge__edges.parquet__color_lrm": { + "digest": "c92dc2b2c78706bf" + }, + "edge__edges.parquet__detail0": "849ae8f80799f4b1", + "edge__edges.parquet__grouped": { + "digest": "74439cac91762e89", + "keys": [ + "edge", + "is_autocrine", + "lrm_count", + "receiving_cell", + "receiving_type", + "score_sum", + "sending_cell", + "sending_type", + "x1", + "x2", + "y1", + "y2" + ], + "n": 223 + }, + "edge__edges.parquet__schema": { + "columns": { + "edge": "string", + "is_autocrine": "bool", + "ligand": "string", + "lrm": "string", + "lrm_id": "int64", + "receiving_cell": "string", + "receiving_type": "string", + "receptor": "string", + "score": "double", + "score_norm": "double", + "sending_cell": "string", + "sending_type": "string", + "x1": "double", + "x2": "double", + "y1": "double", + "y2": "double" + } + }, + "edge__edges.parquet__scores": { + "digest": "17a091898e2af2fb", + "keys": [ + "edge", + "visible_lrm_count", + "visible_score_sum" + ], + "n": 223 + }, + "edge__edges__edge.normalized.product.parquet__catalogue": { + "digest": "556c70f23429ddd8", + "n": 20 + }, + "edge__edges__edge.normalized.product.parquet__color_lrm": { + "digest": "519a5ef75304e432" + }, + "edge__edges__edge.normalized.product.parquet__detail0": "540d31bfbd66002e", + "edge__edges__edge.normalized.product.parquet__grouped": { + "digest": "60404aba71d2d61d", + "keys": [ + "edge", + "is_autocrine", + "lrm_count", + "receiving_cell", + "receiving_type", + "score_sum", + "sending_cell", + "sending_type", + "x1", + "x2", + "y1", + "y2" + ], + "n": 289 + }, + "edge__edges__edge.normalized.product.parquet__schema": { + "columns": { + "edge": "string", + "is_autocrine": "bool", + "ligand": "string", + "lrm": "string", + "lrm_id": "int64", + "receiving_cell": "string", + "receiving_type": "string", + "receptor": "string", + "score": "double", + "score_norm": "double", + "sending_cell": "string", + "sending_type": "string", + "x1": "double", + "x2": "double", + "y1": "double", + "y2": "double" + } + }, + "edge__edges__edge.normalized.product.parquet__scores": { + "digest": "83337511682a043f", + "keys": [ + "edge", + "visible_lrm_count", + "visible_score_sum" + ], + "n": 289 + }, + "edge__edges__edge.raw.minimum.parquet__catalogue": { + "digest": "556c70f23429ddd8", + "n": 20 + }, + "edge__edges__edge.raw.minimum.parquet__color_lrm": { + "digest": "ea178fdb2da7ef01" + }, + "edge__edges__edge.raw.minimum.parquet__detail0": "3efe150623a34474", + "edge__edges__edge.raw.minimum.parquet__grouped": { + "digest": "ba6b68c6da29e63c", + "keys": [ + "edge", + "is_autocrine", + "lrm_count", + "receiving_cell", + "receiving_type", + "score_sum", + "sending_cell", + "sending_type", + "x1", + "x2", + "y1", + "y2" + ], + "n": 155 + }, + "edge__edges__edge.raw.minimum.parquet__schema": { + "columns": { + "edge": "string", + "is_autocrine": "bool", + "ligand": "string", + "lrm": "string", + "lrm_id": "int64", + "receiving_cell": "string", + "receiving_type": "string", + "receptor": "string", + "score": "double", + "score_norm": "double", + "sending_cell": "string", + "sending_type": "string", + "x1": "double", + "x2": "double", + "y1": "double", + "y2": "double" + } + }, + "edge__edges__edge.raw.minimum.parquet__scores": { + "digest": "99276f997c4867c5", + "keys": [ + "edge", + "visible_lrm_count", + "visible_score_sum" + ], + "n": 155 + }, + "gene_list": { + "digest": "09d44724709d2b24", + "n": 5035 + }, + "info_keys": [ + "analysis_sw_version", + "analysis_uuid", + "calibration_uuid", + "cassette_name", + "cassette_uuid", + "chemistry_version", + "experiment_uuid", + "fraction_transcripts_assigned", + "images", + "imported_cell_frac", + "instrument_sn", + "instrument_sw_version", + "major_version", + "minor_version", + "nuclear_transcripts_per_100um", + "num_cells", + "panel_design_id", + "panel_name", + "panel_num_targets_custom", + "panel_num_targets_predesigned", + "panel_organism", + "panel_predesigned_id", + "panel_tissue_type", + "panel_type", + "patch_version", + "pixel_size", + "platform", + "preservation_method", + "region_area", + "region_name", + "roi_uuid", + "run_name", + "run_start_time", + "segmentation_stain", + "segmented_cell_boundary_frac", + "segmented_cell_interior_frac", + "segmented_cell_nuc_expansion_frac", + "segmented_cell_stain_frac", + "slide_id", + "thickness_of_high_quality_decoded_transcripts", + "total_cell_area", + "transcripts_per_100um", + "transcripts_per_cell", + "well_uuid", + "xenium_explorer_files", + "z_step_size" + ], + "pixel_size": 0.2125, + "platform": "xenium", + "tx_bbox": { + "digest": "84b62ff069bb9014", + "keys": [ + "feature_name", + "qv", + "x_location", + "y_location" + ], + "n": 602, + "total": 602 + }, + "tx_bbox_half": { + "digest": "e8f25ce5cb6b26b7", + "keys": [ + "feature_name", + "qv", + "x_location", + "y_location" + ], + "n": 301, + "total": 602 + }, + "tx_full": { + "digest": "3bea25b8419eb09f", + "keys": [ + "feature_name", + "qv", + "x_location", + "y_location" + ], + "n": 1985, + "total": 1985 + }, + "tx_gene0": { + "digest": "e3b0c44298fc1c14", + "n": 0, + "total": 0 + } + }, + "xenium_human_breast_2fov": { + "bounds_full": { + "digest": "2e6916032112a6e5", + "keys": [ + "cell_id", + "vertex_x", + "vertex_y" + ], + "n": 181804, + "n_cells": 7275, + "total": 7275 + }, + "bounds_half": { + "digest": "ff23537fade12766", + "n": 90908, + "n_cells": 3638, + "total": 7275 + }, + "capabilities": { + "has_boundaries": true, + "has_morphology": true, + "has_transcripts": true, + "unit_label": "cell" + }, + "cell_detail_0": "a359455dace01553", + "cell_expression_0": { + "digest": "bcc35f27b554d1f2", + "n": 32 + }, + "cells_all": { + "digest": "278bcced1ed14e24", + "keys": [ + "cell_area", + "cell_id", + "control_codeword_counts", + "control_probe_counts", + "deprecated_codeword_counts", + "nucleus_area", + "total_counts", + "transcript_counts", + "unassigned_codeword_counts", + "x_centroid", + "y_centroid" + ], + "n": 7275 + }, + "cells_schema": { + "columns": { + "cell_area": "float64", + "control_codeword_counts": "int64", + "control_probe_counts": "int64", + "deprecated_codeword_counts": "int64", + "nucleus_area": "float64", + "total_counts": "int64", + "transcript_counts": "int64", + "unassigned_codeword_counts": "int64", + "x_centroid": "float64", + "y_centroid": "float64" + } + }, + "color_gene_set": { + "digest": "7e62d7c4719e9927", + "max": 37.0, + "min": 0.0, + "n": 7275, + "type": "continuous" + }, + "color_meta__transcript_counts": { + "digest": "8f4b807981994590", + "n": 7275, + "type": "continuous" + }, + "color_meta__x_centroid": { + "digest": "8b33aa9839229c15", + "n": 7275, + "type": "continuous" + }, + "color_meta__y_centroid": { + "digest": "46264600ea1f3396", + "n": 7275, + "type": "continuous" + }, + "edge__edges.parquet__catalogue": { + "digest": "556c70f23429ddd8", + "n": 20 + }, + "edge__edges.parquet__color_lrm": { + "digest": "f454e9ec05ba3d82" + }, + "edge__edges.parquet__detail0": "502de7ce02b60495", + "edge__edges.parquet__grouped": { + "digest": "2587c77fd6f6a1b3", + "keys": [ + "edge", + "is_autocrine", + "lrm_count", + "receiving_cell", + "receiving_type", + "score_sum", + "sending_cell", + "sending_type", + "x1", + "x2", + "y1", + "y2" + ], + "n": 44759 + }, + "edge__edges.parquet__schema": { + "columns": { + "edge": "string", + "is_autocrine": "bool", + "ligand": "string", + "lrm": "string", + "lrm_id": "int64", + "receiving_cell": "string", + "receiving_type": "string", + "receptor": "string", + "score": "double", + "score_norm": "double", + "sending_cell": "string", + "sending_type": "string", + "x1": "double", + "x2": "double", + "y1": "double", + "y2": "double" + } + }, + "edge__edges.parquet__scores": { + "digest": "96471c6f8561b653", + "keys": [ + "edge", + "visible_lrm_count", + "visible_score_sum" + ], + "n": 44759 + }, + "gene_list": { + "digest": "bfd0eef30077d962", + "n": 280 + }, + "info_keys": [ + "analysis_sw_version", + "analysis_uuid", + "calibration_uuid", + "cassette_name", + "cassette_uuid", + "experiment_uuid", + "images", + "instrument_sn", + "instrument_sw_version", + "major_version", + "minor_version", + "num_cells", + "panel_design_id", + "panel_name", + "panel_num_targets_custom", + "panel_num_targets_predesigned", + "panel_organism", + "panel_predesigned_id", + "panel_tissue_type", + "patch_version", + "pixel_size", + "platform", + "preservation_method", + "region_name", + "roi_uuid", + "run_name", + "run_start_time", + "segmentation_stain", + "slide_id", + "transcripts_per_100um", + "transcripts_per_cell", + "well_uuid", + "xenium_explorer_files", + "z_step_size" + ], + "pixel_size": 0.2125, + "platform": "xenium", + "tx_bbox": { + "digest": "958f4de8b1dd19b3", + "keys": [ + "feature_name", + "qv", + "x_location", + "y_location" + ], + "n": 200000, + "total": 228952 + }, + "tx_bbox_half": { + "digest": "cfbe3c924fa10464", + "keys": [ + "feature_name", + "qv", + "x_location", + "y_location" + ], + "n": 114476, + "total": 228952 + }, + "tx_full": { + "digest": "4531f4ac79155e19", + "keys": [ + "feature_name", + "qv", + "x_location", + "y_location" + ], + "n": 200000, + "total": 1113950 + }, + "tx_gene0": { + "digest": "6b2cf14148751692", + "keys": [ + "feature_name", + "qv", + "x_location", + "y_location" + ], + "n": 62, + "total": 62 + } + } +} diff --git a/backend/tests/golden_snapshot.py b/backend/tests/golden_snapshot.py new file mode 100644 index 0000000..e4fae51 --- /dev/null +++ b/backend/tests/golden_snapshot.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +""" +Golden-output snapshot for the reader layer. + +This repo has no test suite, so cross-platform work on the readers has nothing to +catch a regression. This script is the minimum viable guard: it exercises every +reader method against the bundled datasets, digests the results, and compares them +to a recorded baseline. + +It calls the readers directly rather than going over HTTP, so no server is needed +and a failure points at the reader instead of the transport. + +Usage +----- + python tests/golden_snapshot.py --record # write the baseline + python tests/golden_snapshot.py # check against it + python tests/golden_snapshot.py -v # show every probe, not just failures + +Run from the ``backend/`` directory. Exits non-zero on any mismatch, so it can be +wired into CI or a pre-commit hook later. + +Determinism +----------- +Every sampled query must be reproducible or the baseline is worthless. The readers +seed their sampling (``duck.SAMPLE_SEED``), so repeated calls return identical rows. +Floats are rounded before hashing to absorb platform FP noise. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import sys +import traceback +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +DATA_ROOT = Path(__file__).resolve().parent.parent.parent / "sample_data" +BASELINE = Path(__file__).resolve().parent / "golden_baseline.json" + +# Datasets to cover. Missing ones are skipped with a note rather than failing, so the +# script still works in a checkout that only has the committed tiny dataset. +DATASETS = ["mouse_ileum_tiny", "xenium_human_breast_2fov", "seqfish_instrument2"] + +FLOAT_PLACES = 4 + + +def _canon(obj): + """Recursively round floats and drop NaN/Inf so digests are stable.""" + if isinstance(obj, float): + if not math.isfinite(obj): + return None + return round(obj, FLOAT_PLACES) + if isinstance(obj, dict): + return {str(k): _canon(v) for k, v in sorted(obj.items(), key=lambda kv: str(kv[0]))} + if isinstance(obj, (list, tuple)): + return [_canon(v) for v in obj] + if hasattr(obj, "item"): # numpy scalar + try: + return _canon(obj.item()) + except Exception: + return str(obj) + if isinstance(obj, (str, int, bool)) or obj is None: + return obj + return str(obj) + + +def digest(obj) -> str: + payload = json.dumps(_canon(obj), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest()[:16] + + +def digest_rows(records) -> str: + """Order-independent digest of a list of records. + + Row order is deliberately not part of the API contract — ``query_grouped`` uses + ``ORDER BY RANDOM()`` and SQL ``GROUP BY`` makes no ordering promise — so hashing + in returned order produces a baseline that fails on every run. Sorting by the + canonical serialization tests the *content* of the result set, which is what we + actually care about not regressing. + """ + if not isinstance(records, (list, tuple)): + return digest(records) + canon = [json.dumps(_canon(r), sort_keys=True, separators=(",", ":")) for r in records] + canon.sort() + return hashlib.sha256("\n".join(canon).encode()).hexdigest()[:16] + + +class Probes: + """Collects {probe_name: value} for one dataset.""" + + def __init__(self): + self.out: dict[str, object] = {} + + def record(self, name, fn): + """Run fn(); store a summary. Exceptions are recorded, not raised, so one + broken method doesn't hide regressions in all the others.""" + try: + self.out[name] = fn() + except Exception as exc: + self.out[name] = f"ERROR: {type(exc).__name__}: {exc}" + + # Summaries keep the baseline small and readable: a count plus a digest catches + # both "wrong number of rows" and "same number, different values". + @staticmethod + def rows(records, total=None): + summary = {"n": len(records), "digest": digest_rows(records)} + if total is not None: + summary["total"] = total + if records: + summary["keys"] = sorted(records[0].keys()) + return summary + + +def probe_spatial(reader, p: Probes) -> None: + p.record("platform", lambda: reader.platform) + p.record("pixel_size", lambda: round(float(reader.pixel_size), 6)) + p.record("capabilities", lambda: _canon(reader.capabilities())) + p.record("info_keys", lambda: sorted(str(k) for k in reader.info().keys())) + + p.record("gene_list", lambda: {"n": len(reader.gene_list()), + "digest": digest(sorted(reader.gene_list()))}) + p.record("cells_schema", lambda: _canon(reader.cells_schema())) + + cells = reader.cells() + p.record("cells_all", lambda: Probes.rows(cells)) + + caps = reader.capabilities() + + # ── transcripts: full, bbox, gene-filtered, sampled ────────────────────── + if caps.get("has_transcripts", True): + full = reader.transcripts(fraction=1.0) + p.record("tx_full", lambda: Probes.rows(full["transcripts"], full["total"])) + + # A bbox covering the lower-left quadrant of the cell centroid extent. + if cells: + xs = [c.get("x_centroid") for c in cells if c.get("x_centroid") is not None] + ys = [c.get("y_centroid") for c in cells if c.get("y_centroid") is not None] + if xs and ys: + bbox = (min(xs), min(ys), (min(xs) + max(xs)) / 2, (min(ys) + max(ys)) / 2) + q = reader.transcripts(bbox=bbox, fraction=1.0) + p.record("tx_bbox", lambda: Probes.rows(q["transcripts"], q["total"])) + h = reader.transcripts(bbox=bbox, fraction=0.5) + p.record("tx_bbox_half", lambda: Probes.rows(h["transcripts"], h["total"])) + + genes = reader.gene_list() + if genes: + g = reader.transcripts(genes=[genes[0]], fraction=1.0) + p.record("tx_gene0", lambda: Probes.rows(g["transcripts"], g["total"])) + + # ── boundaries: full and sampled ───────────────────────────────────────── + if caps.get("has_boundaries", True): + b = reader.cell_boundaries(fraction=1.0) + rows = b["boundaries"] if isinstance(b, dict) else b + total = b.get("total") if isinstance(b, dict) else None + p.record("bounds_full", lambda: { + **Probes.rows(rows, total), + "n_cells": len({r["cell_id"] for r in rows}) if rows else 0, + }) + bq = reader.cell_boundaries(fraction=0.5) + rq = bq["boundaries"] if isinstance(bq, dict) else bq + p.record("bounds_half", lambda: { + "n": len(rq), + "n_cells": len({r["cell_id"] for r in rq}) if rq else 0, + "total": bq.get("total") if isinstance(bq, dict) else None, + "digest": digest_rows(rq), + }) + + # ── per-cell detail + expression ───────────────────────────────────────── + if cells: + cid = cells[0]["cell_id"] + p.record("cell_detail_0", lambda: digest(reader.cell_detail(cid))) + p.record("cell_expression_0", lambda: { + "n": len(reader.cell_expression(cid)), + "digest": digest(reader.cell_expression(cid)), + }) + + # ── color values ───────────────────────────────────────────────────────── + genes = reader.gene_list() + if genes: + cv = reader.color_values("gene_set", None, genes[: min(5, len(genes))]) + p.record("color_gene_set", lambda: { + "type": cv.get("type"), "n": len(cv.get("values", {})), + "min": _canon(cv.get("min")), "max": _canon(cv.get("max")), + "digest": digest(cv.get("values")), + }) + schema = reader.cells_schema().get("columns", {}) + for field in list(schema)[:3]: + cvm = reader.color_values("metadata", field, None) + p.record(f"color_meta__{field}", lambda cvm=cvm: { + "type": cvm.get("type"), "n": len(cvm.get("values", {})), + "digest": digest(cvm.get("values")), + }) + + +def probe_edges(dataset_dir: Path, pixel_size: float, p: Probes) -> None: + from app.readers.edge_reader import EdgeReader + + sources = [] + if (dataset_dir / "edges.parquet").exists(): + sources.append(("edges.parquet", dataset_dir / "edges.parquet")) + edir = dataset_dir / "edges" + if edir.is_dir(): + for f in sorted(edir.glob("*.parquet")): + sources.append((f"edges/{f.name}", f)) + + for label, path in sources: + er = EdgeReader(path, pixel_size=pixel_size) + key = label.replace("/", "__") + p.record(f"edge__{key}__schema", lambda er=er: _canon(er.schema())) + p.record(f"edge__{key}__catalogue", lambda er=er: { + "n": len(er.lrm_catalogue()), "digest": digest_rows(er.lrm_catalogue())}) + grouped = er.query_grouped(density=1.0) + p.record(f"edge__{key}__grouped", lambda g=grouped: Probes.rows(g)) + p.record(f"edge__{key}__scores", lambda er=er: Probes.rows( + er.query_scores(excluded_lrms=[]))) + p.record(f"edge__{key}__color_lrm", lambda er=er: { + "digest": digest(er.edge_color_values("lrm_set", None, None))}) + if grouped: + # query_grouped uses ORDER BY RANDOM(), so grouped[0] is a different edge + # every run. Take the lexicographic minimum instead, or the probe reports + # a spurious failure on each invocation. + eid = min(str(g["edge"]) for g in grouped if g.get("edge") is not None) + p.record(f"edge__{key}__detail0", lambda er=er, eid=eid: + digest(er.edge_detail(eid))) + + +def collect() -> dict: + from app.readers.reader_factory import ReaderFactory + + snap: dict[str, dict] = {} + for name in DATASETS: + d = DATA_ROOT / name + if not d.exists(): + print(f" · {name}: not present, skipped") + continue + if not ReaderFactory.is_dataset(d): + print(f" · {name}: not recognised as a dataset, skipped") + continue + p = Probes() + try: + reader = ReaderFactory.detect(d) + probe_spatial(reader, p) + probe_edges(d, reader.pixel_size, p) + except Exception: + p.out["__fatal__"] = traceback.format_exc(limit=3) + snap[name] = p.out + print(f" · {name}: {len(p.out)} probes") + return snap + + +def compare(old: dict, new: dict, verbose: bool) -> int: + failures = 0 + for ds in sorted(set(old) | set(new)): + if ds not in old: + print(f"\n[NEW DATASET] {ds} — not in baseline (re-record to adopt)") + continue + if ds not in new: + print(f"\n[MISSING] {ds} — in baseline but not produced now") + failures += 1 + continue + o, n = old[ds], new[ds] + keys = sorted(set(o) | set(n)) + diffs = [k for k in keys if o.get(k) != n.get(k)] + if not diffs: + print(f" OK {ds} ({len(keys)} probes identical)") + if verbose: + for k in keys: + print(f" {k} = {json.dumps(n[k])[:90]}") + continue + failures += len(diffs) + print(f" FAIL {ds} ({len(diffs)} of {len(keys)} probes changed)") + for k in diffs: + print(f" ✗ {k}") + print(f" baseline: {json.dumps(o.get(k))[:160]}") + print(f" current : {json.dumps(n.get(k))[:160]}") + return failures + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--record", action="store_true", + help="write the baseline instead of comparing") + ap.add_argument("-v", "--verbose", action="store_true") + args = ap.parse_args() + + print(f"DATA_ROOT = {DATA_ROOT}") + snap = collect() + + if args.record: + BASELINE.write_text(json.dumps(snap, indent=2, sort_keys=True) + "\n") + n = sum(len(v) for v in snap.values()) + print(f"\nrecorded {n} probes across {len(snap)} datasets → {BASELINE.name}") + return 0 + + if not BASELINE.exists(): + print(f"\nno baseline at {BASELINE} — run with --record first") + return 2 + + old = json.loads(BASELINE.read_text()) + print() + failures = compare(old, snap, args.verbose) + if failures: + print(f"\n{failures} probe(s) changed. If intentional, re-run with --record.") + return 1 + print("\nall probes match the baseline.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 6d0549c86db4efdece2b9789986e645c2f94e290 Mon Sep 17 00:00:00 2001 From: Micha Sam Brickman Raredon <42100128+msraredon@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:27:42 -0400 Subject: [PATCH 07/18] feat: seqFISH (Spatial Genomics GenePS) support Adds a fifth platform reader. Xenium is untouched: all 62 of its golden-snapshot probes are byte-identical across the change. "seqFISH" names two unrelated things. The academic Cai-lab method has no standard output layout; this targets the commercial Spatial Genomics GenePS platform, which does. Current v2 layout is fully supported -- cells, transcripts, boundaries, expression, both colour-value modes. Legacy v1 reads cells and transcripts but declares has_boundaries: False, since v1 ships only a label mask and polygonising it would mean either a heavy new dependency or hand-rolled contour tracing; deferred deliberately. The hard part is coordinates. A single seqFISH dataset mixes units, measured on the reference dataset (1000x1000 DAPI at 0.107161 um/px): CellCoordinates center_x 1.82 -> 105.66 microns TranscriptList x 0.00 -> 107.05 microns Boundaries vertices 0 -> 999 pixels Cells and transcripts are divided by pixel_size; boundaries pass through untouched. One global transform in either direction would put cells and their own outlines in different places, presenting as a rendering bug rather than a unit bug. The convention also differs across GenePS software versions, so it cannot be hard-coded: _units_divisor decides per table by comparing that table's extent to the image width, and logs its verdict. On real data the ratios are 0.106 / 0.107 / 0.999 -- two orders of magnitude apart. Verified geometrically rather than by digest alone: every cell centroid falls inside its own polygon, 62/62 on the reference dataset and 36/36 on the synthetic fixture, with zero false positives against a control. Independently, polygon area recomputed from boundary vertices agrees with the reported cell_area to within 3%. Cell identity comes from each GeoJSON feature's id, which equals label. spatialdata-io maps polygons positionally instead and has an open issue about the fragility (scverse/spatialdata-io#249); a silent off-by-one there would draw every outline on the wrong cell. sample_data/make_seqfish.py generates a committable synthetic v2 ROI (400K) and deliberately reproduces the mixed units, so a reader that got them wrong would fail on it. The real reference dataset is public by written permission from Spatial Genomics rather than under an open licence, so it stays gitignored. Three bugs fixed along the way, all pre-existing and none seqFISH-specific: - CellInfoPanel rendered the literal string "undefined um2" for any platform not reporting a cell area -- the optional chain was not paired with a null check the way nucleus_area's is. - Switching datasets kept the previous dataset's activeImage until the image list resolved, so OSD spent that window requesting the old image from the new dataset and logging 404s. Only visible now because image names differ across platforms ("morphology" vs "Roi1_DAPI"). setDataset clears it and OSD init waits. - cell_area is kept in um2 to match Xenium, which never converts it, so the "um2" label in the info panel is true on every platform. Verified end-to-end in the browser across Xenium -> seqFISH -> Xenium with zero console errors and no backend errors. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 120 +- README.md | 22 +- backend/app/main.py | 2 +- backend/app/readers/duck.py | 18 + backend/app/readers/reader_factory.py | 23 +- backend/app/readers/seqfish_reader.py | 546 ++++ backend/tests/golden_baseline.json | 246 ++ backend/tests/golden_snapshot.py | 7 +- frontend/package.json | 2 +- frontend/src/components/CellInfoPanel.jsx | 7 +- frontend/src/components/LayerPanel.jsx | 2 +- frontend/src/components/Viewer.jsx | 3 + frontend/src/store.js | 7 +- sample_data/.gitignore | 3 + sample_data/make_seqfish.py | 163 + .../seqfish_synthetic/Roi1_Boundaries.geojson | 2689 +++++++++++++++++ .../Roi1_CellCoordinates.csv | 37 + .../seqfish_synthetic/Roi1_CellxGene.csv | 37 + sample_data/seqfish_synthetic/Roi1_DAPI.tiff | Bin 0 -> 284248 bytes .../seqfish_synthetic/Roi1_Segmentation.tiff | Bin 0 -> 8356 bytes .../seqfish_synthetic/Roi1_TranscriptList.csv | 2551 ++++++++++++++++ 21 files changed, 6460 insertions(+), 25 deletions(-) create mode 100644 backend/app/readers/seqfish_reader.py create mode 100644 sample_data/make_seqfish.py create mode 100644 sample_data/seqfish_synthetic/Roi1_Boundaries.geojson create mode 100644 sample_data/seqfish_synthetic/Roi1_CellCoordinates.csv create mode 100644 sample_data/seqfish_synthetic/Roi1_CellxGene.csv create mode 100644 sample_data/seqfish_synthetic/Roi1_DAPI.tiff create mode 100644 sample_data/seqfish_synthetic/Roi1_Segmentation.tiff create mode 100644 sample_data/seqfish_synthetic/Roi1_TranscriptList.csv diff --git a/CLAUDE.md b/CLAUDE.md index bc4d7a0..e52b4a4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,7 +9,7 @@ of the project so any Claude instance can contribute immediately. ## What This Is A web-based spatial transcriptomics viewer supporting multiple platforms (Xenium, -MERSCOPE, CosMx) with connectivity layers produced by the lab's NICHESv2 R pipeline. +seqFISH, Visium HD, MERSCOPE, CosMx) with connectivity layers produced by the lab's NICHESv2 R pipeline. Built because Xenium Explorer does not support cell-cell ligand-receptor mechanism (LRM) visualization, and extended to be platform-agnostic. @@ -57,6 +57,7 @@ The backend uses an abstract reader pattern. All platform readers inherit from | Visium HD (10x Genomics) | a `square_???um/` subdirectory | | MERSCOPE (Vizgen) | `cell_by_gene.csv` or `cell_metadata.csv` | | CosMx (Nanostring) | `*_tx_file.csv` | +| seqFISH (Spatial Genomics) | `*_CellCoordinates*.csv` — a glob, so registered **last** | **Coordinate contract**: Every reader converts native coordinates to image pixel space before returning data. The frontend always receives pixel coordinates. @@ -79,6 +80,12 @@ and boundary layers rather than returning empty arrays for them. - CosMx: cells, transcripts, genes, metadata color-values implemented; gene-set color-values stub (requires transcript aggregation per cell); `has_boundaries: False` (boundaries are per-FOV label TIFFs) +- seqFISH (Spatial Genomics GenePS): fully implemented for the current **v2** layout — + cells, transcripts, boundaries, expression, and both color-value modes. Legacy **v1** + reads cells and transcripts but declares `has_boundaries: False`, because v1 ships only + a label mask and polygonising it was deliberately deferred rather than adding a + dependency. See the seqFISH section below — its coordinate handling is unlike any other + reader and is the thing to understand before touching it. **Interface caveat**: `VisiumHDReader.transcripts()` and `.cell_boundaries()` still carry the pre-refactor signature (`limit=` instead of `fraction=`, returning `[]` instead of the @@ -108,6 +115,9 @@ backend/ base_reader.py Abstract base class — SpatialDatasetReader interface reader_factory.py ReaderFactory: auto-detect platform, instantiate reader xenium_reader.py Xenium implementation (inherits SpatialDatasetReader) + seqfish_reader.py seqFISH / Spatial Genomics GenePS; v2 full, v1 partial. + Mixed µm/pixel coordinate handling — see its own section. + duck.py Shared DuckDB query helpers used by the spatial readers visium_hd_reader.py Visium HD implementation — bins as points; partial (see status above) merscope_reader.py MERSCOPE implementation (inherits SpatialDatasetReader) cosmx_reader.py CosMx implementation (inherits SpatialDatasetReader) @@ -117,6 +127,9 @@ backend/ pyramid.py OME-TIFF → DZI; pyvips streaming primary, tifffile+Pillow fallback requirements.txt pinned deps; cffi<2.0 required for pyvips 2.2.3 compatibility Dockerfile + tests/ + golden_snapshot.py Reader regression guard — see Development Workflow + golden_baseline.json Recorded baseline (100 probes / 4 datasets) frontend/ src/ @@ -152,7 +165,10 @@ Caddyfile Reverse proxy + TLS for the cloud deployment; optio deploy.sh One-shot droplet bootstrap (see docs/cloud-deploy.md) upload-data.sh rsync datasets to a deployed server sample_data/ Partially gitignored — default data mount for local dev/demo. - mouse_ileum_tiny is tracked; larger datasets are ignored. + mouse_ileum_tiny and seqfish_synthetic are tracked; larger + and licence-restricted datasets are ignored. + make_edges.py Synthetic edges.parquet generator + make_seqfish.py Synthetic seqFISH v2 ROI generator (committable fixture) r/ Personal analysis scripts with hardcoded paths — a pipeline, not reusable functions. Run in this order: ExportMetaDataforTissuePlex.R dump a Seurat @meta.data to CSV @@ -205,7 +221,11 @@ Real data comes from `export_for_TissuePlex()` in the NICHESv2 R package. User-defined metadata (e.g. from external R analysis) can be loaded without modifying the dataset output by placing files in a `cell-metadata/` subdirectory of the dataset. -Currently implemented in XeniumReader; the pattern should be ported to other readers. +The loader lives on `SpatialDatasetReader`, so it is available to every platform; a reader +opts in by calling `_merge_supplemental()` on its cells table (Xenium and seqFISH do). +All a platform contributes is `_ROOT_CSV_SKIP` / `_ROOT_CSV_SKIP_SUFFIXES` — the list of +its *own* root CSVs, so the loader never ingests platform output as user metadata. CosMx +needs the suffix form because it prefixes every file with the experiment name. ``` dataset_dir/ @@ -237,9 +257,9 @@ write.csv(my_metadata_df, file.path(dataset_dir, "cell-metadata", "metadata.csv" dropdown. Continuous columns get a gradient colormap; string or low-cardinality integer columns get discrete colors. The cell-click info panel also shows the supplemental fields. -`XeniumReader._cells_full()` is cached per reader instance (one Docker request lifecycle). -`_load_supplemental_metadata()` is also cached, so the CSV is only parsed once regardless -of how many color-by requests arrive. +`_cells_full()` is cached per reader instance (one Docker request lifecycle). +`_load_supplemental_metadata()` is also cached on the base class, so the CSV is parsed once +regardless of how many color-by requests arrive. --- @@ -560,6 +580,72 @@ pre-computed server-side. --- +## seqFISH / Spatial Genomics (readers/seqfish_reader.py) + +"seqFISH" names two unrelated things. The academic Cai-lab method has no standard output +layout; **this reader targets the commercial Spatial Genomics GenePS platform**, which +does. One flat directory, every file prefixed with an ROI name, one ROI per dataset folder +(several ROIs in one folder logs a warning and uses the first). + +``` +seqfish_dataset/ + Roi1_CellCoordinates.csv label, area, center_x, center_y + Roi1_CellxGene.csv unnamed first col = label; remaining cols = genes + Roi1_TranscriptList.csv name, x, y, [z] — no `cell`, no `qv` in v2 + Roi1_DAPI.tiff OME-TIFF despite the .tiff extension; often pyramidal + Roi1_Segmentation.tiff integer label mask (unused — v2 uses the GeoJSON) + Roi1_Boundaries.geojson polygons; feature `id` == label +``` + +**A single dataset mixes coordinate systems, and this is the thing to get right.** +Measured on the reference dataset (1000×1000 px DAPI at 0.107161 µm/px = 107.16 µm across): + +| Source | Extent | Units | +|---|---|---| +| `CellCoordinates.csv` `center_x` | 1.82 → 105.66 | **microns** | +| `TranscriptList.csv` `x` | 0.00 → 107.05 | **microns** | +| `Boundaries.geojson` vertices | 0 → 999 | **pixels** | + +Cells and transcripts are divided by `pixel_size`; boundaries pass through untouched. +Applying one transform to everything puts cells and their own outlines in different +places — which reads as a rendering bug rather than a unit bug. Worse, the convention +differs across GenePS software versions, so it cannot be hard-coded. + +`_units_divisor()` therefore decides **per table**, comparing that table's extent to the +image width: a ratio near `pixel_size` means microns, near 1.0 means pixels. On the +reference data the ratios are 0.106 / 0.107 / 0.999 — two orders of magnitude apart. The +verdict is logged on load, so if a dataset ever misdetects it is visible in the backend +output rather than silent. + +The regression test for this is geometric, not a digest: **every cell centroid must fall +inside its own polygon.** 62/62 on the reference dataset and 36/36 on the synthetic +fixture, with zero false positives against a control. Re-run that check after touching +anything in the coordinate path. + +Other things worth knowing: + +- `pixel_size` comes from `PhysicalSizeX` in the **DAPI OME-XML** — not a manifest, unlike + every other platform. Falls back to 0.107 (the documented GenePS value). +- `cell_area` is deliberately left in **µm²** to match Xenium, which never converts it, so + the "µm²" label in `CellInfoPanel` is true on every platform. +- Cell identity comes from each GeoJSON feature's `id`, which equals `label`. + `spatialdata-io` instead maps polygons to cells *positionally* and has an open issue + about the fragility (scverse/spatialdata-io#249); a silent off-by-one there would draw + every outline on the wrong cell. We join on `id` and fall back to position only if + absent. +- GeoJSON rings are closed (first vertex repeated); the reader drops the duplicate because + deck.gl closes polygons itself and Xenium boundaries do not repeat it. +- v2 dropped the transcript→cell assignment column and has no `qv`. Nothing needs them + today, but expression can only come from `CellxGene.csv`, never from transcripts. + +**Test data.** `sample_data/make_seqfish.py` generates a committable synthetic v2 ROI and +deliberately reproduces the mixed units, so a reader that got them wrong would fail on it. +The real reference dataset (`seqfish-2-test-dataset.zip`, scverse CI fixture) is public by +written permission from Spatial Genomics rather than under an open licence — usable +locally, gitignored, and must not be redistributed from this repo. + +--- + ## Spatial Query Path (readers/duck.py) `transcripts()` and `cell_boundaries()` query parquet through DuckDB rather than loading @@ -691,10 +777,24 @@ Tuning knobs live in a `.env.prod` file that is gitignored and must be created b unless it is enabled anyone with the URL can view the data. There is no application-level auth, no user accounts, and no per-dataset permissions. -**No tests, no CI, no linter.** There is no test suite, no `.github/workflows`, and no -ESLint or Python lint configuration in this repo. Changes are verified by running the app. -Be correspondingly careful with refactors that touch the reader interface or the -OSD ↔ deck.gl coordinate bridge, since nothing will catch a regression automatically. +**Regression guard.** `backend/tests/golden_snapshot.py` exercises every reader method +against all local datasets, digests the results, and diffs them against a recorded +baseline (100 probes across 4 datasets). Run it after any reader change: + +```bash +cd backend && python3 tests/golden_snapshot.py # check +cd backend && python3 tests/golden_snapshot.py --record # adopt intentional changes +``` + +Datasets absent from a checkout are skipped, so it works with only the committed fixtures. +Two determinism rules keep it honest: record-list digests are order-independent (because +`query_grouped` uses `ORDER BY RANDOM()`), and sampling is seeded (`duck.SAMPLE_SEED`). +If a probe changes and you cannot explain why, that is the point of the tool. + +There is still **no CI and no linter** — no `.github/workflows`, no ESLint or Python lint +config. The snapshot is a guard, not a test suite: it catches "this changed" but does not +assert correctness. Be correspondingly careful with the OSD ↔ deck.gl coordinate bridge, +which it does not cover at all. --- diff --git a/README.md b/README.md index 8d4a857..c8a36cc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ An interactive spatial transcriptomics viewer for exploring cell-cell communicat ## What it does -Spatial transcriptomics platforms (Xenium, Visium HD, MERSCOPE, CosMx) produce high-resolution images with hundreds of genes measured per cell. NICHESv2 infers which cells are communicating and through which ligand-receptor mechanisms (LRMs). TissuePlex bridges those two outputs: it overlays the NICHESv2 communication graph on the tissue image and lets you explore it interactively. +Spatial transcriptomics platforms (Xenium, seqFISH, Visium HD, MERSCOPE, CosMx) produce high-resolution images with hundreds of genes measured per cell. NICHESv2 infers which cells are communicating and through which ligand-receptor mechanisms (LRMs). TissuePlex bridges those two outputs: it overlays the NICHESv2 communication graph on the tissue image and lets you explore it interactively. **Key capabilities:** @@ -19,6 +19,7 @@ Spatial transcriptomics platforms (Xenium, Visium HD, MERSCOPE, CosMx) produce h - **Multiple edge sets per dataset** — drop several `.parquet` files into an `edges/` folder and flip between scoring approaches on the same tissue without duplicating the image or cell data - **Pan and zoom on high-resolution morphology images** — OME-TIFF tile pyramid with smooth zoom from whole-tissue to single-cell scale - **Multi-channel morphology** — Xenium `morphology_focus/` channels are selectable alongside the top-level morphology image +- **Cross-platform metadata** — the `cell-metadata/` convention works the same way on every platform that supports it, so annotation workflows transfer between Xenium and seqFISH unchanged - **Split-screen comparison** — two independently navigable panels sharing one set of layer controls, with a match-zoom button - **Per-panel rotation** — rotate either panel to any angle to align tissue orientation - **Transcript dot overlay** — per-gene colored dots, filterable by gene species, with hover tooltips @@ -34,11 +35,14 @@ Spatial transcriptomics platforms (Xenium, Visium HD, MERSCOPE, CosMx) produce h | Platform | Vendor | Morphology | Transcripts | Cell segments | Edges | |---|---|:---:|:---:|:---:|:---:| | **Xenium** | 10x Genomics | ✓ | ✓ | ✓ | ✓ | +| **seqFISH** | Spatial Genomics | ✓ | ✓ | ✓ | ✓ | | **Visium HD** | 10x Genomics | ✓ | — | — | ✓ | | **MERSCOPE** | Vizgen | — | ✓ | — | ✓ | | **CosMx** | Nanostring | — | ✓ | — | ✓ | -Xenium is the most complete implementation. The other readers cover cells, transcripts, and metadata coloring; boundary parsing is platform-specific and not yet implemented for them (MERSCOPE stores polygons in HDF5, CosMx in per-FOV label TIFFs). Visium HD renders bins as points rather than polygons and has no per-molecule transcript coordinates. Each reader declares what it supports via a capability flag, and the UI hides layers the platform cannot serve. +Xenium and seqFISH are the complete implementations. seqFISH means the commercial **Spatial Genomics GenePS** output, not the academic seqFISH/seqFISH+ method, which has no standard file layout; the current v2 layout is fully supported, and legacy v1 reads cells and transcripts but not boundaries. + +The other readers cover cells, transcripts, and metadata coloring; boundary parsing is platform-specific and not yet implemented for them (MERSCOPE stores polygons in HDF5, CosMx in per-FOV label TIFFs). Visium HD renders bins as points rather than polygons and has no per-molecule transcript coordinates. Each reader declares what it supports via a capability flag, and the UI hides layers the platform cannot serve. The edge connectivity layer (NICHESv2 output) works with any platform — it is platform-agnostic as long as cell barcodes match. @@ -80,17 +84,25 @@ DATA_PATH=/absolute/path/to/your/datasets docker compose up --build raw_minimum.parquet normalized_product.parquet - visium_hd_run_B/ + seqfish_run_B/ ← Spatial Genomics GenePS; one ROI per folder + Roi1_CellCoordinates.csv ← seqFISH sentinel + Roi1_CellxGene.csv + Roi1_TranscriptList.csv + Roi1_Boundaries.geojson + Roi1_DAPI.tiff + edges.parquet + + visium_hd_run_C/ square_008um/ ← Visium HD sentinel edges.parquet - merscope_run_C/ + merscope_run_D/ cell_by_gene.csv ← MERSCOPE sentinel cell_metadata.csv detected_transcripts.csv edges.parquet - cosmx_run_D/ + cosmx_run_E/ my_experiment_tx_file.csv ← CosMx sentinel edges.parquet ``` diff --git a/backend/app/main.py b/backend/app/main.py index a414261..e0c484f 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,7 @@ from app.routers import tiles, spatial, edges, layers -APP_VERSION = "0.4.0" +APP_VERSION = "0.5.0" app = FastAPI(title="TissuePlex API", version=APP_VERSION) diff --git a/backend/app/readers/duck.py b/backend/app/readers/duck.py index a7536e4..2848acf 100644 --- a/backend/app/readers/duck.py +++ b/backend/app/readers/duck.py @@ -43,11 +43,29 @@ def scan(path: Path) -> str: return "read_parquet('{}')".format(str(path).replace("'", "''")) +def scan_csv(path: Path) -> str: + """SQL FROM-clause fragment that reads a CSV file. + + Platforms that ship CSV instead of parquet (seqFISH) still stream through + DuckDB rather than pandas. CSV has no column statistics so nothing can be + pruned, but the scan is still streamed rather than materialized, which is + what keeps peak memory flat on a multi-GB transcript list. + """ + return "read_csv_auto('{}')".format(str(path).replace("'", "''")) + + def columns(path: Path) -> set[str]: """Column names in a parquet file, read from its footer (no data scan).""" return set(pq.read_schema(path).names) +def csv_columns(path: Path) -> list[str]: + """Column names of a CSV, read from the header row only.""" + with open(path, "r", encoding="utf-8-sig", errors="replace") as fh: + header = fh.readline().rstrip("\r\n") + return [c.strip().strip('"') for c in header.split(",")] + + def bbox_predicate(x_col: str, y_col: str, bbox: tuple) -> tuple[str, list]: """Build a bounding-box WHERE fragment and its bind parameters. diff --git a/backend/app/readers/reader_factory.py b/backend/app/readers/reader_factory.py index af9c9be..dc8d7a8 100644 --- a/backend/app/readers/reader_factory.py +++ b/backend/app/readers/reader_factory.py @@ -6,10 +6,11 @@ could theoretically co-exist in one folder (unlikely in practice). Supported platforms (detection order): - Xenium (10x Genomics) — experiment.xenium + Xenium (10x Genomics) — experiment.xenium Visium HD (10x Genomics) — square_???um/ subdirectory - MERSCOPE (Vizgen) — cell_by_gene.csv or cell_metadata.csv - CosMx (Nanostring) — *_tx_file.csv + MERSCOPE (Vizgen) — cell_by_gene.csv or cell_metadata.csv + CosMx (Nanostring) — *_tx_file.csv + seqFISH (Spatial Genomics) — *_CellCoordinates*.csv (glob; registered last) To add a new platform: define a detector function, a factory function, and call _register(detector, factory) below. @@ -51,6 +52,14 @@ def _is_cosmx(path: Path) -> bool: return any(path.glob("*_tx_file.csv")) +def _is_seqfish(path: Path) -> bool: + # seqFISH ships no manifest or version file, so detection has to be a glob over + # ROI-prefixed filenames. Registered last for that reason — a glob is weaker + # evidence than an exact sentinel and must not shadow the platforms above. + from app.readers.seqfish_reader import SeqfishReader + return bool(SeqfishReader.find_cell_coordinates(path)) + + # ── Factories ───────────────────────────────────────────────────────────────── def _make_xenium(path: Path) -> SpatialDatasetReader: @@ -73,10 +82,16 @@ def _make_cosmx(path: Path) -> SpatialDatasetReader: return CosMxReader(path) +def _make_seqfish(path: Path) -> SpatialDatasetReader: + from app.readers.seqfish_reader import SeqfishReader + return SeqfishReader(path) + + _register(_is_xenium, _make_xenium, "experiment.xenium (Xenium / 10x)") _register(_is_visium_hd, _make_visium_hd, "square_???um/ directory (Visium HD / 10x)") _register(_is_merscope, _make_merscope, "cell_by_gene.csv or cell_metadata.csv (MERSCOPE / Vizgen)") _register(_is_cosmx, _make_cosmx, "*_tx_file.csv (CosMx / Nanostring)") +_register(_is_seqfish, _make_seqfish, "*_CellCoordinates*.csv (seqFISH / Spatial Genomics)") class ReaderFactory: @@ -102,4 +117,4 @@ def is_dataset(path: Path) -> bool: @staticmethod def supported_platforms() -> list[str]: """Names of all registered platforms, in detection-priority order.""" - return ["xenium", "visium_hd", "merscope", "cosmx"] + return ["xenium", "visium_hd", "merscope", "cosmx", "seqfish"] diff --git a/backend/app/readers/seqfish_reader.py b/backend/app/readers/seqfish_reader.py new file mode 100644 index 0000000..ed6fc23 --- /dev/null +++ b/backend/app/readers/seqfish_reader.py @@ -0,0 +1,546 @@ +""" +seqFISH (Spatial Genomics GenePS) dataset reader. + +Note this is the *commercial* platform. "seqFISH" also names the academic +Cai-lab method, which has no standard output layout; that is not what this +reads. + +Expected output layout — current "v2" format +-------------------------------------------- +A flat directory. Every file is prefixed with an ROI name; TissuePlex expects +one ROI per dataset folder. + + _CellCoordinates.csv label, area, center_x, center_y + _CellxGene.csv unnamed first col = label; remaining cols = genes + _TranscriptList.csv name, x, y, [z] + _DAPI.tiff OME-TIFF (OME-XML despite the .tiff extension) + _Segmentation.tiff integer label mask + _Boundaries.geojson cell polygons, feature id == label + +Legacy "v1" format +------------------ + _CellCoordinates_section.csv + _CxG_section.csv + _TranscriptCoordinates_section.csv + _DAPI_section.ome.tiff + _CellMask_section.tiff + (no boundaries file — v1 ships only the label mask) + +v1 transcripts carry a `cell` assignment column that v2 dropped; v2 adds `z`. +v1 has no GeoJSON, so it reports has_boundaries=False until mask polygonisation +is implemented. + +Coordinates — the part that matters +----------------------------------- +A single seqFISH dataset mixes units, verified against the reference dataset: +its DAPI is 1000x1000 px at 0.107161 µm/px (107.16 µm across), and + + CellCoordinates center_x 1.82 -> 105.66 microns + TranscriptList x 0.00 -> 107.05 microns + Boundaries vertices 0 -> 999 pixels + +So cells and transcripts must be divided by pixel_size while boundaries pass +through untouched. Applying one transform to everything puts cells and their own +outlines in different places, which reads as a rendering bug rather than a unit +bug. Worse, the convention differs across GenePS software versions, so it cannot +simply be hard-coded. + +`_units_divisor()` therefore decides per table by comparing the table's extent to +the image width: a ratio near pixel_size means microns, a ratio near 1.0 means +pixels. On the reference dataset those ratios are 0.106 / 0.107 / 0.999, which +separates the cases by two orders of magnitude. The verdict is logged on load. +""" +import json +import math +import re +from pathlib import Path +from typing import Optional + +import pandas as pd + +from app.readers import duck +from app.readers.base_reader import _UNSET, SpatialDatasetReader + +# Fallback when the DAPI OME-XML carries no PhysicalSizeX. This is the documented +# GenePS value and matches the reference dataset (0.107161). +_DEFAULT_PIXEL_SIZE = 0.107 + +_MAX_TRANSCRIPTS = 200_000 + + +class SeqfishReader(SpatialDatasetReader): + + # seqFISH writes only ROI-prefixed CSVs, so exact names cannot be used. + _ROOT_CSV_SKIP_SUFFIXES = ( + "_cellcoordinates.csv", "_cellxgene.csv", "_transcriptlist.csv", + "_transcriptcoordinates.csv", "_cxg.csv", + ) + + def __init__(self, dataset_path: Path): + super().__init__(dataset_path) + self._layout_cache = _UNSET + self._pixel_size_cache: Optional[float] = None + self._image_size_cache: Optional[tuple] = None + self._cxg_cache = _UNSET + self._cells_full_cache = _UNSET + self._divisor_log: set = set() + + # ── Layout discovery ────────────────────────────────────────────────────── + + @staticmethod + def find_cell_coordinates(path: Path) -> list[Path]: + """Every CellCoordinates file in a folder, v2 and v1 naming alike. + + This is also the platform sentinel — `ReaderFactory` calls it to detect + seqFISH, so it must stay cheap and must not raise on odd directories. + """ + try: + return sorted( + f for f in path.glob("*_CellCoordinates*.csv") if f.is_file() + ) + except OSError: + return [] + + def _layout(self) -> dict: + """Resolve the ROI prefix, format variant, and every member file path.""" + if self._layout_cache is not _UNSET: + return self._layout_cache # type: ignore[return-value] + + matches = self.find_cell_coordinates(self.path) + if not matches: + self._layout_cache = {} + return {} + + if len(matches) > 1: + print(f"[seqfish] {self.path.name}: {len(matches)} ROIs present " + f"({', '.join(m.name for m in matches)}); using {matches[0].name}. " + f"TissuePlex expects one ROI per folder — split them to see the rest.") + + chosen = matches[0] + v1 = re.match(r"^(.*)_CellCoordinates_(section\d+)\.csv$", chosen.name) + if v1: + prefix, section = v1.group(1), v1.group(2) + roi = f"{prefix}_{section}" + layout = { + "variant": "v1", + "roi": roi, + "cells": chosen, + "counts": self._first(f"{prefix}_CxG_{section}.csv"), + "transcripts": self._first(f"{prefix}_TranscriptCoordinates_{section}.csv"), + "boundaries": None, # v1 ships no GeoJSON + "mask": self._first(f"{prefix}_CellMask_{section}.tiff"), + "image": (self._first(f"{prefix}_DAPI_{section}.ome.tiff") + or self._first(f"{prefix}_DAPI_{section}.tiff")), + } + else: + roi = chosen.name[: -len("_CellCoordinates.csv")] + layout = { + "variant": "v2", + "roi": roi, + "cells": chosen, + "counts": self._first(f"{roi}_CellxGene.csv"), + "transcripts": self._first(f"{roi}_TranscriptList.csv"), + "boundaries": self._first(f"{roi}_Boundaries.geojson"), + "mask": self._first(f"{roi}_Segmentation.tiff"), + "image": (self._first(f"{roi}_DAPI.tiff") + or self._first(f"{roi}_DAPI.ome.tiff")), + } + self._layout_cache = layout + return layout + + def _first(self, name: str) -> Optional[Path]: + p = self.path / name + return p if p.exists() else None + + # ── Identity ────────────────────────────────────────────────────────────── + + @property + def platform(self) -> str: + return "seqfish" + + @property + def pixel_size(self) -> float: + """µm per image pixel, from the DAPI OME-XML PhysicalSizeX.""" + if self._pixel_size_cache is not None: + return self._pixel_size_cache + self._pixel_size_cache = _DEFAULT_PIXEL_SIZE + img = self._layout().get("image") + if img is not None: + try: + import tifffile + with tifffile.TiffFile(img) as tif: + ome = tif.ome_metadata + if ome: + m = re.search(r'PhysicalSizeX="([0-9.eE+-]+)"', ome) + if m: + val = float(m.group(1)) + if val > 0: + self._pixel_size_cache = val + except Exception as exc: + print(f"[seqfish] could not read PhysicalSizeX from {img.name}: {exc}; " + f"falling back to {_DEFAULT_PIXEL_SIZE} µm/px") + return self._pixel_size_cache + + def _image_size(self) -> Optional[tuple]: + """(width, height) of the DAPI image in pixels, or None.""" + if self._image_size_cache is not None: + return self._image_size_cache + img = self._layout().get("image") + if img is None: + return None + try: + import tifffile + with tifffile.TiffFile(img) as tif: + shape = tif.series[0].shape + h, w = shape[-2], shape[-1] + self._image_size_cache = (int(w), int(h)) + except Exception as exc: + print(f"[seqfish] could not read image dimensions from {img.name}: {exc}") + return None + return self._image_size_cache + + # ── Unit detection ──────────────────────────────────────────────────────── + + def _units_divisor(self, max_x: float, max_y: float, label: str) -> float: + """Return the divisor converting a table's coordinates to image pixels. + + A table in microns spans about `image_width_px * pixel_size`, so its + max/width ratio lands near `pixel_size`. A table already in pixels spans + the image itself, so the ratio lands near 1.0. Pick whichever hypothesis + the observed ratio is closer to, in log space so the comparison is + scale-free. + """ + size = self._image_size() + ps = self.pixel_size + if not size or max_x <= 0 or ps <= 0: + return 1.0 + width, height = size + ratio = max(max_x / width, max_y / height) if height else max_x / width + if ratio <= 0: + return 1.0 + d_um = abs(math.log(ratio) - math.log(ps)) + d_px = abs(math.log(ratio) - math.log(1.0)) + micron = d_um < d_px + if label not in self._divisor_log: + self._divisor_log.add(label) + print(f"[seqfish] {label}: extent ratio {ratio:.4f} vs pixel_size {ps:.6f} " + f"→ treating as {'MICRONS (divide by pixel_size)' if micron else 'PIXELS (no conversion)'}") + return ps if micron else 1.0 + + # ── Experiment metadata ─────────────────────────────────────────────────── + + def info(self) -> dict: + lay = self._layout() + size = self._image_size() + return { + "platform": self.platform, + "format_variant": lay.get("variant"), + "roi": lay.get("roi"), + "pixel_size": self.pixel_size, + "image_width_px": size[0] if size else None, + "image_height_px": size[1] if size else None, + } + + def capabilities(self) -> dict: + lay = self._layout() + return { + "has_morphology": lay.get("image") is not None, + "has_transcripts": lay.get("transcripts") is not None, + # v1 ships only a label mask; polygonising it is not implemented, so + # the layer is declared unavailable rather than returning empty rows. + "has_boundaries": lay.get("boundaries") is not None, + "unit_label": "cell", + } + + # ── Gene catalogue ──────────────────────────────────────────────────────── + + def gene_list(self) -> list[str]: + counts = self._layout().get("counts") + if counts is None: + return [] + try: + cols = duck.csv_columns(counts) + except Exception: + return [] + # First column is the unnamed cell label; the rest are genes. + return [c for c in cols[1:] if c] + + # ── Cells ───────────────────────────────────────────────────────────────── + + def _cells_raw(self) -> Optional[pd.DataFrame]: + cells = self._layout().get("cells") + if cells is None: + return None + df = pd.read_csv(cells) + if "label" not in df.columns: + return None + out = pd.DataFrame({"cell_id": df["label"].astype(str)}) + div = self._units_divisor( + float(df["center_x"].max()), float(df["center_y"].max()), "cells" + ) + out["x_centroid"] = df["center_x"] / div + out["y_centroid"] = df["center_y"] / div + if "area" in df.columns: + # cell_area stays in µm², matching Xenium — which never converts it — so the + # "µm²" label in CellInfoPanel is true on every platform. Area arrives in the + # same space as the centroids, squared: already µm² when the table is in + # microns (div == pixel_size), px² when it is in pixels (div == 1.0). + ps = self.pixel_size + out["cell_area"] = df["area"] * ((ps / div) ** 2) + return out + + def _cells_full(self) -> Optional[pd.DataFrame]: + if self._cells_full_cache is not _UNSET: + return self._cells_full_cache # type: ignore[return-value] + self._cells_full_cache = self._merge_supplemental(self._cells_raw()) + return self._cells_full_cache # type: ignore[return-value] + + def cells(self, bbox: Optional[tuple] = None) -> list[dict]: + df = self._cells_full() + if df is None: + return [] + if bbox: + xmin, ymin, xmax, ymax = bbox + if None not in (xmin, ymin, xmax, ymax): + df = df[(df["x_centroid"] >= xmin) & (df["x_centroid"] <= xmax) & + (df["y_centroid"] >= ymin) & (df["y_centroid"] <= ymax)] + return self._to_records(df) + + def cells_schema(self) -> dict: + df = self._cells_full() + if df is None: + return {"columns": {}} + return {"columns": {c: str(df[c].dtype) for c in df.columns if c != "cell_id"}} + + def cell_detail(self, cell_id: str) -> Optional[dict]: + df = self._cells_full() + if df is None: + return None + row = df[df["cell_id"] == str(cell_id)] + if row.empty: + return None + record = self._to_records(row)[0] + record["expression"] = self.cell_expression(cell_id) + return record + + # ── Transcripts ─────────────────────────────────────────────────────────── + + def transcripts( + self, + bbox: Optional[tuple] = None, + genes: Optional[list[str]] = None, + fraction: float = 1.0, + ) -> dict: + path = self._layout().get("transcripts") + if path is None: + return {"transcripts": [], "total": 0} + + cols = duck.csv_columns(path) + if not {"x", "y"} <= set(cols): + return {"transcripts": [], "total": 0} + src = duck.scan_csv(path) + + # Detect units from the full extent once, then express the bbox in the + # file's own space so the predicate can be pushed into the scan. + with duck.connect() as conn: + mx, my = conn.execute(f"SELECT MAX(x), MAX(y) FROM {src}").fetchone() + div = self._units_divisor(float(mx or 0), float(my or 0), "transcripts") + + conditions: list[str] = [] + params: list = [] + if bbox and None not in bbox: + native = tuple(v * div for v in bbox) + sql, prm = duck.bbox_predicate("x", "y", native) + if sql: + conditions.append(sql) + params.extend(prm) + if genes and "name" in cols: + sql, prm = duck.in_predicate("name", genes) + conditions.append(sql) + params.extend(prm) + where = duck.where_clause(conditions) + + total = int(conn.execute( + f"SELECT COUNT(*) FROM {src} {where}", params).fetchone()[0] or 0) + if total == 0: + return {"transcripts": [], "total": 0} + + fraction = max(0.0001, min(1.0, fraction)) + n = min(round(fraction * total), _MAX_TRANSCRIPTS) + if n <= 0: + return {"transcripts": [], "total": total} + sample = duck.reservoir_sample(n) if n < total else "" + select = ", ".join(f'"{c}"' for c in ("name", "x", "y") if c in cols) + df = conn.execute( + f"SELECT * FROM (SELECT {select} FROM {src} {where}) {sample}", params + ).df() + + df = df.rename(columns={"name": "feature_name", + "x": "x_location", "y": "y_location"}) + df["x_location"] = df["x_location"] / div + df["y_location"] = df["y_location"] / div + return {"transcripts": duck.to_records(df), "total": total} + + # ── Boundaries ──────────────────────────────────────────────────────────── + + def cell_boundaries(self, bbox: Optional[tuple] = None, + fraction: float = 1.0) -> dict: + """Polygon vertices in pixel space, as long-format {cell_id, vertex_x, + vertex_y} rows so the frontend needs no seqFISH-specific handling. + + Cell identity comes from each GeoJSON feature's `id`, which the reference + dataset confirms equals `label`. spatialdata-io instead maps polygons to + cells positionally and has an open issue about the fragility of that; a + silent off-by-one would draw every outline on the wrong cell, so we join + on the id and fall back to position only when it is absent. + """ + path = self._layout().get("boundaries") + if path is None: + return {"boundaries": [], "total": 0} + try: + with open(path) as fh: + gj = json.load(fh) + except Exception as exc: + print(f"[seqfish] could not read {path.name}: {exc}") + return {"boundaries": [], "total": 0} + + features = gj.get("features") or [] + polys: list[tuple] = [] # (cell_id, [(x, y), ...]) + for i, feat in enumerate(features): + geom = feat.get("geometry") or {} + gtype, coords = geom.get("type"), geom.get("coordinates") + if not coords: + continue + fid = feat.get("id") + if fid is None: + fid = (feat.get("properties") or {}).get("label", i + 1) + cid = str(fid) + rings = [coords[0]] if gtype == "Polygon" else \ + [part[0] for part in coords] if gtype == "MultiPolygon" else [] + for ring in rings: + pts = [(float(p[0]), float(p[1])) for p in ring if len(p) >= 2] + # GeoJSON rings repeat the first vertex to close; deck.gl closes + # polygons itself and Xenium boundaries do not repeat, so drop it. + if len(pts) > 1 and pts[0] == pts[-1]: + pts = pts[:-1] + if pts: + polys.append((cid, pts)) + + if not polys: + return {"boundaries": [], "total": 0} + + all_x = [p[0] for _, pts in polys for p in pts] + all_y = [p[1] for _, pts in polys for p in pts] + div = self._units_divisor(max(all_x), max(all_y), "boundaries") + + # A cell qualifies if any vertex is in view, and then all of its vertices + # are returned — same rule as the Xenium reader, so polygons are never + # clipped into torn shapes at the viewport edge. + if bbox and None not in bbox: + xmin, ymin, xmax, ymax = bbox + polys = [ + (cid, pts) for cid, pts in polys + if any(xmin <= x / div <= xmax and ymin <= y / div <= ymax + for x, y in pts) + ] + if not polys: + return {"boundaries": [], "total": 0} + + cell_ids = sorted({cid for cid, _ in polys}) + total = len(cell_ids) + fraction = max(0.0001, min(1.0, fraction)) + n = round(fraction * total) + if n <= 0: + return {"boundaries": [], "total": total} + if n < total: + # Deterministic subset: re-fetching an unchanged viewport must return + # the same cells or the layer flickers. + step = total / n + keep = {cell_ids[min(total - 1, int(i * step))] for i in range(n)} + polys = [(cid, pts) for cid, pts in polys if cid in keep] + + rows = [ + {"cell_id": cid, "vertex_x": x / div, "vertex_y": y / div} + for cid, pts in polys for x, y in pts + ] + return {"boundaries": rows, "total": total} + + # ── Expression ──────────────────────────────────────────────────────────── + + def _cxg(self) -> Optional[pd.DataFrame]: + """Cell × gene counts, indexed by cell_id (string).""" + if self._cxg_cache is not _UNSET: + return self._cxg_cache # type: ignore[return-value] + counts = self._layout().get("counts") + self._cxg_cache = None + if counts is not None: + try: + df = pd.read_csv(counts, index_col=0) + df.index = df.index.astype(str) + df.index.name = "cell_id" + self._cxg_cache = df + except Exception as exc: + print(f"[seqfish] could not read {counts.name}: {exc}") + return self._cxg_cache # type: ignore[return-value] + + def cell_expression(self, cell_id: str) -> dict: + df = self._cxg() + if df is None or str(cell_id) not in df.index: + return {} + row = df.loc[str(cell_id)] + return {g: int(v) for g, v in row.items() + if isinstance(v, (int, float)) and v > 0} + + # ── Color values ────────────────────────────────────────────────────────── + + def color_values(self, mode: str, field: Optional[str] = None, + genes: Optional[list[str]] = None) -> dict: + if mode == "gene_set": + return self._color_values_gene_set(genes or []) + return self._color_values_meta(field or "") + + def _color_values_gene_set(self, genes: list[str]) -> dict: + df = self._cxg() + empty = {"type": "continuous", "values": {}, "min": 0.0, "max": 0.0} + if df is None or not genes: + return empty + cols = [g for g in genes if g in df.columns] + if not cols: + return empty + summed = df[cols].sum(axis=1) + return { + "type": "continuous", + "values": {str(k): float(v) for k, v in summed.items()}, + "min": 0.0, + "max": float(summed.max()) if len(summed) and summed.max() > 0 else 1.0, + } + + def _color_values_meta(self, field: str) -> dict: + df = self._cells_full() + empty = {"type": "continuous", "values": {}, "min": 0.0, "max": 0.0} + if df is None or field not in df.columns: + return empty + col = df[field] + ids = df["cell_id"].astype(str).tolist() + has = col.notna() + # Same rule as the other readers: strings are categorical, and so are + # low-cardinality integers (cluster IDs arrive as ints from Seurat). + categorical = ( + pd.api.types.is_string_dtype(col) or pd.api.types.is_object_dtype(col) + or (pd.api.types.is_integer_dtype(col) and col.nunique() <= 30) + ) + if categorical: + return { + "type": "categorical", + "values": {ids[i]: str(col.iloc[i]) for i in range(len(ids)) if has.iloc[i]}, + "categories": sorted(col[has].astype(str).unique().tolist()), + } + valid = col[has] + if valid.empty: + return empty + return { + "type": "continuous", + "values": {ids[i]: float(col.iloc[i]) for i in range(len(ids)) if has.iloc[i]}, + "min": float(valid.min()), + "max": float(valid.max()), + } diff --git a/backend/tests/golden_baseline.json b/backend/tests/golden_baseline.json index 7d6e6ef..866b685 100644 --- a/backend/tests/golden_baseline.json +++ b/backend/tests/golden_baseline.json @@ -345,6 +345,252 @@ "total": 0 } }, + "seqfish_instrument2": { + "bounds_full": { + "digest": "e93a1e7b407a17dc", + "keys": [ + "cell_id", + "vertex_x", + "vertex_y" + ], + "n": 1147, + "n_cells": 62, + "total": 62 + }, + "bounds_half": { + "digest": "00324d9f4a380a13", + "n": 576, + "n_cells": 31, + "total": 62 + }, + "capabilities": { + "has_boundaries": true, + "has_morphology": true, + "has_transcripts": true, + "unit_label": "cell" + }, + "cell_detail_0": "178c5b055b70bf72", + "cell_expression_0": { + "digest": "cb42bde899c5553c", + "n": 12 + }, + "cells_all": { + "digest": "20ddf74e79b6dafb", + "keys": [ + "cell_area", + "cell_id", + "x_centroid", + "y_centroid" + ], + "n": 62 + }, + "cells_schema": { + "columns": { + "cell_area": "float64", + "x_centroid": "float64", + "y_centroid": "float64" + } + }, + "color_gene_set": { + "digest": "90f5d24654cf35e1", + "max": 69.0, + "min": 0.0, + "n": 62, + "type": "continuous" + }, + "color_meta__cell_area": { + "digest": "053579a6ae0afaf6", + "n": 62, + "type": "continuous" + }, + "color_meta__x_centroid": { + "digest": "fb7ae1f6145429b3", + "n": 62, + "type": "continuous" + }, + "color_meta__y_centroid": { + "digest": "f321b3a07f2159c4", + "n": 62, + "type": "continuous" + }, + "gene_list": { + "digest": "6f1e76b0bd839528", + "n": 12 + }, + "info_keys": [ + "format_variant", + "image_height_px", + "image_width_px", + "pixel_size", + "platform", + "roi" + ], + "pixel_size": 0.107161, + "platform": "seqfish", + "tx_bbox": { + "digest": "6f1eb67b21e4775c", + "keys": [ + "feature_name", + "x_location", + "y_location" + ], + "n": 1750, + "total": 1750 + }, + "tx_bbox_half": { + "digest": "4474c8ca16d0c621", + "keys": [ + "feature_name", + "x_location", + "y_location" + ], + "n": 875, + "total": 1750 + }, + "tx_full": { + "digest": "76f69c03e22fe137", + "keys": [ + "feature_name", + "x_location", + "y_location" + ], + "n": 9051, + "total": 9051 + }, + "tx_gene0": { + "digest": "1526a4d6432a51ab", + "keys": [ + "feature_name", + "x_location", + "y_location" + ], + "n": 370, + "total": 370 + } + }, + "seqfish_synthetic": { + "bounds_full": { + "digest": "fddd6d92608579b7", + "keys": [ + "cell_id", + "vertex_x", + "vertex_y" + ], + "n": 509, + "n_cells": 36, + "total": 36 + }, + "bounds_half": { + "digest": "95dcb82de928fb18", + "n": 255, + "n_cells": 18, + "total": 36 + }, + "capabilities": { + "has_boundaries": true, + "has_morphology": true, + "has_transcripts": true, + "unit_label": "cell" + }, + "cell_detail_0": "a19b5b2640d2fabc", + "cell_expression_0": { + "digest": "232ef4b1b7d7d7ff", + "n": 10 + }, + "cells_all": { + "digest": "3041ba131e9fb3b6", + "keys": [ + "cell_area", + "cell_id", + "x_centroid", + "y_centroid" + ], + "n": 36 + }, + "cells_schema": { + "columns": { + "cell_area": "float64", + "x_centroid": "float64", + "y_centroid": "float64" + } + }, + "color_gene_set": { + "digest": "900101042f1887aa", + "max": 55.0, + "min": 0.0, + "n": 36, + "type": "continuous" + }, + "color_meta__cell_area": { + "digest": "4a1c35b956b246ba", + "n": 36, + "type": "continuous" + }, + "color_meta__x_centroid": { + "digest": "7edb0328310a8096", + "n": 36, + "type": "continuous" + }, + "color_meta__y_centroid": { + "digest": "9ba0a1c6fd2d6107", + "n": 36, + "type": "continuous" + }, + "gene_list": { + "digest": "db5cf5f57e896e74", + "n": 10 + }, + "info_keys": [ + "format_variant", + "image_height_px", + "image_width_px", + "pixel_size", + "platform", + "roi" + ], + "pixel_size": 0.107161, + "platform": "seqfish", + "tx_bbox": { + "digest": "ec4a84b78368cbf6", + "keys": [ + "feature_name", + "x_location", + "y_location" + ], + "n": 365, + "total": 365 + }, + "tx_bbox_half": { + "digest": "4744fb8b1a6cf5f6", + "keys": [ + "feature_name", + "x_location", + "y_location" + ], + "n": 182, + "total": 365 + }, + "tx_full": { + "digest": "973faff05f3a175c", + "keys": [ + "feature_name", + "x_location", + "y_location" + ], + "n": 2550, + "total": 2550 + }, + "tx_gene0": { + "digest": "8081db8d41ee4c2e", + "keys": [ + "feature_name", + "x_location", + "y_location" + ], + "n": 258, + "total": 258 + } + }, "xenium_human_breast_2fov": { "bounds_full": { "digest": "2e6916032112a6e5", diff --git a/backend/tests/golden_snapshot.py b/backend/tests/golden_snapshot.py index e4fae51..e83ce8b 100644 --- a/backend/tests/golden_snapshot.py +++ b/backend/tests/golden_snapshot.py @@ -42,7 +42,12 @@ # Datasets to cover. Missing ones are skipped with a note rather than failing, so the # script still works in a checkout that only has the committed tiny dataset. -DATASETS = ["mouse_ileum_tiny", "xenium_human_breast_2fov", "seqfish_instrument2"] +DATASETS = [ + "mouse_ileum_tiny", # Xenium, committed + "xenium_human_breast_2fov", # Xenium, local only (gitignored, large) + "seqfish_synthetic", # seqFISH, committed — generated by make_seqfish.py + "seqfish_instrument2", # seqFISH, local only (licence-restricted) +] FLOAT_PLACES = 4 diff --git a/frontend/package.json b/frontend/package.json index 1ad5d42..648d8b8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "tissueplex", - "version": "0.4.0", + "version": "0.5.0", "private": true, "scripts": { "dev": "vite", diff --git a/frontend/src/components/CellInfoPanel.jsx b/frontend/src/components/CellInfoPanel.jsx index cb09097..18a4b68 100644 --- a/frontend/src/components/CellInfoPanel.jsx +++ b/frontend/src/components/CellInfoPanel.jsx @@ -83,7 +83,12 @@ export default function CellInfoPanel() { - + {/* Guarded like nucleus_area below: without the null check the optional + chain yields undefined and the concatenation renders "undefined µm²" + on any platform that does not report a cell area. */} + {detail.cell_area != null && ( + + )} {detail.nucleus_area != null && ( )} diff --git a/frontend/src/components/LayerPanel.jsx b/frontend/src/components/LayerPanel.jsx index 10ecbdf..816d0a9 100644 --- a/frontend/src/components/LayerPanel.jsx +++ b/frontend/src/components/LayerPanel.jsx @@ -117,7 +117,7 @@ function DatasetPicker() { <>
Image
+ + {/* Issue #35: integer-coded cluster IDs arrive as ints and would + otherwise be drawn as a gradient. Unchecking forces the reverse, + which is how you get a gradient over a column the auto-rule + called categorical. */} + {isNumericField && ( + + )} )} @@ -326,7 +375,7 @@ function ColorBySection({ unitLabel = "cell" }) { clamp={cellColorClamp} setClamp={setCellColorClamp} accentColor="#6cf" /> )} {mode === "metadata" && field && isCategorical && ( - + )} )} @@ -381,7 +430,16 @@ function ClampableLegend({ label, palette, vmin, vmax, clamp, setClamp, accentCo ); } -function CategoricalLegend({ field, apiBase, dataset }) { +/** + * Editable per-category swatches. + * + * `categories` comes from the store, where panel 0 records whatever the backend + * returned for the active column. This component used to re-POST /color-values + * for itself, which duplicated a request the viewer had already made and — once + * the categorical override existed — would have asked without it, so the legend + * could disagree with the canvas it describes. + */ +function CategoricalLegend({ field, categories = [] }) { const { categoryColorOverrides, setCategoryColorOverride, @@ -389,21 +447,8 @@ function CategoricalLegend({ field, apiBase, dataset }) { resetCategoryColorOverrides, } = useStore(); - const [categories, setCategories] = useState([]); const fileInputRef = useRef(null); - useEffect(() => { - if (!field) return; - fetch(`${apiBase}/spatial/${dataset}/color-values`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ mode: "metadata", field }), - }) - .then((r) => r.json()) - .then((d) => { if (d.type === "categorical") setCategories(d.categories); }) - .catch(() => {}); - }, [apiBase, dataset, field]); - // Resolve display color for a category: override → QUAL_PALETTE → hash // Must mirror the logic in useCellColors.js so legend stays in sync. function resolveColor(cat, i) { @@ -533,6 +578,216 @@ function CategoricalLegend({ field, apiBase, dataset }) { ); } +// ── Metadata filter (issue #45) ─────────────────────────────────────────────── +/** + * Restrict the view to a subset of units by one metadata column. + * + * One component serves both the cell filter and the edge filter — they differ + * only in which endpoint supplies the column list and the distinct values, so + * those arrive as props. The chosen filter is written to the store and travels + * to the backend, which applies it *before* sampling; doing it client-side would + * leave a sample of a subset rather than the subset. + * + * Two shapes, chosen by what the backend says the column is: + * categorical — checkboxes, one per value (this is the "focus on 2–3 cell + * types" case from the issue) + * continuous — inclusive min/max bounds + */ +function MetadataFilterSection({ + scope, columns, filter, setFilter, fetchValues, unitLabel = "cell", +}) { + const [meta, setMeta] = useState(null); // { type, categories, min, max } + const [loading, setLoading] = useState(false); + const field = filter?.field ?? ""; + + // Load the distinct values / range for the selected column. + useEffect(() => { + if (!field) { setMeta(null); return; } + let cancelled = false; + setLoading(true); + fetchValues(field) + .then((d) => { if (!cancelled && d) setMeta(d); }) + .catch(() => { if (!cancelled) setMeta(null); }) + .finally(() => { if (!cancelled) setLoading(false); }); + return () => { cancelled = true; }; + }, [field, fetchValues]); + + const selected = new Set(filter?.values ?? []); + const active = (filter?.values?.length ?? 0) > 0 || + filter?.min != null || filter?.max != null; + + function chooseField(next) { + // Values and bounds belong to the old column; carrying them over would + // silently filter on labels that do not exist in the new one. + setFilter(next ? { field: next, values: null, min: null, max: null } : null); + } + + function toggle(cat) { + const next = new Set(selected); + if (next.has(cat)) next.delete(cat); else next.add(cat); + setFilter({ ...filter, values: next.size ? [...next] : null }); + } + + return ( +
+ + + {field && loading && ( +
loading values…
+ )} + + {field && !loading && meta?.type === "categorical" && ( +
+
+ {(meta.categories ?? []).map((cat) => ( + + ))} +
+
+ + + + {/* No selection is "show everything", not "show nothing" — an empty + allowlist would blank the canvas the moment a column is picked. */} + {selected.size + ? `${selected.size} of ${(meta.categories ?? []).length} shown` + : "all shown"} + +
+
+ )} + + {field && !loading && meta?.type === "continuous" && ( +
+ min + setFilter({ + ...filter, min: e.target.value === "" ? null : parseFloat(e.target.value), + })} + style={{ ...SELECT_STYLE, marginTop: 0, width: 0, flex: 1 }} + /> + max + setFilter({ + ...filter, max: e.target.value === "" ? null : parseFloat(e.target.value), + })} + style={{ ...SELECT_STYLE, marginTop: 0, width: 0, flex: 1 }} + /> +
+ )} + + {active && ( + + )} +
+ ); +} + +function fmtBound(v) { + if (v == null) return ""; + return Math.abs(v) >= 1000 || (v !== 0 && Math.abs(v) < 0.01) + ? v.toExponential(1) : String(Math.round(v * 1000) / 1000); +} + +function CellFilterSection({ unitLabel = "cell" }) { + const { apiBase, dataset, cellFilter, setCellFilter, categoricalOverrides } = useStore(); + const [columns, setColumns] = useState([]); + + useEffect(() => { + fetch(`${apiBase}/spatial/${dataset}/cells/schema`) + .then((r) => (r.ok ? r.json() : null)) + .then((s) => setColumns(s?.columns ? Object.keys(s.columns) : [])) + .catch(() => setColumns([])); + }, [apiBase, dataset]); + + // Honour the same categorical override the color panel uses, so a column the + // user has declared categorical offers checkboxes here rather than a range. + const fetchValues = React.useCallback((field) => { + const categorical = categoricalOverrides[`cell::${field}`] ?? null; + return fetch(`${apiBase}/spatial/${dataset}/color-values`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ mode: "metadata", field, categorical }), + }).then((r) => (r.ok ? r.json() : null)); + }, [apiBase, dataset, categoricalOverrides]); + + return ( + + ); +} + +function EdgeFilterSection() { + const { apiBase, dataset, edgeFile, edgeFilter, setEdgeFilter, categoricalOverrides } = useStore(); + const [columns, setColumns] = useState([]); + const efParam = `?edge_file=${encodeURIComponent(edgeFile)}`; + + useEffect(() => { + fetch(`${apiBase}/edges/${dataset}/schema${efParam}`) + .then((r) => (r.ok ? r.json() : null)) + // Structural and per-LRM columns are not edge attributes to subset on: + // one edge has many LRM rows, so "lrm = X" is a mechanism filter, which + // the LRM checklist below already does properly. + .then((s) => setColumns( + s?.columns + ? Object.keys(s.columns).filter((c) => !EDGE_FILTER_SKIP.has(c)) + : [] + )) + .catch(() => setColumns([])); + }, [apiBase, dataset, efParam]); + + const fetchValues = React.useCallback((field) => { + const categorical = categoricalOverrides[`edge::${field}`] ?? null; + return fetch(`${apiBase}/edges/${dataset}/edge-color-values${efParam}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ mode: "metadata", field, categorical }), + }).then((r) => (r.ok ? r.json() : null)); + }, [apiBase, dataset, efParam, categoricalOverrides]); + + if (!columns.length) return null; + return ( + + ); +} + +const EDGE_FILTER_SKIP = new Set([ + "edge", "sending_cell", "receiving_cell", "x1", "y1", "x2", "y2", + "lrm", "lrm_id", "ligand", "receptor", "score", "score_norm", +]); + // ── Morphology row ──────────────────────────────────────────────────────────── function MorphologyRow() { const { layers, setLayerProp } = useStore(); @@ -1012,6 +1267,7 @@ function EdgeSection() { hiddenLrms, toggleLrm, setAllLrmsVisible, hideAllLrms, edgeColorRange, edgeColorClamp, setEdgeColorClamp, + categoricalOverrides, setCategoricalOverride, } = useStore(); const state = layers.edges ?? { visible: true, opacity: 0.9 }; const [localStrength, setLocalStrength] = useState(edgeMinStrength ?? 0); @@ -1070,9 +1326,31 @@ function EdgeSection() { const { mode, field } = edgeColorBy; const selectedLrmCount = lrmCatalogue.length - hiddenLrms.size; - // Determine if selected metadata column is categorical + // How the selected edge metadata column is typed. Asking the backend rather + // than reading the dtype matters for the same reason it does on the cell side: + // the auto-rule also calls a low-cardinality integer column categorical, and an + // explicit override can flip either way (issue #35). const fieldDtype = field && edgeSchema ? edgeSchema.columns[field] : null; - const isCategorical = fieldDtype === "object" || fieldDtype === "string" || fieldDtype === "bool"; + const isNumericField = !!fieldDtype && /^(int|uint|float|double|Int|UInt|Float)/.test(fieldDtype); + const edgeOverrideKey = `edge::${field}`; + const edgeOverride = categoricalOverrides[edgeOverrideKey] ?? null; + + const [edgeMeta, setEdgeMeta] = useState(null); // { type, categories } + useEffect(() => { + if (mode !== "metadata" || !field) { setEdgeMeta(null); return; } + let cancelled = false; + fetch(`${apiBase}/edges/${dataset}/edge-color-values${efParam}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ mode: "metadata", field, categorical: edgeOverride }), + }) + .then((r) => (r.ok ? r.json() : null)) + .then((d) => { if (!cancelled && d) setEdgeMeta({ type: d.type, categories: d.categories ?? [] }); }) + .catch(() => {}); + return () => { cancelled = true; }; + }, [apiBase, dataset, efParam, mode, field, edgeOverride]); + + const isCategorical = mode === "metadata" && !!field && edgeMeta?.type === "categorical"; return (
@@ -1241,14 +1519,39 @@ function EdgeSection() { )} {mode === "metadata" && ( - + <> + + {field && isNumericField && ( + + )} + )} {/* Palette — only for continuous color modes */} @@ -1274,9 +1577,16 @@ function EdgeSection() { clamp={edgeColorClamp} setClamp={setEdgeColorClamp} accentColor="#f90" /> )} {mode === "metadata" && field && isCategorical && ( - + )} + {/* ── Edge metadata filter (issue #45) ─────────────────────── */} + {/* Distinct from the LRM checklist below: this subsets *edges* by an + attribute of the pair (a curation call, a confidence), whereas the + checklist subsets the mechanisms scored on every edge. */} +
Edge Filter
+ + {/* ── LRM Mechanisms checklist ─────────────────────────────── */} {lrmCatalogue.length > 0 && (
@@ -1333,22 +1643,10 @@ function EdgeSection() { ); } -function EdgeCategoricalLegend({ field, apiBase, dataset, edgeFile = "edges.parquet" }) { - const [categories, setCategories] = useState([]); - - useEffect(() => { - if (!field) return; - const efParam = `?edge_file=${encodeURIComponent(edgeFile)}`; - fetch(`${apiBase}/edges/${dataset}/edge-color-values${efParam}`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ mode: "metadata", field }), - }) - .then((r) => r.json()) - .then((d) => { if (d.type === "categorical") setCategories(d.categories); }) - .catch(() => {}); - }, [apiBase, dataset, field, edgeFile]); - +/** Read-only swatch list. Categories come from EdgeSection, which already asked + * the backend for the column's type — one fetch, one answer, no chance of the + * legend describing a different typing decision than the canvas is using. */ +function EdgeCategoricalLegend({ categories = [] }) { if (!categories.length) return null; return (
diff --git a/frontend/src/components/Viewer.jsx b/frontend/src/components/Viewer.jsx index 1a93ba7..3e3ec55 100644 --- a/frontend/src/components/Viewer.jsx +++ b/frontend/src/components/Viewer.jsx @@ -138,6 +138,7 @@ function ViewerPanel({ panelIndex }) { selectedEdge, setSelectedEdge, setCellColorRange, setEdgeColorRange, cellColorClamp, edgeColorClamp, setEdgeColorClamp, + categoricalOverrides, cellFilter, edgeFilter, setCellColorType, annotationMode, pixelSize, setPixelSize, clearZoomMatch, @@ -537,7 +538,8 @@ function ViewerPanel({ panelIndex }) { total: cellBoundaryTotal, loading: cellBoundariesLoading, } = useCellBoundaries( - apiBase, dataset, viewport, imageSize, cellSegmentsVisible && hasBoundaries, cellBoundaryFraction + apiBase, dataset, viewport, imageSize, cellSegmentsVisible && hasBoundaries, + cellBoundaryFraction, cellFilter ); useEffect(() => { cellPolygonsRef.current = cellPolygons; }, [cellPolygons]); @@ -548,20 +550,38 @@ function ViewerPanel({ panelIndex }) { const { edges, loading: edgesLoading } = useEdges( apiBase, dataset, viewport, imageSize, edgesVisible || tissueGraphVisible, - edgeMinStrength, hiddenLrms, lrmCatalogue, edgeDensity, edgeFile + edgeMinStrength, hiddenLrms, lrmCatalogue, edgeDensity, edgeFile, + cellFilter, edgeFilter ); - const { colorValues, vmin: cellVmin, vmax: cellVmax, loading: cellColorsLoading } = useCellColors( - apiBase, dataset, colorBy, allGenes, selectedGenes, cellColorPalette, cellColorEnabled, cellColorClamp, categoryColorOverrides + // Explicit categorical/continuous choice for the active color-by column, or + // null (auto-detect) when the user has not overridden it — issue #35. + const cellCategorical = categoricalOverrides[`cell::${colorBy?.field}`] ?? null; + const edgeCategorical = categoricalOverrides[`edge::${edgeColorBy?.field}`] ?? null; + + const { + colorValues, vmin: cellVmin, vmax: cellVmax, + type: cellType, categories: cellCategories, loading: cellColorsLoading, + } = useCellColors( + apiBase, dataset, colorBy, allGenes, selectedGenes, cellColorPalette, + cellColorEnabled, cellColorClamp, categoryColorOverrides, cellCategorical ); // Only update shared store ranges from panel 0 to avoid redundant updates useEffect(() => { if (panelIndex === 0) setCellColorRange(cellVmin, cellVmax); }, [cellVmin, cellVmax]); // eslint-disable-line + // The backend is the authority on whether a column is categorical, so report + // the type it actually returned rather than letting the panel re-derive it + // from the schema dtype — the two disagreed for low-cardinality integers. + useEffect(() => { + if (panelIndex === 0) setCellColorType(cellType, cellCategories); + }, [cellType, cellCategories]); // eslint-disable-line + const edgeColorEnabled = edgeColorBy.mode !== "default"; const { colorValues: edgeColorValues, vmin: edgeVmin, vmax: edgeVmax, p95: edgeP95, loading: edgeColorsLoading } = useEdgeColors( - apiBase, dataset, edgeColorBy, hiddenLrms, lrmCatalogue, edgeColorPalette, edgeColorEnabled, edgeColorClamp, edges, edgeFile + apiBase, dataset, edgeColorBy, hiddenLrms, lrmCatalogue, edgeColorPalette, + edgeColorEnabled, edgeColorClamp, edges, edgeFile, edgeCategorical ); useEffect(() => { if (panelIndex === 0) setEdgeColorRange(edgeVmin, edgeVmax); diff --git a/frontend/src/hooks/useCellBoundaries.js b/frontend/src/hooks/useCellBoundaries.js index 5d07c7b..84f38b2 100644 --- a/frontend/src/hooks/useCellBoundaries.js +++ b/frontend/src/hooks/useCellBoundaries.js @@ -19,8 +19,30 @@ import { useState, useEffect, useRef } from "react"; const TARGET_CELLS = 5_000; const SEED_TOTAL = 50_000; // conservative first-probe estimate +/** + * Serialise a metadata filter (issue #45) into query params. + * + * Returns "" when there is nothing to constrain, so the URL is byte-identical to + * the pre-filter one and no cached response is missed. The filter is sent to the + * server rather than applied to the response because sampling happens server-side: + * filtering afterwards would leave a fraction of a fraction on screen. + */ +function filterParams(filter) { + if (!filter?.field) return ""; + const p = new URLSearchParams(); + const hasValues = Array.isArray(filter.values) && filter.values.length > 0; + if (!hasValues && filter.min == null && filter.max == null) return ""; + p.set("filter_field", filter.field); + if (hasValues) for (const v of filter.values) p.append("filter_values", v); + if (filter.min != null) p.set("filter_min", filter.min); + if (filter.max != null) p.set("filter_max", filter.max); + if (filter.includeMissing) p.set("filter_missing", "true"); + return `&${p.toString()}`; +} + export function useCellBoundaries( - apiBase, dataset, viewport, imageSize, enabled = true, fraction = null + apiBase, dataset, viewport, imageSize, enabled = true, fraction = null, + filter = null ) { const [cells, setCells] = useState([]); const [total, setTotal] = useState(0); @@ -31,6 +53,23 @@ export function useCellBoundaries( const abortRef = useRef(null); const prevTotalRef = useRef(SEED_TOTAL); // running estimate of cells in viewport + // Serialised once so it can be both spliced into the URL and used as an effect + // dependency — the filter arrives as an object whose identity changes on every + // render, which would otherwise refetch continuously. + const filterQS = filterParams(filter); + + // One-shot recalibration. + // + // In auto mode the fraction is picked from `prevTotalRef`, the total the *last* + // fetch saw. Applying a metadata filter (or switching dataset) changes that + // total out from under the estimate, and nothing else would trigger another + // fetch — so the layer would sit showing a tenth of an already-small subset + // until the user happened to pan. Bumping this counter re-runs the fetch once + // with the corrected fraction; `calibratedRef` keys it to the current request + // so it can converge rather than oscillate. + const [recalibrate, setRecalibrate] = useState(0); + const calibratedRef = useRef(null); + useEffect(() => { if (!enabled || !dataset) { setLoading(false); @@ -62,6 +101,7 @@ export function useCellBoundaries( } else { url += `?${fracParam}`; } + url += filterQS; const res = await fetch(url, { signal: ctrl.signal }); if (!res.ok) { setCells([]); setTotal(0); return; } const data = await res.json(); @@ -74,6 +114,23 @@ export function useCellBoundaries( if (totalCells > 0) prevTotalRef.current = totalCells; setTotal(totalCells); + // If that estimate was badly wrong, correct it now rather than waiting + // for the user to pan. Only in auto mode — an explicit slider value is + // the user's decision, not an estimate. Once per request key. + // The 1.2 threshold is what makes the panel's sample readout honest: the + // panel derives its percentage from the *current* total, so anything + // looser leaves it advertising a fraction the canvas is not drawing at. + // Total is a pre-sample count for a fixed bbox and filter, so the second + // fetch computes the same fraction and the loop settles after one pass. + if (fraction === null && totalCells > 0) { + const better = Math.min(1.0, TARGET_CELLS / totalCells); + const key = `${url}|${totalCells}`; + if (better > eff * 1.2 && calibratedRef.current !== key) { + calibratedRef.current = key; + setRecalibrate((c) => c + 1); + } + } + if (!Array.isArray(rows)) { setCells([]); return; } // Group flat vertex list by cell_id → polygon arrays @@ -94,7 +151,7 @@ export function useCellBoundaries( }, 200); return () => clearTimeout(timerRef.current); - }, [apiBase, dataset, viewport?.xmin, viewport?.ymin, viewport?.xmax, viewport?.ymax, enabled, fraction]); + }, [apiBase, dataset, viewport?.xmin, viewport?.ymin, viewport?.xmax, viewport?.ymax, enabled, fraction, filterQS, recalibrate]); // Abort in-flight request on unmount useEffect(() => { diff --git a/frontend/src/hooks/useCellColors.js b/frontend/src/hooks/useCellColors.js index 07aa2c0..236aec7 100644 --- a/frontend/src/hooks/useCellColors.js +++ b/frontend/src/hooks/useCellColors.js @@ -25,6 +25,7 @@ import { geneColor } from "../utils/geneColor"; * Modes: * gene_set — POST with selected genes; returns continuous sum * metadata — POST with field; backend auto-detects continuous vs. categorical + * unless `categorical` overrides it (issue #35) * * Returns: * colorValues Map or null when disabled @@ -34,7 +35,7 @@ import { geneColor } from "../utils/geneColor"; * categoryColors Map for categorical legend * loading */ -export function useCellColors(apiBase, dataset, colorBy, allGenes, selectedGenes, palette, enabled, clamp, categoryColorOverrides) { +export function useCellColors(apiBase, dataset, colorBy, allGenes, selectedGenes, palette, enabled, clamp, categoryColorOverrides, categorical = null) { const [result, setResult] = useState({ colorValues: null, type: "continuous", vmin: 0, vmax: 0, categories: [], categoryColors: new Map(), @@ -90,7 +91,7 @@ export function useCellColors(apiBase, dataset, colorBy, allGenes, selectedGenes try { const body = mode === "gene_set" ? { mode: "gene_set", genes: genesToSend } - : { mode: "metadata", field }; + : { mode: "metadata", field, categorical }; const res = await fetch(`${apiBase}/spatial/${dataset}/color-values`, { method: "POST", @@ -119,7 +120,7 @@ export function useCellColors(apiBase, dataset, colorBy, allGenes, selectedGenes } }, 400); return () => clearTimeout(timerRef.current); - }, [apiBase, dataset, colorBy?.mode, colorBy?.field, allGenes, selectedGenes, enabled]); // eslint-disable-line + }, [apiBase, dataset, colorBy?.mode, colorBy?.field, allGenes, selectedGenes, enabled, categorical]); // eslint-disable-line // ── Effect 2: apply clamp + palette to continuous data (no fetch, no debounce) ── // Fires immediately when rawCont, clamp, or palette changes so slider drags diff --git a/frontend/src/hooks/useEdgeColors.js b/frontend/src/hooks/useEdgeColors.js index 6e1a849..618fcb4 100644 --- a/frontend/src/hooks/useEdgeColors.js +++ b/frontend/src/hooks/useEdgeColors.js @@ -28,7 +28,8 @@ export function useEdgeColors( apiBase, dataset, edgeColorBy, hiddenLrms, lrmCatalogue, palette, enabled, clamp, edges, // array from useEdges — used for client-side lrm_set coloring - edgeFile = "edges.parquet" // which edge-source parquet the metadata fetch reads + edgeFile = "edges.parquet", // which edge-source parquet the metadata fetch reads + categorical = null // issue #35 override: null = auto-detect, true/false = forced ) { // Appended to the metadata edge-color-values request (lrm_set is client-side only). const efParam = `?edge_file=${encodeURIComponent(edgeFile)}`; @@ -99,7 +100,7 @@ export function useEdgeColors( const res = await fetch(`${apiBase}/edges/${dataset}/edge-color-values${efParam}`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ mode: "metadata", field }), + body: JSON.stringify({ mode: "metadata", field, categorical }), signal: ctrl.signal, }); if (!res.ok) throw new Error(`HTTP ${res.status}`); @@ -131,7 +132,7 @@ export function useEdgeColors( } }, 400); return () => clearTimeout(timerRef.current); - }, [apiBase, dataset, edgeColorBy?.mode, edgeColorBy?.field, enabled, efParam]); // eslint-disable-line + }, [apiBase, dataset, edgeColorBy?.mode, edgeColorBy?.field, enabled, efParam, categorical]); // eslint-disable-line // ── Effect 2: apply clamp + palette to continuous metadata (no fetch, no debounce) ── useEffect(() => { diff --git a/frontend/src/hooks/useEdges.js b/frontend/src/hooks/useEdges.js index 32a9618..a1d0624 100644 --- a/frontend/src/hooks/useEdges.js +++ b/frontend/src/hooks/useEdges.js @@ -27,13 +27,36 @@ import { useState, useEffect, useRef, useMemo } from "react"; const DEBOUNCE_MS = 400; +/** + * Normalise a store filter into the request-body shape the backend expects + * (issue #45), or undefined when nothing is constrained. + */ +function filterBody(filter) { + if (!filter?.field) return undefined; + const hasValues = Array.isArray(filter.values) && filter.values.length > 0; + if (!hasValues && filter.min == null && filter.max == null) return undefined; + return { + field: filter.field, + values: hasValues ? filter.values : null, + min: filter.min ?? null, + max: filter.max ?? null, + include_missing: !!filter.includeMissing, + }; +} + export function useEdges( apiBase, dataset, viewport, imageSize, enabled, minStrength, hiddenLrms, lrmCatalogue, density = 1.0, - edgeFile = "edges.parquet" + edgeFile = "edges.parquet", cellFilter = null, edgeFilter = null ) { // Which edge-source parquet to query; appended to every /edges request. const efParam = `?edge_file=${encodeURIComponent(edgeFile)}`; + + // Serialised so the structural effect can depend on filter *content* rather + // than object identity, which changes on every render. + const cellFilterBody = filterBody(cellFilter); + const edgeFilterBody = filterBody(edgeFilter); + const filterKey = JSON.stringify([cellFilterBody ?? null, edgeFilterBody ?? null]); // ── Structural state ────────────────────────────────────────────────────── const [structuralEdges, setStructuralEdges] = useState([]); const [loadingStructural, setLoadingStructural] = useState(false); @@ -69,6 +92,11 @@ export function useEdges( const { xmin, ymin, xmax, ymax } = viewport; const body = { xmin, ymin, xmax, ymax, density: Math.max(0.01, Math.min(1.0, density)) }; if (minStrength != null && minStrength > 0) body.min_strength = minStrength; + // Metadata filters go to the server so they apply before the density + // sample; filtering the response instead would sample first and leave a + // fraction of the subset. + if (cellFilterBody) body.cell_filter = cellFilterBody; + if (edgeFilterBody) body.edge_filter = edgeFilterBody; try { const res = await fetch(`${apiBase}/edges/${dataset}/query-grouped${efParam}`, { @@ -86,7 +114,7 @@ export function useEdges( }, DEBOUNCE_MS); return () => clearTimeout(structTimerRef.current); - }, [apiBase, dataset, viewport, imageSize, enabled, minStrength, density, efParam]); // eslint-disable-line + }, [apiBase, dataset, viewport, imageSize, enabled, minStrength, density, efParam, filterKey]); // eslint-disable-line // ── Effect 2: score fetch ────────────────────────────────────────────────── // Runs when viewport OR hiddenLrms changes. diff --git a/frontend/src/store.js b/frontend/src/store.js index 78e6016..3387334 100644 --- a/frontend/src/store.js +++ b/frontend/src/store.js @@ -27,6 +27,10 @@ export const useStore = create((set, get) => ({ edgeFile: "edges.parquet", lrmCatalogue: [], hiddenLrms: new Set(), selectedEdge: null, edgeColorRange: { vmin: null, vmax: null }, edgeColorClamp: { low: null, high: null }, + // Column names are dataset-specific, so a categorical override or an active + // filter naming a column the new dataset does not have would either do nothing + // or 400 on every viewport change. + categoricalOverrides: {}, cellFilter: null, edgeFilter: null, }), setActiveImage: (activeImage) => set({ activeImage }), @@ -36,8 +40,47 @@ export const useStore = create((set, get) => ({ setEdgeFile: (edgeFile) => set({ edgeFile, lrmCatalogue: [], hiddenLrms: new Set(), selectedEdge: null, edgeColorRange: { vmin: null, vmax: null }, edgeColorClamp: { low: null, high: null }, + // The edge filter names a column of the edge table, which differs between + // edge sources; the cell filter is unaffected because cells are shared. + edgeFilter: null, }), + // ── Categorical / continuous override (issue #35) ───────────────────────── + // Keyed "cell::" / "edge::" → true | false. Absent means + // auto-detect, which is what the backend does when `categorical` is null. + // Seurat writes cluster IDs as integers, so dtype alone routes them to a + // viridis gradient; this is how the user says "these are twenty categories". + categoricalOverrides: {}, + setCategoricalOverride: (scope, field, value) => set((s) => { + const next = { ...s.categoricalOverrides }; + if (value === null || value === undefined) delete next[`${scope}::${field}`]; + else next[`${scope}::${field}`] = value; + return { categoricalOverrides: next }; + }), + + // ── Metadata subsetting (issue #45) ─────────────────────────────────────── + // A filter is { field, values: string[] | null, min, max, includeMissing }. + // null means no filter. `values` is a categorical allowlist; min/max an + // inclusive numeric range. Applied server-side before sampling, so narrowing + // to a rare cluster shows all of it rather than a sample of a sample. + // + // cellFilter also governs edges: an edge is drawn only when BOTH endpoints + // survive it. edgeFilter is independent and applies to the edge table itself. + cellFilter: null, + setCellFilter: (f) => set({ cellFilter: f }), + edgeFilter: null, + setEdgeFilter: (f) => set({ edgeFilter: f }), + + // Resolved type of the active cell color-by column, reported by panel 0 so the + // LayerPanel can render the matching legend. The panel used to guess from the + // schema dtype, which disagreed with the backend for low-cardinality integers: + // the canvas drew discrete colors while the panel showed a gradient with two + // sliders that did nothing. + cellColorType: "continuous", + cellColorCategories: [], + setCellColorType: (type, categories) => + set({ cellColorType: type, cellColorCategories: categories ?? [] }), + // ── Platform capabilities (fetched from /spatial/{dataset}/info) ────────── // null = not yet loaded; object = { has_morphology, has_transcripts, has_boundaries, unit_label } platformCapabilities: null, diff --git a/sample_data/mouse_ileum_tiny/cell-metadata/example_clusters.csv b/sample_data/mouse_ileum_tiny/cell-metadata/example_clusters.csv index d220439..0b69f01 100644 --- a/sample_data/mouse_ileum_tiny/cell-metadata/example_clusters.csv +++ b/sample_data/mouse_ileum_tiny/cell-metadata/example_clusters.csv @@ -1,37 +1,37 @@ -,cluster,region,pseudotime -aaamobki-1,1,mid,0.4035 -aaclkaod-1,1,mid,0.3969 -bhakoonb-1,1,mid,0.4145 -bpefijoo-1,1,mid,0.3881 -ckfandjp-1,1,mid,0.4027 -dgcpicgh-1,1,mid,0.4054 -djgiipfb-1,1,mid,0.4114 -dkcgpkmh-1,1,mid,0.3911 -egeelggc-1,1,mid,0.3986 -ehjojgpl-1,1,mid,0.4064 -fkhfgimb-1,1,mid,0.4293 -gackndbe-1,1,mid,0.4185 -ghfdfbpc-1,1,mid,0.4345 -gjklkjjk-1,1,mid,0.424 -gpihhicj-1,1,mid,0.4417 -hakcibka-1,0,crypt,0.0452 -hbpoaeic-1,0,crypt,0.0341 -hhmlkgel-1,0,crypt,0.0079 -hkhcepfl-1,0,crypt,0.0 -ibclpflk-1,0,crypt,0.043 -ifjkhhkf-1,3,villus,0.9792 -jdchjdgi-1,3,villus,0.976 -jedneilm-1,3,villus,0.999 -jpbefbck-1,4,villus,1.0 -kecdfjaf-1,3,villus,0.9832 -kfefndgm-1,3,villus,0.9705 -kggmilif-1,3,villus,0.9673 -kkonlcio-1,3,villus,0.9596 -koaolmni-1,0,crypt,0.0416 -lfenejfi-1,0,crypt,0.0441 -niehkpen-1,0,crypt,0.0238 -ohmibdle-1,0,crypt,0.0337 -oinmeidp-1,0,crypt,0.0293 -ojeggnjp-1,3,villus,0.9824 -olbjkpjc-1,3,villus,0.9555 -omjmdimk-1,3,villus,0.9666 +,cluster,region,pseudotime,seurat_clusters +aaamobki-1,1,mid,0.4035,0 +aaclkaod-1,1,mid,0.3969,1 +bhakoonb-1,1,mid,0.4145,2 +bpefijoo-1,1,mid,0.3881,3 +ckfandjp-1,1,mid,0.4027,4 +dgcpicgh-1,1,mid,0.4054,5 +djgiipfb-1,1,mid,0.4114,6 +dkcgpkmh-1,1,mid,0.3911,7 +egeelggc-1,1,mid,0.3986,8 +ehjojgpl-1,1,mid,0.4064,9 +fkhfgimb-1,1,mid,0.4293,10 +gackndbe-1,1,mid,0.4185,11 +ghfdfbpc-1,1,mid,0.4345,0 +gjklkjjk-1,1,mid,0.424,1 +gpihhicj-1,1,mid,0.4417,2 +hakcibka-1,0,crypt,0.0452,3 +hbpoaeic-1,0,crypt,0.0341,4 +hhmlkgel-1,0,crypt,0.0079,5 +hkhcepfl-1,0,crypt,0.0,6 +ibclpflk-1,0,crypt,0.043,7 +ifjkhhkf-1,3,villus,0.9792,8 +jdchjdgi-1,3,villus,0.976,9 +jedneilm-1,3,villus,0.999,10 +jpbefbck-1,4,villus,1.0,11 +kecdfjaf-1,3,villus,0.9832,0 +kfefndgm-1,3,villus,0.9705,1 +kggmilif-1,3,villus,0.9673,2 +kkonlcio-1,3,villus,0.9596,3 +koaolmni-1,0,crypt,0.0416,4 +lfenejfi-1,0,crypt,0.0441,5 +niehkpen-1,0,crypt,0.0238,6 +ohmibdle-1,0,crypt,0.0337,7 +oinmeidp-1,0,crypt,0.0293,8 +ojeggnjp-1,3,villus,0.9824,9 +olbjkpjc-1,3,villus,0.9555,10 +omjmdimk-1,3,villus,0.9666,11