Refactor server gateway health checks and add chain data download

.env.example

@@ -377,6 +377,11 @@ SERVER_AUTH_AUDIT_LOG=true
 # TRAEFIK (Reverse Proxy)
 # =============================================================================
 
+# Internal URL the backend uses to verify a server route is reachable through
+# Traefik before reporting status=running. Points to Traefik's load-balancer
+# port inside the container network. Leave empty to skip the probe.
+TRAEFIK_INTERNAL_URL=http://traefik:80
+
 # Let's Encrypt contact email (used in infrastructure/traefik/traefik.yml)
 TRAEFIK_ACME_EMAIL=admin@nukelab.org
 

backend/AGENTS.md

@@ -13,7 +13,7 @@ All files under `backend/` except generated artifacts (`.venv-dev`, `__pycache__
 - Python 3.13; formatting and linting configured in `pyproject.toml`.
 - `app/main.py` is the ASGI entry point.
 - `app/api/` owns route definitions; `app/services/` owns business logic; `app/models/` owns SQLAlchemy models; `app/db/` owns session/connection logic; `app/core/` owns cross-cutting utilities; `app/middleware/` owns ASGI middleware; `app/container/` owns container-runtime orchestration; `app/tasks.py` and `app/worker.py` own Celery.
-- `app/container/` is a driver layer: `driver.py` defines the `ContainerDriver` ABC + `ContainerDriverError` (plain-data returns only — no runtime objects escape), `docker_driver.py` is the Docker/Podman implementation, `factory.py` selects the driver via `CONTAINER_RUNTIME` (default `docker`), `client.py` is a compatibility shim for legacy imports/test seams, and `spawner.py` (server lifecycle) talks only to driver methods. To add a runtime (e.g. Kubernetes): implement `ContainerDriver` (synthesizing the documented return shapes, e.g. Docker-stats for `get_container_stats`) and register it in `factory.py`. Container health is driver-level config, not image metadata: `docker_driver.py` injects a uniform `Healthcheck` (`/usr/local/bin/nukelab-healthcheck.sh`) into every create config, because OCI images drop Dockerfile `HEALTHCHECK` and Kubernetes ignores it — a k8s driver must translate the same definition into pod liveness/startup probes and surface failures as the same `State.Health.Status` shape `HealthCheckService` consumes.
+- `app/container/` is a driver layer: `driver.py` defines the `ContainerDriver` ABC + `ContainerDriverError` (plain-data returns only — no runtime objects escape), `docker_driver.py` is the Docker/Podman implementation, `factory.py` selects the driver via `CONTAINER_RUNTIME` (default `docker`), `client.py` is a compatibility shim for legacy imports/test seams, and `spawner.py` (server lifecycle) talks only to driver methods. To add a runtime (e.g. Kubernetes): implement `ContainerDriver` (synthesizing the documented return shapes, e.g. Docker-stats for `get_container_stats`) and register it in `factory.py`. Container health is driver-level config, not image metadata: `docker_driver.py` injects a uniform `Healthcheck` (`/usr/local/bin/nukelab-healthcheck.sh`) into every create config, because OCI images drop Dockerfile `HEALTHCHECK` and Kubernetes ignores it — a k8s driver must translate the same definition into pod liveness/startup probes and surface failures as the same `State.Health.Status` shape `HealthCheckService` consumes. `spawner.py` performs two readiness probes before marking a server `running`: (1) the container's own `/health` endpoint over the Docker network alias, and (2) the public server path through the internal Traefik load balancer (`TRAEFIK_INTERNAL_URL`, default `http://traefik:80`) with a `healthy` body check. This ensures the browser-facing route exists before the frontend is told to redirect.
 - `app/api/search.py` — grouped, permission-scoped search at `/api/search/`; optional `group` query parameter scopes the response to a single group; groups the user lacks read permission for are omitted from the response (never 403).
 - `app/api/tokens.py` owns `VALID_TOKEN_SCOPES`, the source of truth for API-token scopes; the frontend `AVAILABLE_SCOPES` in `frontend/src/components/settings/tokens-page.tsx` must stay in sync (unknown scopes are rejected with 422).
 - `app/services/gpu_allocator.py` owns exclusive GPU device reservations (table `gpu_allocations`, active only when `GPU_DEVICES` lists CDI device names); every code path that stops or deletes a server must release its devices, and every GPU spawn path must allocate first.

backend/app/config.py

@@ -217,6 +217,12 @@ def gpu_device_list(self) -> list[str]:
     container_readiness_timeout: int = 60  # seconds
     container_readiness_interval: float = 1.0  # seconds between probes
 
+    # Internal URL the backend uses to verify that Traefik has picked up a new
+    # server route before reporting status=running. Should point to Traefik's
+    # load-balancer port inside the container network (e.g. http://traefik:80).
+    # Leave empty to skip the Traefik readiness probe.
+    traefik_internal_url: str = "http://traefik:80"
+
     registration_enabled: bool = True
     max_servers_per_user: int = 10
 

backend/app/container/docker_driver.py

@@ -541,6 +541,7 @@ async def wait_for_container_ready(
         health_url: str,
         timeout: int | None = None,
         interval: float | None = None,
+        body_contains: str | None = None,
     ) -> bool:
         """Wait until the container responds successfully on health_url.
 
@@ -566,6 +567,16 @@ async def wait_for_container_ready(
                 async with aiohttp.ClientSession(timeout=timeout_obj) as session:
                     async with session.get(health_url) as resp:
                         if resp.status == 200:
+                            if body_contains:
+                                text = await resp.text()
+                                if body_contains not in text:
+                                    logger.debug(
+                                        "Container %s returned 200 but body did not contain %r",
+                                        container_name,
+                                        body_contains,
+                                    )
+                                    await asyncio.sleep(interval)
+                                    continue
                             logger.info("Container %s is ready", container_name)
                             return True
             except Exception as e:

backend/app/container/driver.py

@@ -112,8 +112,16 @@ async def wait_for_container_ready(
         health_url: str,
         timeout: int | None = None,
         interval: float | None = None,
+        body_contains: str | None = None,
     ) -> bool:
-        """Wait until the container responds successfully on health_url."""
+        """Wait until the container responds successfully on health_url.
+
+        Args:
+            body_contains: If set, the response body must contain this string
+                (case-sensitive) for the probe to be considered successful.
+                Useful to distinguish the container's /health body from a
+                fallback SPA shell served by the reverse proxy.
+        """
 
     @abstractmethod
     async def list_containers(self, filters: dict | None = None) -> list[dict]:

backend/app/container/spawner.py

@@ -325,14 +325,38 @@ async def spawn(
                     container_name,
                 )
 
+            # Wait for Traefik to publish the server route. Without this, the
+            # frontend can redirect before Traefik knows about the container and
+            # land on the SPA user-gateway fallback or a 503. Probe the public
+            # path through Traefik's internal load-balancer address and check the
+            # body to distinguish the container's /health response from a
+            # fallback HTML shell.
+            traefik_ready = True
+            if settings.traefik_internal_url:
+                traefik_health_url = (
+                    f"{settings.traefik_internal_url.rstrip('/')}{route_prefix}/health"
+                )
+                traefik_ready = await container_client.wait_for_container_ready(
+                    container_name,
+                    traefik_health_url,
+                    timeout=settings.container_readiness_timeout,
+                    interval=settings.container_readiness_interval,
+                    body_contains="healthy",
+                )
+                if not traefik_ready:
+                    logger.warning(
+                        "Traefik route for %s not ready within timeout; continuing",
+                        container_name,
+                    )
+
             # Determine primary volume_id from volume_mounts if provided
             primary_volume_id = None
             if volume_mounts:
                 # Find primary mount or use first mount
                 primary = next((m for m in volume_mounts if m.get("is_primary")), volume_mounts[0])
                 primary_volume_id = primary.get("volume_id")
 
-            # Create server record. Reflect the result of the readiness probe in
+            # Create server record. Reflect the result of the readiness probes in
             # health_status so the API shows an accurate initial state instead of
             # the model default "unknown".
             server = Server(
@@ -344,7 +368,7 @@ async def spawn(
                 image=image,
                 volume_id=uuid.UUID(primary_volume_id) if primary_volume_id else None,
                 status="running",
-                health_status="healthy" if ready else "unhealthy",
+                health_status="healthy" if (ready and traefik_ready) else "unhealthy",
                 allocated_cpu=cpu,
                 allocated_memory=memory,
                 allocated_disk=disk,

backend/tests/container/test_spawner.py

@@ -664,19 +664,35 @@ async def test_spawn_returns_server_with_url(self, fresh_spawner):
 
     @pytest.mark.asyncio
     async def test_spawn_waits_for_container_ready(self, fresh_spawner):
-        """spawn should wait for container readiness before returning."""
+        """spawn should wait for container and Traefik readiness before returning."""
+        from app.container.spawner import settings
+
         user_id = str(uuid_mod.uuid4())
-        with mock.patch("app.container.spawner.settings.public_url", "http://test"):
+        with (
+            mock.patch("app.container.spawner.settings.public_url", "http://test"),
+            mock.patch(
+                "app.container.spawner.settings.traefik_internal_url",
+                "http://traefik",
+            ),
+        ):
             server = await fresh_spawner.spawn(
                 user_id=user_id,
                 username="testuser",
                 server_name="srv1",
             )
 
         health_alias = f"srv-{str(server.id)[:8]}"
-        fresh_spawner.container_client.wait_for_container_ready.assert_awaited_once_with(
-            health_alias,
-            f"http://{health_alias}:8080/health",
+        calls = fresh_spawner.container_client.wait_for_container_ready.await_args_list
+        assert mock.call(health_alias, f"http://{health_alias}:8080/health") in calls
+        assert (
+            mock.call(
+                mock.ANY,
+                "http://traefik/user/testuser/srv1/health",
+                timeout=settings.container_readiness_timeout,
+                interval=settings.container_readiness_interval,
+                body_contains="healthy",
+            )
+            in calls
         )
 
     @pytest.mark.asyncio

compose.yml

@@ -159,6 +159,7 @@ services:
             - REDIS_PASSWORD=${REDIS_PASSWORD:-}
             - REDIS_DB=${REDIS_DB:-0}
             - DOCKER_NETWORK=${DOCKER_NETWORK:-nukelab-network}
+            - TRAEFIK_INTERNAL_URL=${TRAEFIK_INTERNAL_URL:-http://traefik:80}
             - LOG_LEVEL=${LOG_LEVEL:-INFO}
             - LOG_FORMAT=${LOG_FORMAT:-json}
             - DEV_RELOAD=${DEV_RELOAD:-true}

environments/radiation-transport/Dockerfile

@@ -22,9 +22,10 @@ ARG KDSOURCE_BRANCH="v0.2.2"
 ARG PYNE_BRANCH="scikit-build-core"
 ARG ALARA_BRANCH="main"
 
-# Define whether to download Geant4 data and cross-section data
+# Define whether to download Geant4 data, cross-section data, and depletion chain data
 ARG DOWNLOAD_GEANT4_DATA=ON
 ARG DOWNLOAD_CROSS_SECTION_DATA=ON
+ARG DOWNLOAD_CHAIN_DATA=ON
 
 ###########################################################################
 #                              Initial Setup                              #
@@ -213,13 +214,16 @@ RUN git clone -b ${OPENMC_BRANCH} https://github.com/openmc-dev/openmc.git && \
     cd .. && pip install --no-cache-dir . && \
     rm -rf /tmp/openmc
 
-# Download cross-section data using the system shell
+# Download cross-section and depletion chain data using the system shell
 ARG DOWNLOAD_CROSS_SECTION_DATA
+ARG DOWNLOAD_CHAIN_DATA
 ENV OPENMC_DATA_DIR=/opt/openmc_data
 ENV OPENMC_CROSS_SECTIONS=${OPENMC_DATA_DIR}/lib80x_hdf5/cross_sections.xml
-COPY --chmod=755 download_cross_sections.sh .
+ENV OPENMC_CHAIN_FILE=${OPENMC_DATA_DIR}/chain/chain_endfb80_thermal.xml
+COPY --chmod=755 download_cross_sections.sh download_chain_files.sh .
 RUN ./download_cross_sections.sh ${DOWNLOAD_CROSS_SECTION_DATA} ${OPENMC_DATA_DIR} && \
-    rm download_cross_sections.sh
+    ./download_chain_files.sh ${DOWNLOAD_CHAIN_DATA} ${OPENMC_DATA_DIR} && \
+    rm download_cross_sections.sh download_chain_files.sh
 
 # Install PyNE
 ARG PYNE_BRANCH

environments/radiation-transport/download_chain_files.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: 2023-2026 NukeHub Developers
+# SPDX-License-Identifier: BSD-2-Clause
+
+# Download OpenMC depletion chain files.
+#
+# Usage:
+#   download_chain_files.sh <ON|OFF> <target_directory>
+#
+# When the first argument is ON, chain XML files are downloaded into
+# <target_directory>/chain/. The default OpenMC chain file environment
+# variable should point to one of these files.
+
+set -euo pipefail
+
+download_chain_data=${1:-OFF}
+chain_data_dir=${2:-/opt/openmc_data}
+
+if [ "$download_chain_data" == "ON" ]; then
+    mkdir -p "${chain_data_dir}/chain"
+    cd "${chain_data_dir}/chain"
+
+    # ENDF/B-VIII.0 depletion chains (thermal and fast spectrum).
+    # These pair with the LANL ENDF/B-VIII.0 (lib80x) cross-section library.
+    wget -O chain_endfb80_thermal.xml "https://anl.box.com/shared/static/nyezmyuofd4eqt6wzd626lqth7wvpprr.xml"
+    wget -O chain_endfb80_fast.xml "https://anl.box.com/shared/static/x3kp739hr5upmeqpbwx9zk9ep04fnmtg.xml"
+
+    # ENDF/B-VIII.1 depletion chains.
+    wget -O chain_endfb81_thermal.xml "https://anl.box.com/shared/static/q6ev8pl7xct179ke7kq148smde8gzni6.xml"
+    wget -O chain_endfb81_fast.xml "https://anl.box.com/shared/static/n0pkqe66uotskoljr93szvjyvtvycgze.xml"
+fi

environments/radiation-transport/download_cross_sections.sh

@@ -2,24 +2,46 @@
 # SPDX-FileCopyrightText: 2023-2026 NukeHub Developers
 # SPDX-License-Identifier: BSD-2-Clause
 
+# Download OpenMC cross-section data libraries.
+#
+# Usage:
+#   download_cross_sections.sh <ON|OFF> <target_directory>
+#
+# When the first argument is ON, libraries are downloaded and extracted into
+# <target_directory>/.
+
+set -euo pipefail
+
 # Assign command line arguments to variables
-download_cross_section_data=$1
-cross_section_data_lib=$2
+download_cross_section_data=${1:-OFF}
+cross_section_data_lib=${2:-/opt/openmc_data}
 
 if [ "$download_cross_section_data" == "ON" ]; then
-    mkdir -p ${cross_section_data_lib}
-    cd ${cross_section_data_lib}
+    mkdir -p "${cross_section_data_lib}"
+    cd "${cross_section_data_lib}"
+
     # Function to download and extract data
     download_and_extract() {
         local url=$1
-        local filename=$(basename $url)
+        local filename
+        filename=$(basename "$url")
         mkdir -p tmp
         cd tmp
-        wget $url
+        wget "$url"
         cd ..
-        tar -Jxvf tmp/$filename
+        tar -Jxvf "tmp/${filename}"
         rm -rf tmp
     }
+
+    # Current libraries are the LANL-based data sets distributed for use with
+    # MCNP/OpenMC. They extract to mcnp_endfb70, mcnp_endfb71, and lib80x_hdf5.
+    #
+    # Newer official OpenMC-produced libraries are available from
+    # https://openmc.org/data/ and may be substituted here. For example:
+    #   - ENDF/B-VIII.0 official: https://anl.box.com/shared/static/uhbxlrx7hvxqw27psymfbhi7bx7s6u6a.xz
+    #   - ENDF/B-VIII.1 official: https://anl.box.com/shared/static/6qr7jezzihkj9p9esl5jn19qgpujyjyz.xz
+    # If switching libraries, update OPENMC_CROSS_SECTIONS in the Dockerfile to
+    # match the extracted directory name and verify with a test build.
     download_and_extract "https://anl.box.com/shared/static/t25g7g6v0emygu50lr2ych1cf6o7454b.xz"
     download_and_extract "https://anl.box.com/shared/static/d359skd2w6wrm86om2997a1bxgigc8pu.xz"
     download_and_extract "https://anl.box.com/shared/static/nd7p4jherolkx4b1rfaw5uqp58nxtstr.xz"

environments/radiation-transport/nuke-env.sh

@@ -6,7 +6,25 @@
 # conda environment in login/interactive shells so users get the right Python
 # and the MOAB/OpenMC/PyNE toolchain.
 if [ -n "${NUKE_DIR:-}" ] && [ -d "${NUKE_DIR}/bin" ]; then
+    # Conda activation replaces PATH with the env's bin directory. The Dockerfile
+    # adds nuclear-code bin directories to PATH; preserve them across activation
+    # so openmc, geant4-config, njoy, etc. remain available in terminals.
+    _nuke_tool_path="${NUKE_DIR}/bin"
+    _nuke_tool_path="${_nuke_tool_path}:${MOAB_ROOT:-/opt/moab}/bin"
+    _nuke_tool_path="${_nuke_tool_path}:${DOUBLE_DOWN_ROOT:-/opt/double-down}/lib"
+    _nuke_tool_path="${_nuke_tool_path}:${GEANT4_ROOT:-/opt/geant4}/bin"
+    _nuke_tool_path="${_nuke_tool_path}:${DAGMC_ROOT:-/opt/dagmc}/bin"
+    _nuke_tool_path="${_nuke_tool_path}:${LIBMESH_ROOT:-/opt/libmesh}/bin"
+    _nuke_tool_path="${_nuke_tool_path}:${NJOY2016_ROOT:-/opt/njoy2016}/bin"
+    _nuke_tool_path="${_nuke_tool_path}:${OPENMC_ROOT:-/opt/openmc}/bin"
+    _nuke_tool_path="${_nuke_tool_path}:${KDSOURCE_ROOT:-/opt/kdsource}/bin"
+    _nuke_tool_path="${_nuke_tool_path}:${ALARA_ROOT:-/opt/alara}/bin"
+
     conda activate "${NUKE_DIR}" > /dev/null 2>&1 || true
+
+    export PATH="${_nuke_tool_path}${PATH:+:${PATH}}"
+    unset _nuke_tool_path
+
     # The nuke env replaces PATH with its own bin directory. Keep the base
     # conda tools (node, yarn, npm) available in login/terminal sessions.
     if [ -d "/opt/conda/bin" ] && [[ ":${PATH}:" != *":/opt/conda/bin:"* ]]; then

frontend/public/sw.js.tpl

@@ -23,6 +23,11 @@ function shouldBypass(request, url) {
   for (const prefix of BYPASS_PATHS) {
     if (pathname.startsWith(prefix)) return true;
   }
+  // Visualizer reverse-proxy routes are served by the IDE container's trame
+  // servers and must not be served the cached SPA shell. They normally live
+  // under /user/.../visualizer/, but bypass them anywhere they appear as a
+  // defense-in-depth measure.
+  if (pathname.includes('/visualizer/')) return true;
   return false;
 }