From d81578a6f59b652e5c76b77cc6c8a64d8e3428b0 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 22 Sep 2026 12:22:54 +0800 Subject: [PATCH] Use published modules on Github npm --- .dockerignore | 7 + .gitignore | 7 + .mise/config.coverage.toml | 13 +- .mise/config.maint.toml | 142 ++++++- .mise/config.rust.toml | 28 +- .mise/config.toml | 23 +- .mise/cov-branch-assert.jq | 41 +- CLAUDE.md | 30 ++ Cargo.lock | 11 + Cargo.toml | 2 + Dockerfile | 3 + Dockerfile.nanoserver | 4 + README.md | 2 +- .../ast-grep/rules/new-fn-returns-option.yaml | 62 +++ .../rules/org-identity-lives-in-et-org.yaml | 38 ++ config/jscpd-baseline.json | 36 +- .../model-modules/model-eye1/pkg/package.json | 2 +- .../model-face1/pkg/package.json | 2 +- .../model-har-motion1/pkg/package.json | 2 +- .../model-modules/model-llm1/pkg/package.json | 2 +- .../model-speech1/pkg/package.json | 2 +- generated/dart-rest/lib/clients/modules.dart | 11 +- .../api/modules/get_module_file.py | 28 +- generated/python-rest/pkg/et_rest_client.js | 6 +- generated/python-ws/pkg/et_ws.js | 6 +- generated/rust-rest/src/lib.rs | 11 +- generated/specs/rest.yaml | 11 +- generated/zig-rest/src/et_rest_client.zig | 9 +- libs/edge-toolkit/src/config.rs | 69 +++- libs/edge-toolkit/tests/staged_module_dirs.rs | 93 +++++ libs/org/Cargo.toml | 14 + libs/org/src/lib.rs | 70 ++++ libs/org/tests/identity.rs | 33 ++ libs/ws-runner-common/src/lib.rs | 34 +- .../tests/fetch_main_field.rs | 4 +- .../tests/module_file_missing.rs | 94 +++++ pnpm-lock.yaml | 6 +- pnpm-workspace.yaml | 6 + services/modules/src/lib.rs | 70 +++- services/modules/src/routes.rs | 11 +- services/modules/tests/api_modules.rs | 29 +- .../modules/tests/configure_missing_root.rs | 33 +- services/modules/tests/declared_names.rs | 79 ++++ .../dart-comm1/pkg/et_ws_dart_comm1.js | 2 +- .../ws-modules/dart-comm1/pkg/package.json | 2 +- .../dart-data1/pkg/et_ws_dart_data1.js | 2 +- .../ws-modules/dart-data1/pkg/package.json | 2 +- .../dart-math1/pkg/et_ws_dart_math1.js | 2 +- .../ws-modules/dart-math1/pkg/package.json | 2 +- .../ws-modules/dotnet-data1/pkg/package.json | 2 +- .../ws-modules/dotnet-math1/pkg/package.json | 2 +- services/ws-modules/face-detection/Cargo.toml | 1 + services/ws-modules/face-detection/src/lib.rs | 2 +- services/ws-modules/har1/Cargo.toml | 1 + services/ws-modules/har1/src/lib.rs | 6 +- .../ws-modules/java-data1/pkg/package.json | 2 +- .../ws-modules/java-math1/pkg/package.json | 2 +- services/ws-modules/js-data1/package.json | 2 +- services/ws-modules/js-data1/pkg/package.json | 2 +- .../ws-modules/js-math1/pkg/et_ws_js_math1.js | 2 +- services/ws-modules/js-math1/pkg/package.json | 2 +- .../ws-modules/kotlin-data1/pkg/package.json | 2 +- .../ws-modules/kotlin-math1/pkg/package.json | 2 +- services/ws-modules/llm1/Cargo.toml | 1 + services/ws-modules/llm1/src/lib.rs | 2 +- services/ws-modules/math1-sender/Cargo.toml | 3 - .../ws-modules/pydata1/pkg/et_ws_pydata1.js | 9 +- services/ws-modules/pydata1/pkg/package.json | 4 +- .../ws-modules/pydemo1/pkg/et_ws_pydemo1.js | 11 +- services/ws-modules/pydemo1/pkg/package.json | 12 +- .../ws-modules/pyeye1/pkg/et_ws_pyeye1.js | 9 +- .../ws-modules/pyface1/pkg/et_ws_pyface1.js | 9 +- .../ws-modules/pymath1/pkg/et_ws_pymath1.js | 5 +- .../ws-modules/pyo3-math1/pkg/package.json | 2 +- .../pyspeech1/pkg/et_ws_pyspeech1.js | 7 +- .../ws-modules/pyspeech1/pkg/package.json | 6 +- .../ws-modules/rcomm1/pkg/et_ws_rcomm1.js | 6 +- services/ws-modules/rcomm1/pkg/package.json | 2 +- .../ws-modules/rdata1/pkg/et_ws_rdata1.js | 6 +- services/ws-modules/rdata1/pkg/package.json | 2 +- .../ws-modules/rmath1/pkg/et_ws_rmath1.js | 6 +- services/ws-modules/rmath1/pkg/package.json | 2 +- .../wasi_graphics_info/__init__.py | 6 +- .../wasi-math1-sender/pkg/package.json | 2 +- services/ws-modules/zig-data1/build.zig | 56 +-- services/ws-modules/zig-data1/build.zig.zon | 3 + services/ws-modules/zig-except1/build.zig | 55 +-- services/ws-modules/zig-except1/build.zig.zon | 3 + services/ws-modules/zig-math1/build.zig | 47 +-- services/ws-modules/zig-math1/build.zig.zon | 3 + services/ws-modules/zig-shared/build.zig | 96 +++++ services/ws-modules/zig-shared/build.zig.zon | 6 + services/ws-pyo3-runner/Cargo.toml | 1 + services/ws-pyo3-runner/src/hub_module.rs | 13 +- services/ws-pyo3-runner/tests/modules.rs | 14 +- services/ws-server/Dockerfile | 9 +- services/ws-server/static/app.js | 27 +- services/ws-server/static/package.json | 4 +- services/ws-test-server/src/lib.rs | 6 +- services/ws-wasi-runner/Cargo.toml | 1 + .../ws-wasi-runner/src/host/wasi_keyvalue.rs | 51 ++- services/ws-wasi-runner/src/lib.rs | 6 +- services/ws-wasi-runner/tests/modules.rs | 12 +- .../ws-wasi-runner/tests/otel_propagation.rs | 2 +- services/ws-web-runner/Cargo.toml | 1 + services/ws-web-runner/tests/modules.rs | 15 +- utilities/cli/Cargo.toml | 1 + .../src/deployment_types/docker_compose.rs | 157 +++++--- utilities/cli/src/deployment_types/k3s.rs | 147 ++++---- utilities/cli/src/deployment_types/mise.rs | 352 +++++++++++++----- utilities/cli/src/deployment_types/mod.rs | 32 +- .../src/deployment_types/scenario_image.rs | 4 +- utilities/cli/src/input.rs | 2 +- utilities/cli/src/lib.rs | 59 ++- utilities/cli/src/module_package_json/mod.rs | 69 +++- utilities/cli/tests/module_package_json.rs | 92 ++++- utilities/cli/tests/scenario_generation.rs | 132 ++++--- .../output/default/Dockerfile.dockerignore | 7 + .../local/output/default/compose.yaml | 1 + verification/local/output/default/k3s.yaml | 2 + verification/local/output/default/mise.toml | 6 +- .../Dockerfile.dockerignore | 7 + .../facility-security-scenario/compose.yaml | 1 + .../facility-security-scenario/k3s.yaml | 2 + .../facility-security-scenario/mise.toml | 9 +- .../output/math1/Dockerfile.dockerignore | 7 + verification/local/output/math1/compose.yaml | 8 +- verification/local/output/math1/k3s.yaml | 8 +- verification/local/output/math1/mise.toml | 12 +- .../output/pyo3-math1/Dockerfile.dockerignore | 7 + .../local/output/pyo3-math1/compose.yaml | 8 +- verification/local/output/pyo3-math1/k3s.yaml | 8 +- .../local/output/pyo3-math1/mise.toml | 12 +- .../output/wasi-math1/Dockerfile.dockerignore | 7 + .../local/output/wasi-math1/compose.yaml | 8 +- verification/local/output/wasi-math1/k3s.yaml | 8 +- .../local/output/wasi-math1/mise.toml | 12 +- .../output/default/Dockerfile.dockerignore | 7 + .../published/output/default/compose.yaml | 1 + .../published/output/default/k3s.yaml | 2 + .../published/output/default/mise.toml | 17 +- verification/published/output/default/npmrc | 2 + .../output/math1/Dockerfile.dockerignore | 7 + .../published/output/math1/compose.yaml | 8 +- verification/published/output/math1/k3s.yaml | 8 +- verification/published/output/math1/mise.toml | 26 +- verification/published/output/math1/npmrc | 2 + .../output/pyo3-math1/Dockerfile.dockerignore | 7 + .../published/output/pyo3-math1/compose.yaml | 8 +- .../published/output/pyo3-math1/k3s.yaml | 8 +- .../published/output/pyo3-math1/mise.toml | 26 +- .../published/output/pyo3-math1/npmrc | 2 + .../output/wasi-math1/Dockerfile.dockerignore | 7 + .../published/output/wasi-math1/compose.yaml | 8 +- .../published/output/wasi-math1/k3s.yaml | 8 +- .../published/output/wasi-math1/mise.toml | 26 +- .../published/output/wasi-math1/npmrc | 2 + 157 files changed, 2395 insertions(+), 745 deletions(-) create mode 100644 config/ast-grep/rules/new-fn-returns-option.yaml create mode 100644 config/ast-grep/rules/org-identity-lives-in-et-org.yaml create mode 100644 libs/edge-toolkit/tests/staged_module_dirs.rs create mode 100644 libs/org/Cargo.toml create mode 100644 libs/org/src/lib.rs create mode 100644 libs/org/tests/identity.rs create mode 100644 libs/ws-runner-common/tests/module_file_missing.rs create mode 100644 services/modules/tests/declared_names.rs create mode 100644 services/ws-modules/zig-shared/build.zig create mode 100644 services/ws-modules/zig-shared/build.zig.zon create mode 100644 verification/published/output/default/npmrc create mode 100644 verification/published/output/math1/npmrc create mode 100644 verification/published/output/pyo3-math1/npmrc create mode 100644 verification/published/output/wasi-math1/npmrc diff --git a/.dockerignore b/.dockerignore index 3671d9dc..8090f9b2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -52,6 +52,13 @@ lcov.info # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# Lock files a generated scenario leaves behind when it is run. +# `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario +# drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather +# than part of it: `regen-verification` never writes one, the drift check never reads one, and what it pins is +# whatever the machine that ran it happened to resolve. One rule here rather than a `.gitignore` generated into +# each scenario, so a scenario added later is covered without anything having to be written into it. +verification/**/*.lock # No rule for the per-scenario credential file, deliberately. # Under verification/ it is committed like every other generated artifact: the password is derived from an # input this repository already carries, so the file reproduces from public material rather than keeping a diff --git a/.gitignore b/.gitignore index 38ec2da6..263c0089 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,13 @@ coverage-python.xml # Dart pub tool dir, regenerated by `dart pub get`; one global rule for the whole workspace (root + members). **/.dart_tool/ services/ws-server/storage/ +# Lock files a generated scenario leaves behind when it is run. +# `lockfile = true` makes mise write one beside whichever config it resolved tools for, so running a scenario +# drops `mise.lock` into that scenario's output directory. It is a by-product of running the deployment rather +# than part of it: `regen-verification` never writes one, the drift check never reads one, and what it pins is +# whatever the machine that ran it happened to resolve. One rule here rather than a `.gitignore` generated into +# each scenario, so a scenario added later is covered without anything having to be written into it. +verification/**/*.lock # No rule for the per-scenario credential file, deliberately. # Under verification/ it is committed like every other generated artifact: the password is derived from an # input this repository already carries, so the file reproduces from public material rather than keeping a diff --git a/.mise/config.coverage.toml b/.mise/config.coverage.toml index 96d3036f..8f87d8ad 100644 --- a/.mise/config.coverage.toml +++ b/.mise/config.coverage.toml @@ -41,6 +41,13 @@ # gap in one pipeline be answered by the other. native_cov_libs = "edge-toolkit et-otlp path test-helpers test-otlp ws-runner-common" wasm_cov_libs = "wasi-guest web" +# The third kind: a crate whose source is `const` and `macro_rules!` only. +# llvm-cov emits no records at all for one -- there is no executable code to instrument, so its tests can pass +# while the report stays empty. +# Naming it here rather than in the native list is what tells the assertion its empty report is expected, and +# the assertion then holds it to the opposite invariant: records appearing means the crate has grown code, and +# it has to move to the native list to be held to 100% like everything else. +const_cov_libs = "org" # The wasm coverage builds run on nightly (-Zno-profiler-runtime) with the wasm-capable conda clang on PATH. # It is set env-wide here because this env loads only under the coverage workflow; per-command RUSTFLAGS and @@ -198,7 +205,8 @@ if [ ! -f "$report" ]; then echo "Run the coverage task that produces it first; a missing report is a failure, not a skip." >&2 exit 1 fi -shortfall="$(jaq -r --arg libs "$usage_libs" -f .mise/cov-branch-assert.jq "$report")" +nocode="${usage_nocode:-}" +shortfall="$(jaq -r --arg libs "$usage_libs" --arg nocode "$nocode" -f .mise/cov-branch-assert.jq "$report")" if [ -n "$shortfall" ]; then echo "cov-branch-assert: libs/ must be at 100% branch coverage, and is not:" >&2 echo "$shortfall" >&2 @@ -210,6 +218,7 @@ shell = "{{ vars.task_shell }}" usage = """ arg "" help="llvm-cov JSON summary (--format=text) to assert over" arg "" help="Space-separated libs/ directory names that must be at 100% branch coverage" +arg "[nocode]" help="Space-separated libs/ directory names that must contribute no records at all" """ # Reports on the profile data cargo-llvm-cov already wrote; it does not re-run the tests. @@ -236,7 +245,7 @@ coreutils mkdir -p target/cov report=target/cov/native-libs.json eval "$(cargo llvm-cov show-env --sh)" cargo llvm-cov report --json --summary-only --output-path "$report" -mise run _cov-branch-assert "$report" "{{ vars.native_cov_libs }}" +mise run _cov-branch-assert "$report" "{{ vars.native_cov_libs }}" "{{ vars.const_cov_libs }}" """ shell = "{{ vars.task_shell }}" diff --git a/.mise/config.maint.toml b/.mise/config.maint.toml index 9e85b302..96fdbf52 100644 --- a/.mise/config.maint.toml +++ b/.mise/config.maint.toml @@ -14,6 +14,9 @@ # asset via the `http:et-rp-wasm` mise tool entry in config.toml. # - release-rust-crates: version-bumps, tags and publishes every # publishable workspace crate to crates.io via cargo-release. +# - publish-module-packages: publishes the owner-scoped module packages to +# GitHub Packages, which is where a published deployment stages the +# modules the hub serves. # # Not in ALL_LANGS (maint isn't a language); invoked manually: # @@ -21,6 +24,7 @@ # MISE_ENV=maint mise run publish-eye1-to-hf-cache # MISE_ENV=maint mise run publish-rp-wasm-to-release # MISE_ENV=maint mise run release-rust-crates patch +# MISE_ENV=maint mise run publish-module-packages [tools] # Rust-native HTTP client replacing the host `curl`. @@ -818,9 +822,10 @@ description = "Publish one batch of five workspace crates to crates.io (dry run # # The crates list is ordered so every crate's workspace dependencies sit in an earlier or the same batch. # Only crossing a batch boundary matters: cargo-release topologically orders whatever one invocation selects, -# so order within a batch is its problem, not this list's. Crates carrying -# `[package.metadata.release] release = false` (the browser modules) and the `int-` crates are absent because -# neither is published. +# so order within a batch is its problem, not this list's. The `int-` crates are absent because they are not +# published, as are the browser modules carrying `[package.metadata.release] release = false` -- which is not +# every browser module. A module that a published scenario serves has to be on crates.io, so those carry no +# such marker and appear here alongside the libraries. # # Rebuild the order from `cargo metadata` when the workspace graph changes -- NOT from the crate list a # `cargo release --workspace` run prints, which is its version-bump order and put a dev-dependency @@ -837,11 +842,11 @@ description = "Publish one batch of five workspace crates to crates.io (dry run run = """ crates=( et-path edge-toolkit et-test-helpers et-otlp et-test-otlp - et-web et-rest-client et-ws-runner-common et-ws-wasm-agent et-ws-comm1 - et-ws-data1 et-ws-math1 et-ws-wasi-comm1 et-ws-wasi-data1 et-ws-wasi-math1 - et-modules-service et-storage-service et-websockify-service et-ws-service et-ws-test-server - et-ws-pyo3-runner et-ws-server et-ws-wasi-runner et-ws-web-runner et-cli - et-onnx + et-web et-rest-client et-ws-runner-common et-ws-wasm-agent et-wasi-guest + et-ws-comm1 et-ws-data1 et-ws-math1 et-ws-math1-sender et-ws-wasi-comm1 + et-ws-wasi-data1 et-ws-wasi-math1 et-ws-wasi-math1-sender et-modules-service et-storage-service + et-websockify-service et-ws-service et-ws-test-server et-ws-pyo3-runner et-ws-server + et-ws-wasi-runner et-ws-web-runner et-cli et-onnx et-repo-check ) per_batch=5 # `set --` then `$#` counts the array without the `${` + `#` pair, which mise's Tera pass reads as a comment. @@ -867,3 +872,124 @@ usage = """ arg "" help="Which batch of five to publish, counting from 1" flag "--execute" help="Actually publish; omit to print the plan and change nothing" """ + +[tasks.publish-module-packages] +description = "Publish the scoped module packages to GitHub Packages (dry run without --execute)" +# A published deployment stages its modules from an npm registry. +# That is the only route the hub has to a module it cannot find on disk. GitHub Packages will only accept a +# package scoped to the repository owner, which is why each `package.json` here is named `@edge-toolkit/...`; +# the hub drops that scope when it names the module, so nothing a deployment references changes because of it. +# +# The registry and credential reach pnpm through a generated npmrc rather than the ambient one, so a +# maintainer's own npm login is never consulted and `~/.npmrc` is left alone. mise reads user-level npm config +# and `NPM_CONFIG_*` but deliberately neutralises a project `.npmrc`, so `NPM_CONFIG_USERCONFIG` is the one +# handle that works for both this publish and the `npm:` installs a generated deployment performs. The file +# holds `${GITHUB_TOKEN}` by reference -- npm expands it at read time -- so no token is written to disk. +# +# An already-published version is skipped rather than retried. The registry rejects re-publishing a version, +# and a run that stopped halfway has to be safe to repeat. +run = """ +: "${GITHUB_TOKEN:=$(gh auth token)}" +export GITHUB_TOKEN +# Refuse a token that cannot publish, before anything is packed or uploaded. +# The registry answers a missing scope with `E403 permission_denied: The token provided does not match +# expected scopes`, from inside pnpm and after the first package is already built -- and on a list that +# publishes in dependency order, failing midway can leave a dependent uploaded without its dependency. A +# classic token advertises its scopes in this header; a fine-grained one sends no such header, so only a +# header that is present and lacks the scope is treated as a definite no. +if [ "${usage_execute:-}" = "true" ]; then + scopes=$(gh api -i /user 2>/dev/null | rg -i '^x-oauth-scopes:' || true) + case "$scopes" in + *write:packages*) ;; + "") echo "token advertises no scopes; letting the registry decide" ;; + *) + echo "GITHUB_TOKEN lacks the write:packages scope that publishing needs." >&2 + echo "Grant it with: gh auth refresh -s write:packages" >&2 + echo "Scopes seen: $scopes" >&2 + exit 1 + ;; + esac +fi +# Directories that may hold a module to publish, discovered rather than listed. +# A list naming every module by hand goes stale the moment one is added, and the manifest already says +# whether a directory is publishable: a scoped name means yes, anything else means the registry would reject +# it. A module whose `pkg/` has not been built is simply absent, so this publishes whatever the tree holds. +# +# The order puts what nothing depends on first -- the model bundles and the generated clients, then the agent +# and the page, then the modules that name those. npm does not check a dependency at publish time, so this is +# not what makes a publish succeed; it is so that no dependent is ever on the registry without the thing it +# needs, for anyone installing while a run is part way through. +candidates=( + data/model-modules/*/pkg + generated/python-ws/pkg + generated/python-rest/pkg + services/ws-wasm-agent/pkg + services/ws-server/static + services/ws-modules/*/pkg +) +modules=() +for dir in "${candidates[@]}"; do + [ -f "$dir/package.json" ] || continue + name=$(jaq -r '.name // ""' "$dir/package.json") + case "$name" in + @edge-toolkit/*) ;; + *) + echo "skipping $dir: name '$name' is not scoped, so the registry would reject it" + continue + ;; + esac + # `--only` narrows the run to the named modules, matched on the name without the scope. + # Publishing every built module at once is rarely what a maintainer wants: a version cannot be replaced + # once it is up, and the set includes the model bundles, one of which is large enough that pushing it by + # accident is its own problem. Naming what to publish keeps a run to what was intended. + if [ -n "${usage_only:-}" ]; then + bare=${name#@edge-toolkit/} + case ",${usage_only}," in + *",$bare,"*) ;; + *) continue ;; + esac + fi + modules+=("$dir") +done +# `set --` then `$#` counts the array without the `${` + `#` pair, which mise's Tera pass reads as a comment. +set -- "${modules[@]}" +if [ "$#" -eq 0 ]; then + echo "no built, scoped modules found -- build the modules first" >&2 + exit 1 +fi +npmrc=target/publish-npmrc +coreutils mkdir -p target +registry_line='@edge-toolkit:registry=https://npm.pkg.github.com' +# `${GITHUB_TOKEN}` has to reach the file unexpanded, for npm to resolve when it reads it. +# Expanding it here would write the credential itself into the npmrc on disk, which is the one thing +# referencing it by name avoids. +# shellcheck disable=SC2016 +token_line='//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}' +printf '%s\\n%s\\n' "$registry_line" "$token_line" >"$npmrc" +NPM_CONFIG_USERCONFIG="$PWD/$npmrc" +export NPM_CONFIG_USERCONFIG +for module in "${modules[@]}"; do + if [ ! -f "$module/package.json" ]; then + echo "no package.json in $module -- build the module first" >&2 + exit 1 + fi + name=$(jaq -r .name "$module/package.json") + version=$(jaq -r .version "$module/package.json") + if pnpm view "$name@$version" version >/dev/null 2>&1; then + echo "$name@$version already published; skipping" + continue + fi + if [ "${usage_execute:-}" = "true" ]; then + echo "publishing $name@$version from $module" + pnpm publish "$module" --no-git-checks + else + echo "would publish $name@$version from $module" + pnpm publish "$module" --no-git-checks --dry-run + fi +done +""" +shell = "{{ vars.task_shell }}" +usage = """ +flag "--execute" help="Actually publish; omit to print the plan and change nothing" +flag "--only " help="Comma-separated module names to publish; omit for every built module" +""" diff --git a/.mise/config.rust.toml b/.mise/config.rust.toml index 08f444e5..b7548d3d 100644 --- a/.mise/config.rust.toml +++ b/.mise/config.rust.toml @@ -22,85 +22,85 @@ run = "cargo build -p int-wasm-cov-wrapper" depends = ["build-wasm-cov-wrapper"] description = "Build the data1 workflow WASM module" dir = "services/ws-modules/data1" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-math1-module] depends = ["build-wasm-cov-wrapper"] description = "Build the math1 FedAvg WASM module" dir = "services/ws-modules/math1" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-math1-sender-module] depends = ["build-wasm-cov-wrapper"] description = "Build the math1-sender trigger WASM module" dir = "services/ws-modules/math1-sender" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-comm1-module] depends = ["build-wasm-cov-wrapper"] description = "Build the comm1 workflow WASM module" dir = "services/ws-modules/comm1" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-pic-viewer-module] depends = ["build-wasm-cov-wrapper"] description = "Build the pic-viewer broadcast picture viewer WASM module" dir = "services/ws-modules/pic-viewer" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-except1-module] depends = ["build-wasm-cov-wrapper"] description = "Build the except1 exception-handling demo WASM module" dir = "services/ws-modules/except1" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-sensor1-module] depends = ["build-wasm-cov-wrapper"] description = "Build the sensor1 workflow WASM module" dir = "services/ws-modules/sensor1" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-audio1-module] depends = ["build-wasm-cov-wrapper"] description = "Build the audio1 workflow WASM module" dir = "services/ws-modules/audio1" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-video1-module] depends = ["build-wasm-cov-wrapper"] description = "Build the video1 workflow WASM module" dir = "services/ws-modules/video1" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-bluetooth-module] depends = ["build-wasm-cov-wrapper"] description = "Build the bluetooth workflow WASM module" dir = "services/ws-modules/bluetooth" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-geolocation-module] depends = ["build-wasm-cov-wrapper"] description = "Build the geolocation workflow WASM module" dir = "services/ws-modules/geolocation" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-graphics-info-module] depends = ["build-wasm-cov-wrapper"] description = "Build the graphics info workflow WASM module" dir = "services/ws-modules/graphics-info" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-speech-recognition-module] depends = ["build-wasm-cov-wrapper"] description = "Build the speech recognition workflow WASM module" dir = "services/ws-modules/speech-recognition" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-nfc-module] depends = ["build-wasm-cov-wrapper"] description = "Build the nfc workflow WASM module" dir = "services/ws-modules/nfc" -run = "{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.no_opt }}{{ vars.web_cov_feat }}" +run = "{{ vars.web_pack_cov }}{{ vars.no_opt }}{{ vars.web_cov_feat }}" [tasks.build-ws-wasi-data1-module] depends = ["build-et-cli"] diff --git a/.mise/config.toml b/.mise/config.toml index d0c00f44..db929a84 100644 --- a/.mise/config.toml +++ b/.mise/config.toml @@ -566,11 +566,25 @@ web_cov_feat = '{% if env?.ET_TEST_COVERAGE == "true" %} -- --features et-web/co # Instruments our crates, never dependency cdylibs. Empty unless ET_TEST_COVERAGE is set, so normal builds get # no wrapper (no fingerprint churn). The wrapper binary is built by the build-wasm-cov-wrapper task dependency. web_cov_wrapper = '{% if env?.ET_TEST_COVERAGE == "true" %}{{ vars.a_web_cov_wrapper_env }}{% endif %}' +# Owner scope every module package is published under, as wasm-pack's flag for it. +# GitHub Packages only accepts a package scoped to the repository owner, so an unscoped module cannot be +# published at all. One var rather than the flag repeated per module, so the scope is stated once and a new +# module build picks it up by using this like every other. It costs nothing on a module that is never +# published: the hub drops the scope when it names a module, so a scoped and an unscoped build of the same +# module are served under one name, at one URL, and are indistinguishable to a deployment. Defined above its +# consumers because mise resolves `[vars]` in file order -- a var referenced before it is defined is simply +# absent, which surfaces as `Field ... is not defined` rather than as an empty string. +wasm_scope = "--scope edge-toolkit " +# The module this repository's server serves at `/`, named as its own package.json declares it. +# The server has no default: which module is a deployment's front page is that deployment's business, and a +# name defaulted there would be one project's carried by every other. Stated once here because all three +# ws-server tasks need it and a copy per task is a copy that drifts. +hub_root_module = "@edge-toolkit/et-ws-server-static" # Override-free head of a browser module's wasm-pack build: coverage wrapper prefix plus fixed build args. # Used by the two modules that also run et-cli module-package-json (har1, face-detection). It excludes no_opt / # web_cov_feat -- those are appended inline at the call site because no_opt carries a per-platform override that a # nested var would freeze at this file's value. -web_pack_cov = '{{ vars.web_cov_wrapper }}wasm-pack build . --target web' +web_pack_cov = '{{ vars.web_cov_wrapper }}wasm-pack build . --target web {{ vars.wasm_scope }}' # Extra flag for the wasm-pack module builds; empty so wasm-opt runs. # config.windows.toml overrides it to --no-opt where wasm-opt can't execute. no_opt = "" @@ -1786,7 +1800,7 @@ dir = "services/ws-wasm-agent" # so other globally-loaded tasks -- prefetch-ci's transitive deps via `prefetch:node` -- can reference it # without forcing `rust` into MISE_ENV. A bash `shell` for the same base-config PATH-dropout reason # `prefetch:rust` carries one. -run = "wasm-pack build . --target web {{ vars.no_opt }}" +run = "wasm-pack build . --target web {{ vars.wasm_scope }}{{ vars.no_opt }}" shell = "{{ vars.task_shell }}" [tasks."prefetch:node"] @@ -1907,10 +1921,12 @@ run = "git diff --exit-code -- generated services/ws-wasi-runner/src/bindings.rs # (with `--clear`, the screen simply blanks). The command has no shell features, so program mode is correct # on every OS. This is upstream watchexec#918 (open); drop --shell=none once it is fixed. Verified on 2.5.1. [tasks.ws-server] +alias = "hub" description = "Run the WebSocket server, restarting on .rs/.toml changes" run = "watchexec --restart --clear --exts rs,toml -i '**/tests/**' --shell=none -- cargo run -p et-ws-server" [tasks.ws-server.env] +MODULES_ROOT = "{{ vars.hub_root_module }}" # Must match ZO_ROOT_USER_PASSWORD in config/o2.env; the server authenticates its OTLP exports as that root user. OTLP_AUTH_PASSWORD = "Complexpass#123" # skipcq: SCT-A000 -- local dev-only credential, committed deliberately OTLP_AUTH_USERNAME = "root@example.com" @@ -1919,6 +1935,7 @@ OTLP_AUTH_USERNAME = "root@example.com" description = "Run the WebSocket server without OTLP/o2 observability (OpenTelemetry init is skipped)" # Same as ws-server but sets no OTLP_* env, so Config.otlp deserialises to None and et_otlp::init is skipped. # Use this when you don't want the o2/OpenObserve stack (`mise run o2`) running alongside the server. +env = { MODULES_ROOT = "{{ vars.hub_root_module }}" } run = "watchexec --restart --clear --exts rs,toml -i '**/tests/**' --shell=none -- cargo run -p et-ws-server" [tasks.ws-server-offline-demo] @@ -1926,7 +1943,7 @@ description = "Run the WebSocket server for a hotspot demo: no OTLP, no watchexe # Demo the server from a machine that shares its own Wi-Fi hotspot and has no internet uplink. # Sets no OTLP_* env, runs cargo directly (no watchexec restart-on-change), and `--offline` makes cargo # resolve and build from the crates already in the local cache instead of touching the network. -env = { NET_LOG_INTERFACE = "bridge100" } +env = { MODULES_ROOT = "{{ vars.hub_root_module }}", NET_LOG_INTERFACE = "bridge100" } run = "cargo run --offline -p et-ws-server" [tasks.ws-wasi-runner] diff --git a/.mise/cov-branch-assert.jq b/.mise/cov-branch-assert.jq index 1d13de5b..c04f1e53 100644 --- a/.mise/cov-branch-assert.jq +++ b/.mise/cov-branch-assert.jq @@ -1,19 +1,32 @@ -# Names every libs/ crate in $libs that an llvm-cov JSON summary does not show at 100% branch coverage. +# Names the libs/ crates an llvm-cov JSON summary shows as wrong. +# Wrong means one of two things: a crate in $libs that is not at 100% branch coverage, or a crate in $nocode +# that the same summary shows any records for at all. # That JSON (`--format=text`, confusingly) carries a per-file `summary.branches` object, so `.data[0].files[]` # already holds everything this reads. A crate with no records at all is reported rather than passing # vacuously: a crate dropping out of the report is how coverage silently stops being measured. -($libs | split(" ")) as $names +# $nocode inverts that for the crates that are constants and macros only, which llvm-cov cannot record even +# when their tests run -- an empty report is expected there, and records appearing mean the crate grew code. +($libs | split(" ") | map(select(length > 0))) as $names +| ($nocode | split(" ") | map(select(length > 0))) as $constants | .data[0].files as $files -| [ $names[] - | . as $n - | ("/libs/" + $n + "/") as $dir - | ($files | map(select(.filename | contains($dir)))) as $hit - | if ($hit | length) == 0 - then "libs/" + $n + ": no records in the report -- this run never exercised the crate" - else ($hit[] - | select(.summary.branches.covered < .summary.branches.count) - | "libs/" + $n + ": " + .filename + " " - + (.summary.branches.covered | tostring) + "/" - + (.summary.branches.count | tostring) + " branches") - end ] +| [ ( $names[] + | . as $n + | ("/libs/" + $n + "/") as $dir + | ($files | map(select(.filename | contains($dir)))) as $hit + | if ($hit | length) == 0 + then "libs/" + $n + ": no records in the report -- this run never exercised the crate" + else ($hit[] + | select(.summary.branches.covered < .summary.branches.count) + | "libs/" + $n + ": " + .filename + " " + + (.summary.branches.covered | tostring) + "/" + + (.summary.branches.count | tostring) + " branches") + end ), + ( $constants[] + | . as $n + | ("/libs/" + $n + "/") as $dir + | ($files | map(select(.filename | contains($dir)))) as $hit + | if ($hit | length) == 0 + then empty + else "libs/" + $n + ": has coverable code now -- move it to native_cov_libs and cover it" + end ) ] | .[] diff --git a/CLAUDE.md b/CLAUDE.md index 79884a07..1b640630 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -645,6 +645,14 @@ protects. The bar is therefore the whole of it -- every line and every branch, n lets one untested helper hide behind a well-covered neighbour. Services and utilities are held to no such number; they are covered on their merits, which is exactly why the shared layer underneath has to be total. +**A new crate under `libs/` joins the checked set in the same change that creates it.** The bar is only +enforced over the crates named in `.mise/config.coverage.toml`'s `[vars]` -- `native_cov_libs` for everything +`cargo-llvm-cov` instruments directly, `wasm_cov_libs` for the crates that only ever compile to wasm and whose +covmaps come out of the wasm-side tasks. A crate belongs to exactly one of the two. Left out of both, it is +not held to anything: the lane stays green while the newest shared code in the repo is the least covered, and +nothing reports that, because an unnamed crate is indistinguishable from one that does not exist. Adding the +directory name is one word, and it belongs beside the `Cargo.toml` that introduces the crate. + Write the tests with the code, not after it. A new function in `libs/` lands in the same change as the tests that cover it, and a new branch in an existing one lands with the case that takes it. Finding the gap from a red coverage lane instead means the change is already written, reviewed and pushed -- the most expensive moment @@ -1540,6 +1548,28 @@ Use `#[expect(...)]` rather than `#[allow(...)]` (the workspace denies `unfulfil restriction lints whose pattern is intentional in a given spot -- prefer a justified `#[expect(..., reason = "...")]` over contorting the code to dodge the lint. +## `Option` needs sign-off, and a function returning one is almost always wrong + +**Do not reach for `Option` without asking first.** An `Option` usually marks the spot where an error was +thrown away. `None` says "there isn't one" and nothing else: not which input was wrong, not which file was +missing, not which subprocess failed. The caller is then left to invent an explanation, and the one it +invents is a guess -- which is how a missing tool surfaces three layers up as an empty list rather than as +the sentence naming the tool. + +**A function returning `Option` is the sharpest form of this**, because the return type is where the reason +would have gone. `Result` carries it; `Option` discards it at exactly the moment it was known. If a function +can fail, it returns `Result` with an error that says what happened. `Option` is for a value that is +legitimately absent without anything having gone wrong -- and that case is rarer than it looks, which is why +`new-fn-returns-option` rejects a new one and holds the existing returns in an explicit list. A name joining +that list is a decision to record, not a default to fall into. + +The same applies to a field. `Option` for "unset means work it out" is a third state that every reader +has to resolve; a `#[serde(default = "...")]` naming the function that computes the default says the same +thing with two states and no `unwrap_or_else` at the use site. `ModulesConfig::mise_discover` was written +the first way and is now the second. + +Where an `Option` is genuinely right, say so at the site: what absence means, and why it is not a failure. + ## Naming conventions - **`.map_err` wrappers must be named `map_*`.** Extension methods that diff --git a/Cargo.lock b/Cargo.lock index d500d5e9..5a6f8a35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4305,6 +4305,7 @@ dependencies = [ "clap-markdown", "command-error", "edge-toolkit", + "et-org", "et-path", "et-test-helpers", "et-ws-test-server", @@ -4390,6 +4391,10 @@ dependencies = [ "onnx-extractor", ] +[[package]] +name = "et-org" +version = "0.1.0" + [[package]] name = "et-otlp" version = "0.1.0" @@ -4634,6 +4639,7 @@ dependencies = [ name = "et-ws-face-detection" version = "0.1.0" dependencies = [ + "et-org", "et-web", "et-ws-wasm-agent", "js-sys", @@ -4688,6 +4694,7 @@ dependencies = [ name = "et-ws-har1" version = "0.1.0" dependencies = [ + "et-org", "et-web", "et-ws-wasm-agent", "js-sys", @@ -4706,6 +4713,7 @@ dependencies = [ name = "et-ws-llm1" version = "0.1.0" dependencies = [ + "et-org", "et-web", "et-ws-wasm-agent", "js-sys", @@ -4799,6 +4807,7 @@ dependencies = [ "base64 0.23.1", "command-error", "edge-toolkit", + "et-org", "et-otlp", "et-rest-client", "et-ws-runner-common", @@ -5024,6 +5033,7 @@ dependencies = [ "bytemuck", "command-error", "edge-toolkit", + "et-org", "et-otlp", "et-rest-client", "et-test-helpers", @@ -5088,6 +5098,7 @@ dependencies = [ "deno_resolver", "deno_runtime", "edge-toolkit", + "et-org", "et-otlp", "et-rest-client", "et-ws-runner-common", diff --git a/Cargo.toml b/Cargo.toml index 45bd9533..a2d742d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ rust-version = "1.91.1" members = [ "libs/edge-toolkit", "libs/et-otlp", + "libs/org", "libs/path", "libs/test-helpers", "libs/test-otlp", @@ -88,6 +89,7 @@ deno_resolver = "0.85" deno_runtime = { version = "0.262", features = ["transpile", "hmr"] } edge-toolkit = { path = "libs/edge-toolkit", version = "0.2.0" } et-modules-service = { path = "services/modules", version = "0.1.0" } +et-org = { path = "libs/org", version = "0.1.0" } et-otlp = { path = "libs/et-otlp", version = "0.1.0" } et-path = { path = "libs/path", version = "0.1.0" } et-rest-client = { path = "generated/rust-rest", version = "0.1.0", default-features = false } diff --git a/Dockerfile b/Dockerfile index 891c0693..3e1de23f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -376,6 +376,9 @@ EOF ENV MISE_CACHE_DIR=/tmp/mise-cache +# The module this image serves at `/`, named as its own package.json declares it. +ENV MODULES_ROOT="@edge-toolkit/et-ws-server-static" + # Numeric uid, matching the useradd above, so a host inspecting the image resolves it without our passwd file. USER 10001 EXPOSE 8080 8443 diff --git a/Dockerfile.nanoserver b/Dockerfile.nanoserver index 8e81bfe9..37acc6ba 100644 --- a/Dockerfile.nanoserver +++ b/Dockerfile.nanoserver @@ -523,5 +523,9 @@ FROM precompile AS server RUN mise exec -- cargo build --release -p et-ws-server && ` copy target\x86_64-pc-windows-gnullvm\release\et-ws-server.exe C:\mise\bin\ && ` if exist target rmdir /s /q target + +# The module this image serves at `/`, named as its own package.json declares it. +ENV MODULES_ROOT="@edge-toolkit/et-ws-server-static" + EXPOSE 8080 8443 CMD ["et-ws-server"] diff --git a/README.md b/README.md index ab7f6d4e..79e12a01 100644 --- a/README.md +++ b/README.md @@ -237,7 +237,7 @@ Then start the fetch the ONNX models and run the server ```bash mise run download-models mise run build-modules-all -mise run ws-server +mise run hub ``` Scan the QR-Code with a smart-phone camera and open the URL. diff --git a/config/ast-grep/rules/new-fn-returns-option.yaml b/config/ast-grep/rules/new-fn-returns-option.yaml new file mode 100644 index 00000000..72dbb4f5 --- /dev/null +++ b/config/ast-grep/rules/new-fn-returns-option.yaml @@ -0,0 +1,62 @@ +id: new-fn-returns-option +language: Rust +severity: error +message: | + This function returns `Option`, and a new one may not. The return type is where the reason a call failed + would have gone: `Result` carries it, `Option` discards it at the moment it was known. `None` says "there + isn't one" and nothing else -- not which input was wrong, not which file was missing, not which subprocess + failed -- so the caller invents an explanation, and a missing tool surfaces three layers up as an empty + list rather than as the sentence naming the tool. + + Return `Result` with an error that says what happened. `Option` is for a value that is legitimately absent + without anything having gone wrong, which is rarer than it looks. + + The list below is every function that returned `Option` when this rule was written. It is a record of what + is already here, not a pattern to copy: a name joins it only with the operator's sign-off, and the point of + naming them one by one is that adding to it is a decision somebody makes rather than a default anything + falls into. Prefer emptying it -- a function that leaves the list has had its failures given names. +rule: + all: + - kind: function_item + - has: + field: return_type + regex: "^Option<" + - not: + any: + - has: + field: name + regex: ^(agent_id|agent_session|assigned_agent_id|default_connect_ack_timeout)$ + - has: + field: name + regex: ^(default_connection_timeout|extract_speech_event_transcript|find_fn)$ + - has: + field: name + regex: ^(find_http_pyodide_install|find_npm_modules_path_in|find_site_packages_in)$ + - has: + field: name + regex: ^(js_bool_field|js_nested_object|js_number_field|load_stored_agent_id)$ + - has: + field: name + regex: ^(mise_current_tools|mise_npm_modules_path|mise_npm_package_path|mise_where)$ + - has: + field: name + regex: ^(module_package_json|next_message|optional_duration|optional_string)$ + - has: + field: name + regex: ^(origin_slug|otlp|parse_capture_notification|project_repository)$ + - has: + field: name + regex: ^(read_cargo_package|read_package_json|read_package_name)$ + - has: + field: name + regex: ^(read_pyproject_package|resolve_inherited|stored_model|wait_for_agent_id)$ + - has: + field: name + regex: ^(wait_for_marker|wait_for_pointer|wait_for_relayed_span|wait_for_span)$ + - has: + field: name + regex: ^(workspace_dir)$ + # Signatures spread over several lines, which is how they escaped the first sweep for this list. + - has: + field: name + regex: ^(queue_direct|socks5_handshake|wait_for_list_agents_response)$ diff --git a/config/ast-grep/rules/org-identity-lives-in-et-org.yaml b/config/ast-grep/rules/org-identity-lives-in-et-org.yaml new file mode 100644 index 00000000..8701f291 --- /dev/null +++ b/config/ast-grep/rules/org-identity-lives-in-et-org.yaml @@ -0,0 +1,38 @@ +id: org-identity-lives-in-et-org +language: Rust +severity: error +message: | + This constant spells out the organisation name or the crate prefix. Both belong to `et-org`, which is the + one place either literal is written: take the value from `et_org::ORG`, `et_org::CRATE_PREFIX`, + `et_org::NPM_SCOPE`, `et_org::REPOSITORY_URL` or `et_org::IMAGE_REGISTRY`, or build a longer string from + them, rather than writing the name out again here. + + These strings are the same fact wearing different clothes -- the npm scope, the container registry, the + repository URL and the crate prefix all move together if the project is renamed -- so a copy living outside + `et-org` is a copy that gets missed. `et-org` writes the organisation exactly once and assembles the rest + from it, which is what makes that rename a one-line change. + + A `concat!` cannot take a constant, so a call site that has to embed the URL at compile time uses + `et_org::repository_url!()`; `et_org::org!()` is there for the same reason. + + This does not apply to a specific name that merely starts with the prefix. `et-ws-server` names one crate + and `et-path-prefix-probe` names one test fixture; neither is the project's identity, and neither changes + because the organisation did. Only the bare prefix `"et-"` and any string carrying the organisation name + are covered. + + Tests are deliberately not exempt from the definition rule, but asserting a literal is not defining one: a + test that pinned expected output against `et_org::NPM_SCOPE` would keep passing if that constant were + wrong, so those assertions stay spelled out. +rule: + all: + # Quoted: the `: ` in the pattern would otherwise read as a nested YAML mapping. + - pattern: "const $NAME: $TYPE = $VALUE;" + - has: + kind: string_literal + # The organisation anywhere in the value, or the bare crate prefix as the whole of it. + # A longer name that merely opens with the prefix is somebody's actual name, not the project's + # identity, so it is left alone. + regex: 'edge-toolkit|^"et-"$' +ignores: + # The one sanctioned home. Every other file builds on what it exports. + - libs/org/src/lib.rs diff --git a/config/jscpd-baseline.json b/config/jscpd-baseline.json index cc450ede..24cecbcd 100644 --- a/config/jscpd-baseline.json +++ b/config/jscpd-baseline.json @@ -2,7 +2,6 @@ "version": 1, "fingerprints": { "006f44d440c27cda": 1, - "01a4e90eede80cb9": 1, "023513f38a87a1e6": 1, "025e751ba88e4efb": 1, "02c3bb68a9cb1430": 1, @@ -25,29 +24,29 @@ "10b898032781bd2b": 1, "12bdba43f82ce591": 1, "136c57b4180f7238": 1, - "14c932221a47ee8b": 1, "151943582f4d3a1c": 1, "15f330210837288e": 1, "161f873e1c1bf887": 1, "162b7eccbfa52a74": 1, + "16acb47f836cc1f8": 1, "16ff0dbe3f3bf18c": 1, - "1722046a9fb68d8c": 1, "1a07ddf368451c0a": 1, "1a1867cca4e1d4e8": 1, "1a1e1611e4db4656": 1, "1a5d031b56ae4d6c": 1, "1b2ff1d957de5f84": 1, - "1b88564119751a30": 1, "1b9c78cc42fc26b6": 1, "1bd6f5989d1961e5": 1, "1c8cf13e10b67513": 1, + "1cb36741b93ca325": 1, "1ce17b70419170cb": 1, "1e540a5d293c0201": 1, "1ef5e997a2b1f9ca": 1, "1f8c3b477a9abb84": 1, "20149cb81e6e030d": 1, - "204f3cdfa2071c3b": 1, + "20b0690adee22fd3": 1, "220baca8675f61fd": 1, + "22a83dffee6efe5a": 1, "249241d8c264bc63": 1, "24d94784c469f23a": 1, "25bfced4b2a97152": 1, @@ -73,13 +72,13 @@ "389aea3e8662fb9b": 1, "3a075ed2f78fa99e": 1, "3aec07afda461f58": 1, + "3afe9805243f94d2": 1, "3c0efbffa8bb6f9b": 1, "3d7d16d66b899144": 1, "3de59029c7b786b0": 1, - "3e0979bbb8c28ddc": 1, - "3e434b9269573a1c": 1, "3e5aadc3e77078b6": 1, "3e6e51f257b8453d": 1, + "406b435c57117487": 1, "4268c1cf96611f46": 1, "42d9fd6761886c9c": 1, "43336c4ecfeca2a6": 1, @@ -97,10 +96,8 @@ "51d9869c79c352cd": 1, "521c8a862b4eacb3": 1, "524a6a96d3fbfb6b": 1, - "526e1a73331a8724": 1, "52973adcc596dd52": 1, "533bdecb47a87756": 1, - "53a6c52fb08ae691": 1, "54b576854edda79b": 1, "54f0cec8830d66e4": 1, "55a3b3883b483d42": 1, @@ -109,7 +106,6 @@ "57abbbbe99d0f40f": 1, "582ab0d252910ed1": 1, "583f38ee93d4fa73": 1, - "59aaafa19255814f": 1, "5a38489b585cef78": 1, "5a5b6a5ae3b87eea": 1, "5cfb0f0873663a20": 1, @@ -135,10 +131,13 @@ "6ab191050bb8943d": 1, "6b7ef9701aa8a94b": 1, "6bad88d8a503b406": 1, + "6d80cc58c007a238": 1, "6f84b76a00168521": 1, + "70f2dd70bb5a794f": 1, "71c439c66817fc57": 1, "71d4c22b912832b9": 1, "72295cb3aacc3d6c": 1, + "726487c2d82f5f69": 1, "728c97d5724db2c3": 1, "73ffadbd8a93c577": 1, "7530634a13dc6308": 1, @@ -147,7 +146,6 @@ "773fa53d9003b640": 1, "781cc393068d5440": 1, "791347abafa6f64e": 1, - "7928760720f08c6d": 1, "7ac63870d0087e3b": 1, "7aee849465ea91ab": 1, "7ca55da801def6b2": 1, @@ -161,14 +159,14 @@ "8390cc689d64efcb": 1, "84b3a552aa5c21c2": 1, "84e106866c27d5cb": 1, + "8605ad40d87649a2": 1, "87566acb3f2550bf": 1, - "875ab91a8c15e035": 1, "879bfe322b3d676b": 1, "87f1eade46af4d27": 1, "8996a093b7bb3b54": 1, "89bb6ca7b46aa5cd": 1, "8acea4476f04e4a0": 2, - "8bcb05f9d0a15152": 1, + "8b5b2be949793716": 1, "8beb95ee363cd6e3": 1, "8c912ef640675113": 1, "8cfbef0da485b326": 1, @@ -185,9 +183,7 @@ "98f11864904716c0": 1, "99e862abc592b065": 1, "9a8dea5abd17445a": 1, - "9a8fed48470f9c3f": 1, "9b6b5ce6c7b4646c": 1, - "9bee686b5043aedd": 1, "9e835c94361d6917": 1, "a03ab32ade9684de": 1, "a0438c241cc4a042": 1, @@ -196,7 +192,6 @@ "a3111223b027f569": 1, "a3a2b355d83e7c60": 1, "a41e4aadea155813": 1, - "a4cb07095b808af8": 1, "a518b5ae71e6e140": 1, "a52b3254cc78b001": 1, "a56b9fc117032c88": 1, @@ -235,10 +230,8 @@ "c09857b2be5aa4e1": 1, "c10d9673e6b919c8": 1, "c16ccb530445d1a2": 1, - "c1739adb5d43837b": 1, "c1f303564ce2be53": 1, "c304192584ec08b5": 1, - "c3b5adc9a5233cc5": 1, "c479ded037f23e52": 1, "c48839dee3d0d4b9": 1, "c4cbfbcff13bc988": 1, @@ -269,13 +262,14 @@ "d7954652649a6f2c": 1, "d7ad53a1ef1dc59c": 1, "d8a0ad2f6e337e91": 1, + "da812f4e54fefbf1": 1, "da8f06b38d34738e": 1, "dd56a7d95c286b3e": 1, + "ddca47a4caded0fd": 1, "e25456ad9cc7f9f9": 1, "e4b9e7313fc22777": 1, "e517097fd06d5cb5": 1, "e5481f802d5a9f84": 1, - "e6747813e4dffbec": 1, "e67876c933d8a3b6": 1, "e78731e14c433e07": 1, "e797378988ced77b": 1, @@ -288,7 +282,7 @@ "edfb18c73a139873": 1, "eeb29c82a26e0952": 1, "ef69568ef92f1b79": 1, - "f0ecf845ae38d979": 1, + "f05cf69593a671da": 1, "f1f99624666ed51c": 1, "f20e3a3f28708194": 1, "f2167d54ff8b56ea": 1, @@ -297,6 +291,7 @@ "f32c2b14cbf25a04": 1, "f3eba887cab8ffc1": 1, "f4e5cc8c67e3dad7": 1, + "f4ef8dd0f5d3cb89": 1, "f5104a6be7ba929d": 1, "f538dacb09de8e14": 1, "f773b3122d6cb21a": 1, @@ -306,7 +301,6 @@ "f99b1a47383f6735": 1, "f9a76d82860ee95f": 1, "faad1b4efc432727": 1, - "fae2a874410e4fbd": 1, "fb0ad916b8f16dee": 1, "fb78fa42d28ddc83": 1, "fb8747afe01f9463": 1, diff --git a/data/model-modules/model-eye1/pkg/package.json b/data/model-modules/model-eye1/pkg/package.json index 87a83c97..12acef26 100644 --- a/data/model-modules/model-eye1/pkg/package.json +++ b/data/model-modules/model-eye1/pkg/package.json @@ -1,6 +1,6 @@ { "description": "MediaPipe FaceLandmarker model bundle", - "name": "et-model-eye1", + "name": "@edge-toolkit/et-model-eye1", "version": "0.1.0", "files": [ "face_landmarker.task" diff --git a/data/model-modules/model-face1/pkg/package.json b/data/model-modules/model-face1/pkg/package.json index 26c89a47..aa13c0bd 100644 --- a/data/model-modules/model-face1/pkg/package.json +++ b/data/model-modules/model-face1/pkg/package.json @@ -1,6 +1,6 @@ { "description": "Face detection model", - "name": "et-model-face1", + "name": "@edge-toolkit/et-model-face1", "version": "0.1.0", "files": [ "video_cv.onnx" diff --git a/data/model-modules/model-har-motion1/pkg/package.json b/data/model-modules/model-har-motion1/pkg/package.json index 0a9530a8..187b1148 100644 --- a/data/model-modules/model-har-motion1/pkg/package.json +++ b/data/model-modules/model-har-motion1/pkg/package.json @@ -1,6 +1,6 @@ { "description": "HAR motion model", - "name": "et-model-har-motion1", + "name": "@edge-toolkit/et-model-har-motion1", "version": "0.1.0", "files": [ "har-motion1.onnx" diff --git a/data/model-modules/model-llm1/pkg/package.json b/data/model-modules/model-llm1/pkg/package.json index c02a4af7..e3d67188 100644 --- a/data/model-modules/model-llm1/pkg/package.json +++ b/data/model-modules/model-llm1/pkg/package.json @@ -1,7 +1,7 @@ { "description": "SmolLM2-135M-Instruct q4f16 ONNX weights and tokenizer for llm1", "license": "Apache-2.0", - "name": "et-model-llm1", + "name": "@edge-toolkit/et-model-llm1", "version": "0.1.0", "files": [ "config.json", diff --git a/data/model-modules/model-speech1/pkg/package.json b/data/model-modules/model-speech1/pkg/package.json index bd5ac822..9dc8f3f1 100644 --- a/data/model-modules/model-speech1/pkg/package.json +++ b/data/model-modules/model-speech1/pkg/package.json @@ -1,7 +1,7 @@ { "description": "FP16 ONNX speech detection model", "license": "MIT", - "name": "et-model-speech1", + "name": "@edge-toolkit/et-model-speech1", "version": "0.1.0", "files": ["speech1.onnx"] } diff --git a/generated/dart-rest/lib/clients/modules.dart b/generated/dart-rest/lib/clients/modules.dart index b3297d77..6997f011 100644 --- a/generated/dart-rest/lib/clients/modules.dart +++ b/generated/dart-rest/lib/clients/modules.dart @@ -17,10 +17,15 @@ abstract class Modules { /// Fetch a file from a module's bundled static assets. /// - /// `path` is resolved relative to the module's bundle root; an unknown. - /// module or missing file returns 404. + /// `path` is resolved relative to the module's bundle root; an unknown module or missing file returns 404. /// - /// [name] - Module name. + /// Both path parameters can themselves contain `/`. A module is served under the name its `package.json`. + /// declares, which carries an owner scope (`@scope/name`) for anything published to a registry, and `path`. + /// addresses sub-directories of the bundle. A client that percent-encodes each parameter as one path segment. + /// turns those slashes into `%2F` and asks for something no server serves, so build the request path rather. + /// than passing the values through a per-segment encoder. + /// + /// [name] - Module name, as its package.json declares it -- may be scoped. /// /// [path] - Path of the file within the module bundle. @GET('/modules/{name}/{path}') diff --git a/generated/python-rest/et_rest_client/api/modules/get_module_file.py b/generated/python-rest/et_rest_client/api/modules/get_module_file.py index a9d790f6..42af588c 100644 --- a/generated/python-rest/et_rest_client/api/modules/get_module_file.py +++ b/generated/python-rest/et_rest_client/api/modules/get_module_file.py @@ -55,8 +55,18 @@ def sync_detailed( ) -> Response[Any]: """Fetch a file from a module's bundled static assets. - `path` is resolved relative to the module's bundle root; an unknown - module or missing file returns 404. + `path` is resolved relative to the module's bundle root; an unknown module or missing file returns + 404. + + Both path parameters can themselves contain `/`. A module is served under the name its + `package.json` + declares, which carries an owner scope (`@scope/name`) for anything published to a registry, and + `path` + addresses sub-directories of the bundle. A client that percent-encodes each parameter as one path + segment + turns those slashes into `%2F` and asks for something no server serves, so build the request path + rather + than passing the values through a per-segment encoder. Args: name (str): @@ -90,8 +100,18 @@ async def asyncio_detailed( ) -> Response[Any]: """Fetch a file from a module's bundled static assets. - `path` is resolved relative to the module's bundle root; an unknown - module or missing file returns 404. + `path` is resolved relative to the module's bundle root; an unknown module or missing file returns + 404. + + Both path parameters can themselves contain `/`. A module is served under the name its + `package.json` + declares, which carries an owner scope (`@scope/name`) for anything published to a registry, and + `path` + addresses sub-directories of the bundle. A client that percent-encodes each parameter as one path + segment + turns those slashes into `%2F` and asks for something no server serves, so build the request path + rather + than passing the values through a per-segment encoder. Args: name (str): diff --git a/generated/python-rest/pkg/et_rest_client.js b/generated/python-rest/pkg/et_rest_client.js index e5a9e7ef..2dcd08aa 100644 --- a/generated/python-rest/pkg/et_rest_client.js +++ b/generated/python-rest/pkg/et_rest_client.js @@ -1,5 +1,5 @@ // et_rest_client.js — helper exposed to Pyodide-based ws-modules that need -// the generated REST client. Mounted at /modules/et-rest-client/ by +// the generated REST client. Mounted at /modules/@edge-toolkit/et-rest-client/ by // et-modules-service (each consumer declares `et-rest-client = "*"` in // `[tool.ws-module.dependencies]`). // @@ -8,9 +8,11 @@ // `--py3-none-any.whl` convention so version bumps don't // require touching every consumer. +// skipcq: JS-0833 -- committed ES module; the analyzer's script-mode parse is a false positive export async function installWheel(pyodide) { const pkg = await fetch(new URL("package.json", import.meta.url)).then((r) => r.json()); - const wheel = `${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; + const distribution = pkg.name.split("/").pop(); + const wheel = `${distribution.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; const bytes = new Uint8Array(await fetch(new URL(wheel, import.meta.url)).then((r) => r.arrayBuffer())); pyodide.FS.writeFile(`/tmp/${wheel}`, bytes); pyodide.runPython(`import sys\nsys.path.insert(0, "/tmp/${wheel}")`); diff --git a/generated/python-ws/pkg/et_ws.js b/generated/python-ws/pkg/et_ws.js index 4526737d..a4be6272 100644 --- a/generated/python-ws/pkg/et_ws.js +++ b/generated/python-ws/pkg/et_ws.js @@ -1,5 +1,5 @@ // et_ws.js — helper exposed to Pyodide-based ws-modules that need the -// generated Pydantic models. Mounted at /modules/et-ws/ by et-modules-service +// generated Pydantic models. Mounted at /modules/@edge-toolkit/et-ws/ by et-modules-service // (each consumer declares `et-ws = "*"` in `[tool.ws-module.dependencies]`). // // The wheel ships next to this file in pkg/. Consumers import `installWheel` @@ -7,9 +7,11 @@ // `--py3-none-any.whl` convention so version bumps don't // require touching every consumer. +// skipcq: JS-0833 -- committed ES module; the analyzer's script-mode parse is a false positive export async function installWheel(pyodide) { const pkg = await fetch(new URL("package.json", import.meta.url)).then((r) => r.json()); - const wheel = `${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; + const distribution = pkg.name.split("/").pop(); + const wheel = `${distribution.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; const bytes = new Uint8Array(await fetch(new URL(wheel, import.meta.url)).then((r) => r.arrayBuffer())); pyodide.FS.writeFile(`/tmp/${wheel}`, bytes); pyodide.runPython(`import sys\nsys.path.insert(0, "/tmp/${wheel}")`); diff --git a/generated/rust-rest/src/lib.rs b/generated/rust-rest/src/lib.rs index 013dbb26..d21a574d 100644 --- a/generated/rust-rest/src/lib.rs +++ b/generated/rust-rest/src/lib.rs @@ -258,13 +258,18 @@ impl Client { } /**Fetch a file from a module's bundled static assets - `path` is resolved relative to the module's bundle root; an unknown - module or missing file returns 404. + `path` is resolved relative to the module's bundle root; an unknown module or missing file returns 404. + + Both path parameters can themselves contain `/`. A module is served under the name its `package.json` + declares, which carries an owner scope (`@scope/name`) for anything published to a registry, and `path` + addresses sub-directories of the bundle. A client that percent-encodes each parameter as one path segment + turns those slashes into `%2F` and asks for something no server serves, so build the request path rather + than passing the values through a per-segment encoder. Sends a `GET` request to `/modules/{name}/{path}` Arguments: - - `name`: Module name + - `name`: Module name, as its package.json declares it -- may be scoped - `path`: Path of the file within the module bundle */ pub async fn get_module_file<'a>( diff --git a/generated/specs/rest.yaml b/generated/specs/rest.yaml index 23034c96..a14fb89f 100644 --- a/generated/specs/rest.yaml +++ b/generated/specs/rest.yaml @@ -48,13 +48,18 @@ paths: - modules summary: Fetch a file from a module's bundled static assets. description: |- - `path` is resolved relative to the module's bundle root; an unknown - module or missing file returns 404. + `path` is resolved relative to the module's bundle root; an unknown module or missing file returns 404. + + Both path parameters can themselves contain `/`. A module is served under the name its `package.json` + declares, which carries an owner scope (`@scope/name`) for anything published to a registry, and `path` + addresses sub-directories of the bundle. A client that percent-encodes each parameter as one path segment + turns those slashes into `%2F` and asks for something no server serves, so build the request path rather + than passing the values through a per-segment encoder. operationId: get_module_file parameters: - in: path name: name - description: Module name + description: Module name, as its package.json declares it -- may be scoped required: true schema: type: string diff --git a/generated/zig-rest/src/et_rest_client.zig b/generated/zig-rest/src/et_rest_client.zig index f0c993f1..50b22472 100644 --- a/generated/zig-rest/src/et_rest_client.zig +++ b/generated/zig-rest/src/et_rest_client.zig @@ -587,8 +587,13 @@ pub fn head_fileRaw(client: *Client, agent_id: []const u8, filename: []const u8) // Fetch a file from a module's bundled static assets. // // Description: -// `path` is resolved relative to the module's bundle root; an unknown -// module or missing file returns 404. +// `path` is resolved relative to the module's bundle root; an unknown module or missing file returns 404. +// +// Both path parameters can themselves contain `/`. A module is served under the name its `package.json` +// declares, which carries an owner scope (`@scope/name`) for anything published to a registry, and `path` +// addresses sub-directories of the bundle. A client that percent-encodes each parameter as one path segment +// turns those slashes into `%2F` and asks for something no server serves, so build the request path rather +// than passing the values through a per-segment encoder. // pub fn get_module_file(client: *Client, name: []const u8, path: []const u8) !void { var raw = try get_module_fileRaw(client, name, path); diff --git a/libs/edge-toolkit/src/config.rs b/libs/edge-toolkit/src/config.rs index af638195..f6ec0863 100644 --- a/libs/edge-toolkit/src/config.rs +++ b/libs/edge-toolkit/src/config.rs @@ -311,6 +311,53 @@ pub fn mise_where(tool: &str) -> Option { path.is_dir().then_some(path) } +/// Module directories for every npm package the mise config active in the current directory declares. +/// +/// This is what lets a deployment say which modules it serves exactly once, in its `[tools]` table, instead +/// of repeating the list as paths it cannot know: mise reports where it put each package, and the layout +/// probing below turns that into the directory holding the `package.json`. +/// +/// Scoped to `--current`, so it answers with the tools of the config in scope rather than everything ever +/// installed on the machine -- two deployments on one host each see their own modules. It follows that the +/// process has to run where that config applies, which for a generated deployment is the directory holding +/// its `mise.toml`. +/// +/// An empty list is returned rather than an error for every failure: mise absent, a config that declares no +/// npm tools, output that will not parse. A deployment that expected modules gets the same outcome as one +/// that asked for none, which the caller reports as a module that cannot be found. +#[must_use] +pub fn mise_staged_module_dirs() -> Vec { + let Some(payload) = mise_current_tools() else { + return Vec::new(); + }; + staged_module_dirs_from_tool_list(&payload) +} + +/// The module directories named by a `mise ls --current --json` payload. +/// +/// Split from [`mise_staged_module_dirs`] so the shape of the payload can be exercised without a `mise` to +/// produce it. Every failure is a silent omission -- output that will not parse, a listing that is not an +/// object, a tool of some other backend, an entry with no version or no install path, and an install whose +/// directory holds the package in no layout this knows. A deployment that expected a module gets the same +/// outcome as one that never asked for it, which the caller reports as a module it cannot find. +#[must_use] +pub fn staged_module_dirs_from_tool_list(tool_list_json: &[u8]) -> Vec { + let Ok(listed) = serde_json::from_slice::(tool_list_json) else { + return Vec::new(); + }; + let Some(tools) = listed.as_object() else { + return Vec::new(); + }; + tools + .iter() + .filter_map(|(tool, versions)| { + let package = tool.strip_prefix("npm:")?; + let install = versions.get(0)?.get("install_path")?.as_str()?; + find_npm_modules_path_in(Path::new(install), package).map(|dir| dir.join(package)) + }) + .collect() +} + /// Returns the directory containing `` for an `npm:` mise install. /// /// I.e. the `node_modules` directory you'd point `MODULES_PATHS` at. Calls @@ -387,15 +434,23 @@ pub fn mise_python_site_packages() -> Vec { if !mise_is_available() { return Vec::new(); } - // `output_checked` errors on both a spawn failure and a non-zero exit, so the `mise ls` best-effort - // path collapses to a single fallible call -- no separate `status.success()` filter. - let Ok(output) = std::process::Command::new("mise") - .args(["ls", "--current", "--json"]) - .output_checked() - else { + let Some(listed) = mise_current_tools() else { return Vec::new(); }; - site_packages_from_tool_list(&output.stdout) + site_packages_from_tool_list(&listed) +} + +/// What `mise` reports installed for the config in scope, as the raw `mise ls --current --json` payload. +/// +/// Shared by the callers that pick different tools out of the same listing, so the subprocess and its +/// failure handling are written once. `output_checked` errors on both a spawn failure and a non-zero exit, +/// so a best-effort caller collapses to one fallible call with no separate `status.success()` filter. +fn mise_current_tools() -> Option> { + let output = std::process::Command::new("mise") + .args(["ls", "--current", "--json"]) + .output_checked() + .ok()?; + Some(output.stdout) } /// The `site-packages` directories named by a `mise ls --current --json` payload. diff --git a/libs/edge-toolkit/tests/staged_module_dirs.rs b/libs/edge-toolkit/tests/staged_module_dirs.rs new file mode 100644 index 00000000..fcbd94c3 --- /dev/null +++ b/libs/edge-toolkit/tests/staged_module_dirs.rs @@ -0,0 +1,93 @@ +//! Turning a `mise ls --current --json` payload into the module directories the hub serves. +//! +//! Every rejection here is silent by design: a deployment names its modules as `[tools]`, and one that is missing, of +//! another backend, or laid out in a way this does not recognise simply contributes nothing. The hub then reports it +//! as a module it cannot find, which is the diagnosis the operator needs -- as opposed to the server refusing to start +//! because one entry in a tool listing was not what it expected. +#![cfg(test)] + +use edge_toolkit::config::{mise_staged_module_dirs, staged_module_dirs_from_tool_list}; +use fs_err as fs; +use tempfile::TempDir; + +/// Build an npm install whose package really sits at `/node_modules/`. +fn staged_install(root: &TempDir, install: &str, package: &str) -> std::path::PathBuf { + let install_dir = root.path().join(install); + fs::create_dir_all(install_dir.join("node_modules").join(package)).unwrap(); + install_dir +} + +#[test] +fn a_tool_list_that_is_not_json_yields_no_directories() { + // An older mise, a wrapper that printed a warning first, a truncated pipe. The hub has to come away with nothing + // rather than fail to start, so it serves what it can and says what is missing. + assert!(staged_module_dirs_from_tool_list(b"mise: not a tool list").is_empty()); + assert!( + staged_module_dirs_from_tool_list(b"").is_empty(), + "an empty body is not valid JSON either" + ); +} + +#[test] +fn a_listing_that_is_not_an_object_yields_no_directories() { + // Valid JSON, wrong shape: mise answers with a map of tool id to versions, and anything else is a version of mise + // this does not know how to read. + assert!(staged_module_dirs_from_tool_list(b"[]").is_empty()); + assert!(staged_module_dirs_from_tool_list(b"\"a string\"").is_empty()); +} + +#[test] +fn only_npm_tools_whose_package_is_present_contribute_directories() { + // A mixed tool set, which is what a real config produces. Only the npm tool whose install actually holds the + // package contributes: a tool of another backend is not a module, and an npm install missing its package would put + // a directory with no `package.json` in front of the module scan. + let root = TempDir::new().unwrap(); + let present = staged_install(&root, "math1-install", "et-ws-math1"); + let missing = root.path().join("bare-install"); + fs::create_dir_all(&missing).unwrap(); + + let tool_list = serde_json::json!({ + "npm:et-ws-math1": [{ "install_path": present }], + "npm:et-ws-absent": [{ "install_path": missing }], + "cargo:et-ws-server": [{ "install_path": present }], + }); + let found = staged_module_dirs_from_tool_list(&serde_json::to_vec(&tool_list).unwrap()); + + assert_eq!(found, vec![present.join("node_modules").join("et-ws-math1")]); +} + +#[test] +fn a_scoped_package_resolves_to_its_own_directory() { + // The scope is part of the path, so the package dir is `node_modules/@scope/name` -- the scope directory itself + // holds no `package.json` and would be served as nothing. + let root = TempDir::new().unwrap(); + let install = staged_install(&root, "scoped-install", "@edge-toolkit/et-ws-math1"); + + let tool_list = serde_json::json!({ "npm:@edge-toolkit/et-ws-math1": [{ "install_path": install }] }); + let found = staged_module_dirs_from_tool_list(&serde_json::to_vec(&tool_list).unwrap()); + + assert_eq!( + found, + vec![install.join("node_modules").join("@edge-toolkit/et-ws-math1")] + ); +} + +#[test] +fn an_entry_with_no_version_or_no_install_path_is_skipped() { + // Both shapes mise can produce for a tool it knows of but has not installed. + let tool_list = serde_json::json!({ + "npm:et-ws-never-installed": [], + "npm:et-ws-no-path": [{ "active": true }], + "npm:et-ws-path-not-a-string": [{ "install_path": 42_i32 }], + }); + + assert!(staged_module_dirs_from_tool_list(&serde_json::to_vec(&tool_list).unwrap()).is_empty()); +} + +#[test] +fn no_mise_to_ask_contributes_no_directories() { + // The other half of the lookup: with no `mise` to run there is no listing to interpret, and a deployment that + // staged nothing has to reach the same empty answer as one whose listing named nothing. An empty PATH hides the + // binary from the spawn, and `with_empty_path` puts PATH back so sibling tests in this binary still find it. + assert!(et_test_helpers::with_empty_path(mise_staged_module_dirs).is_empty()); +} diff --git a/libs/org/Cargo.toml b/libs/org/Cargo.toml new file mode 100644 index 00000000..698bb849 --- /dev/null +++ b/libs/org/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "et-org" +publish = true +description = "The organisation name and crate prefix this project publishes under" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +doctest = false + +[lints] +workspace = true diff --git a/libs/org/src/lib.rs b/libs/org/src/lib.rs new file mode 100644 index 00000000..834024a0 --- /dev/null +++ b/libs/org/src/lib.rs @@ -0,0 +1,70 @@ +//! Who this project is, as the strings that carry its identity into registries, images and manifests. +//! +//! Everything here is one of two facts -- the organisation the project publishes under, and the prefix its +//! own crates and modules carry -- or a name built from one of them. They are collected into a crate of +//! their own because they are the same fact wearing different clothes: the npm scope, the container +//! registry, the repository URL and the crate prefix all move together if the project is ever renamed, and +//! a copy of any of them living somewhere else is a copy that gets missed. `org-identity-lives-in-et-org` +//! keeps them here, so a new spelling has to be built from these rather than written out again. +//! +//! The organisation name itself is written exactly once, in [`ORG`]'s macro, and every longer string is +//! assembled from it. That is what makes the rule enforceable rather than aspirational: there is a single +//! literal to protect. + +/// The organisation name as a literal, for the constants below to build on. +/// +/// A macro rather than a `const` because `concat!` joins literals at compile time and cannot take one: a +/// `const` would have to be spelled out again in every string that embeds it, which is the duplication this +/// crate exists to prevent. +#[macro_export] +macro_rules! org { + () => { + "edge-toolkit" + }; +} + +/// The organisation this project publishes under. +pub const ORG: &str = org!(); + +/// The prefix every crate and served module of this project's own carries. +/// +/// What separates a module of ours from a third-party one in a dependency list: ours is published under the +/// owner scope and has to be named as the registry knows it, whereas somebody else's is already published +/// under exactly the name it is declared by. +pub const CRATE_PREFIX: &str = "et-"; + +/// The npm scope as a literal, for a `concat!` that has to embed it. +/// +/// [`NPM_SCOPE`] is the constant to prefer. This exists because `concat!` takes literals and not constants, +/// so a module building one of its own URLs at compile time -- `/modules//...` -- cannot use +/// one, and writing the scope out at that call site is what this crate exists to stop. +#[macro_export] +macro_rules! npm_scope { + () => { + concat!("@", $crate::org!(), "/") + }; +} + +/// The npm scope this project's packages are published under, including the trailing separator. +/// +/// GitHub Packages accepts nothing else -- a package not scoped to the repository owner is rejected -- and +/// the name it accepts is the name the hub serves the module under, so it reaches URLs too. +pub const NPM_SCOPE: &str = npm_scope!(); + +/// This repository's URL as a literal, for a `concat!` that has to embed it. +/// +/// [`REPOSITORY_URL`] is the constant to prefer. This exists because `concat!` takes literals and not +/// constants, so a generated file that builds a line around the URL at compile time cannot use one -- and +/// writing the URL out at that call site is exactly what this crate exists to stop. +#[macro_export] +macro_rules! repository_url { + () => { + concat!("https://github.com/", $crate::org!(), "/core") + }; +} + +/// This repository, as the URL that identifies it to anything reading a published artifact. +pub const REPOSITORY_URL: &str = repository_url!(); + +/// The container registry this project's own images are published under. +pub const IMAGE_REGISTRY: &str = concat!("ghcr.io/", org!(), "/core"); diff --git a/libs/org/tests/identity.rs b/libs/org/tests/identity.rs new file mode 100644 index 00000000..cfbebb9a --- /dev/null +++ b/libs/org/tests/identity.rs @@ -0,0 +1,33 @@ +//! The assembled identity strings, pinned to the exact text registries and manifests are matched against. +//! +//! Worth asserting despite being constants: each is built by `concat!` from one literal, so a missing +//! separator or a stray segment is a compile-time success and a runtime mismatch -- a scope without its +//! trailing slash still compiles, and then every package name built from it is wrong. +#![cfg(test)] + +use et_org::{CRATE_PREFIX, IMAGE_REGISTRY, NPM_SCOPE, ORG, REPOSITORY_URL}; + +#[test] +fn the_identity_strings_are_what_registries_are_matched_against() { + assert_eq!(ORG, "edge-toolkit"); + assert_eq!(CRATE_PREFIX, "et-"); + assert_eq!(NPM_SCOPE, "@edge-toolkit/"); + assert_eq!(REPOSITORY_URL, "https://github.com/edge-toolkit/core"); + assert_eq!(IMAGE_REGISTRY, "ghcr.io/edge-toolkit/core"); +} + +#[test] +fn every_assembled_string_carries_the_organisation_it_was_built_from() { + // The point of assembling them: renaming the organisation has to move all of these together, and a + // constant that stopped containing it would be one that had been written out by hand again. + for assembled in [NPM_SCOPE, REPOSITORY_URL, IMAGE_REGISTRY] { + assert!(assembled.contains(ORG), "{assembled} does not carry {ORG}"); + } +} + +#[test] +fn the_scope_ends_with_its_separator_so_a_package_name_can_follow_it() { + // `NPM_SCOPE` is concatenated directly onto a bare module name, so the separator has to be part of it. + assert!(NPM_SCOPE.ends_with('/')); + assert_eq!(format!("{NPM_SCOPE}et-ws-math1"), "@edge-toolkit/et-ws-math1"); +} diff --git a/libs/ws-runner-common/src/lib.rs b/libs/ws-runner-common/src/lib.rs index 78d83238..b51d2f18 100644 --- a/libs/ws-runner-common/src/lib.rs +++ b/libs/ws-runner-common/src/lib.rs @@ -12,6 +12,7 @@ use std::time::{Duration, SystemTime}; use edge_toolkit::ws::{ClientMessage, ConnectStatus, ServerMessage}; +use et_rest_client::ClientInfo as _; use futures_util::{SinkExt as _, StreamExt as _}; use retry_policies::policies::ExponentialBackoff; use retry_policies::{RetryDecision, RetryPolicy}; @@ -354,6 +355,35 @@ async fn fetch_package_json_bytes( reason = "the single attempt fetch_package_json_bytes retries; separate so the retry loop stays readable" )] async fn try_fetch_package_json(client: &et_rest_client::Client, module_name: &str) -> Result, BootstrapError> { - let response = client.get_module_file(module_name, "package.json").await?; - collect_byte_stream(response.into_inner()).await + fetch_module_file(client, module_name, "package.json").await +} + +/// Fetch one file from the directory a module is served out of. +/// +/// Built on the raw client rather than the typed `get_module_file`, which percent-encodes each argument as a +/// single path segment. A module is served under the name it publishes under, so the name carries the owner +/// scope and holds a `/` and an `@`; encoded they become `%2F` and `%40`, and the request then asks for a +/// module no hub serves -- which the hub answers with a 400 rather than the file. The same reason keeps the +/// web runner's own module loader off the typed call. +pub async fn fetch_module_file( + client: &et_rest_client::Client, + module_name: &str, + path: &str, +) -> Result, BootstrapError> { + let url = format!("{}/modules/{module_name}/{path}", client.baseurl()); + let response = client.client().get(url).send().await?.error_for_status()?; + Ok(response.bytes().await?.to_vec()) +} + +/// Whether a [`fetch_module_file`] failure was the hub saying it serves no such file. +/// +/// A caller for whom absence is an answer rather than a fault needs to tell the two apart, and going off the +/// typed client means there is no generated `ErrorResponse` variant to match on -- the status is what is +/// left. Anything else, including a hub that could not be reached at all, stays an error. +#[must_use] +pub fn is_module_file_missing(err: &BootstrapError) -> bool { + let BootstrapError::Stream(err) = err else { + return false; + }; + err.status().is_some_and(|status| status.as_u16() == 404) } diff --git a/libs/ws-runner-common/tests/fetch_main_field.rs b/libs/ws-runner-common/tests/fetch_main_field.rs index 102184de..894b29a4 100644 --- a/libs/ws-runner-common/tests/fetch_main_field.rs +++ b/libs/ws-runner-common/tests/fetch_main_field.rs @@ -92,8 +92,10 @@ async fn fetch_main_field_gives_up_once_the_wait_window_is_spent() { .await .unwrap_err(); + // `Stream` rather than `Rest`: the fetch goes through the raw client, because the typed one encodes its + // arguments as single path segments and a module name carries the scope it publishes under. assert!( - matches!(&error, BootstrapError::Rest(_)), + matches!(&error, BootstrapError::Stream(_)), "expected the last fetch failure to be reported, got: {error:?}" ); } diff --git a/libs/ws-runner-common/tests/module_file_missing.rs b/libs/ws-runner-common/tests/module_file_missing.rs new file mode 100644 index 00000000..ce4e6329 --- /dev/null +++ b/libs/ws-runner-common/tests/module_file_missing.rs @@ -0,0 +1,94 @@ +//! Covers `is_module_file_missing`, which decides whether a failed fetch was an answer or a fault. +//! +//! A caller asking the hub for an optional asset treats "no such file" as a result and everything else as an error, +//! so the three outcomes are pinned here: the hub answering 404, the hub answering something else, and a failure that +//! never reached a hub at all. Getting the last two wrong turns a broken deployment into a silent empty value, which +//! surfaces far from the cause. +#![cfg(test)] + +use std::io::{Read as _, Write as _}; + +use et_test_helpers::reserve_port; +use et_ws_runner_common::{BootstrapError, fetch_module_file, is_module_file_missing}; + +/// Answer one request on `port` with `status`, then stop. +/// +/// Deliberately not an HTTP library: the point is to choose the status of a single response, which a few bytes down a +/// `TcpStream` does without pulling a server framework into a low-level test. +#[expect( + clippy::single_call_fn, + reason = "the stub hub is its own step; keeping it out of fetch_failure leaves that reading as the request" +)] +fn hub_answering(port: u16, status: &'static str) -> std::thread::JoinHandle<()> { + let listener = std::net::TcpListener::bind(("127.0.0.1", port)).unwrap(); + std::thread::spawn(move || { + if let Some(Ok(mut stream)) = listener.incoming().next() { + // Read only so the client can finish sending; nothing here inspects the request. + let mut request = [0_u8; 1024]; + let _read = stream.read(&mut request); + let response = format!("HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); + let _written = stream.write_all(response.as_bytes()); + let _flushed = stream.flush(); + } + }) +} + +/// The error from asking a hub on `port` for a file, with `port` answering `status`. +async fn fetch_failure(status: &'static str) -> BootstrapError { + let port = reserve_port(); + let hub = hub_answering(port, status); + + let error = fetch_module_file( + &et_rest_client::Client::new(&format!("http://127.0.0.1:{port}")), + "@edge-toolkit/et-ws-math1", + "mnist-12.onnx", + ) + .await + .unwrap_err(); + + hub.join().unwrap(); + error +} + +#[tokio::test] +async fn a_404_from_the_hub_is_the_file_being_absent() { + assert!(is_module_file_missing(&fetch_failure("404 Not Found").await)); +} + +#[tokio::test] +async fn another_status_from_the_hub_stays_an_error() { + // A hub that answered at all but refused is a fault, not an absence: treating it as "no such file" would hand the + // caller an empty value for a module the hub does serve. + assert!(!is_module_file_missing( + &fetch_failure("500 Internal Server Error").await + )); +} + +#[tokio::test] +async fn a_hub_that_was_never_reached_stays_an_error() { + // Nothing is listening, so the failure carries no status at all -- the request never got far enough to be answered. + // That is the case where reading absence into the error would hide an unreachable hub. + let port = reserve_port(); + + let error = fetch_module_file( + &et_rest_client::Client::new(&format!("http://127.0.0.1:{port}")), + "@edge-toolkit/et-ws-math1", + "package.json", + ) + .await + .unwrap_err(); + + assert!(matches!(&error, BootstrapError::Stream(_)), "got: {error:?}"); + assert!(!is_module_file_missing(&error)); +} + +#[test] +fn a_failure_that_is_not_a_request_at_all_stays_an_error() { + // The remaining arm: a module whose `package.json` parsed but named no entry point never involved a status, so + // there is nothing for the 404 test to read. + let error = BootstrapError::PackageJsonMissingMain { + module: "@edge-toolkit/et-ws-math1".to_string(), + }; + + assert!(!is_module_file_missing(&error)); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed27efbd..6d119f3d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,8 +20,8 @@ importers: services/ws-server/static: dependencies: - et-ws-wasm-agent: - specifier: link:../../ws-wasm-agent/pkg + '@edge-toolkit/et-ws-wasm-agent': + specifier: workspace:* version: link:../../ws-wasm-agent/pkg onnxruntime-web: specifier: '*' @@ -30,6 +30,8 @@ importers: specifier: '*' version: 4.2.3 + services/ws-wasm-agent/pkg: {} + packages: '@aws-sdk/checksums@3.1000.26': diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 191ddcd5..6005f457 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,6 +7,12 @@ # Some of their deps are loaded via mise installed tools, which are checked separately. packages: - services/ws-server/static + # The one generated `pkg/` that is a member, because `static` depends on it and is published. + # A `link:` specifier ships verbatim and means nothing to whoever installs the published package, so the + # dependency is declared `workspace:*` instead -- which pnpm resolves to this directory locally and rewrites + # to a real version on publish. Being a member is what makes that specifier resolvable. It adds nothing to + # the scanned lockfile: the agent's own `package.json` declares no dependencies. + - services/ws-wasm-agent/pkg # js-data1 is a real workspace member, not a served pkg/ dir. # It declares published npm deps (@aws-sdk/client-s3, esbuild) that must land in the root lockfile for # osv-scanner; its bundled output under pkg/ is generated, not a member. diff --git a/services/modules/src/lib.rs b/services/modules/src/lib.rs index 5aba6679..8dcd9c70 100644 --- a/services/modules/src/lib.rs +++ b/services/modules/src/lib.rs @@ -19,28 +19,62 @@ pub use self::routes::list_modules_handler; pub struct ModulesConfig { #[serde(default = "default_modules_folders")] pub paths: Vec, - #[serde_inline_default(String::from("et-ws-server-static"))] + /// Name of the module served at `/`, exactly as its `package.json` declares it. + /// + /// No default, because which module is a deployment's front page is a property of that deployment and + /// not of this server: a name defaulted here would be one project's, and every other one would be + /// carrying it around as dead configuration. Unset serves nothing at `/` and is not an error -- a + /// deployment whose agents are headless runners has no page to put there, and demanding one would make + /// every such deployment name a module it never loads. + #[serde(default)] pub root: String, + /// Also serve whatever the mise config in scope staged, on top of `paths`. + /// + /// A deployment that installs its modules as `[tools]` has already said which ones it serves. Repeating + /// that as a list of directories would be the same set written twice, in a form nothing can write down -- + /// where mise puts a package is decided per backend and platform when it installs. + /// + /// Defaults to whether mise is there to ask, because mise being on `PATH` is exactly what makes a staged + /// module findable: a deployment that provisioned its modules some other way has no mise to consult and + /// gets nothing extra, and one that did needs to declare nothing. Set it to `false` to serve only + /// `paths` on a host that does have mise -- a config whose tool set mixes modules with development + /// tooling wants that, since discovery cannot tell one from the other. + #[serde(default = "edge_toolkit::config::mise_is_available")] + pub mise_discover: bool, } impl ModulesConfig { + /// Config that serves exactly `paths`, which is what a caller naming directories outright wants. #[must_use] pub const fn new(paths: Vec, root: String) -> Self { - Self { paths, root } + Self { + paths, + root, + mise_discover: false, + } } } +/// The package name a `package.json` declares, exactly as written. fn read_package_name(package_json: &std::path::Path) -> Option { let content = fs::read_to_string(package_json).ok()?; let value: serde_json::Value = serde_json::from_str(&content).ok()?; - value.get("name")?.as_str().map(str::to_string) + Some(value.get("name")?.as_str()?.to_string()) } /// Scan all configured module paths and return a sorted list of `(name, pkg_dir)` pairs. #[must_use] pub fn list_modules(config: &ModulesConfig) -> Vec<(String, PathBuf)> { let mut modules: Vec<(String, PathBuf)> = Vec::new(); - for path in &config.paths { + let mut paths = config.paths.clone(); + if config.mise_discover { + paths.extend(edge_toolkit::config::mise_staged_module_dirs()); + // A directory reached both ways would otherwise be served under two routes, which is a startup + // error rather than a duplicate listing. + paths.sort(); + paths.dedup(); + } + for path in &paths { let pkg_dir = path.join("pkg"); if pkg_dir.is_dir() { let name = read_package_name(&pkg_dir.join("package.json")) @@ -60,7 +94,7 @@ pub fn list_modules(config: &ModulesConfig) -> Vec<(String, PathBuf)> { // backend lays out `node_modules/.aube/node_modules/` as a symlink farm, so the symlink-following // variant is required to discover those packages. let entry_path = entry.path(); - if entry_path.is_dir() && !config.paths.contains(&entry_path) { + if entry_path.is_dir() && !paths.contains(&entry_path) { let pkg_dir = entry_path.join("pkg"); if pkg_dir.is_dir() { let name = read_package_name(&pkg_dir.join("package.json")) @@ -89,25 +123,35 @@ pub fn list_modules(config: &ModulesConfig) -> Vec<(String, PathBuf)> { /// Register `GET /modules/` (JSON list), `GET /modules/{name}/...` (static files), and `GET /` (root module). /// +/// An unset `config.root` serves nothing at `/`, leaving the module routes above as the whole surface. +/// /// # Panics -/// Panics if no discovered module is named `config.root` -- server config is fatal early so the operator sees the -/// misconfiguration at startup. +/// Panics if `config.root` names a module none of the scanned `config.paths` provides -- naming a front page +/// that isn't there is a config error, and it is fatal early so the operator sees it at startup rather than +/// as a 404 much later. #[expect( clippy::panic, - reason = "missing root module is a config error; failing fast at startup is intentional" + reason = "a root module named but absent is a config error; failing fast at startup is intentional" )] pub fn configure(cfg: &mut web::ServiceConfig, config: &ModulesConfig) { let modules = list_modules(config); - let root_module_dir = modules.iter().find(|(name, _)| name == &config.root).map_or_else( - || panic!("Root module '{}' not found", config.root), - |(_, path)| path.clone(), - ); - let _routed = cfg.route("/modules/", web::get().to(list_modules_handler)); for (name, pkg_dir) in &modules { let _served = cfg.service(Files::new(&format!("/modules/{name}"), pkg_dir)); } + + if config.root.is_empty() { + return; + } + let root_module_dir = modules.iter().find(|(name, _)| name == &config.root).map_or_else( + || { + let served: Vec<&str> = modules.iter().map(|(name, _)| name.as_str()).collect(); + panic!("Root module '{}' not found; serving {served:?}", config.root) + }, + |(_, path)| path.clone(), + ); + // Registered last: `Files::new("/")` matches everything, so anything mounted after it is unreachable. let _root_served = cfg.service( Files::new("/", root_module_dir) .index_file("index.html") diff --git a/services/modules/src/routes.rs b/services/modules/src/routes.rs index 4424ebd1..00c91c79 100644 --- a/services/modules/src/routes.rs +++ b/services/modules/src/routes.rs @@ -40,15 +40,20 @@ pub async fn list_modules_handler(config: web::Data) -> HttpRespo /// Fetch a file from a module's bundled static assets. /// -/// `path` is resolved relative to the module's bundle root; an unknown -/// module or missing file returns 404. +/// `path` is resolved relative to the module's bundle root; an unknown module or missing file returns 404. +/// +/// Both path parameters can themselves contain `/`. A module is served under the name its `package.json` +/// declares, which carries an owner scope (`@scope/name`) for anything published to a registry, and `path` +/// addresses sub-directories of the bundle. A client that percent-encodes each parameter as one path segment +/// turns those slashes into `%2F` and asks for something no server serves, so build the request path rather +/// than passing the values through a per-segment encoder. #[cfg(feature = "openapi-spec")] #[utoipa::path( get, path = "/modules/{name}/{path}", tag = "modules", params( - ("name" = String, Path, description = "Module name"), + ("name" = String, Path, description = "Module name, as its package.json declares it -- may be scoped"), ("path" = String, Path, description = "Path of the file within the module bundle") ), responses( diff --git a/services/modules/tests/api_modules.rs b/services/modules/tests/api_modules.rs index 98fa81a4..0f9d7a43 100644 --- a/services/modules/tests/api_modules.rs +++ b/services/modules/tests/api_modules.rs @@ -7,7 +7,11 @@ use et_modules_service::{ModulesConfig, configure}; #[actix_rt::test] async fn list_modules_api() { - let config = ModulesConfig::default(); + // The default search paths, but the root named outright: the server has no default for which module is + // a deployment's front page, so every caller says which one it means. + let mut config = ModulesConfig::default(); + config.root = "@edge-toolkit/et-ws-server-static".to_string(); + let config = config; let app = test::init_service( App::new() .app_data(web::Data::new(AgentRegistry::<()>::default())) @@ -19,22 +23,25 @@ async fn list_modules_api() { let req = test::TestRequest::get().uri("/modules/").to_request(); let resp: Vec = test::call_and_read_body_json(&app, req).await; - // Modules whose `pkg/` is built by an always-loaded task (et-ws-wasm-agent - // in the base config) or shipped as a static directory (et-ws-server-static - // and the rclone-downloaded model). These are unconditionally expected. - assert!(resp.contains(&"et-ws-server-static".to_string())); - assert!(resp.contains(&"et-ws-wasm-agent".to_string())); - assert!(resp.contains(&"et-model-har-motion1".to_string())); + // Modules whose `pkg/` is built by an always-loaded task (the wasm agent in + // the base config) or shipped as a static directory (the page module and the + // rclone-downloaded model). These are unconditionally expected. + // + // Named with the owner scope because that is what their `package.json` declares and what the server + // serves: it reshapes no name, so what a registry would accept is what a request has to ask for. + assert!(resp.contains(&"@edge-toolkit/et-ws-server-static".to_string())); + assert!(resp.contains(&"@edge-toolkit/et-ws-wasm-agent".to_string())); + assert!(resp.contains(&"@edge-toolkit/et-model-har-motion1".to_string())); // The remaining modules each live in a per-language env: their // `build-ws-*-module` task is loaded only when MISE_ENV includes that // env, so the `pkg/` won't exist (and the module won't be listed) when // CI narrows MISE_ENV. Gate each assertion on the matching env. for (module, language) in [ - ("et-ws-comm1", Language::Rust), - ("et-ws-data1", Language::Rust), - ("et-ws-har1", Language::Js), - ("et-ws-face-detection", Language::Js), + ("@edge-toolkit/et-ws-comm1", Language::Rust), + ("@edge-toolkit/et-ws-data1", Language::Rust), + ("@edge-toolkit/et-ws-har1", Language::Js), + ("@edge-toolkit/et-ws-face-detection", Language::Js), ] { if mise_env_includes(language) { assert!(resp.contains(&module.to_string()), "missing {module}: {resp:?}"); diff --git a/services/modules/tests/configure_missing_root.rs b/services/modules/tests/configure_missing_root.rs index bf64b058..21187b7c 100644 --- a/services/modules/tests/configure_missing_root.rs +++ b/services/modules/tests/configure_missing_root.rs @@ -1,8 +1,10 @@ -//! Covers `configure`'s fail-fast panic when `config.root` names a module that isn't among the -//! scanned `config.paths` -- a misconfiguration that must surface at startup, not as a silent 404. +//! Covers what `configure` does about `config.root`: a name none of the scanned `config.paths` provides is a +//! misconfiguration that must surface at startup rather than as a silent 404, while naming nothing at all is +//! the ordinary case for a deployment with no front page and must serve the module routes regardless. #![cfg(test)] -use actix_web::{App, test}; +use actix_web::http::StatusCode; +use actix_web::{App, test, web}; use et_modules_service::{ModulesConfig, configure}; #[actix_rt::test] @@ -13,3 +15,28 @@ async fn configure_panics_when_root_module_is_missing() { let _app = test::init_service(App::new().configure(|cfg| configure(cfg, &config))).await; } + +#[actix_rt::test] +async fn an_unset_root_serves_the_module_routes_and_nothing_at_the_root() { + let tmp = tempfile::tempdir().unwrap(); + let config = ModulesConfig::new(vec![tmp.path().to_path_buf()], String::default()); + + let app = test::init_service( + App::new() + .app_data(web::Data::new(config.clone())) + .configure(|cfg| configure(cfg, &config)), + ) + .await; + + let listing = test::call_service(&app, test::TestRequest::get().uri("/modules/").to_request()).await; + assert_eq!( + listing.status(), + StatusCode::OK, + "the module listing is the whole surface" + ); + + // Nothing is mounted at `/`, so the request falls through to actix's own default rather than an index + // page -- which is the point: no front page was asked for, and none was invented. + let root = test::call_service(&app, test::TestRequest::get().uri("/").to_request()).await; + assert_eq!(root.status(), StatusCode::NOT_FOUND, "no module was named for /"); +} diff --git a/services/modules/tests/declared_names.rs b/services/modules/tests/declared_names.rs new file mode 100644 index 00000000..78d6004b --- /dev/null +++ b/services/modules/tests/declared_names.rs @@ -0,0 +1,79 @@ +//! A module is served under the name its `package.json` declares, whatever that name is. +//! +//! The server reshapes nothing. It has no notion of whose scope is whose, so it cannot privilege one +//! project's packages over another's -- a deployment run by anyone gets the same treatment, and the name a +//! module is published under is the name it is served, resolved and referred to by throughout. +#![cfg(test)] + +use std::path::PathBuf; + +use et_modules_service::{ModulesConfig, list_modules}; +use fs_err as fs; +use tempfile::TempDir; + +/// Write `//package.json` declaring `name`. +fn write_module(root: &TempDir, dir: &str, name: &str) { + let module_dir = root.path().join(dir); + fs::create_dir_all(&module_dir).unwrap(); + fs::write( + module_dir.join("package.json"), + format!(r#"{{"name":"{name}","version":"0.1.0"}}"#), + ) + .unwrap(); +} + +fn discovered(root: &TempDir) -> Vec { + let config = ModulesConfig::new(vec![root.path().to_path_buf()], "unused-root".to_string()); + let mut names: Vec = list_modules(&config).into_iter().map(|(name, _)| name).collect(); + names.sort(); + names +} + +#[test] +fn a_scoped_name_is_served_exactly_as_declared() { + let root = TempDir::new().unwrap(); + write_module(&root, "math1", "@edge-toolkit/et-ws-math1"); + + assert_eq!(discovered(&root), vec!["@edge-toolkit/et-ws-math1".to_string()]); +} + +#[test] +fn no_scope_is_privileged_over_another() { + // The whole point: this server is not one project's. Two scopes go in, both come out untouched, and + // nothing here knows which of them published the server it is running. + let root = TempDir::new().unwrap(); + write_module(&root, "ours", "@edge-toolkit/et-ws-math1"); + write_module(&root, "theirs", "@huggingface/transformers"); + + assert_eq!( + discovered(&root), + vec![ + "@edge-toolkit/et-ws-math1".to_string(), + "@huggingface/transformers".to_string(), + ] + ); +} + +#[test] +fn an_unscoped_name_is_served_exactly_as_declared() { + let root = TempDir::new().unwrap(); + write_module(&root, "agent", "et-ws-wasm-agent"); + + assert_eq!(discovered(&root), vec!["et-ws-wasm-agent".to_string()]); +} + +#[test] +fn a_configured_path_that_does_not_exist_is_skipped_rather_than_breaking_the_scan() { + // A deployment naming a directory that is not there is a configuration error, but it must not take the + // rest of the scan with it: the modules that are present still get served. + let root = TempDir::new().unwrap(); + write_module(&root, "math1", "@edge-toolkit/et-ws-math1"); + + let config = ModulesConfig::new( + vec![PathBuf::from("/definitely/not/a/directory"), root.path().to_path_buf()], + "unused-root".to_string(), + ); + let names: Vec = list_modules(&config).into_iter().map(|(name, _)| name).collect(); + + assert_eq!(names, vec!["@edge-toolkit/et-ws-math1".to_string()]); +} diff --git a/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js b/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js index 69fa484c..04001d48 100644 --- a/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js +++ b/services/ws-modules/dart-comm1/pkg/et_ws_dart_comm1.js @@ -17,7 +17,7 @@ export async function run() { } // Dart @JS() interop resolves against globalThis, so expose the wasm-agent // classes there for the duration of the call. - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; globalThis.WsClient = WsClient; diff --git a/services/ws-modules/dart-comm1/pkg/package.json b/services/ws-modules/dart-comm1/pkg/package.json index 23b0bef4..ef4ffef5 100644 --- a/services/ws-modules/dart-comm1/pkg/package.json +++ b/services/ws-modules/dart-comm1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-dart-comm1", + "name": "@edge-toolkit/et-ws-dart-comm1", "type": "module", "description": "dart comm1", "version": "0.1.0", diff --git a/services/ws-modules/dart-data1/pkg/et_ws_dart_data1.js b/services/ws-modules/dart-data1/pkg/et_ws_dart_data1.js index 62b96390..16446f38 100644 --- a/services/ws-modules/dart-data1/pkg/et_ws_dart_data1.js +++ b/services/ws-modules/dart-data1/pkg/et_ws_dart_data1.js @@ -18,7 +18,7 @@ export async function run() { // Dart @JS() interop resolves against globalThis, so expose the wasm-agent // classes there for the duration of the call. The REST round-trip uses dio's // browser adapter directly and needs no globals. - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; globalThis.WsClient = WsClient; diff --git a/services/ws-modules/dart-data1/pkg/package.json b/services/ws-modules/dart-data1/pkg/package.json index 39a6317f..f3526a77 100644 --- a/services/ws-modules/dart-data1/pkg/package.json +++ b/services/ws-modules/dart-data1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-dart-data1", + "name": "@edge-toolkit/et-ws-dart-data1", "type": "module", "description": "dart data1", "version": "0.1.0", diff --git a/services/ws-modules/dart-math1/pkg/et_ws_dart_math1.js b/services/ws-modules/dart-math1/pkg/et_ws_dart_math1.js index 4daaeba8..ed9911a2 100644 --- a/services/ws-modules/dart-math1/pkg/et_ws_dart_math1.js +++ b/services/ws-modules/dart-math1/pkg/et_ws_dart_math1.js @@ -18,7 +18,7 @@ export async function run() { // Dart @JS() interop resolves against globalThis, so expose the wasm-agent // classes there for the duration of the call. The FedAvg kernel itself is // pure local computation and needs no further globals. - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; globalThis.WsClient = WsClient; diff --git a/services/ws-modules/dart-math1/pkg/package.json b/services/ws-modules/dart-math1/pkg/package.json index 93a26baa..40af63d9 100644 --- a/services/ws-modules/dart-math1/pkg/package.json +++ b/services/ws-modules/dart-math1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-dart-math1", + "name": "@edge-toolkit/et-ws-dart-math1", "type": "module", "description": "dart math1", "version": "0.1.0", diff --git a/services/ws-modules/dotnet-data1/pkg/package.json b/services/ws-modules/dotnet-data1/pkg/package.json index 80515f19..67a0dcb0 100644 --- a/services/ws-modules/dotnet-data1/pkg/package.json +++ b/services/ws-modules/dotnet-data1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-dotnet-data1", + "name": "@edge-toolkit/et-ws-dotnet-data1", "type": "module", "description": "dotnet data 1", "version": "0.1.0", diff --git a/services/ws-modules/dotnet-math1/pkg/package.json b/services/ws-modules/dotnet-math1/pkg/package.json index 5612b887..0a41f7e0 100644 --- a/services/ws-modules/dotnet-math1/pkg/package.json +++ b/services/ws-modules/dotnet-math1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-dotnet-math1", + "name": "@edge-toolkit/et-ws-dotnet-math1", "type": "module", "description": "dotnet math1", "version": "0.1.0", diff --git a/services/ws-modules/face-detection/Cargo.toml b/services/ws-modules/face-detection/Cargo.toml index 7e0601f0..8a4842f2 100644 --- a/services/ws-modules/face-detection/Cargo.toml +++ b/services/ws-modules/face-detection/Cargo.toml @@ -16,6 +16,7 @@ et-model-face1 = "*" onnxruntime-web = "*" [dependencies] +et-org.workspace = true et-web.workspace = true et-ws-wasm-agent.workspace = true js-sys.workspace = true diff --git a/services/ws-modules/face-detection/src/lib.rs b/services/ws-modules/face-detection/src/lib.rs index 5b627553..42ca57db 100644 --- a/services/ws-modules/face-detection/src/lib.rs +++ b/services/ws-modules/face-detection/src/lib.rs @@ -36,7 +36,7 @@ use wasm_bindgen_futures::{JsFuture, spawn_local}; use web_sys::MediaStreamConstraints; use web_sys::{CanvasRenderingContext2d, HtmlCanvasElement, HtmlVideoElement, ImageData, MediaStream}; -const FACE_MODEL_PATH: &str = "/modules/et-model-face1/video_cv.onnx"; +const FACE_MODEL_PATH: &str = concat!("/modules/", et_org::npm_scope!(), "et-model-face1/video_cv.onnx"); const FACE_INPUT_WIDTH: usize = 640; const FACE_INPUT_HEIGHT: usize = 608; const FACE_INPUT_WIDTH_F64: f64 = FACE_INPUT_WIDTH as f64; diff --git a/services/ws-modules/har1/Cargo.toml b/services/ws-modules/har1/Cargo.toml index c5322aef..8eee71ac 100644 --- a/services/ws-modules/har1/Cargo.toml +++ b/services/ws-modules/har1/Cargo.toml @@ -16,6 +16,7 @@ et-model-har-motion1 = "*" onnxruntime-web = "*" [dependencies] +et-org.workspace = true et-web.workspace = true et-ws-wasm-agent.workspace = true js-sys.workspace = true diff --git a/services/ws-modules/har1/src/lib.rs b/services/ws-modules/har1/src/lib.rs index c5e6ceef..6f8f8836 100644 --- a/services/ws-modules/har1/src/lib.rs +++ b/services/ws-modules/har1/src/lib.rs @@ -31,7 +31,11 @@ use wasm_bindgen::prelude::*; use wasm_bindgen_futures::JsFuture; use web_sys::Event; -const HAR_MODEL_PATH: &str = "/modules/et-model-har-motion1/har-motion1.onnx"; +const HAR_MODEL_PATH: &str = concat!( + "/modules/", + et_org::npm_scope!(), + "et-model-har-motion1/har-motion1.onnx" +); const HAR_SEQUENCE_LENGTH: usize = 150; const HAR_FEATURE_COUNT: usize = 8; const HAR_FEAT_INPUT_SIZE: usize = 36; diff --git a/services/ws-modules/java-data1/pkg/package.json b/services/ws-modules/java-data1/pkg/package.json index 2f0f5e74..bcaf0b11 100644 --- a/services/ws-modules/java-data1/pkg/package.json +++ b/services/ws-modules/java-data1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-java-data1", + "name": "@edge-toolkit/et-ws-java-data1", "type": "module", "description": "java data 1", "version": "0.1.0", diff --git a/services/ws-modules/java-math1/pkg/package.json b/services/ws-modules/java-math1/pkg/package.json index 326d1583..8d55c7b9 100644 --- a/services/ws-modules/java-math1/pkg/package.json +++ b/services/ws-modules/java-math1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-java-math1", + "name": "@edge-toolkit/et-ws-java-math1", "type": "module", "description": "java math1", "version": "0.1.0", diff --git a/services/ws-modules/js-data1/package.json b/services/ws-modules/js-data1/package.json index bb32fa05..b1b51b3f 100644 --- a/services/ws-modules/js-data1/package.json +++ b/services/ws-modules/js-data1/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-js-data1", + "name": "@edge-toolkit/et-ws-js-data1", "version": "0.1.0", "description": "JS data1 twin using AWS SDK v3 (S3)", "license": "Apache-2.0 OR MIT", diff --git a/services/ws-modules/js-data1/pkg/package.json b/services/ws-modules/js-data1/pkg/package.json index 47f1af0d..a0ad903b 100644 --- a/services/ws-modules/js-data1/pkg/package.json +++ b/services/ws-modules/js-data1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-js-data1", + "name": "@edge-toolkit/et-ws-js-data1", "type": "module", "description": "data1 twin using AWS SDK v3 (S3)", "version": "0.1.0", diff --git a/services/ws-modules/js-math1/pkg/et_ws_js_math1.js b/services/ws-modules/js-math1/pkg/et_ws_js_math1.js index e4a50579..93f6c88c 100644 --- a/services/ws-modules/js-math1/pkg/et_ws_js_math1.js +++ b/services/ws-modules/js-math1/pkg/et_ws_js_math1.js @@ -73,7 +73,7 @@ export default async function init() {} export async function run() { log("entered run()"); - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const proto = window.location.protocol === "https:" ? "wss:" : "ws:"; const wsUrl = `${proto}//${window.location.host}/ws`; diff --git a/services/ws-modules/js-math1/pkg/package.json b/services/ws-modules/js-math1/pkg/package.json index 7a48871c..062be6ca 100644 --- a/services/ws-modules/js-math1/pkg/package.json +++ b/services/ws-modules/js-math1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-js-math1", + "name": "@edge-toolkit/et-ws-js-math1", "type": "module", "description": "js math1", "version": "0.1.0", diff --git a/services/ws-modules/kotlin-data1/pkg/package.json b/services/ws-modules/kotlin-data1/pkg/package.json index c94b0a12..6b69a6d3 100644 --- a/services/ws-modules/kotlin-data1/pkg/package.json +++ b/services/ws-modules/kotlin-data1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-kotlin-data1", + "name": "@edge-toolkit/et-ws-kotlin-data1", "type": "module", "description": "kotlin data1", "version": "0.1.0", diff --git a/services/ws-modules/kotlin-math1/pkg/package.json b/services/ws-modules/kotlin-math1/pkg/package.json index eed2ca29..7e74a78c 100644 --- a/services/ws-modules/kotlin-math1/pkg/package.json +++ b/services/ws-modules/kotlin-math1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-kotlin-math1", + "name": "@edge-toolkit/et-ws-kotlin-math1", "type": "module", "description": "kotlin math1", "version": "0.1.0", diff --git a/services/ws-modules/llm1/Cargo.toml b/services/ws-modules/llm1/Cargo.toml index 7f0239ca..51e538fa 100644 --- a/services/ws-modules/llm1/Cargo.toml +++ b/services/ws-modules/llm1/Cargo.toml @@ -17,6 +17,7 @@ test = false et-model-llm1 = "*" [dependencies] +et-org.workspace = true et-web.workspace = true et-ws-wasm-agent.workspace = true js-sys.workspace = true diff --git a/services/ws-modules/llm1/src/lib.rs b/services/ws-modules/llm1/src/lib.rs index d9194f89..22ef51d3 100644 --- a/services/ws-modules/llm1/src/lib.rs +++ b/services/ws-modules/llm1/src/lib.rs @@ -35,7 +35,7 @@ const LOCAL_MODEL_PATH: &str = "/modules/"; /// Weight precision to load. Matches the single `onnx/model_q4f16.onnx` file the fetch task downloads. const MODEL_DTYPE: &str = "q4f16"; /// Directory the module's build task vendors the ORT wasm runtime into, from transformers.js's own pin. -const ORT_WASM_DIR: &str = "/modules/et-ws-llm1/ort"; +const ORT_WASM_DIR: &str = concat!("/modules/", et_org::npm_scope!(), "et-ws-llm1/ort"); /// System turn prepended to every request, kept short because a 135M-parameter model follows little else. const SYSTEM_PROMPT: &str = "You are a concise, helpful assistant running locally on an edge device."; /// Generation cap per reply. Small enough that a CPU-fallback device still answers in a sensible time. diff --git a/services/ws-modules/math1-sender/Cargo.toml b/services/ws-modules/math1-sender/Cargo.toml index ad2c2127..2fd464c8 100644 --- a/services/ws-modules/math1-sender/Cargo.toml +++ b/services/ws-modules/math1-sender/Cargo.toml @@ -32,6 +32,3 @@ wasm-bindgen-test.workspace = true [lints] workspace = true - -[package.metadata.release] -release = false diff --git a/services/ws-modules/pydata1/pkg/et_ws_pydata1.js b/services/ws-modules/pydata1/pkg/et_ws_pydata1.js index 814efa40..ec0a646b 100644 --- a/services/ws-modules/pydata1/pkg/et_ws_pydata1.js +++ b/services/ws-modules/pydata1/pkg/et_ws_pydata1.js @@ -49,7 +49,7 @@ export default async function init() { // the generated client; pyodide-http rewires httpx to use the browser's // fetch()), plus two local wheels -- pydata1 itself (next to this shim) // and the generated et-rest-client wheel served by its own ws-module - // mount at /modules/et-rest-client/. Going through micropip for the + // mount at /modules/@edge-toolkit/et-rest-client/. Going through micropip for the // local wheels would make it look up "et-rest-client" on PyPI, which we // deliberately don't publish. Pyodide unvendors `ssl` from the stdlib // (loaded on demand via loadPackage) and our generated httpx-based @@ -60,7 +60,7 @@ export default async function init() { await micropip.install("attrs"); await micropip.install("pyodide-http"); - const { installWheel: installEtRestClient } = await import("/modules/et-rest-client/et_rest_client.js"); + const { installWheel: installEtRestClient } = await import("/modules/@edge-toolkit/et-rest-client/et_rest_client.js"); await installEtRestClient(pyodide); const injectWheel = async (wheelName) => { @@ -70,7 +70,8 @@ export default async function init() { }; const pkg = await fetch(new URL("package.json", import.meta.url)).then((r) => r.json()); moduleVersion = pkg.version; - const ownWheel = `${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; + const distribution = pkg.name.split("/").pop(); + const ownWheel = `${distribution.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; await injectWheel(ownWheel); // Start Pyodide coverage before importing so import-time lines count (no-op unless the runner set the gate). @@ -90,7 +91,7 @@ export async function run() { const wsHost = loc?.host ?? "localhost:8080"; const wsUrl = globalThis.__ET_WS_URL || `${wsProto}//${wsHost}/ws`; - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; const client = new WsClient(new WsClientConfig(wsUrl)); diff --git a/services/ws-modules/pydata1/pkg/package.json b/services/ws-modules/pydata1/pkg/package.json index 5fe123dc..c19f10f8 100644 --- a/services/ws-modules/pydata1/pkg/package.json +++ b/services/ws-modules/pydata1/pkg/package.json @@ -1,12 +1,12 @@ { "dependencies": { - "et-rest-client": "*", + "@edge-toolkit/et-rest-client": "*", "pyodide": "*" }, "description": "Python data 1", "license": "Apache-2.0 OR MIT", "main": "et_ws_pydata1.js", - "name": "et-ws-pydata1", + "name": "@edge-toolkit/et-ws-pydata1", "type": "module", "version": "0.1.2" } diff --git a/services/ws-modules/pydemo1/pkg/et_ws_pydemo1.js b/services/ws-modules/pydemo1/pkg/et_ws_pydemo1.js index 45e0e676..5b4e2af5 100644 --- a/services/ws-modules/pydemo1/pkg/et_ws_pydemo1.js +++ b/services/ws-modules/pydemo1/pkg/et_ws_pydemo1.js @@ -71,12 +71,12 @@ async function initializePythonRuntime() { await pyodide.pyimport("micropip").install("pydantic"); setPreparationStatus("Loading the demo Python modules..."); await Promise.all([ - installModuleWheel("/modules/et-ws-pyeye1/"), - installModuleWheel("/modules/et-ws-pyspeech1/"), + installModuleWheel("/modules/@edge-toolkit/et-ws-pyeye1/"), + installModuleWheel("/modules/@edge-toolkit/et-ws-pyspeech1/"), installModuleWheel(new URL(".", import.meta.url)), ]); setPreparationStatus("Installing the WebSocket Python support module..."); - const { installWheel: installEtWs } = await import("/modules/et-ws/et_ws.js"); + const { installWheel: installEtWs } = await import("/modules/@edge-toolkit/et-ws/et_ws.js"); await installEtWs(pyodide); if (globalThis.__etPyCov) await globalThis.__etPyCov.start(pyodide, "pydemo1"); try { @@ -113,7 +113,8 @@ async function installModuleWheel(baseUrl) { if (!response.ok) throw new Error(`Unable to load ${pkgUrl}: HTTP ${response.status}`); return response.json(); }); - const wheelName = `${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; + const distribution = pkg.name.split("/").pop(); + const wheelName = `${distribution.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; const response = await fetch(new URL(wheelName, resolvedBaseUrl)); if (!response.ok) throw new Error(`Unable to load ${wheelName}: HTTP ${response.status}`); const path = `/tmp/${wheelName}`; @@ -478,7 +479,7 @@ function createPanelBadge(text) { } async function connectClient(state) { - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"; state.client = new wasmAgent.WsClient(new wasmAgent.WsClientConfig(`${protocol}//${window.location.host}/ws`)); diff --git a/services/ws-modules/pydemo1/pkg/package.json b/services/ws-modules/pydemo1/pkg/package.json index b3a5871f..50123215 100644 --- a/services/ws-modules/pydemo1/pkg/package.json +++ b/services/ws-modules/pydemo1/pkg/package.json @@ -1,18 +1,18 @@ { "dependencies": { + "@edge-toolkit/et-model-eye1": "*", + "@edge-toolkit/et-model-speech1": "*", + "@edge-toolkit/et-ws": "*", + "@edge-toolkit/et-ws-pyeye1": "*", + "@edge-toolkit/et-ws-pyspeech1": "*", "@mediapipe/tasks-vision": "*", - "et-model-eye1": "*", - "et-model-speech1": "*", - "et-ws": "*", - "et-ws-pyeye1": "*", - "et-ws-pyspeech1": "*", "onnxruntime-web": "*", "pyodide": "*" }, "description": "Python eye and speech detection demo", "license": "Apache-2.0 OR MIT", "main": "et_ws_pydemo1.js", - "name": "et-ws-pydemo1", + "name": "@edge-toolkit/et-ws-pydemo1", "type": "module", "version": "0.1.0" } diff --git a/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js b/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js index eb6208cb..8ce50c0c 100644 --- a/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js +++ b/services/ws-modules/pyeye1/pkg/et_ws_pyeye1.js @@ -47,9 +47,10 @@ export default async function init() { }; const pkg = await fetch(new URL("package.json", import.meta.url), { cache: "no-cache" }).then((r) => r.json()); - await installLocalWheel(`${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`); - // et-ws is its own ws-module mounted at /modules/et-ws/; delegate its wheel install to its shim. - const { installWheel: installEtWs } = await import("/modules/et-ws/et_ws.js"); + const distribution = pkg.name.split("/").pop(); + await installLocalWheel(`${distribution.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`); + // et-ws is its own ws-module mounted at /modules/@edge-toolkit/et-ws/; delegate its wheel install to its shim. + const { installWheel: installEtWs } = await import("/modules/@edge-toolkit/et-ws/et_ws.js"); await installEtWs(pyodide); if (globalThis.__etPyCov) await globalThis.__etPyCov.start(pyodide, "pyeye1"); @@ -81,7 +82,7 @@ export async function run() { function platformFor(state) { return { connect_ws: async () => { - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"; diff --git a/services/ws-modules/pyface1/pkg/et_ws_pyface1.js b/services/ws-modules/pyface1/pkg/et_ws_pyface1.js index 991a142a..d3328f14 100644 --- a/services/ws-modules/pyface1/pkg/et_ws_pyface1.js +++ b/services/ws-modules/pyface1/pkg/et_ws_pyface1.js @@ -48,13 +48,14 @@ export default async function init() { }; const pkg = await fetch(new URL("package.json", import.meta.url)).then((r) => r.json()); - const pyfaceWheel = `${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; + const distribution = pkg.name.split("/").pop(); + const pyfaceWheel = `${distribution.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; await installLocalWheel(pyfaceWheel); // The generated et-ws Pydantic-models wheel is its own ws-module mounted - // at /modules/et-ws/. We declare it in [tool.ws-module.dependencies] and + // at /modules/@edge-toolkit/et-ws/. We declare it in [tool.ws-module.dependencies] and // delegate wheel install to its shim -- version lives in its own // package.json so a bump there doesn't require touching this file. - const { installWheel: installEtWs } = await import("/modules/et-ws/et_ws.js"); + const { installWheel: installEtWs } = await import("/modules/@edge-toolkit/et-ws/et_ws.js"); await installEtWs(pyodide); // Start Pyodide coverage before importing so import-time lines count (no-op unless the runner set the gate). if (globalThis.__etPyCov) await globalThis.__etPyCov.start(pyodide, "pyface1"); @@ -77,7 +78,7 @@ export async function run() { let state = null; try { - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"; diff --git a/services/ws-modules/pymath1/pkg/et_ws_pymath1.js b/services/ws-modules/pymath1/pkg/et_ws_pymath1.js index 6438658a..e3d3d0ef 100644 --- a/services/ws-modules/pymath1/pkg/et_ws_pymath1.js +++ b/services/ws-modules/pymath1/pkg/et_ws_pymath1.js @@ -31,7 +31,8 @@ export default async function init() { pyodide = await globalThis.loadPyodide({ indexURL: PYODIDE_BASE_PATH }); const pkg = await fetch(new URL("package.json", import.meta.url)).then((r) => r.json()); - const wheelName = `${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; + const distribution = pkg.name.split("/").pop(); + const wheelName = `${distribution.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`; const bytes = new Uint8Array(await fetch(new URL(wheelName, import.meta.url)).then((r) => r.arrayBuffer())); pyodide.FS.writeFile(`/tmp/${wheelName}`, bytes); pyodide.runPython(`import sys\nsys.path.insert(0, "/tmp/${wheelName}")`); @@ -53,7 +54,7 @@ export async function run() { const wsHost = loc?.host ?? "localhost:8080"; const wsUrl = globalThis.__ET_WS_URL || `${wsProto}//${wsHost}/ws`; - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; const client = new WsClient(new WsClientConfig(wsUrl)); diff --git a/services/ws-modules/pyo3-math1/pkg/package.json b/services/ws-modules/pyo3-math1/pkg/package.json index 8e616796..8739417c 100644 --- a/services/ws-modules/pyo3-math1/pkg/package.json +++ b/services/ws-modules/pyo3-math1/pkg/package.json @@ -1,5 +1,5 @@ { - "name": "et-ws-pyo3-math1", + "name": "@edge-toolkit/et-ws-pyo3-math1", "type": "module", "description": "pyo3 math1", "version": "0.1.0", diff --git a/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js b/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js index 58709089..b225b916 100644 --- a/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js +++ b/services/ws-modules/pyspeech1/pkg/et_ws_pyspeech1.js @@ -30,8 +30,9 @@ export default async function init() { }; const pkg = await fetch(new URL("package.json", import.meta.url)).then((response) => response.json()); - await installLocalWheel(`${pkg.name.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`); - const { installWheel: installEtWs } = await import("/modules/et-ws/et_ws.js"); + const distribution = pkg.name.split("/").pop(); + await installLocalWheel(`${distribution.replace(/-/g, "_")}-${pkg.version}-py3-none-any.whl`); + const { installWheel: installEtWs } = await import("/modules/@edge-toolkit/et-ws/et_ws.js"); await installEtWs(pyodide); if (globalThis.__etPyCov) await globalThis.__etPyCov.start(pyodide, "pyspeech1"); py = pyodide.pyimport("pyspeech1"); @@ -60,7 +61,7 @@ export async function run() { runtime = state; try { - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const protocol = window.location.protocol === "https:" ? "wss:" : "ws:"; state.client = new wasmAgent.WsClient(new wasmAgent.WsClientConfig(`${protocol}//${window.location.host}/ws`)); diff --git a/services/ws-modules/pyspeech1/pkg/package.json b/services/ws-modules/pyspeech1/pkg/package.json index 1db93bf9..02d53c17 100644 --- a/services/ws-modules/pyspeech1/pkg/package.json +++ b/services/ws-modules/pyspeech1/pkg/package.json @@ -1,14 +1,14 @@ { "dependencies": { - "et-model-speech1": "*", - "et-ws": "*", + "@edge-toolkit/et-model-speech1": "*", + "@edge-toolkit/et-ws": "*", "onnxruntime-web": "*", "pyodide": "*" }, "description": "Python microphone speech detection", "license": "Apache-2.0 OR MIT", "main": "et_ws_pyspeech1.js", - "name": "et-ws-pyspeech1", + "name": "@edge-toolkit/et-ws-pyspeech1", "type": "module", "version": "0.1.0" } diff --git a/services/ws-modules/rcomm1/pkg/et_ws_rcomm1.js b/services/ws-modules/rcomm1/pkg/et_ws_rcomm1.js index e5d68c52..1d734bfc 100644 --- a/services/ws-modules/rcomm1/pkg/et_ws_rcomm1.js +++ b/services/ws-modules/rcomm1/pkg/et_ws_rcomm1.js @@ -6,8 +6,8 @@ // list (R selects the peer from it) and logs inbound messages. All sequencing and message composition happen in // module.R. webR is vendored under pkg/webr/ and served at the path below. -const WEBR_BASE_URL = "/modules/et-ws-rcomm1/webr/"; -const R_SOURCE_URL = "/modules/et-ws-rcomm1/module.R"; +const WEBR_BASE_URL = "/modules/@edge-toolkit/et-ws-rcomm1/webr/"; +const R_SOURCE_URL = "/modules/@edge-toolkit/et-ws-rcomm1/module.R"; let webR = null; @@ -31,7 +31,7 @@ export async function run() { // Expose the agent WebSocket to R on globalThis.__etAgent. R drives it (state, agent_id, send, disconnect) via // webr::eval_js; the shim creates it, connects, and keeps the latest agent list for R's peer selection. async function setupAgent() { - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; const loc = typeof location !== "undefined" ? location : null; diff --git a/services/ws-modules/rcomm1/pkg/package.json b/services/ws-modules/rcomm1/pkg/package.json index 96a53bd9..db2c698d 100644 --- a/services/ws-modules/rcomm1/pkg/package.json +++ b/services/ws-modules/rcomm1/pkg/package.json @@ -2,7 +2,7 @@ "description": "R comm 1", "license": "Apache-2.0 OR MIT", "main": "et_ws_rcomm1.js", - "name": "et-ws-rcomm1", + "name": "@edge-toolkit/et-ws-rcomm1", "type": "module", "version": "0.1.0" } diff --git a/services/ws-modules/rdata1/pkg/et_ws_rdata1.js b/services/ws-modules/rdata1/pkg/et_ws_rdata1.js index f440710b..10fd197d 100644 --- a/services/ws-modules/rdata1/pkg/et_ws_rdata1.js +++ b/services/ws-modules/rdata1/pkg/et_ws_rdata1.js @@ -6,8 +6,8 @@ // round-trip (httr2 over the /websockify relay), verification -- happens in module.R. webR is vendored under // pkg/webr/ and served at the path below. -const WEBR_BASE_URL = "/modules/et-ws-rdata1/webr/"; -const R_SOURCE_URL = "/modules/et-ws-rdata1/module.R"; +const WEBR_BASE_URL = "/modules/@edge-toolkit/et-ws-rdata1/webr/"; +const R_SOURCE_URL = "/modules/@edge-toolkit/et-ws-rdata1/module.R"; let webR = null; @@ -31,7 +31,7 @@ export async function run() { // Expose the agent WebSocket to R on globalThis.__etAgent. R drives it (connect state, agent_id, disconnect) // via webr::eval_js; the shim only creates and connects it. async function setupAgent() { - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; const loc = typeof location !== "undefined" ? location : null; diff --git a/services/ws-modules/rdata1/pkg/package.json b/services/ws-modules/rdata1/pkg/package.json index 08d47fab..df313db3 100644 --- a/services/ws-modules/rdata1/pkg/package.json +++ b/services/ws-modules/rdata1/pkg/package.json @@ -2,7 +2,7 @@ "description": "R data 1", "license": "Apache-2.0 OR MIT", "main": "et_ws_rdata1.js", - "name": "et-ws-rdata1", + "name": "@edge-toolkit/et-ws-rdata1", "type": "module", "version": "0.1.0" } diff --git a/services/ws-modules/rmath1/pkg/et_ws_rmath1.js b/services/ws-modules/rmath1/pkg/et_ws_rmath1.js index 7dddb021..ad6d1d79 100644 --- a/services/ws-modules/rmath1/pkg/et_ws_rmath1.js +++ b/services/ws-modules/rmath1/pkg/et_ws_rmath1.js @@ -7,8 +7,8 @@ // /websockify relay), the FedAvg kernel -- happens in module.R. webR is vendored under pkg/webr/ and served // at the path below. -const WEBR_BASE_URL = "/modules/et-ws-rmath1/webr/"; -const R_SOURCE_URL = "/modules/et-ws-rmath1/module.R"; +const WEBR_BASE_URL = "/modules/@edge-toolkit/et-ws-rmath1/webr/"; +const R_SOURCE_URL = "/modules/@edge-toolkit/et-ws-rmath1/module.R"; let webR = null; @@ -32,7 +32,7 @@ export async function run() { // Expose the agent WebSocket to R on globalThis.__etAgent. R drives it (connect state, agent_id, disconnect) // via webr::eval_js; the shim only creates and connects it, and captures the math1-input pointer broadcast. async function setupAgent() { - const wasmAgent = await import("/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"); + const wasmAgent = await import("/modules/@edge-toolkit/et-ws-wasm-agent/et_ws_wasm_agent.js"); await wasmAgent.default(); const { WsClient, WsClientConfig } = wasmAgent; const loc = typeof location !== "undefined" ? location : null; diff --git a/services/ws-modules/rmath1/pkg/package.json b/services/ws-modules/rmath1/pkg/package.json index e1ca0831..b6b9dd6f 100644 --- a/services/ws-modules/rmath1/pkg/package.json +++ b/services/ws-modules/rmath1/pkg/package.json @@ -2,7 +2,7 @@ "description": "R math 1", "license": "Apache-2.0 OR MIT", "main": "et_ws_rmath1.js", - "name": "et-ws-rmath1", + "name": "@edge-toolkit/et-ws-rmath1", "type": "module", "version": "0.1.0" } diff --git a/services/ws-modules/wasi-graphics-info/wasi_graphics_info/__init__.py b/services/ws-modules/wasi-graphics-info/wasi_graphics_info/__init__.py index 931443a2..1c39ae56 100644 --- a/services/ws-modules/wasi-graphics-info/wasi_graphics_info/__init__.py +++ b/services/ws-modules/wasi-graphics-info/wasi_graphics_info/__init__.py @@ -395,8 +395,10 @@ def _mnist_inference() -> dict: _log("loading mnist-12.onnx") # The model file is a sibling static asset, served from pkg/ by # et-modules-service. We treat it as a read-only wasi:keyvalue bucket - # backed by the module's static-asset directory (`/modules//`). - module_assets = store.open("modules/et-ws-wasi-graphics-info") + # backed by the module's static-asset directory (`/modules//`), + # named the way this module's package.json declares it -- owner scope + # and all, since that is what the hub serves it under. + module_assets = store.open("modules/@edge-toolkit/et-ws-wasi-graphics-info") model_value = module_assets.get("mnist-12.onnx") if model_value is None: raise RuntimeError("mnist-12.onnx not found in modules bucket") diff --git a/services/ws-modules/wasi-math1-sender/pkg/package.json b/services/ws-modules/wasi-math1-sender/pkg/package.json index 78e83a67..16a349ab 100644 --- a/services/ws-modules/wasi-math1-sender/pkg/package.json +++ b/services/ws-modules/wasi-math1-sender/pkg/package.json @@ -1,6 +1,6 @@ { "main": "et_ws_wasi_math1_sender.wasm", - "name": "et-ws-wasi-math1-sender", + "name": "@edge-toolkit/et-ws-wasi-math1-sender", "repository": { "type": "git", "url": "https://github.com/edge-toolkit/core" diff --git a/services/ws-modules/zig-data1/build.zig b/services/ws-modules/zig-data1/build.zig index ecbe88b8..9bb74637 100644 --- a/services/ws-modules/zig-data1/build.zig +++ b/services/ws-modules/zig-data1/build.zig @@ -1,67 +1,25 @@ const std = @import("std"); const zon = @import("build.zig.zon"); - -const npm_name = blk: { - const s = @tagName(zon.name); - var buf: [s.len]u8 = s[0..s.len].*; - for (&buf) |*c| if (c.* == '_') { - c.* = '-'; - }; - break :blk buf; -}; - -const name = @tagName(zon.name); -const wasm_install_path = "../pkg/" ++ name ++ ".wasm"; +const shared = @import("zig_shared"); pub fn build(b: *std.Build) void { - const target = b.resolveTargetQuery(.{ - .cpu_arch = .wasm32, - .os_tag = .freestanding, - }); - const optimize = b.standardOptimizeOption(.{}); - - const root_module = b.createModule(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); + const module = shared.addWasmModule(b, zon, .{}); // Generated REST client (path-pinned, lives under generated/zig-rest/). // The single `extern fn js_rest_request` it relies on is satisfied by // the worker shim in pkg/. const rest_module = b.createModule(.{ .root_source_file = b.path("../../../generated/zig-rest/src/et_rest_client.zig"), - .target = target, - .optimize = optimize, + .target = module.target, + .optimize = module.optimize, }); - root_module.addImport("et_rest_client", rest_module); + module.root_module.addImport("et_rest_client", rest_module); - const lib = b.addExecutable(.{ - .name = name, - .root_module = root_module, - }); - lib.entry = .disabled; - lib.rdynamic = true; - root_module.addCSourceFile(.{ .file = b.path("src/util.c") }); + module.root_module.addCSourceFile(.{ .file = b.path("src/util.c") }); // C++ compiles through the same clang, but freestanding wasm32 has no libc++: keep exceptions and RTTI // off so nothing references the missing C++ runtime (unwind tables, type_info, __cxa_* symbols). - root_module.addCSourceFile(.{ + module.root_module.addCSourceFile(.{ .file = b.path("src/util.cpp"), .flags = &.{ "-fno-exceptions", "-fno-rtti" }, }); - - const install = b.addInstallFile(lib.getEmittedBin(), wasm_install_path); - b.getInstallStep().dependOn(&install.step); - - const pkg_json = std.json.Stringify.valueAlloc(b.allocator, .{ - .name = &npm_name, - .type = "module", - .description = zon.description, - .version = zon.version, - .license = zon.license, - .main = zon.main, - }, .{ .whitespace = .indent_2 }) catch unreachable; - const wf = b.addWriteFile("package.json", pkg_json); - const install_pkg_json = b.addInstallFile(wf.getDirectory().path(b, "package.json"), "../pkg/package.json"); - b.getInstallStep().dependOn(&install_pkg_json.step); } diff --git a/services/ws-modules/zig-data1/build.zig.zon b/services/ws-modules/zig-data1/build.zig.zon index f4b7dd39..43465e88 100644 --- a/services/ws-modules/zig-data1/build.zig.zon +++ b/services/ws-modules/zig-data1/build.zig.zon @@ -5,5 +5,8 @@ .license = "Apache-2.0 or MIT", .main = "et_ws_zig_data1.js", .fingerprint = 0x8c5646f08b8cd4b1, + .dependencies = .{ + .zig_shared = .{ .path = "../zig-shared" }, + }, .paths = .{""}, } diff --git a/services/ws-modules/zig-except1/build.zig b/services/ws-modules/zig-except1/build.zig index cb077cd1..d7793aa9 100644 --- a/services/ws-modules/zig-except1/build.zig +++ b/services/ws-modules/zig-except1/build.zig @@ -1,61 +1,20 @@ const std = @import("std"); const zon = @import("build.zig.zon"); - -const npm_name = blk: { - const s = @tagName(zon.name); - var buf: [s.len]u8 = s[0..s.len].*; - for (&buf) |*c| if (c.* == '_') { - c.* = '-'; - }; - break :blk buf; -}; - -const name = @tagName(zon.name); -const wasm_install_path = "../pkg/" ++ name ++ ".wasm"; +const shared = @import("zig_shared"); pub fn build(b: *std.Build) void { - // exception_handling is added to zig's baseline wasm CPU because -mcpu baseline is appended after any - // per-file cflags, so a plain -mexception-handling cflag on exceptions.cpp would be stripped again and - // its __builtin_wasm_throw would fail with "needs target feature exception-handling". Zig itself emits - // no EH instructions; only the exception-enabled C++ TU uses the feature. - const target = b.resolveTargetQuery(.{ - .cpu_arch = .wasm32, + // exception_handling joins the target rather than arriving as a cflag on exceptions.cpp, because + // `-mcpu baseline` is appended after per-file cflags and would strip it again -- its + // __builtin_wasm_throw then fails with "needs target feature exception-handling". Zig itself emits no EH + // instructions; only the exception-enabled C++ TU uses the feature. + const module = shared.addWasmModule(b, zon, .{ .cpu_features_add = std.Target.wasm.featureSet(&.{.exception_handling}), - .os_tag = .freestanding, - }); - const optimize = b.standardOptimizeOption(.{}); - - const root_module = b.createModule(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, }); - const lib = b.addExecutable(.{ - .name = name, - .root_module = root_module, - }); - lib.entry = .disabled; - lib.rdynamic = true; // Exception-enabled C++ TU: real wasm exception-handling instructions plus the minimal runtime defined in // the file itself. See src/exceptions.cpp for the model and its catch (...)-only constraint. - root_module.addCSourceFile(.{ + module.root_module.addCSourceFile(.{ .file = b.path("src/exceptions.cpp"), .flags = &.{ "-fwasm-exceptions", "-mexception-handling", "-fno-rtti" }, }); - - const install = b.addInstallFile(lib.getEmittedBin(), wasm_install_path); - b.getInstallStep().dependOn(&install.step); - - const pkg_json = std.json.Stringify.valueAlloc(b.allocator, .{ - .name = &npm_name, - .type = "module", - .description = zon.description, - .version = zon.version, - .license = zon.license, - .main = zon.main, - }, .{ .whitespace = .indent_2 }) catch unreachable; - const wf = b.addWriteFile("package.json", pkg_json); - const install_pkg_json = b.addInstallFile(wf.getDirectory().path(b, "package.json"), "../pkg/package.json"); - b.getInstallStep().dependOn(&install_pkg_json.step); } diff --git a/services/ws-modules/zig-except1/build.zig.zon b/services/ws-modules/zig-except1/build.zig.zon index df69472d..17d1781b 100644 --- a/services/ws-modules/zig-except1/build.zig.zon +++ b/services/ws-modules/zig-except1/build.zig.zon @@ -5,5 +5,8 @@ .license = "Apache-2.0 or MIT", .main = "et_ws_zig_except1.js", .fingerprint = 0x7e0344ac53a4bcec, + .dependencies = .{ + .zig_shared = .{ .path = "../zig-shared" }, + }, .paths = .{""}, } diff --git a/services/ws-modules/zig-math1/build.zig b/services/ws-modules/zig-math1/build.zig index c389b8d6..f1fc1146 100644 --- a/services/ws-modules/zig-math1/build.zig +++ b/services/ws-modules/zig-math1/build.zig @@ -1,50 +1,7 @@ const std = @import("std"); const zon = @import("build.zig.zon"); - -const npm_name = blk: { - const s = @tagName(zon.name); - var buf: [s.len]u8 = s[0..s.len].*; - for (&buf) |*c| if (c.* == '_') { - c.* = '-'; - }; - break :blk buf; -}; - -const name = @tagName(zon.name); -const wasm_install_path = "../pkg/" ++ name ++ ".wasm"; +const shared = @import("zig_shared"); pub fn build(b: *std.Build) void { - const target = b.resolveTargetQuery(.{ - .cpu_arch = .wasm32, - .os_tag = .freestanding, - }); - const optimize = b.standardOptimizeOption(.{}); - - const root_module = b.createModule(.{ - .root_source_file = b.path("src/main.zig"), - .target = target, - .optimize = optimize, - }); - - const lib = b.addExecutable(.{ - .name = name, - .root_module = root_module, - }); - lib.entry = .disabled; - lib.rdynamic = true; - - const install = b.addInstallFile(lib.getEmittedBin(), wasm_install_path); - b.getInstallStep().dependOn(&install.step); - - const pkg_json = std.json.Stringify.valueAlloc(b.allocator, .{ - .name = &npm_name, - .type = "module", - .description = zon.description, - .version = zon.version, - .license = zon.license, - .main = zon.main, - }, .{ .whitespace = .indent_2 }) catch unreachable; - const wf = b.addWriteFile("package.json", pkg_json); - const install_pkg_json = b.addInstallFile(wf.getDirectory().path(b, "package.json"), "../pkg/package.json"); - b.getInstallStep().dependOn(&install_pkg_json.step); + _ = shared.addWasmModule(b, zon, .{}); } diff --git a/services/ws-modules/zig-math1/build.zig.zon b/services/ws-modules/zig-math1/build.zig.zon index 5c6d9b9c..fdfa4cf5 100644 --- a/services/ws-modules/zig-math1/build.zig.zon +++ b/services/ws-modules/zig-math1/build.zig.zon @@ -5,5 +5,8 @@ .license = "Apache-2.0 or MIT", .main = "et_ws_zig_math1.js", .fingerprint = 0x50849fc82a52afc5, + .dependencies = .{ + .zig_shared = .{ .path = "../zig-shared" }, + }, .paths = .{""}, } diff --git a/services/ws-modules/zig-shared/build.zig b/services/ws-modules/zig-shared/build.zig new file mode 100644 index 00000000..c7d358ba --- /dev/null +++ b/services/ws-modules/zig-shared/build.zig @@ -0,0 +1,96 @@ +//! The build every Zig ws-module runs, and the npm name each is served and published under. +//! +//! Imported as a path dependency rather than by relative path, because Zig refuses an `@import` that escapes +//! the importing module's own directory. Carries no build of its own: the `build` function exists so this is +//! a valid dependency, and does nothing. +//! +//! Not a module directory itself. It holds no `pkg/` and no `package.json`, which is what the hub's scan +//! requires of anything under `services/ws-modules/`, so the scan passes over it rather than serving it. + +const std = @import("std"); + +pub fn build(b: *std.Build) void { + _ = b; +} + +/// The scoped npm package name for a module whose `build.zig.zon` name is `tag`. +/// +/// Two things happen to the zon name. Underscores become dashes, because a zon name is a Zig identifier and +/// so cannot hold a dash, while the npm name is expected to. Then the repository owner's scope goes on the +/// front, which is the only shape the registry these are published to accepts -- it rejects an unscoped +/// package outright. The hub drops that scope again when it names a module, so what is served, what a +/// scenario names, and what a deployment refers to are all unchanged by carrying it. +pub fn scopedNpmName(comptime tag: []const u8) []const u8 { + // A `comptime` block that yields the name, rather than one that returns it: this is called from + // `addWasmModule`, which is an ordinary runtime function, and a `return` from inside a `comptime` block + // there fails with "function called at runtime cannot return value at comptime". + return comptime blk: { + var buf: [tag.len]u8 = tag[0..tag.len].*; + for (&buf) |*c| if (c.* == '_') { + c.* = '-'; + }; + break :blk "@edge-toolkit/" ++ buf; + }; +} + +/// What a module's own build script needs back in order to add its extra sources and imports. +pub const WasmModule = struct { + root_module: *std.Build.Module, + target: std.Build.ResolvedTarget, + optimize: std.builtin.OptimizeMode, +}; + +/// Per-module departures from the standard freestanding-wasm build. +pub const Options = struct { + /// CPU features to add on top of zig's wasm baseline. + /// + /// Needed because `-mcpu baseline` is appended after any per-file cflags, so a feature asked for as a + /// cflag on one C++ file is stripped again -- it has to be part of the resolved target instead. + cpu_features_add: std.Target.Cpu.Feature.Set = std.Target.Cpu.Feature.Set.empty, +}; + +/// Build `src/main.zig` as a freestanding wasm binary and write the `pkg/` the hub serves. +/// +/// Installs both halves of what a module directory is: the wasm binary, and the `package.json` naming it. +/// The returned module is still open for a caller to add C sources or imports to -- the install step reads +/// the executable's emitted binary lazily, so adding to it afterwards is what the two modules that need +/// extra sources already do. +pub fn addWasmModule(b: *std.Build, comptime zon: anytype, options: Options) WasmModule { + const name = @tagName(zon.name); + const target = b.resolveTargetQuery(.{ + .cpu_arch = .wasm32, + .cpu_features_add = options.cpu_features_add, + .os_tag = .freestanding, + }); + const optimize = b.standardOptimizeOption(.{}); + + const root_module = b.createModule(.{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + }); + + const lib = b.addExecutable(.{ + .name = name, + .root_module = root_module, + }); + lib.entry = .disabled; + lib.rdynamic = true; + + const install = b.addInstallFile(lib.getEmittedBin(), "../pkg/" ++ name ++ ".wasm"); + b.getInstallStep().dependOn(&install.step); + + const pkg_json = std.json.Stringify.valueAlloc(b.allocator, .{ + .name = scopedNpmName(name), + .type = "module", + .description = zon.description, + .version = zon.version, + .license = zon.license, + .main = zon.main, + }, .{ .whitespace = .indent_2 }) catch unreachable; + const wf = b.addWriteFile("package.json", pkg_json); + const install_pkg_json = b.addInstallFile(wf.getDirectory().path(b, "package.json"), "../pkg/package.json"); + b.getInstallStep().dependOn(&install_pkg_json.step); + + return .{ .root_module = root_module, .target = target, .optimize = optimize }; +} diff --git a/services/ws-modules/zig-shared/build.zig.zon b/services/ws-modules/zig-shared/build.zig.zon new file mode 100644 index 00000000..574686b5 --- /dev/null +++ b/services/ws-modules/zig-shared/build.zig.zon @@ -0,0 +1,6 @@ +.{ + .name = .et_ws_zig_shared, + .version = "0.1.0", + .fingerprint = 0x915f9c107afec67f, + .paths = .{""}, +} diff --git a/services/ws-pyo3-runner/Cargo.toml b/services/ws-pyo3-runner/Cargo.toml index 6d6968fd..b53ae853 100644 --- a/services/ws-pyo3-runner/Cargo.toml +++ b/services/ws-pyo3-runner/Cargo.toml @@ -41,6 +41,7 @@ pyo3-build-config.workspace = true backon = { workspace = true, features = ["tokio-sleep"] } base64.workspace = true command-error.workspace = true +et-org.workspace = true et-ws-test-server.workspace = true rstest.workspace = true diff --git a/services/ws-pyo3-runner/src/hub_module.rs b/services/ws-pyo3-runner/src/hub_module.rs index f6aafae2..11bf0f39 100644 --- a/services/ws-pyo3-runner/src/hub_module.rs +++ b/services/ws-pyo3-runner/src/hub_module.rs @@ -14,7 +14,7 @@ use std::path::PathBuf; -use et_ws_runner_common::{collect_byte_stream, derive_http_base, fetch_main_field}; +use et_ws_runner_common::{derive_http_base, fetch_main_field}; use tracing::Instrument as _; use crate::error::RunnerError; @@ -66,20 +66,15 @@ pub async fn fetch_if_absent( })) } -/// Download one of a module's published files. -/// -/// Returns `BootstrapError` rather than the caller's error type so the REST failure converts through the `From` -/// impl the shared runner crate already carries, and `?` does the work at both levels. +/// Download one of a module's published files, inside a span naming the module and the file it fetched. async fn fetch_module_file( rest: &et_rest_client::Client, module_name: &str, main: &str, ) -> Result, et_ws_runner_common::BootstrapError> { - let response = rest - .get_module_file(module_name, main) + et_ws_runner_common::fetch_module_file(rest, module_name, main) .instrument(tracing::info_span!("fetch_module", module = module_name, file = %main)) - .await?; - collect_byte_stream(response.into_inner()).await + .await } /// Derive the name a fetched file is compiled under from the hub's `main` entry. diff --git a/services/ws-pyo3-runner/tests/modules.rs b/services/ws-pyo3-runner/tests/modules.rs index d60d1622..a52865c1 100644 --- a/services/ws-pyo3-runner/tests/modules.rs +++ b/services/ws-pyo3-runner/tests/modules.rs @@ -328,15 +328,17 @@ async fn run_exchange( /// expected weights for the canonical input. The runner is long-lived, so it is killed once the /// exchange resolves (mirroring `module_behaves`). /// -/// This is the hub-fetch case, and the only one: `et-ws-pyo3-math1` is a published ws-module rather than a -/// file under `python/`, and the name is not a legal Python identifier, so it cannot resolve on `sys.path` at -/// all. `spawn_runner` still sets `PYO3_PYTHONPATH`, which is the point -- a configured python path that does -/// not happen to contain the module must fall through to the hub rather than fail. Every other case here -/// names a module under `python/` and so still takes the local import. +/// This is the hub-fetch case, and the only one: the module is a published ws-module rather than a file under +/// `python/`, so it is asked for by the scoped name its `package.json` declares -- which the hub serves it +/// under, and which is not a legal Python identifier, so it cannot resolve on `sys.path` at all. +/// `spawn_runner` still sets `PYO3_PYTHONPATH`, which is the point -- a configured python path that does not +/// happen to contain the module must fall through to the hub rather than fail. Every other case here names a +/// module under `python/` and so still takes the local import. #[tokio::test(flavor = "current_thread")] async fn math1_stores_verified_model() -> Result<(), Box> { let server = et_ws_test_server::start(); - let mut runner = spawn_runner("et-ws-pyo3-math1", &server.ws_url); + let module = format!("{}et-ws-pyo3-math1", et_org::NPM_SCOPE); + let mut runner = spawn_runner(&module, &server.ws_url); let outcome = et_ws_test_server::math1::drive_math1_exchange(&server.ws_url, server.storage_dir.path(), EXCHANGE_BUDGET) .await; diff --git a/services/ws-server/Dockerfile b/services/ws-server/Dockerfile index 3af92d6a..f23a7a55 100644 --- a/services/ws-server/Dockerfile +++ b/services/ws-server/Dockerfile @@ -7,9 +7,9 @@ # The hub image: et-ws-server plus only what the hub UI itself cannot start without. # That is the `et-ws-server-static` root module (the page served at `/`) and the `et-ws-wasm-agent` browser client -# that page imports from `/modules/et-ws-wasm-agent/`. No workflow module, no model weights and no npm runtime -# dependency is baked in: which of those a deployment serves is a per-cluster decision, so `et-cli` provisions them -# on top of this image rather than every image carrying every module. +# that page imports from `/modules/@edge-toolkit/et-ws-wasm-agent/`. No workflow module, no model weights and no +# npm runtime dependency is baked in: which of those a deployment serves is a per-cluster decision, so `et-cli` +# provisions them on top of this image rather than every image carrying every module. # # MODULES_PATHS is deliberately left unset. Unset, the server falls back to `default_modules_folders()`, which # resolves each default against the working directory -- `/app` in this image -- so the two bundled paths resolve @@ -134,6 +134,9 @@ mkdir -p /app/storage /app/services/ws-server/storage chown -R 10001:10001 /app EOF +# The module this image serves at `/`, named as its own package.json declares it. +ENV MODULES_ROOT="@edge-toolkit/et-ws-server-static" + # Numeric uid, matching the useradd above, so a host inspecting the image resolves it without our passwd file. USER 10001 diff --git a/services/ws-server/static/app.js b/services/ws-server/static/app.js index 181e1757..4e7d23b1 100644 --- a/services/ws-server/static/app.js +++ b/services/ws-server/static/app.js @@ -1,5 +1,15 @@ +// The names this page's own modules are served under, which are the names they publish under. +// The server reshapes nothing, so a module is asked for by the name its `package.json` declares -- scope +// included. This page belongs to the same project as those modules, so it is entitled to know the scope; +// the server it runs on is not, and does not. +const MODULE_SCOPE = "@edge-toolkit/"; +const AGENT_MODULE = `${MODULE_SCOPE}et-ws-wasm-agent`; +const AGENT_BASE = `/modules/${AGENT_MODULE}`; + +// Imported dynamically rather than with a static `import`: the scoped URL pushes that statement past the +// line limit, and there is no way to break it because a static import specifier cannot be a variable. // skipcq: JS-0833 -- committed ES module; the analyzer's script-mode parse is a false positive -import init, { initTracing, WsClient, WsClientConfig } from "/modules/et-ws-wasm-agent/et_ws_wasm_agent.js"; +const { default: init, initTracing, WsClient, WsClientConfig } = await import(`${AGENT_BASE}/et_ws_wasm_agent.js`); // Bump this string on every meaningful app.js edit. index.html loads this file via a plain, non-cache-busted //