diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca19207..0aab7bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,8 @@ name: ci # + marketplace.json/plugins/ sync + version bumps on release PRs # + post-merge version backstop on pushes to main # link-check: CI-02: the tools/check-links.sh sweep -# compile-check: EVAL-02 compile the baked JCSMP samples verbatim +# jcsmp-sample-check: EVAL-02 compile the baked JCSMP samples verbatim +# python-sample-check: check the baked Python samples against the latest solace-pubsubplus # trigger-evals: lightweight skill-triggering check; disabled until CI may hold an API key on: # Deliberately no `edited` type: running on title/body edits either re-runs @@ -50,7 +51,7 @@ jobs: # missing license are warnings only. Also validate the Claude # plugin/marketplace manifests. validate: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 timeout-minutes: 10 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -99,7 +100,7 @@ jobs: # CI-02: sweep the plugins/ tree for broken and soft-404 documentation links. link-check: - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 timeout-minutes: 15 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -108,8 +109,8 @@ jobs: # EVAL-02: compile the baked JCSMP samples verbatim against the live-resolved # sol-jcsmp. Pure compile gate, no API budget and no credential. - compile-check: - runs-on: ubuntu-latest + jcsmp-sample-check: + runs-on: ubuntu-26.04 timeout-minutes: 25 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -118,8 +119,23 @@ jobs: distribution: temurin java-version: 11 cache: maven - - name: Compile baked samples (EVAL-02 hard gate) - run: ./plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/compile.sh + - name: Check baked JCSMP samples (EVAL-02 hard gate) + run: ./plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/check-samples.sh + + # Check the baked Python samples against the latest solace-pubsubplus from PyPI: + # py_compile, then import each sample and build its API objects against a + # resolvable host without a broker. Python has no compile step, so this is the + # Python counterpart of jcsmp-sample-check. No API budget and no credential. + python-sample-check: + runs-on: ubuntu-26.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.11' + - name: Check baked Python samples + run: ./plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check-samples.sh # Lightweight skill-triggering check. For every case in each plugin's # evals/trigger-evals.json, run the prompt with only that plugin loaded and @@ -138,7 +154,7 @@ jobs: # re-enable the job once a key is allowed. trigger-evals: if: false - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 timeout-minutes: 120 # per model leg; 57 cases x 3 runs (~50 min observed) strategy: fail-fast: false # each model is an independent check diff --git a/CLAUDE.md b/CLAUDE.md index 7a753f4..db331da 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -16,7 +16,7 @@ This repository is a Claude Code plugin marketplace (`solace-agent-plugins`) of | `tools/` | CI check scripts, the two eval runners, and `output-evals.schema.json`. | | `.github/workflows/ci.yml` | The only CI workflow. | -Nothing in the repository is generated or vendored. The Eclipse files under `references/jcsmp/evals/compile-fixture/` are IDE metadata for the compile fixture; leave them alone. +Nothing in the repository is generated or vendored. The Eclipse files under `references/jcsmp/evals/sample-check/` are IDE metadata for the JCSMP sample check; leave them alone. ## Commands @@ -27,7 +27,8 @@ gh skill publish --dry-run # skills against the Agent Skil claude plugin validate . --strict # marketplace and plugin manifests ./tools/check-plugin-versions.sh --sync-only # marketplace.json matches plugins/ ./tools/check-links.sh plugins # broken and soft-404 doc links -./plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/compile.sh # compile the baked JCSMP samples +./plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/check-samples.sh # check the baked JCSMP samples (compile and config smoke) +./plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check-samples.sh # check the baked Python samples (python3 and PyPI access) claude --plugin-dir plugins/solace-messaging-skills # load the plugin from the clone ``` @@ -47,7 +48,7 @@ The trigger runner has no single-case mode; set `TRIGGER_EVAL_RUNS=1` for a quic - Link documentation to the live `docs.solace.com` `.md` URL. Do not copy doc content into a skill. Every link under `plugins/` must pass `check-links.sh`. - A new skill needs cases in both `trigger-evals.json` (positive and near-miss negative prompts) and `output-evals.json` (validate the latter against `tools/output-evals.schema.json`). - A new, removed, or renamed plugin directory needs the matching `marketplace.json` change in the same pull request. -- A new baked JCSMP sample must be added to the sample roster in `compile.sh`, or the compile check does not cover it. +- A new baked JCSMP sample must be added to the sample roster in `jcsmp/evals/sample-check/check-samples.sh`, or the sample check does not cover it. A new baked Python sample goes into the roster in `python/evals/sample-check/check-samples.sh`; `check_samples.py` picks it up from there. - Do not put exact counts (of cases, skills, or graders) in READMEs or skill docs. They go stale. ## Gotchas / do not diff --git a/plugins/solace-messaging-skills/evals/README.md b/plugins/solace-messaging-skills/evals/README.md index f02ec65..d24e029 100644 --- a/plugins/solace-messaging-skills/evals/README.md +++ b/plugins/solace-messaging-skills/evals/README.md @@ -49,9 +49,10 @@ Each `turnN.jsonl` in the work directory is the raw `--output-format stream-json - The `claude` CLI (Claude Code) on your PATH. - `jq` and `curl` on your PATH. - `mvn` (Apache Maven) with a JDK 11 or newer on your PATH. Only the cases that compile generated code need it, and the runner checks for it only when such a case is selected. +- `python3` with the `venv` module and `pip`, for the cases that generate Solace Python projects. A generated Python project installs `solace-pubsubplus` into its own virtual environment, and its `verify.sh` runs that environment's interpreter. - An exported credential, either `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`. An existing `claude` login does not satisfy this: the runner uses a throwaway config directory so that only the plugin under test is loaded, and that directory has no login. Claude seat holders can mint a token with `claude setup-token`. - `timeout` from GNU coreutils (or Homebrew's `gtimeout`) on your PATH, to cap each `mvn compile` and `verify.sh` run at 600 seconds. Optional: without it those steps run unbounded, and the runner prints a warning. On macOS, install it with `brew install coreutils`. -- Network access to `repo1.maven.org` (Maven Central metadata and dependencies) and `docs.solace.com` (the skills fetch documentation pages). +- Network access to `repo1.maven.org` (Maven Central metadata and dependencies), `pypi.org` and `files.pythonhosted.org` (the `solace-pubsubplus` release metadata and package), and `docs.solace.com` (the skills fetch documentation pages). - Optional: the four `OUTPUT_EVAL_BROKER_*` variables for the live round trip against a real broker (see below). - On macOS, `caffeinate` to keep the machine awake for a full leg. - On Windows, Git Bash. The repository's `.gitattributes` checks shell scripts out with LF endings, which bash and the byte-identity grader on `verify.sh` both need. A checkout made with `core.autocrlf=true` before that file existed keeps its CRLF copies, and git reports the tree clean because it treats them as unchanged, so `git pull` and `git checkout -- .` do not rewrite them. Re-clone, or delete the tracked scripts and check them out again: `git ls-files -z '*.sh' | xargs -0 rm -f && git checkout -- .`. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/SKILL.md b/plugins/solace-messaging-skills/skills/solace-application-development/SKILL.md index b2c0cf5..41b40aa 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/SKILL.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/SKILL.md @@ -44,12 +44,12 @@ Non-negotiable rules that apply across every API. Apply all. 2. **WebFetch-on-demand doc grounding**: This skill bundles no documentation. When a step needs documentation content to ground a generation or design decision, WebFetch the live canonical page on demand (its `docs.solace.com` `.md` URL, the same link the reference files carry), then quote or summarize the fetched page. Do not answer from memory and do not paraphrase guidance the page does not contain. Do not restate doc content in the skill; link the live `.md` URL. Never substitute another channel for that WebFetch: grounding taken from a docs chatbot, a search tool, or decompiling a jar does not satisfy this invariant, and a page never fetched in the session must never be cited as grounding. A short list of references stays live for the same reason it always has (the JCSMP Javadoc and the Python API reference, the JCSMP and Python API Release Notes, Maven Central and PyPI, the tutorials, and the GitHub samples) because those have no `docs.solace.com` `.md` form. Each API's entry file lists its own live exceptions. 3. **No hardcoded versions**: Never pin a library version anywhere in skill content. The generated build resolves the latest release of every dependency at generation time from the API's AUTHORITATIVE registry: the Maven repository metadata for JCSMP, the PyPI JSON `info.version` for Python. Each API's entry file names the exact coordinate and carries the exact lookup command. Never read a JCSMP version from the legacy `search.maven.org/solrsearch` index: it lags the repository metadata and reports stale versions. Security floors (a minimum version below which known CVEs live) are enforced as checks on the resolved result, never as pins. 4. **AI-assisted disclaimer header**: Every source file generated under this skill starts with this exact line: `AI-assisted code. Review before production use.` Directly below it: `See the verification checklist: solace-verification-checklist.md`. This applies to EVERY generated source file, including custom builds, web apps, and embedded shapes that adapt the reference samples outside the canonical generator. Build files (the pom, `requirements.txt`) and other non-source artifacts are exempt; the bundled reference samples themselves carry no header. -5. **Verification artifacts, always**: Any session that generates Solace code from this skill writes a tailored `solace-verification-checklist.md` beside the code as generation output, copies the API's own bundled verify script (plus its generated `verify-hooks.sh`) into the project, and ends in a real verify run or the compile-only fallback with the exact handed-back commands. The API's entry file names its verify script. An API that does not bundle a verify script yet (Python, until its `verify.sh` lands) ends in the compile-only fallback every time, and its implement-mode file defines that fallback; never claim a verified run without the script. The `VERIFY:` markers are non-optional in generated Solace messaging code, whatever the app shape and whatever the API. An improvised check (for example curl against the app's own HTTP API) may TRIGGER traffic but never renders the verdict; the verdict comes from the markers and the verify exit code. +5. **Verification artifacts, always**: Any session that generates Solace code from this skill writes a tailored `solace-verification-checklist.md` beside the code as generation output, copies the API's own bundled verify script (plus its generated `verify-hooks.sh`) into the project, and ends in a real verify run or the compile-only fallback with the exact handed-back commands. The API's entry file names its verify script. Never claim a verified run without the script. The `VERIFY:` markers are non-optional in generated Solace messaging code, whatever the app shape and whatever the API. An improvised check (for example curl against the app's own HTTP API) may TRIGGER traffic but never renders the verdict; the verdict comes from the markers and the verify exit code. 6. **Live-broker heads-up**: Immediately before starting ANY process that will connect to a live broker — verify.sh, `java -jar`, a `mvn` run, a `python` run, a run script, anything — print one line that names the broker host and states the actual broker-side effects of this run (the queues it provisions, the connections it opens, the messages it publishes), scaled to the real app. Environment discovery is never consent and never an answer: a running local broker, an existing config file, or found credentials are facts to report, not answers to consume; report them, then still ask which broker the developer wants to target. ## Reference Files (read on-demand only) - `references/jcsmp.md`: the JCSMP entry file (mode detection, the JCSMP coordinate, and the live JCSMP doc links); read this for any JCSMP request. -- `references/jcsmp/`: all JCSMP API content (mode files, the reference samples, the verify script, and the compile fixture), reached through `references/jcsmp.md`. +- `references/jcsmp/`: all JCSMP API content (mode files, the reference samples, the verify script, and the sample check), reached through `references/jcsmp.md`. - `references/python.md`: the Python entry file (mode detection, the `solace-pubsubplus` coordinate, and the live Python doc links); read this for any Solace Python API request. -- `references/python/`: all Python API content (mode files and the reference samples), reached through `references/python.md`. +- `references/python/`: all Python API content (mode files, the reference samples, the verify script, and the sample check), reached through `references/python.md`. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/.gitignore b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/.gitignore similarity index 100% rename from plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/.gitignore rename to plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/.gitignore diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/ConfigSmokeTest.java b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/ConfigSmokeTest.java similarity index 96% rename from plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/ConfigSmokeTest.java rename to plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/ConfigSmokeTest.java index 64757a6..976f394 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/ConfigSmokeTest.java +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/ConfigSmokeTest.java @@ -1,5 +1,5 @@ /* - * Runtime smoke test for SolaceConnectionConfig, driven by compile.sh. + * Runtime smoke test for SolaceConnectionConfig, driven by check-samples.sh. * * NOT a reference sample: compiling the helper never executes its config.json * parser or CLI-args fallback, so this tiny driver is compiled against the @@ -9,7 +9,7 @@ * value, blank CLI arg). * * One case per JVM invocation: the parser reads a CWD-relative config.json and - * the fail-fast paths throw/exit, so compile.sh drives each case from its own + * the fail-fast paths throw/exit, so check-samples.sh drives each case from its own * working directory and checks the exit code. The case name is argv[0]; any * connection arguments the case needs are built here, not taken from argv. */ diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/compile.sh b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/check-samples.sh similarity index 99% rename from plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/compile.sh rename to plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/check-samples.sh index cab3887..a307688 100755 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/compile.sh +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/check-samples.sh @@ -23,7 +23,7 @@ set -euo pipefail # Resolve this script's own directory so sample/pom paths are independent of CWD. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" POM="${SCRIPT_DIR}/pom.xml" -# references/jcsmp/ lives two levels up from compile-fixture/ (evals/ -> jcsmp/). +# references/jcsmp/ lives two levels up from sample-check/ (evals/ -> jcsmp/). REFERENCES_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" # Per-sample source paths. Each new baked sample a later per-pattern plan adds is # a ONE-LINE append here, mirroring this shape: diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/pom.xml b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/pom.xml similarity index 91% rename from plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/pom.xml rename to plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/pom.xml index eb9016d..68c5a1d 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/pom.xml +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/pom.xml @@ -5,13 +5,13 @@ 4.0.0 com.solace.samples - jcsmp-compile-fixture + jcsmp-sample-check 0.0.1-SNAPSHOT jar UTF-8 11 - RESOLVED_AT_GENERATION_TIME diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md index a13aa14..af910ec 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/implement-mode.md @@ -164,7 +164,7 @@ Invoke the matching stage after generating each class, passing the connection pa 1. After generating the Subscriber: `./verify.sh consumer `. It waits for `VERIFY: QUEUE_BOUND` in the Subscriber output (queue provisioned, subscribed, flow started). 2. After generating the Publisher: `./verify.sh publisher `. NOTE: this stage starts the Subscriber first to bind the queue, so it requires BOTH classes; it waits for `VERIFY: PUBLISH_ACKED` in the Publisher's own output (connect, then publish ACK). -3. At the end: `./verify.sh roundtrip `. It starts the Subscriber, runs the Publisher, and waits for `VERIFY: MESSAGE_RECEIVED` in the Subscriber output (a published PERSISTENT message lands on the queue and is consumed and ACKed). +3. At the end: `./verify.sh roundtrip `. It starts the Subscriber, runs the Publisher, and waits for `VERIFY: PUBLISH_ACKED` in the Publisher output and then a new `VERIFY: MESSAGE_RECEIVED` in the Subscriber output (a published PERSISTENT message lands on the queue and is consumed and ACKed; a message an earlier run left on the durable queue does not count). The stages above (`consumer`, `publisher`, `roundtrip`) verify the guaranteed pub/sub leaves. `verify.sh` also accepts three further stage names for the other leaves: `direct` (the Direct Pub/Sub leaf), `direct-request-reply` (the Request-Reply (Direct) leaf), and `guaranteed-request-reply` (the Request-Reply (Guaranteed) leaf). All are runnable. The two request-reply stages are replier-first single round-trips: each starts the Replier first (the readiness gate on `VERIFY: SUBSCRIBED`), then runs the Requestor in the foreground and passes on `VERIFY: REPLY_RECEIVED` in the Requestor log (with `VERIFY: REQUEST_RECEIVED` in the Replier log as richer evidence). The `guaranteed-request-reply` stage differs only in that the Replier provisions a durable request queue (an endpoint-management denial classifies as an environment failure, exit 2) and the Requestor uses a temporary reply queue plus a blocking `flow.receive` rather than the direct `Requestor` convenience. @@ -182,13 +182,13 @@ PASS_MARKER='VERIFY: MESSAGE_RECEIVED' # marker proving the round trip Generate `verify-hooks.sh` with ONLY these four assignments. The app stage imports exactly these four names from a child shell and ignores everything else in the file, so an extra variable, a `cd`, or a function never reaches the script's own state; keep the file to the four lines all the same. Give a network trigger its own timeout (the `--max-time` above): the stage also bounds the trigger externally and force-kills one that never returns, but a self-timing trigger leaves cleaner evidence. -The `app` stage starts `START_CMD` in the background and captures its output, waits for `VERIFY: CONNECTED` and then `READY_MARKER` (an environment signature or a timeout before that classifies as exit 2 or exit 1), runs `TRIGGER_CMD` under a bounded deadline (a trigger that never returns is force-killed, and the marker watch still decides the verdict), waits for `PASS_MARKER` in the app's captured output, then stops the app with SIGINT and classifies. Set `READY_MARKER`/`PASS_MARKER` to the chosen leaf's own markers. A curl inside `TRIGGER_CMD` is exactly the right use of curl: it triggers, and the markers judge. On the canonical two-class shape, generate `verify-hooks.sh` too, carrying the classic commands (`START_CMD='mvn -q exec:java@subscriber'`, `TRIGGER_CMD='mvn -q exec:java@publisher'`) so the file documents the same contract everywhere; the fixed stages do not read it. +The `app` stage starts `START_CMD` in the background and captures its output, waits for `VERIFY: CONNECTED` and then `READY_MARKER` (an environment signature or a timeout before that classifies as exit 2 or exit 1), runs `TRIGGER_CMD` under a bounded deadline (a trigger that never returns is force-killed, and the marker watch still decides the verdict), waits for a new `PASS_MARKER` in the app's captured output (one already there before the trigger does not count), then stops the app with SIGINT and classifies. Set `READY_MARKER`/`PASS_MARKER` to the chosen leaf's own markers. A curl inside `TRIGGER_CMD` is exactly the right use of curl: it triggers, and the markers judge. On the canonical two-class shape, generate `verify-hooks.sh` too, carrying the classic commands (`START_CMD='mvn -q exec:java@subscriber'`, `TRIGGER_CMD='mvn -q exec:java@publisher'`) so the file documents the same contract everywhere; the fixed stages do not read it. ### Interpret the exit code -- **Exit 0 (stage passes).** The stage milestone appeared and the run tore down cleanly (for the fixed stages that includes the graceful-shutdown proof line and a clean JVM exit, 0 or 130). Advance to the next stage. -- **Exit 1 (code failure).** Compile failed, a marker never appeared, a process died before its milestone, or a run hung and had to be force-killed. Enter the bounded fix loop below. -- **Exit 2 (environment failure).** A doc-traceable JCSMP connection/auth signature appeared (the broker is unreachable or the credentials are wrong). STOP. Do NOT enter the fix loop. The code is not the problem. +- **Exit 0 (stage passes).** The stage milestone appeared in this run and the run tore down cleanly (for the fixed stages that includes a foreground role that exited with status 0 on its own, the graceful-shutdown proof line, and a clean JVM exit, 0 or 130). Advance to the next stage. +- **Exit 1 (code failure).** Compile failed, a marker never appeared, a process died before its milestone, the foreground role exited non-zero, or a run hung and had to be force-killed. Enter the bounded fix loop below. An exit 1 before any process starts is a missing precondition instead (a usage error, or a missing `pom.xml` or `verify-hooks.sh`), and stderr names it: fix that and re-run, without counting it as a fix-loop attempt. +- **Exit 2 (environment failure).** A doc-traceable JCSMP connection/auth signature appeared (the broker is unreachable or the credentials are wrong), or the long-running role was still connecting when the readiness deadline passed and never printed `VERIFY: CONNECTED` (an unreachable host, or a firewall that blocks the port). STOP. Do NOT enter the fix loop. The code is not the problem. **Bounded fix loop (exit 1 only).** Diagnose the captured output, fix the code, and re-run the same stage, up to 3 automatic attempts per stage. After the third failure, stop fixing: summarize exactly what you tried across the attempts and what the captured output shows, then hand the decision to the developer. Report the evidence, not just "it failed". diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/scripts/verify.sh b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/scripts/verify.sh index 3e915fc..c21f173 100755 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/scripts/verify.sh +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/scripts/verify.sh @@ -24,16 +24,22 @@ # foreground and lets it exit on its own, watches each process's OWN captured # output for a fixed VERIFY: marker line, sends SIGINT to the subscriber to # exercise its graceful-shutdown hook, and classifies the outcome by exit code: -# 0 = stage passed (marker seen, the shutdown hook ran, AND the subscriber -# exited cleanly. A clean exit is exit code 0 OR 130: the JVM reports 130 +# 0 = stage passed (marker seen in this run, the foreground role exited with +# status 0 on its own, the shutdown hook ran, AND the subscriber exited +# cleanly. A clean exit is exit code 0 OR 130: the JVM reports 130 # (128+SIGINT) when it terminates AFTER running its shutdown hooks, which -# is the expected, graceful outcome here) +# is the expected, graceful outcome here. The app stage instead passes on +# READY_MARKER, then a new PASS_MARKER after the trigger, and a bounded +# teardown; the app's shutdown is not checked) # 1 = code failure (compile failed, a marker never appeared, a process died -# before its milestone, the shutdown hook did not run, or a run hung and -# had to be force-killed) +# before its milestone, the foreground role exited non-zero, the shutdown +# hook did not run, or a run hung and had to be force-killed), OR a +# precondition miss before any process starts (a usage error, or a missing +# pom.xml or verify-hooks.sh); stderr names which # 2 = environment failure (broker/credentials: a doc-traceable JCSMP error -# signature appeared in either process log; the code is not the problem, -# so the agent must NOT enter the fix loop) +# signature appeared in either process log, or the long-running role was +# still connecting at the readiness deadline with no VERIFY: CONNECTED; the +# code is not the problem, so the agent must NOT enter the fix loop) # # SIGINT delivery under `mvn exec:java` (the D-06 mechanism, why `set -m`): # in a NON-interactive shell without job control, a command started in the @@ -215,7 +221,8 @@ echo "── Live-run heads-up: stage '$STAGE' starts processes that connect to # stage waits for a role-specific milestone in that role's OWN log: # publisher → VERIFY: PUBLISH_ACKED in PUB_LOG (the publish ACK callback fired) # consumer → VERIFY: QUEUE_BOUND in SUB_LOG (flow.start() bound the durable queue) -# roundtrip → VERIFY: MESSAGE_RECEIVED in SUB_LOG (a published message was consumed + ACKed) +# roundtrip → VERIFY: PUBLISH_ACKED in PUB_LOG, then a NEW VERIFY: MESSAGE_RECEIVED in SUB_LOG +# (a published message was consumed + ACKed; a stale queued one does not count) # Reaching any stage marker already proves the connect happened for that process. # These strings form a fixed contract with the VERIFY: markers that # implement-mode.md Step 4 instructs the generated app to println. The two MUST @@ -223,7 +230,7 @@ echo "── Live-run heads-up: stage '$STAGE' starts processes that connect to case "$STAGE" in publisher) ;; # waits for VERIFY: PUBLISH_ACKED in PUB_LOG consumer) ;; # waits for VERIFY: QUEUE_BOUND in SUB_LOG - roundtrip) ;; # waits for VERIFY: MESSAGE_RECEIVED in SUB_LOG + roundtrip) ;; # waits for VERIFY: PUBLISH_ACKED, then a new VERIFY: MESSAGE_RECEIVED in SUB_LOG direct) ;; # direct pub/sub stage direct-request-reply) ;; # direct request-reply stage guaranteed-request-reply) ;; # guaranteed request-reply stage @@ -278,6 +285,32 @@ TIMEOUT_S=30 # marker-watch deadline (Claude's discretion per CONTEXT) PUBLISH_WAIT_S=30 # foreground-publisher deadline before escalating to a code failure SHUTDOWN_WAIT_S=15 # bounded post-SIGINT wait before escalating to SIGTERM/SIGKILL +# A long-running role that is still alive at the readiness deadline and never +# printed VERIFY: CONNECTED is still connecting: an unreachable host or a blocked +# port, where the connect attempt (and its retries) outlasts TIMEOUT_S without +# logging an error. That is the environment, not the code, so exit 2. +classify_stalled_connect() { # + if [ "$status" -ne 2 ] && [ "$3" -eq 0 ] && kill -0 "$1" 2>/dev/null && ! grep -qF "VERIFY: CONNECTED" "$2"; then + status=2 + CONNECT_STALLED=1 + fi +} + +# Print the count of in once it has stopped changing for 2 seconds +# (bounded by TIMEOUT_S), so messages an earlier run left on a durable queue, which +# arrive just after the bind, are all counted before the publisher starts. +settled_count() { # + local n m settle_deadline=$(( SECONDS + TIMEOUT_S )) + n=$(grep -cF "$2" "$1" || true) + while [ "$SECONDS" -lt "$settle_deadline" ]; do + sleep 2 + m=$(grep -cF "$2" "$1" || true) + [ "$m" -eq "$n" ] && break + n=$m + done + echo "$n" +} + # ── Bounded long-running shutdown (shared by the env-failure teardown and STAGE D) ─ # Send SIGINT to the process GROUP of the PID passed as $1, poll up to # SHUTDOWN_WAIT_S for the JVM to exit on its own, then escalate SIGTERM then @@ -379,6 +412,7 @@ trap cleanup EXIT status=1 # default to code failure until a stage milestone is confirmed DIRECT_ZERO_RECEIPT=0 # set by run_direct_stage when the at-most-once message dropped post-gate (D-05) RR_NO_REPLY=0 # set by run_request_reply_stage when the replier subscribed but no reply came back +CONNECT_STALLED=0 # set when the long-running role was still connecting at the readiness deadline # ── run_guaranteed_stage: the guaranteed pub/sub flow (publisher|consumer|roundtrip) ─ # The entire STAGE A..D body is lifted here VERBATIM (GEN-06 byte-stability): no @@ -428,6 +462,7 @@ while [ "$SECONDS" -lt "$deadline" ]; do fi sleep 1 done +classify_stalled_connect "$SUB_PID" "$SUB_LOG" "$bound" # If the broker/credentials are wrong the subscriber already proved it; tear it # down (bounded, same SIGINT/SIGTERM/SIGKILL escalation as STAGE D) and classify @@ -448,7 +483,13 @@ else # (subscriber-first ordering, D-07-R11). The wait is BOUNDED: a publisher that # hangs past PUBLISH_WAIT_S is escalated and classified as a code failure so # the unattended script NEVER hangs. + published=0 + stale=0 if { [ "$STAGE" = "publisher" ] || [ "$STAGE" = "roundtrip" ]; } && [ "$bound" -eq 1 ]; then + # A receipt already in the log is a message an earlier run left on the durable + # queue, delivered at bind time; the round trip must see a receipt beyond these. + stale=$(settled_count "$SUB_LOG" "VERIFY: MESSAGE_RECEIVED") + [ "$stale" -eq 0 ] || echo "── $stale message(s) left on the queue by an earlier run arrived before the publisher started; they do not count ──" >&2 echo "── Running publisher (mvn -q exec:java@publisher), watching for: VERIFY: PUBLISH_ACKED ──" set -m if [ "$TWO_PROJECT" -eq 1 ]; then @@ -479,32 +520,37 @@ else if [ "$pub_hung" -eq 1 ]; then # The publisher did not exit on its own within the deadline; escalate - # and treat the hang as a code failure (status stays 1). + # and record the hang (pub_rc 124), which fails the stage. echo "── Publisher did not exit within ${PUBLISH_WAIT_S}s; escalating (SIGTERM, then SIGKILL) ──" >&2 kill -TERM "$PUB_PID" 2>/dev/null || true sleep 2 kill -KILL "$PUB_PID" 2>/dev/null || true wait "$PUB_PID" 2>/dev/null || true + pub_rc=124 # synthetic "timed out" code; never a clean exit else - wait "$PUB_PID" 2>/dev/null || true + pub_rc=0 + wait "$PUB_PID" 2>/dev/null || pub_rc=$? fi if grep -qE "$ENV_SIGNATURES" "$PUB_LOG"; then status=2 # environment failure (broker/credentials), bypass the fix loop - elif [ "$STAGE" = "publisher" ] && grep -qF "VERIFY: PUBLISH_ACKED" "$PUB_LOG"; then - status=0 # the publish ACK callback fired + elif [ "$pub_rc" -eq 0 ] && grep -qF "VERIFY: PUBLISH_ACKED" "$PUB_LOG"; then + published=1 # the publish ACK callback fired and the publisher exited 0 + if [ "$STAGE" = "publisher" ]; then status=0; fi fi fi # ── STAGE C: confirm receipt on the SUBSCRIBER's log (roundtrip only) ──────── - if [ "$STAGE" = "roundtrip" ] && [ "$status" -ne 2 ]; then + # One receipt more than before the publisher ran. A publisher that never acked, + # exited non-zero, or hung cannot pass the round trip. + if [ "$STAGE" = "roundtrip" ] && [ "$published" -eq 1 ]; then rt_deadline=$(( SECONDS + TIMEOUT_S )) while [ "$SECONDS" -lt "$rt_deadline" ]; do if grep -qE "$ENV_SIGNATURES" "$SUB_LOG"; then status=2 break fi - if grep -qF "VERIFY: MESSAGE_RECEIVED" "$SUB_LOG"; then + if [ "$(grep -cF "VERIFY: MESSAGE_RECEIVED" "$SUB_LOG" || true)" -gt "$stale" ]; then status=0 # a published message was consumed + ACKed break fi @@ -598,6 +644,7 @@ run_direct_stage() { fi sleep 1 done + classify_stalled_connect "$SUB_PID" "$SUB_LOG" "$subscribed" if [ "$status" -eq 2 ]; then # Broker/credentials are wrong; tear the subscriber down (bounded) and @@ -639,8 +686,10 @@ run_direct_stage() { sleep 2 kill -KILL "$PUB_PID" 2>/dev/null || true wait "$PUB_PID" 2>/dev/null || true + pub_rc=124 # synthetic "timed out" code; never a clean exit else - wait "$PUB_PID" 2>/dev/null || true + pub_rc=0 + wait "$PUB_PID" 2>/dev/null || pub_rc=$? fi # ── Pass condition: at least one VERIFY: MESSAGE_RECEIVED in the subscriber ─ @@ -652,6 +701,9 @@ run_direct_stage() { status=2 # environment failure (broker/credentials), bypass the fix loop break fi + if [ "$pub_rc" -ne 0 ]; then + break # the publisher exited non-zero or hung (code failure) + fi if grep -qF "VERIFY: MESSAGE_RECEIVED" "$SUB_LOG"; then status=0 # at least one direct message was received (D-04 pass) break @@ -666,7 +718,7 @@ run_direct_stage() { # clean, but not one message of the burst arrived) stays status 1; the case-1 print is # annotated with the Direct at-most-once caveat below (D-05). An env match # already set status 2 above. - if [ "$status" -ne 0 ] && [ "$status" -ne 2 ]; then + if [ "$status" -ne 0 ] && [ "$status" -ne 2 ] && [ "$pub_rc" -eq 0 ]; then DIRECT_ZERO_RECEIPT=1 fi @@ -775,6 +827,7 @@ run_request_reply_stage() { fi sleep 1 done + classify_stalled_connect "$SUB_PID" "$SUB_LOG" "$subscribed" if [ "$status" -eq 2 ]; then # Broker/credentials are wrong; tear the replier down (bounded) and classify @@ -819,8 +872,10 @@ run_request_reply_stage() { sleep 2 kill -KILL "$PUB_PID" 2>/dev/null || true wait "$PUB_PID" 2>/dev/null || true + pub_rc=124 # synthetic "timed out" code; never a clean exit else - wait "$PUB_PID" 2>/dev/null || true + pub_rc=0 + wait "$PUB_PID" 2>/dev/null || pub_rc=$? fi # ── Pass condition: VERIFY: REPLY_RECEIVED in the requestor's OWN log, AND ─ @@ -828,14 +883,14 @@ run_request_reply_stage() { # match in either log maps to status 2. if grep -qE "$ENV_SIGNATURES" "$SUB_LOG" || grep -qE "$ENV_SIGNATURES" "$PUB_LOG"; then status=2 # environment failure (broker/credentials), bypass the fix loop - elif grep -qF "VERIFY: REPLY_RECEIVED" "$PUB_LOG" && grep -qF "VERIFY: REQUEST_RECEIVED" "$SUB_LOG"; then + elif [ "$pub_rc" -eq 0 ] && grep -qF "VERIFY: REPLY_RECEIVED" "$PUB_LOG" && grep -qF "VERIFY: REQUEST_RECEIVED" "$SUB_LOG"; then status=0 # the request landed on the replier and the correlated reply came back fi # A clean requestor run with no reply (subscription was confirmed, the requestor # ran and self-exited, but no VERIFY: REPLY_RECEIVED) stays status 1; the case-1 # print is annotated with the no-reply caveat below. An env match already set 2. - if [ "$status" -ne 0 ] && [ "$status" -ne 2 ]; then + if [ "$status" -ne 0 ] && [ "$status" -ne 2 ] && [ "$pub_rc" -ne 124 ] && ! grep -qF "VERIFY: REPLY_RECEIVED" "$PUB_LOG"; then RR_NO_REPLY=1 fi @@ -933,6 +988,10 @@ run_app_stage() { # response never came back. Reusing PUB_PID (not a new name) keeps the # EXIT trap's group-kill coverage over an orphaned trigger. sleep 2 + # A PASS_MARKER already in the log (a backlog the app consumed at startup) + # does not count; the trigger must produce a new one. + local seen + seen=$(settled_count "$SUB_LOG" "$PASS_MARKER") echo "── Running trigger (TRIGGER_CMD from verify-hooks.sh), watching for: $PASS_MARKER ──" set -m bash -c "$TRIGGER_CMD" >"$PUB_LOG" 2>&1 & @@ -957,7 +1016,7 @@ run_app_stage() { status=2 break fi - if grep -qF "$PASS_MARKER" "$SUB_LOG"; then + if [ "$(grep -cF "$PASS_MARKER" "$SUB_LOG" || true)" -gt "$seen" ]; then status=0 # the pass marker appeared in the app's own captured output break fi @@ -986,6 +1045,13 @@ case "$STAGE" in *) echo "unknown stage: $STAGE" >&2; usage ;; esac +# A connection error can be logged only after the stage gave up (for example during +# the SIGINT teardown, while the API was still retrying the connect), so look for +# the environment signatures again before classifying a code failure. +if [ "$status" -eq 1 ] && { grep -qE "$ENV_SIGNATURES" "$SUB_LOG" || grep -qE "$ENV_SIGNATURES" "$PUB_LOG"; }; then + status=2 +fi + # ── Print the captured evidence, exit classified (the EXIT trap removes the ──── # temp logs, V8) ───────────────────────────────────────────────────────────────── echo "── Captured subscriber output ──────────────────────────────────────" @@ -999,17 +1065,25 @@ case "$status" in if [ "$STAGE" = "app" ]; then echo "PASS (app): the readiness and pass markers were observed in the app's own captured output, and the app was stopped with a bounded teardown." else - echo "PASS ($STAGE): the stage milestone was observed in the role's own log, the subscriber's shutdown hook ran ('$SHUTDOWN_LINE'), and the subscriber exited cleanly (exit 0 or 130)." + echo "PASS ($STAGE): the stage milestone was observed in the role's own log, the foreground role exited with status 0, the subscriber's shutdown hook ran ('$SHUTDOWN_LINE'), and the subscriber exited cleanly (exit 0 or 130)." + fi + ;; + 2) + if [ "$CONNECT_STALLED" -eq 1 ]; then + echo "ENV FAILURE ($STAGE): the long-running role was still connecting to ${HOST:-the broker in config.json} after ${TIMEOUT_S}s and never printed VERIFY: CONNECTED. The host is unreachable or a firewall blocks the port (connect retries can stretch this). Check the host and port. NOT entering the fix loop." >&2 + else + echo "ENV FAILURE ($STAGE): a JCSMP connection/auth signature matched in a process log (broker or credentials are wrong, not the code). NOT entering the fix loop." >&2 fi ;; - 2) echo "ENV FAILURE ($STAGE): a JCSMP connection/auth signature matched in a process log (broker or credentials are wrong, not the code). NOT entering the fix loop." >&2 ;; *) if [ "$DIRECT_ZERO_RECEIPT" -eq 1 ]; then echo "CODE FAILURE ($STAGE): the subscription was confirmed (VERIFY: SUBSCRIBED) and the direct publisher ran clean, but not one message of the burst was received (VERIFY: MESSAGE_RECEIVED never appeared). Direct messaging is at-most-once: a whole burst dropped after a confirmed subscription points at the topic subscription, the topic string, or a slow-consumer egress discard, not a transient race. Check that the subscriber's subscription matches the publisher's topic." >&2 elif [ "$RR_NO_REPLY" -eq 1 ]; then echo "CODE FAILURE ($STAGE): the replier subscribed (VERIFY: SUBSCRIBED) and the requestor ran and self-exited, but no correlated reply came back (VERIFY: REPLY_RECEIVED never appeared in the requestor log). Check the requestor's request topic against the replier's subscription, the reply-to handling (the requestor's createRequestor / the replier's sendReply onto getReplyTo), and whether VERIFY: REQUEST_RECEIVED appeared in the replier log: its absence means the request never reached the replier, its presence means the reply was lost on the way back." >&2 else - echo "CODE FAILURE ($STAGE): the stage milestone was not observed, a process died before its milestone, the shutdown hook did not run, or a run hung and had to be force-killed." >&2 + fg_note="" + [ "$STAGE" = "app" ] || [ "${pub_rc:-0}" -eq 0 ] || fg_note=" The foreground role exited with status $pub_rc (124: it did not exit on its own and was force-killed)." + echo "CODE FAILURE ($STAGE): the stage milestone was not observed, a process died before its milestone, the foreground role exited non-zero, the shutdown hook did not run, or a run hung and had to be force-killed.$fg_note" >&2 fi ;; esac diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/python.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/python.md index a726fa3..b546b85 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/python.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/python.md @@ -12,7 +12,7 @@ Determine the user's intent and enter the appropriate mode. Routing is keyed on |---|---|---| | "Help me choose the Solace messaging pattern for a new Python app" / "Topic or queue?" / "What delivery semantics should I use?" | **Design** | Read `python/design-mode.md` | | Build or generate a NEW app or messaging component with NO valid design contract yet ("Build me a Solace Python publisher/consumer", "Generate a Python project with solace-pubsubplus", a web app / dashboard / service, or any embedded shape where the Python messaging layer lives inside a larger application) | **Design first** | Read `python/design-mode.md`. A prompt that already states the design resolves on its fully-specified path in ONE confirm; a bare prompt walks the tree. The confirmed summary then feeds Implement. | -| A build request WITH a valid design contract (see the design-contract gate below) | **Implement** | Read `python/implement-mode.md` | +| A build request WITH a valid design contract (see the design-contract gate below) | **Implement** | Read `python/implement-mode.md`; for an embedded shape generate the messaging layer per its leaf rules and verify with the `app` stage of `verify.sh` (its Step 5) | | An edit to an EXISTING app that changes the messaging topology (the topology rule below) | **Design first** | Re-enter `python/design-mode.md` to re-confirm the affected summary fields, then Implement applies the change | | A mechanical edit to an EXISTING app with no topology change (reconnect handling, payload format, logging, renames, ack tuning) | **Implement** | Read `python/implement-mode.md`; no design pass | | "My Python app is throwing on connect" / "Why is my receiver not binding the queue?" | **Debug** | Debug mode is not yet available in this release. Redirect the user to the canonical Solace Python API documentation: [Python API Home](https://docs.solace.com/API/Messaging-APIs/Python-API/python-home.md). The redirect is the ENTIRE debug answer: acknowledge the problem in one line, link the page, and stop. Do not generate debugging guidance from memory. That prohibition covers root-cause hypotheses, likely causes, interim checks or "things worth checking in the meantime", workarounds, and configuration or code suggestions. Restating this rule does not license adding such guidance after the redirect. | @@ -40,7 +40,7 @@ The package's `requires_python` value on PyPI is an install floor, not a support ## Verify script -The Python API does not bundle a verify script yet. Every Implement run ends in the compile-only fallback that `python/implement-mode.md` Step 5 defines, and never claims a verified run (SKILL.md Invariant 5). +The Python API bundles `python/scripts/verify.sh`, a stage-dispatched run-and-observe script (`verify.sh ...`) that Implement mode copies into every generated project and runs against a reachable broker (`python/implement-mode.md` Step 5). It is a separate script from the JCSMP one: it runs each project's `.venv` interpreter, not Maven. Without real broker values, Implement mode ends in the compile-only fallback of the same step and never claims a verified run (SKILL.md Invariant 5). ## Canonical doc links @@ -77,3 +77,5 @@ Live exceptions (these have no `docs.solace.com` `.md` form; use the live URL di - `python/python-solace-connection-config.py`: the shared connection-config helper every generated app uses. - `python/python-direct-publisher-sample.py` and `python/python-direct-subscriber-sample.py`: the direct pub/sub samples (bounded-buffer publish with a publish failure listener; direct topic subscription with discard detection). - `python/python-direct-requestor-blocking-sample.py`, `python/python-direct-requestor-sample.py`, and `python/python-direct-replier-sample.py`: the direct request-reply samples (blocking and non-blocking requestor; replier answering with `replier.reply()`). +- `python/verification-checklist.md`: the master verification checklist template that implement-mode Step 6 tailors into every generated project. +- `python/scripts/verify.sh`: the bundled verify script that Implement mode copies into every generated project; the `app` stage drives web and embedded shapes through a generated `verify-hooks.sh`. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check-samples.sh b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check-samples.sh new file mode 100755 index 0000000..5de6288 --- /dev/null +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check-samples.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# +# Sample check for the baked Solace Python samples. Python has no compile step, so the +# gate installs the latest solace-pubsubplus, runs py_compile on every baked sample, +# and runs check_samples.py, which imports each sample, builds its API objects without a +# broker, and exercises the shared connection helper. The script's exit code IS the +# gate: exit 0 only if every check passes. +# +# Runnable locally with one command from any working directory. Resolves the +# solace-pubsubplus version from PyPI at run time, so no version number is committed +# anywhere (SKILL.md Invariant 3). All work happens in a throwaway temp directory, so +# the repository tree is left unmodified. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# references/python/ lives two levels up from sample-check/ (evals/ -> python/). +REFERENCES_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" +PYTHON="${PYTHON:-python3}" + +# Sample roster: each baked sample and the name a generated project gives it +# (implement-mode.md Step 4). A new baked sample is a one-line append here. +SAMPLES=( + "python-solace-connection-config.py:solace_connection_config.py" + "python-guaranteed-publisher-sample.py:guaranteed_publisher.py" + "python-guaranteed-subscriber-sample.py:guaranteed_subscriber.py" + "python-direct-publisher-sample.py:direct_publisher.py" + "python-direct-subscriber-sample.py:direct_subscriber.py" + "python-direct-requestor-blocking-sample.py:direct_requestor_blocking.py" + "python-direct-requestor-sample.py:direct_requestor.py" + "python-direct-replier-sample.py:direct_replier.py" +) + +# Existence guard: fail early if any input is missing, including a new roster entry +# whose sample is not yet committed. +for entry in "${SAMPLES[@]}"; do + src="${REFERENCES_DIR}/${entry%%:*}" + [[ -f "$src" ]] || { echo "ERROR: required input not found: $src" >&2; exit 1; } +done +for f in "${SCRIPT_DIR}/check_samples.py" "${REFERENCES_DIR}/scripts/verify.sh"; do + [[ -f "$f" ]] || { echo "ERROR: required input not found: $f" >&2; exit 1; } +done + +# (1) Resolve the latest solace-pubsubplus release from the PyPI JSON API, the same +# source implement-mode.md Step 3 uses. On any failure, exit non-zero rather than +# guessing a version. +echo "Resolving the latest solace-pubsubplus version from PyPI..." +VERSION="$(curl -s --max-time 20 https://pypi.org/pypi/solace-pubsubplus/json \ + | "$PYTHON" -c 'import json, sys; print(json.load(sys.stdin)["info"]["version"])' 2>/dev/null || true)" +if [[ -z "${VERSION// /}" ]]; then + echo "ERROR: could not resolve the latest solace-pubsubplus version from PyPI" >&2 + echo " (network failure or unreachable API). Not guessing a version." >&2 + exit 1 +fi +echo "Resolved solace-pubsubplus version: ${VERSION}" + +# (2) Build a throwaway working tree and virtual environment. +WORK_DIR="$(mktemp -d)" +cleanup() { rm -rf "$WORK_DIR"; } +trap cleanup EXIT + +"$PYTHON" -m venv "${WORK_DIR}/.venv" +VENV_PY="${WORK_DIR}/.venv/bin/python" +echo "Installing solace-pubsubplus==${VERSION} with $("$VENV_PY" --version)..." +"$VENV_PY" -m pip install -q --disable-pip-version-check "solace-pubsubplus==${VERSION}" + +# (3) Copy the samples under their generated names. Content is verbatim; only the +# file name changes, so the samples import each other as a generated project does. +for entry in "${SAMPLES[@]}"; do + cp "${REFERENCES_DIR}/${entry%%:*}" "${WORK_DIR}/${entry##*:}" +done +cp "${SCRIPT_DIR}/check_samples.py" "${WORK_DIR}/check_samples.py" + +# (4) py_compile every sample. +echo "Running py_compile on every baked sample..." +( cd "$WORK_DIR" && "$VENV_PY" -m py_compile ./*.py ) +echo "PASS: py_compile" + +# (5) Import each sample and build its API objects without a broker, then exercise +# the connection helper. +echo "Building the API objects against tcp://localhost:55555 without a broker..." +( cd "$WORK_DIR" && "$VENV_PY" check_samples.py ) + +# (6) The bundled verify script must at least parse. +bash -n "${REFERENCES_DIR}/scripts/verify.sh" +echo "PASS: scripts/verify.sh parses" + +echo "PASS: the baked Python samples against solace-pubsubplus ${VERSION}" diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check_samples.py b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check_samples.py new file mode 100644 index 0000000..5bc89c1 --- /dev/null +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check_samples.py @@ -0,0 +1,122 @@ +"""Sample check for the baked Solace Python samples, run by check-samples.sh. + +check-samples.sh copies every sample into the working directory under its generated name and +runs this file with the virtual environment's interpreter. Without a broker, it: + +1. imports each sample module (each keeps its __main__ guard, so nothing connects), +2. calls each sample's setup_solace(), which builds the MessagingService and the + role's publisher, receiver, or request-reply objects, then teardown_solace() on the + never-connected service, and +3. exercises the shared solace_connection_config helper's config.json, command-line + fallback, and fail-fast paths. + +build() resolves the host, so every build uses tcp://localhost:55555, which resolves +whether or not a broker listens there. Any failure raises, and the non-zero exit fails +the gate. +""" + +import glob +import importlib +import json +import os +import tempfile +import threading + +# Every sample check-samples.sh copied in, so a new roster entry is covered without a +# second list to keep in step. The shared helper has no setup_solace() of its own. +NOT_SAMPLES = {"check_samples", "solace_connection_config"} +MODULES = sorted( + name + for name in (os.path.splitext(path)[0] for path in glob.glob("*.py")) + if name not in NOT_SAMPLES +) + +HOST_KEY = "solace.messaging.transport.host" +VPN_KEY = "solace.messaging.service.vpn-name" +USER_KEY = "solace.messaging.authentication.basic.username" +PASSWORD_KEY = "solace.messaging.authentication.basic.password" + +PROPERTIES = { + HOST_KEY: "tcp://localhost:55555", + VPN_KEY: "smoke-vpn", + USER_KEY: "smoke-user", + PASSWORD_KEY: "smoke-pass", +} + + +def build_samples() -> None: + if not MODULES: + raise RuntimeError("no sample modules found in the working directory") + for name in MODULES: + module = importlib.import_module(name) + state = module.setup_solace(dict(PROPERTIES), threading.Event()) + module.teardown_solace(state) + print(f"PASS: {name} builds its service and role objects without a broker", flush=True) + + +def expect_exit(call, case: str) -> None: + try: + call() + except SystemExit as exit_error: + if exit_error.code == 1: + return + raise AssertionError(f"{case}: exit code {exit_error.code}, expected 1") from exit_error + raise AssertionError(f"{case}: expected the helper to exit") + + +def expect_value_error(call, case: str) -> None: + try: + call() + except ValueError: + return + raise AssertionError(f"{case}: expected a ValueError") + + +def check_config_helper() -> None: + from solace_connection_config import load_service_properties + + start_dir = os.getcwd() + with tempfile.TemporaryDirectory() as work: + os.chdir(work) + try: + def write_config(config: dict) -> None: + with open("config.json", "w", encoding="utf-8") as config_file: + json.dump(config, config_file) + + # a valid config.json round-trips every key, including a pass-through key + write_config({**PROPERTIES, "solace.messaging.tls.trust-store-path": "/smoke/trust"}) + loaded = load_service_properties([], "smoke") + assert loaded[HOST_KEY] == PROPERTIES[HOST_KEY], loaded + assert loaded["solace.messaging.tls.trust-store-path"] == "/smoke/trust", loaded + + # config.json wins over command-line arguments + loaded = load_service_properties(["tcp://other:55555", "v", "u"], "smoke") + assert loaded[HOST_KEY] == PROPERTIES[HOST_KEY], loaded + + # a missing required key fails fast + write_config({VPN_KEY: "smoke-vpn", USER_KEY: "smoke-user"}) + expect_value_error(lambda: load_service_properties([], "smoke"), "missing-key") + + # a present but blank required value fails fast + write_config({**PROPERTIES, HOST_KEY: " "}) + expect_value_error(lambda: load_service_properties([], "smoke"), "blank-value") + + # with no config.json, the command-line arguments are the source, and an + # omitted password is sent as an empty string + os.remove("config.json") + loaded = load_service_properties(["tcp://localhost:55555", "smoke-vpn", "smoke-user"], "smoke") + assert loaded[USER_KEY] == "smoke-user" and loaded[PASSWORD_KEY] == "", loaded + + # a blank command-line argument fails fast + expect_value_error(lambda: load_service_properties(["", "smoke-vpn", "smoke-user"], "smoke"), "args-blank") + + # neither source: print the usage line and exit 1 + expect_exit(lambda: load_service_properties([], "smoke"), "no-source") + finally: + os.chdir(start_dir) + print("PASS: solace_connection_config file, fallback, and fail-fast paths", flush=True) + + +if __name__ == "__main__": + build_samples() + check_config_helper() diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-direct-pubsub.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-direct-pubsub.md index 53488c5..b2dc851 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-direct-pubsub.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-direct-pubsub.md @@ -35,4 +35,4 @@ The publisher only connects and publishes a short bounded burst of DIRECT messag ## The marker contract -These three `VERIFY:` marker strings (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: MESSAGE_RECEIVED`) are the same strings the JCSMP Direct Pub/Sub leaf emits, character for character. They are read PER PROCESS: the subscriber output carries `SUBSCRIBED` and `MESSAGE_RECEIVED`, and both scripts print `CONNECTED`. Until the Python verify script lands, the developer reads them from the two terminals (`implement-mode.md` Step 5); the verify script will read the same strings, so never change one. +These three `VERIFY:` marker strings (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: MESSAGE_RECEIVED`) are the same strings the JCSMP Direct Pub/Sub leaf emits, character for character. They are read PER PROCESS: the subscriber output carries `SUBSCRIBED` and `MESSAGE_RECEIVED`, and both scripts print `CONNECTED`. `verify.sh` reads these same strings in each process's own output (`implement-mode.md` Step 5), so never change one. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-guaranteed-pubsub.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-guaranteed-pubsub.md index aaa19cf..826e4b9 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-guaranteed-pubsub.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-guaranteed-pubsub.md @@ -36,7 +36,7 @@ queue = Queue.durable_exclusive_queue(queue_name) # one active consumer / When the `Access type` field reads exclusive, keep the `durable_exclusive_queue` line: only one bound receiver is active at a time, and the receiver state listener reports the passive standby. When it reads non-exclusive, use the `durable_non_exclusive_queue` line instead, so several subscriber processes bound to the same queue share the messages. Emit exactly one of the two lines. Ground the choice in [Topic Endpoints and Queues](https://docs.solace.com/Get-Started/topic-endpoints-queues.md). -**Fan-out.** On the fan-out leaf, every consuming service gets its own durable queue with the same topic subscription, so each service receives its own full copy of every event. Generate one subscriber script per consuming service: `guaranteed_subscriber.py` for the first service, and `guaranteed_subscriber_.py` for each further service, where `` is a lowercase Python identifier (letters, digits, and underscores, for example `billing` or `audit_log`) so the import smoke can import the module, each with its own queue name from the design summary and an exclusive durable queue. Hand back one run command per subscriber in Step 5, and add each script to the import smoke. +**Fan-out.** On the fan-out leaf, every consuming service gets its own durable queue with the same topic subscription, so each service receives its own full copy of every event. Generate one subscriber script per consuming service: `guaranteed_subscriber.py` for the first service, and `guaranteed_subscriber_.py` for each further service, where `` is a lowercase Python identifier (letters, digits, and underscores, for example `billing` or `audit_log`) so the import smoke can import the module, each with its own queue name from the design summary and an exclusive durable queue. The `verify.sh` stages drive `guaranteed_subscriber.py` only, so hand back one run command per further subscriber in Step 5, and add each script to the import smoke. ## Publisher script (`guaranteed_publisher.py`) @@ -52,4 +52,4 @@ The publisher only connects and publishes; it does NOT provision the queue or th ## The marker contract -These four `VERIFY:` marker strings (`VERIFY: CONNECTED`, `VERIFY: PUBLISH_ACKED`, `VERIFY: QUEUE_BOUND`, `VERIFY: MESSAGE_RECEIVED`) are the same strings the JCSMP guaranteed pub/sub leaf emits, character for character. They are read PER PROCESS: the subscriber output carries `QUEUE_BOUND` and `MESSAGE_RECEIVED`, the publisher output carries `PUBLISH_ACKED`, and both print `CONNECTED`. Until the Python verify script lands, the developer reads them from the two terminals (`implement-mode.md` Step 5); the verify script will read the same strings, so never change one. +These four `VERIFY:` marker strings (`VERIFY: CONNECTED`, `VERIFY: PUBLISH_ACKED`, `VERIFY: QUEUE_BOUND`, `VERIFY: MESSAGE_RECEIVED`) are the same strings the JCSMP guaranteed pub/sub leaf emits, character for character. They are read PER PROCESS: the subscriber output carries `QUEUE_BOUND` and `MESSAGE_RECEIVED`, the publisher output carries `PUBLISH_ACKED`, and both print `CONNECTED`. `verify.sh` reads these same strings in each process's own output (`implement-mode.md` Step 5), so never change one. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-mode.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-mode.md index 5f5d465..fb95d3a 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-mode.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-mode.md @@ -6,7 +6,7 @@ Implement mode is NOT only for the canonical shape. A web app, a dashboard, a se Work through the steps in order. Read this file's grounding links only as each step needs them. Link the canonical doc and state the decision in one line, never paraphrase doc content. Resolve every dependency version at generation time; pin nothing in skill content (SKILL.md Invariant 3). -**Coverage in this release.** Implement mode generates all five Python leaves on all three doors. The Python API does not bundle a verify script yet, so every run ends in the compile-only fallback of Step 5. +**Coverage in this release.** Implement mode generates all five Python leaves on all three doors and verifies them with the bundled `scripts/verify.sh` (Step 5). ## Generation outputs: the file contract @@ -16,8 +16,9 @@ Every Implement run writes ALL of these as generation output, on every app shape 2. `requirements.txt` with the `solace-pubsubplus` version resolved per Step 3. 3. `config.example.json` (placeholders) and a `.gitignore` that ignores `config.json` and the virtual environment (Step 4). 4. A tailored `solace-verification-checklist.md` at each project root (tailoring rules in Step 6; the WRITE happens here at generation time, like the scripts). +5. A copy of the bundled Python `scripts/verify.sh` at each project root, plus a generated `verify-hooks.sh` beside it (Step 5), so the verification commands resolve from inside the project. -There is no `verify.sh` and no `verify-hooks.sh` in a Python project yet. Do not copy the JCSMP `verify.sh` into a Python project: it drives Maven and cannot run Python scripts. +Copy the Python `verify.sh` from `references/python/scripts/`, never the JCSMP one: the JCSMP script drives Maven and cannot run Python scripts. ## Step 0: Ask Quickstart, Solace Suggested, or Custom, before any broker details @@ -93,7 +94,7 @@ Generation is keyed on the `Pattern` leaf read in Step 2. Read ONLY the leaf fil **Every leaf also generates the shared connection helper.** Read `python-solace-connection-config.py` and generate it as `solace_connection_config.py` (a module name needs underscores). Each script builds its connection dict by calling `load_service_properties(args, APP_NAME)` from that module and passes it to `MessagingService.builder().from_properties(...)`, rather than setting the host, VPN, username, or password inline. -**Fixed file names.** Generate each script under the name its sample's docstring gives, next to `solace_connection_config.py`: `guaranteed_publisher.py` and `guaranteed_subscriber.py` (the fan-out leaf adds one subscriber script per further consuming service, named in `implement-guaranteed-pubsub.md`), `direct_publisher.py` and `direct_subscriber.py`, and `direct_replier.py` with `direct_requestor_blocking.py` (or `direct_requestor.py` on the non-blocking variant). The run commands in Step 5 and the checklist address these names; the names of functions, constants, and the `APP_NAME` value inside each script stay your discretion. +**Fixed file names.** Generate each script under the name its sample's docstring gives, next to `solace_connection_config.py`: `guaranteed_publisher.py` and `guaranteed_subscriber.py` (the fan-out leaf adds one subscriber script per further consuming service, named in `implement-guaranteed-pubsub.md`), `direct_publisher.py` and `direct_subscriber.py`, and `direct_replier.py` with `direct_requestor_blocking.py` (or `direct_requestor.py` on the non-blocking variant). `verify.sh` (Step 5) finds the scripts by these names; the names of functions, constants, and the `APP_NAME` value inside each script stay your discretion. **Disclaimer header on EVERY generated source file.** Stamp the top of each generated `.py` file with these two comment lines, before the module docstring and before any import: @@ -104,9 +105,9 @@ Generation is keyed on the `Pattern` leaf read in Step 2. Read ONLY the leaf fil Stamp the header AT WRITE TIME, as the first content of each new file. `requirements.txt`, `config.example.json`, and `.gitignore` are not source and carry no header. The reference samples themselves carry NO disclaimer. This rule is SKILL.md Invariant 4. -**Embedded and web-app shapes.** When the Python messaging layer sits inside a larger application, the chosen leaf's wiring still governs the messaging layer: the same connection helper, the same service, receiver, and publisher listeners, the same `VERIFY:` markers emitted through a stdout-reaching `trace(...)`, and the disclaimer header on every generated file, messaging or not. The `VERIFY:` markers are NOT demo harness. +**Embedded and web-app shapes.** When the Python messaging layer sits inside a larger application, the chosen leaf's wiring still governs the messaging layer: the same connection helper, the same service, receiver, and publisher listeners, the same `VERIFY:` markers emitted through a stdout-reaching `trace(...)`, and the disclaimer header on every generated file, messaging or not. The `VERIFY:` markers are NOT demo harness; they stay in the messaging layer whatever the shape, because the `app` verify stage (Step 5) greps for them in the app's captured output. -**Write the verification artifacts in this same step.** Alongside the scripts, write the tailored `solace-verification-checklist.md` at each project root (Step 6), `config.example.json`, and `.gitignore`. A session that ends early still leaves the checklist in the project. +**Write the verification artifacts in this same step.** Alongside the scripts, write the tailored `solace-verification-checklist.md` at each project root (Step 6), `config.example.json`, `.gitignore`, a copy of the bundled `scripts/verify.sh` at each project root, and the generated `verify-hooks.sh` (Step 5 defines its contract). A session that ends early still leaves the checklist and the verify entry points in the project. Make the `verify.sh` copy with a read-and-redirect, `cat "/references/python/scripts/verify.sh" > verify.sh && chmod +x verify.sh`, where the skill directory is the one announced when this skill loaded. Do not use `cp` on the bundled file: Claude Code protects the plugin directory and blocks a `cp` that names a file inside it, and a blocked copy pushes the script through a retype that must stay byte-identical. Never edit the bundled script itself. ### Concurrency: threads, not asyncio, and no multiprocessing @@ -153,13 +154,19 @@ Generate two small project files next to the scripts (on Solace Suggested, on Cu Tell the developer they opt in to the config file by copying `config.example.json` to `config.json` and filling in their real broker values. Both scripts then read it from the working directory, so they run from the project root. -## Step 5: Check the project, then hand back the run commands +## Step 5: Check the project, then verify with verify.sh -The Python API does not bundle a verify script yet, so Implement mode ends in the compile-only fallback on every run, whatever the contents of `config.json` (SKILL.md Invariant 5). Never fabricate a run result and never call a check below a verified round trip. +Verification is a `verify.sh` run, on every door and every app shape. Step 4 copied `scripts/verify.sh` into each project root and generated `verify-hooks.sh` beside it, so every command below resolves from inside the project. An improvised check (curl against the app's own HTTP API, a manual browser click) may TRIGGER traffic, but it NEVER renders the verdict: the verdict is the `VERIFY:` markers plus the script's exit code, which is what separates "your code is broken" (exit 1) from "your broker, credentials, or trust store are wrong" (exit 2). Never fabricate a run result. + +**Live-broker heads-up (every run, every launcher).** Immediately before starting ANY process that will connect to a live broker (a `verify.sh` stage, a `.venv/bin/python` run, a launcher script), print ONE line that names the broker host and states the actual broker-side effects of this run: the durable queues the subscribers provision, the connections it opens, and the messages or requests the foreground role sends, scaled to the generated scripts (SKILL.md Invariant 6). + +### Check first: compile, import, and conformance + +Run these checks on every run, before any `verify.sh` stage. On the compile-only fallback below they are the whole check. 1. **Compile check.** Run `.venv/bin/python -m py_compile` on every generated `.py` file. A failure is a code failure: fix it and re-run, up to 3 attempts, then report the evidence and hand the decision to the developer. 2. **Import smoke.** From the project root, run `.venv/bin/python -c "import solace_connection_config, guaranteed_publisher, guaranteed_subscriber"` with the module names of the chosen leaf (for example `direct_publisher, direct_subscriber` or `direct_replier, direct_requestor_blocking`), adding every further generated module (for example each fan-out subscriber). On separate projects, run the checks in each project root against that project's modules. Each script keeps its `if __name__ == "__main__":` guard, so the import resolves every `solace.messaging` name the scripts use without connecting to a broker. A failure (for example an `ImportError` on a misspelled module path) is a code failure under the same fix loop. -3. **Conformance checks.** From the project root, run these three checks and record each result in the checklist. A missing header or a missing checklist is a code failure under the same fix loop; a version mismatch is reported to the developer, not fixed silently. +3. **Conformance checks.** From the project root, run these three checks and record each result in the checklist. A missing header or a missing checklist is a code failure under the same fix loop; a version mismatch is reported to the developer, not fixed silently. The `verify.sh` preflight repeats them as warnings on every run. ```bash # every generated .py file starts with the disclaimer header @@ -170,36 +177,69 @@ The Python API does not bundle a verify script yet, so Implement mode ends in th grep -qxF "solace-pubsubplus==$(curl -s https://pypi.org/pypi/solace-pubsubplus/json | python3 -c 'import json, sys; print(json.load(sys.stdin)["info"]["version"])')" requirements.txt || echo "VERSION DRIFT" ``` -4. **Report and hand back.** Report the compile, import, and conformance results, then hand the developer the exact commands to run against their own broker from the project root. The long-running role always starts first: the subscriber, so the durable queue (guaranteed) or the topic subscription (direct) exists before the first publish, and the replier, so the request topic subscription exists before the first request. For the Guaranteed Pub/Sub leaves: +### The canonical two-script shape: the fixed stages - ``` - # terminal 1: provisions the queue, binds, and waits (Ctrl-C stops it) - .venv/bin/python guaranteed_subscriber.py [ [password]] - # terminal 2: publishes, waits for the broker's acknowledgement, and exits - .venv/bin/python guaranteed_publisher.py [ [password]] - ``` +`verify.sh` checks every script with `py_compile`, starts the long-running role FIRST with the project's own `.venv` interpreter (the subscriber, so the durable queue or the topic subscription exists before the first publish; the replier, so the request topic subscription exists before the first request), waits for its readiness marker, runs the foreground role and lets it exit on its own, watches each process's own output for the stage's `VERIFY:` marker, sends SIGINT to the long-running role to exercise its graceful shutdown, and returns an exit code that classifies the outcome. It finds the scripts by the fixed file names of Step 4, so never rename a generated script. + +| `Pattern` leaf | Stages, in order | Each stage waits for | +|----------------|------------------|----------------------| +| The three Guaranteed Pub/Sub leaves | `consumer`, then `publisher`, then `roundtrip` | `VERIFY: QUEUE_BOUND` in the subscriber output; `VERIFY: PUBLISH_ACKED` in the publisher output (this stage starts the subscriber first, so it needs both scripts); `VERIFY: PUBLISH_ACKED` in the publisher output and then a new `VERIFY: MESSAGE_RECEIVED` in the subscriber output (a message an earlier run left on the durable queue does not count) | +| `Direct Pub/Sub` | `direct` | `VERIFY: SUBSCRIBED` in the subscriber output as the readiness gate, then at least one `VERIFY: MESSAGE_RECEIVED` from the publisher's short burst | +| `Request-Reply (Direct)` | `direct-request-reply` | `VERIFY: SUBSCRIBED` in the replier output as the readiness gate, then `VERIFY: REPLY_RECEIVED` in the requestor output and `VERIFY: REQUEST_RECEIVED` in the replier output | - For the Direct Pub/Sub leaf, the same two terminals run `direct_subscriber.py` (subscribes and waits; Ctrl-C stops it), then `direct_publisher.py` (publishes a short burst and exits). For the Request-Reply (Direct) leaf, they run `direct_replier.py` (subscribes to the request topic and waits; Ctrl-C stops it), then `direct_requestor_blocking.py` or `direct_requestor.py` (sends, waits for the reply, and exits). +The `direct-request-reply` stage runs `direct_requestor_blocking.py`, or `direct_requestor.py` when the project has the non-blocking variant instead. The Python API has no guaranteed request-reply, so there is no `guaranteed-request-reply` stage. On the fan-out leaf, the stages drive `guaranteed_subscriber.py` only; hand back one command per further subscriber script (`guaranteed_subscriber_.py`), each started in its own terminal before the publisher, and tell the developer that each prints its own `VERIFY: MESSAGE_RECEIVED`. - On the fan-out leaf, hand back one subscriber command per subscriber script (`guaranteed_subscriber.py` and each `guaranteed_subscriber_.py`), each in its own terminal and all started before the publisher; every subscriber prints its own markers. On separate projects (Solace Suggested, or the Custom decoupled-projects knob), hand back each command from its own project directory with that project's `.venv/bin/python`. +Invoke each stage from the project root in one of these forms: - The arguments are needed only when there is no `config.json`, and they are never offered on a path that drops the command-line fallback. Tell the developer what a passing run prints, per leaf: +- **With `config.json`** (always the form for a `tcps://` host, because the trust store path has no argument form): `./verify.sh `. Each script reads `config.json` from the working directory. +- **With connection arguments** (a plaintext host and no `config.json`): `./verify.sh [password]`, the same contract the scripts' command-line fallback uses. +- **On separate projects** (Solace Suggested, or the Custom decoupled-projects knob): `./verify.sh --subscriber-dir --publisher-dir ` on the pub/sub leaves, or `./verify.sh direct-request-reply --replier-dir --requestor-dir `. Run it from either project; the directories are relative to the working directory. It passes NO connection details on the command line: each script runs from inside its own project and reads that project's `config.json`, with that project's `.venv`. + +### Every other shape: the `app` stage and `verify-hooks.sh` + +A web app, an embedded service, or any single-process shape that the fixed stages cannot drive is verified with the `app` stage: `./verify.sh app`. The stage sources the generated `verify-hooks.sh`, which carries the ONLY app-specific facts (how to start the app and how to cause one publish), while the script keeps the universal logic: marker watching, timeouts, environment-signature classification, and the 0/1/2 exit contract. Generation writes `verify-hooks.sh` because the generator just wrote the app and knows both commands: + +```bash +# verify-hooks.sh: generated with the project; verify.sh's app stage sources it. +START_CMD='.venv/bin/python app.py' # starts the app (long-running) +TRIGGER_CMD='curl -s --max-time 10 -X POST localhost:8081/api/orders -d "{\"demo\":1}"' # causes exactly one publish +READY_MARKER='VERIFY: QUEUE_BOUND' # marker proving the consuming side is live +PASS_MARKER='VERIFY: MESSAGE_RECEIVED' # marker proving the round trip +``` - - Guaranteed Pub/Sub: the subscriber prints `VERIFY: CONNECTED` and then `VERIFY: QUEUE_BOUND`; the publisher prints `VERIFY: CONNECTED` and `VERIFY: PUBLISH_ACKED` and exits with status 0; the subscriber then prints `VERIFY: MESSAGE_RECEIVED`. - - Direct Pub/Sub: the subscriber prints `VERIFY: CONNECTED` and then `VERIFY: SUBSCRIBED`; the publisher prints `VERIFY: CONNECTED` and exits with status 0; the subscriber then prints `VERIFY: MESSAGE_RECEIVED`. - - Request-Reply (Direct): the replier prints `VERIFY: CONNECTED` and then `VERIFY: SUBSCRIBED`; the requestor prints `VERIFY: CONNECTED`; the replier prints `VERIFY: REQUEST_RECEIVED`; the requestor prints `VERIFY: REPLY_RECEIVED` and exits with status 0. +Generate `verify-hooks.sh` with ONLY these four assignments. The app stage imports exactly these four names from a child shell and ignores everything else in the file; keep the file to the four lines all the same. Give a network trigger its own timeout (the `--max-time` above): the stage also bounds the trigger and force-kills one that never returns, but a self-timing trigger leaves cleaner evidence. Set `READY_MARKER` and `PASS_MARKER` to the chosen leaf's own markers. On the canonical two-script shape, generate `verify-hooks.sh` too, carrying the classic commands (for example `START_CMD='.venv/bin/python guaranteed_subscriber.py'` and `TRIGGER_CMD='.venv/bin/python guaranteed_publisher.py'`), so the file documents the same contract everywhere; the fixed stages do not read it. - On every leaf, Ctrl-C on the long-running role prints `Shutdown signal received` and it exits with status 0. +The `app` stage starts `START_CMD` in the background, waits for `READY_MARKER` in the app's captured output, runs `TRIGGER_CMD` under a bounded deadline, waits for a new `PASS_MARKER` (one already in the output before the trigger does not count), then stops the app with SIGINT and classifies. A curl inside `TRIGGER_CMD` is exactly the right use of curl: it triggers, and the markers judge. -**When the developer asks you to run it.** If the developer explicitly asks you to run the scripts against their broker, print the live-broker heads-up first (SKILL.md Invariant 6: name the host, every durable queue the subscribers provision, one connection per script, and the messages or requests the foreground role sends, scaled to the generated scripts). Then start each long-running role in the background with its output captured, wait for its readiness marker (`VERIFY: QUEUE_BOUND` on the guaranteed leaves, `VERIFY: SUBSCRIBED` on the direct leaves) from each, run the foreground role, wait for the pass marker (`VERIFY: MESSAGE_RECEIVED` in each subscriber's output, or `VERIFY: REPLY_RECEIVED` in the requestor's output), and stop the long-running roles with SIGINT. Report the markers you observed as evidence of a manual run. It is not a verify verdict: record it in the checklist as a manual run, and never describe it as a passed verify stage. +### Interpret the exit code -**Environment failure on a manual run: do NOT modify the code.** When the captured output shows that the broker is unreachable, the credentials or VPN are wrong, the TLS handshake or the trust store fails, or the client may not create the queue (the subscriber's `Could not bind to queue` error), the code is not the problem. Report the matched output line as evidence and STOP: do not enter the fix loop, and do not change the code. The fix is the developer's broker, credentials, or client permissions. Fix the code only when the output shows a code failure (an exception from the scripts' own logic, or a marker that never appears after a clean connect and bind), up to 3 attempts, then hand the decision to the developer. +- **Exit 0 (stage passes).** For the fixed stages: the stage milestone appeared in this run, the foreground role exited with status 0 on its own, the long-running role printed `Shutdown signal received`, and it exited with status 0. For the `app` stage: `READY_MARKER` appeared, a new `PASS_MARKER` followed the trigger, and the app was stopped with a bounded teardown (its own shutdown is not checked). Advance to the next stage. +- **Exit 1 (code failure).** `py_compile` failed, a marker never appeared, a process died before its milestone, the foreground role exited non-zero, the shutdown line did not print, or a run hung and had to be force-killed. Enter the bounded fix loop below. An exit 1 before any process starts is a missing precondition instead (a usage error, or a missing script, `.venv`, `config.json`, or `verify-hooks.sh`), and stderr names it: fix that and re-run, without counting it as a fix-loop attempt. +- **Exit 2 (environment failure).** A Solace Python API connection, authentication, or TLS signature appeared (the broker is unreachable, the credentials or VPN are wrong, or the trust store is missing or holds the wrong CA), or the subscriber could not bind its queue because the client may not create it (`Could not bind to queue`), or the long-running role was still connecting when the readiness deadline passed and never printed `VERIFY: CONNECTED` (an unreachable host, or a firewall that blocks the port). STOP. Do NOT enter the fix loop. The code is not the problem. -**Close every run with the next steps, stated plainly.** Tell the developer, in order: (1) copy `config.example.json` to `config.json` and fill in their real broker values; on any `tcps://` host (always on Solace Suggested and with the Custom secure-session knob, and on Quickstart or on Custom with the secure-session knob unticked when the Step 1 answer was secure), also put the broker's CA certificate into a trust store directory and set `solace.messaging.tls.trust-store-path` to it, as `solace-suggested-mode.md` Step 1 explains. On a Quickstart run, or a Custom run with the secure-session knob unticked, whose Step 1 answer was plaintext, still add one line: if the host later changes to `tcps://`, the project needs a trust store directory set in `solace.messaging.tls.trust-store-path` in `config.json`, or the connection fails with "Failed to load trust store"; (2) run the long-running role, then the foreground role, with the commands above, and check the markers. Step 4 already wrote the tailored `solace-verification-checklist.md`; invite the developer to add their own items in its Your additional items section and to record any skill-undelivered concerns under Developer-owned items. +**Bounded fix loop (exit 1 only).** Diagnose the captured output, fix the code, and re-run the same stage, up to 3 automatic attempts per stage. After the third failure, stop fixing: summarize exactly what you tried and what the captured output shows, then hand the decision to the developer. Fixes apply without a confirmation gate, on every door. + +**Environment-failure path (exit 2).** Report the matched line from the captured output as evidence, and do NOT modify the code. The fix is the developer's broker, credentials, trust store, or client permissions. + +### When to run verify.sh, and when to hand it back + +**Run it from `config.json` with real values.** Run the stages for real, with NO confirmation prompt, ONLY when a `config.json` is present at every project root involved AND its values are real, meaning they are NOT the `config.example.json` placeholders (`tcp://HOST:55555` or `tcps://HOST:55443`, `YOUR_VPN`, `YOUR_USERNAME`, `YOUR_PASSWORD`, `PATH_TO_TRUST_STORE_DIRECTORY`). Filling in real values is the opt-in; a `config.json` that still carries a placeholder is not. Use the `config.json` form of the stage command (single project) or the separate-projects form, and run the leaf's stages in the table's order. Print the live-broker heads-up immediately before the run, naming the host from `config.json`; the real-values `config.json` is the consent, the heads-up is not a prompt. The developer can also ask you to run `verify.sh` with connection arguments they give for a plaintext host, but only when no `config.json` exists: the scripts prefer `config.json`, so `verify.sh` refuses connection arguments when one is present. + +**Honesty rule (no fabricated runs).** When `config.json` is absent or still holds placeholders, do NOT run the scripts and do NOT fabricate a result. Report the compile, import, and conformance results, and hand back the exact `verify.sh` commands for the developer to run from the project root, for example on the Guaranteed Pub/Sub leaves: + +``` +./verify.sh consumer +./verify.sh publisher +./verify.sh roundtrip +``` + +Add the connection arguments only for a plaintext host with no `config.json`, and never on a path that drops the command-line fallback. On separate projects, hand back the separate-projects form. For an `app`-stage shape, the handed-back command is `./verify.sh app` after the developer fills in the app's config. + +**Close every run with the next steps, stated plainly.** Tell the developer, in order: (1) copy `config.example.json` to `config.json` (in each project on separate projects) and fill in their real broker values; on any `tcps://` host (always on Solace Suggested and with the Custom secure-session knob, and on Quickstart or on Custom with the secure-session knob unticked when the Step 1 answer was secure), also put the broker's CA certificate into a trust store directory and set `solace.messaging.tls.trust-store-path` to it, as `solace-suggested-mode.md` Step 1 explains. On a Quickstart run, or a Custom run with the secure-session knob unticked, whose Step 1 answer was plaintext, still add one line: if the host later changes to `tcps://`, the project needs a trust store directory set in `solace.messaging.tls.trust-store-path` in `config.json`, or the connection fails with "Failed to load trust store"; (2) then just ask you to verify, and you will run `verify.sh` for them against their broker (naming the host and side effects first). Make the fill-in-`config.json`-then-ask-me path the recommended one; if they would rather run it themselves, point them at the handed-back commands. Step 4 already wrote the tailored `solace-verification-checklist.md`; invite the developer to add their own items in its Your additional items section and to record any skill-undelivered concerns under Developer-owned items. ### Traps to avoid -- **Improvised verdicts.** A connect that does not raise, or a publish call that returns, proves nothing about the round trip. Judge only by the markers in each script's output. +- **Improvised verdicts.** A connect that does not raise, a publish call that returns, or a curl against the app's own API proves nothing about the round trip. Trigger with whatever fits; judge ONLY by the markers and the `verify.sh` exit code. - **Publishing direct-to-queue instead of pub/sub.** Publish to a topic; let the durable queue carry a topic subscription. See [Creating Queues with the Solace Python API](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Create-Queues.md). - **DIRECT delivery where Guaranteed is required.** Use the persistent publisher for the guaranteed pub/sub journey. See [Publishing Persistent Messages](https://docs.solace.com/API/API-Developer-Guide-Python/Python-PM-Publish.md). - **Seconds instead of milliseconds.** `RetryStrategy.parametrized_retry(retries, retry_interval)` takes its interval in MILLISECONDS per the API reference; a value of `3` means 3 milliseconds. The sample code on the Messaging Service page comments `parametrized_retry(20, 3)` as "over an interval of 3 seconds"; that comment is wrong, and the API reference wins. Whenever a generated script sets a retry interval, write it in milliseconds, as the samples' named constants do. @@ -207,66 +247,14 @@ The Python API does not bundle a verify script yet, so Implement mode ends in th - **Hardcoded versions.** Resolve the `solace-pubsubplus` version at generation time (Step 3); never carry a version from memory, from an old `requirements.txt`, or from the upstream samples repository. - **Wildcard imports.** Keep every import explicit, as the samples do; never write `from solace.messaging... import *`. - **The system interpreter.** Run and check everything with `.venv/bin/python`, so the project uses the version it installed. -- **Running the publisher before the subscriber has provisioned the queue.** A persistent message published before the queue subscribes to its topic is dropped and never received. By default the broker still ACKs it, so the publisher can print `VERIFY: PUBLISH_ACKED` for a message no subscriber gets; the publisher sees a NACK only when the client profile rejects messages that match no subscription. Judge the round trip by `VERIFY: MESSAGE_RECEIVED` in the subscriber output, never by `PUBLISH_ACKED` alone. Run the subscriber first. +- **Running the publisher before the subscriber has provisioned the queue.** A persistent message published before the queue subscribes to its topic is dropped and never received. By default the broker still ACKs it, so the publisher can print `VERIFY: PUBLISH_ACKED` for a message no subscriber gets; the publisher sees a NACK only when the client profile rejects messages that match no subscription. Judge the round trip by `VERIFY: MESSAGE_RECEIVED` in the subscriber output, never by `PUBLISH_ACKED` alone. Run the subscriber first; `verify.sh` enforces that order. - **Treating a sample as a spec that overrides the approved design.** See "Adapting a sample to a variant of its leaf" above. ## Step 6: Tailor the checklist (written at Step 4) and report -Step 4 writes the tailored `solace-verification-checklist.md` into the generated project as generation output; this step defines its content and adds the report in chat. The Python API has no master checklist template yet; until it lands, write the file from the list below. Resolve each item for the generated code: keep only the items for the chosen leaf (a leaf in parentheses marks a leaf-specific item), and move an item the generation did not deliver to Your responsibility. On Solace Suggested, and for each ticked Custom knob, an item under Your responsibility that the door or the knob delivered (the TLS secure session, DMQ eligibility on published messages, separate projects, and the HA reconnect values only when the HA-failover question was answered yes) moves to Delivered by this generation. The broker-side dead message queue always stays under Your responsibility, because the skill never provisions it. Write it as a Markdown task list (`- [ ]`), ticking only what this run actually checked. Each item in the first two groups carries one canonical doc link, as the master JCSMP template does; the round-trip and conformance items carry none because they check skill conformance, not Solace guidance. - -**Delivered by this generation** - -- Basic-auth connection through the shared `solace_connection_config.py` helper; credentials live only in a gitignored `config.json`. [Messaging Service](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Messaging-Service.md) -- Service reconnection and interruption listeners registered before `connect()`. [Messaging Service](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Messaging-Service.md) -- (Guaranteed Pub/Sub) Persistent publish to a topic with a publish receipt listener; a NACK is logged with the message id. [Publishing Persistent Messages](https://docs.solace.com/API/API-Developer-Guide-Python/Python-PM-Publish.md) -- (Guaranteed Pub/Sub) Capacity-bounded back pressure on the publisher (`on_back_pressure_wait`). [Publishing Persistent Messages](https://docs.solace.com/API/API-Developer-Guide-Python/Python-PM-Publish.md) -- (Guaranteed Pub/Sub) Message time-to-live set once on the publisher's message template. [Publishing Persistent Messages](https://docs.solace.com/API/API-Developer-Guide-Python/Python-PM-Publish.md) -- (Guaranteed Pub/Sub) Durable queue plus topic subscription provisioned at start (`CREATE_ON_START`), with the access type from the design summary (one durable queue per consuming service on fan-out); with the Custom admin-provisioned-queue knob, the subscriber binds the administrator's queue instead. [Creating Queues with the Solace Python API](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Create-Queues.md) -- (Guaranteed Pub/Sub) Client acknowledgement only after processing. [Consuming Persistent Messages](https://docs.solace.com/API/API-Developer-Guide-Python/Python-PM-Receive.md) -- (Direct Pub/Sub) Direct publish to a topic with a publish failure listener and capacity-bounded back pressure (`on_back_pressure_wait`). [Publishing Direct Messages](https://docs.solace.com/API/API-Developer-Guide-Python/Python-DM-Publish.md) -- (Direct Pub/Sub) Direct topic subscription with capacity-bounded back pressure (`on_back_pressure_drop_latest`) and discard detection on received messages. [Consuming Direct Messages](https://docs.solace.com/API/API-Developer-Guide-Python/Python-DM-Receive.md) -- (Request-Reply (Direct)) Requests sent through a `RequestReplyMessagePublisher` with a positive reply timeout, and a replier that answers with `replier.reply()` and ignores a message with no reply-to. [Request-Reply Messaging in the Solace Python API](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Request-Reply.md) -- Graceful SIGINT/SIGTERM shutdown: `terminate(grace_period)` and then `disconnect()` on every exit path. [Consuming Persistent Messages](https://docs.solace.com/API/API-Developer-Guide-Python/Python-PM-Receive.md) - -**Your responsibility (not delivered here)** - -- TLS secure session (`tcps://` with certificate validation set explicitly in code). Quickstart, and Custom with the secure-session knob unticked, open a TLS session only when the host is `tcps://`, and then validate only through the API defaults, which a `config.json` key can turn off; neither sets the certificate checks in code. [Messaging Service](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Messaging-Service.md) -- (Any `tcps://` host: Solace Suggested, the Custom secure-session knob, or Quickstart or Custom without that knob with a `tcps://` host) A trust store directory that holds the broker's CA certificate as a PEM file, named by `solace.messaging.tls.trust-store-path` in `config.json`; the Python API has no default trust store, so the connection fails without it. [Using Client Certificates with Solace Cloud](https://docs.solace.com/Cloud/ght_client_certs.md) -- (Custom admin-provisioned-queue knob) The queue and its topic subscription provisioned on the broker by an administrator, with the access type the design summary records. [Creating Queues with the Solace Python API](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Create-Queues.md) -- Credentials handled safely: `config.json` is never committed, and credentials are not passed as command-line arguments, where they land in shell history and process listings (the handed-back run commands accept them only for convenience). [Messaging Service](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Messaging-Service.md) -- Authentication stronger than basic username/password (for example a client certificate or Kerberos) where the environment requires it; this journey generates basic username/password only. [Messaging Service](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Messaging-Service.md) -- (Guaranteed Pub/Sub) Dead message queue eligibility (`PERSISTENT_DMQ_ELIGIBLE`) set explicitly on published messages. [Publishing Persistent Messages](https://docs.solace.com/API/API-Developer-Guide-Python/Python-PM-Publish.md) -- (Guaranteed Pub/Sub) A dead message queue provisioned on the broker with a max-redelivery count or a time-to-live; the eligibility flag alone does nothing without it. [Configuring Dead Message Queues](https://docs.solace.com/Messaging/Guaranteed-Msg/Setting-Dead-Msg-Queues.md) -- Reconnect tuning for HA failover; Quickstart leaves connect and reconnect behavior at the API defaults, and Custom without the HA knob keeps the samples' baseline of 20 retries 3000 ms apart. [Solace C API Best Practices](https://docs.solace.com/API/API-Developer-Guide/C-API-Best-Practices.md) -- Separate projects or deployments per role (skill guidance: a real deployment lifts each script into its own service or project; no Solace doc page states it). -- (Guaranteed Pub/Sub) `respect-ttl` enabled on the queue, without which the broker ignores the message time-to-live. [Configuring Queues](https://docs.solace.com/Messaging/Guaranteed-Msg/Configuring-Queues.md) -- (Guaranteed Pub/Sub) An administrator-provisioned queue, or the endpoint-create permission for this client, in production. [Creating Queues with the Solace Python API](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-Create-Queues.md) -- A Python version in active or security support and OpenSSL installed on Linux for TLS ([Supported Environments](https://docs.solace.com/API/API-Developer-Guide-Python/Python-API-supported-Environments.md)), and no `multiprocessing` in the application ([solace-pubsubplus on PyPI](https://pypi.org/project/solace-pubsubplus/)). - -**Verified by the round-trip** (developer-run until the Python verify script lands) - -- (Guaranteed Pub/Sub) Every subscriber prints `VERIFY: CONNECTED` and `VERIFY: QUEUE_BOUND`. -- (Guaranteed Pub/Sub) The publisher prints `VERIFY: CONNECTED` and `VERIFY: PUBLISH_ACKED`, then exits with status 0. -- (Guaranteed Pub/Sub) Every subscriber prints `VERIFY: MESSAGE_RECEIVED` (`PUBLISH_ACKED` alone does not prove delivery). -- (Direct Pub/Sub) The subscriber prints `VERIFY: CONNECTED` and `VERIFY: SUBSCRIBED`; the publisher prints `VERIFY: CONNECTED` and exits with status 0; the subscriber then prints `VERIFY: MESSAGE_RECEIVED`. -- (Request-Reply (Direct)) The replier prints `VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, and `VERIFY: REQUEST_RECEIVED`; the requestor prints `VERIFY: CONNECTED` and `VERIFY: REPLY_RECEIVED`, then exits with status 0. -- Ctrl-C on each long-running role (subscriber or replier) prints `Shutdown signal received` and it exits with status 0. - -**Generation conformance** - -- The disclaimer header is the first two lines of every generated `.py` file (Step 5 header check). -- `requirements.txt` pins the `solace-pubsubplus` version that PyPI reported as `info.version` at generation time, or the developer's version when PyPI was unreachable (Step 5 drift check). -- This tailored checklist exists at the project root (Step 5 checklist check). -- `py_compile` and the import smoke passed (Step 5), with the result recorded here. -- Comments follow their code (Step 4). -- Every page named in the design summary's Grounding docs field was WebFetched in this session (`none fetched` is an honest value; an unfetched citation is not). -- Verification: record `verify script not yet bundled for Python; round trip handed back`, or `manual run` with the observed markers when the developer asked you to run it. - -**Developer-owned items (not delivered by this skill)**: any item the skill does not deliver, recorded as developer-owned rather than skill-satisfied. A consented deviation is recorded here. - -**Your additional items**: open space for the developer to keep growing the list. - -The emitted checklist is an artifact, not source, so it carries NO disclaimer header. Report the same resolution in chat: what landed under each group, the compile, import, and conformance results, and that the round trip is handed back. +Step 4 writes the tailored `solace-verification-checklist.md` into the generated project as generation output; this step defines its tailoring and adds the report in chat. Read `verification-checklist.md` (the master template) and resolve each item for the generated code: keep only the items for the chosen leaf (a leaf in parentheses marks a leaf-specific item), and move an item the generation did not deliver to Your responsibility. On Solace Suggested, and for each ticked Custom knob, an item under Your responsibility that the door or the knob delivered (the TLS secure session, DMQ eligibility on published messages, separate projects, and the HA reconnect values only when the HA-failover question was answered yes) moves to Delivered by this generation. The broker-side dead message queue always stays under Your responsibility, because the skill never provisions it. Write it as a Markdown task list (`- [ ]`), ticking only what this run actually checked. Keep the master template's groups and doc links, drop its template introduction and the leaf names in parentheses from the emitted copy, and keep its two closing sections (Developer-owned items, where a consented deviation is recorded, and Your additional items). On separate projects, write ONE copy into EACH project root. + +The emitted checklist is an artifact, not source, so it carries NO disclaimer header. Report the same resolution in chat: what landed under each group and the compile, import, and conformance results. Record each `verify.sh` stage that ran and its exit code in the checklist's Generation conformance group; on the compile-only fallback, record that the run was handed back instead. ## Grounding references diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-request-reply.md b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-request-reply.md index 5552d16..8bb33e5 100644 --- a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-request-reply.md +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/implement-request-reply.md @@ -41,4 +41,4 @@ The requestor connects, sends its request (or requests), collects the reply, and ## The marker contract -These four `VERIFY:` marker strings (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: REQUEST_RECEIVED`, `VERIFY: REPLY_RECEIVED`) are the same strings the JCSMP Request-Reply (Direct) leaf emits, character for character. They are read PER PROCESS: the replier output carries `SUBSCRIBED` and `REQUEST_RECEIVED`, the requestor output carries `REPLY_RECEIVED`, and both print `CONNECTED`. Until the Python verify script lands, the developer reads them from the two terminals (`implement-mode.md` Step 5); the verify script will read the same strings, so never change one. +These four `VERIFY:` marker strings (`VERIFY: CONNECTED`, `VERIFY: SUBSCRIBED`, `VERIFY: REQUEST_RECEIVED`, `VERIFY: REPLY_RECEIVED`) are the same strings the JCSMP Request-Reply (Direct) leaf emits, character for character. They are read PER PROCESS: the replier output carries `SUBSCRIBED` and `REQUEST_RECEIVED`, the requestor output carries `REPLY_RECEIVED`, and both print `CONNECTED`. `verify.sh` reads these same strings in each process's own output (`implement-mode.md` Step 5), so never change one. diff --git a/plugins/solace-messaging-skills/skills/solace-application-development/references/python/scripts/verify.sh b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/scripts/verify.sh new file mode 100755 index 0000000..327ddb2 --- /dev/null +++ b/plugins/solace-messaging-skills/skills/solace-application-development/references/python/scripts/verify.sh @@ -0,0 +1,627 @@ +#!/usr/bin/env bash +# verify.sh: run-and-observe verification for the generated Solace Python projects +# (guaranteed pub/sub, direct pub/sub, and direct request-reply). +# +# Usage (three modes): +# Single project (both scripts in the working directory, one .venv): +# verify.sh [ [pass]] +# With no connection arguments, each script reads ./config.json (the only way +# to reach a tcps:// host, because the trust store path has no argument form). +# Connection arguments are refused when ./config.json exists, because the +# scripts prefer config.json and would connect to a different broker. +# Separate projects (Solace Suggested, or the Custom decoupled-projects knob; +# each project has its own .venv and reads its own gitignored config.json): +# verify.sh --subscriber-dir --publisher-dir +# verify.sh direct-request-reply --replier-dir --requestor-dir +# Any other app shape (web app, embedded service; the app reads its own config): +# verify.sh app +# Sources ./verify-hooks.sh (generated with the project) for the ONLY +# app-specific facts: START_CMD, TRIGGER_CMD, READY_MARKER, PASS_MARKER. +# +# What it does: checks every generated script with py_compile, starts the +# long-running role (subscriber or replier) FIRST with the project's own +# .venv interpreter, waits for its readiness marker, runs the foreground role +# (publisher or requestor) and lets it exit on its own, watches each process's +# OWN captured output for the fixed VERIFY: marker lines, sends SIGINT to the +# long-running role to exercise its graceful shutdown, and classifies the +# outcome by exit code: +# 0 = stage passed. For the fixed stages: the marker was seen in this run, the +# foreground role exited with status 0 on its own, the shutdown line +# printed, AND the long-running role exited with status 0. For the app +# stage: READY_MARKER, then a new PASS_MARKER after the trigger, and a +# bounded teardown (the app's shutdown is not checked). +# 1 = code failure (py_compile failed, a marker never appeared, a process died +# before its milestone, the foreground role exited non-zero, the shutdown +# line never printed, or a run hung and had to be force-killed), OR a +# precondition miss before any process starts (a usage error, or a missing +# script, .venv, config.json, or verify-hooks.sh). stderr names which. +# 2 = environment failure (a Solace Python API connection, authentication, or +# TLS signature appeared in a process log, or the long-running role was +# still connecting at the readiness deadline with no VERIFY: CONNECTED; the +# code is not the problem, so the agent must NOT enter the fix loop) +# +# The Python API has no guaranteed request-reply, so there is no +# guaranteed-request-reply stage. +# +# Script resolution: the script does NOT read a build file. It relies on the fixed +# script names that implement-mode.md Step 4 gives every generated project +# (guaranteed_subscriber.py, direct_replier.py, ...), so the same script works +# across generations whatever the function and constant names inside each file. +# On the fan-out leaf it drives guaranteed_subscriber.py only. +# +# This is run-and-observe, NOT a test framework. It does not stand up a broker. +# +# Windows: run it under WSL or Git Bash; it also finds .venv/Scripts/python.exe. +# +# Credential handling (accepted Quickstart tradeoff): connection arguments are a +# pass-through, mirroring the generated scripts' own command-line fallback, so on a +# shared host the password may appear in process listings. With config.json (and +# always in separate-projects mode) no credential appears on the command line. The +# script never echoes the password, never enables shell command tracing, and never +# evals its arguments. + +set -euo pipefail + +# ── Arguments ────────────────────────────────────────────────────────────────── +# The --subscriber-dir/--replier-dir flag names the LONG-RUNNING role's project (the +# SIGINT target); --publisher-dir/--requestor-dir names the FOREGROUND role's +# project. Both directories default to "." so the single-project path needs no flag. +usage() { + cat >&2 <<'EOF' +usage: + # Single project (scripts read ./config.json; pass connection arguments only when there is no config.json): + verify.sh [ [pass]] + + # Separate projects (each project reads its own gitignored config.json): + verify.sh --subscriber-dir --publisher-dir + verify.sh direct-request-reply --replier-dir --requestor-dir + + # Any other app shape (web app, embedded service; reads ./verify-hooks.sh): + verify.sh app +EOF + exit 1 +} + +STAGE="${1:-}" +[ -z "$STAGE" ] && usage +shift || true + +SUB_DIR="." # long-running role's project (subscriber, or replier for request-reply) +PUB_DIR="." # foreground role's project (publisher, or requestor for request-reply) +TWO_PROJECT=0 +SUB_SET=0; PUB_SET=0 +CONN_ARGS=() +while [ "$#" -gt 0 ]; do + case "$1" in + --subscriber-dir|--replier-dir) + [ -n "${2:-}" ] || { echo "$1 needs a directory value" >&2; usage; } + SUB_DIR="$2"; SUB_SET=1; TWO_PROJECT=1; shift 2 ;; + --publisher-dir|--requestor-dir) + [ -n "${2:-}" ] || { echo "$1 needs a directory value" >&2; usage; } + PUB_DIR="$2"; PUB_SET=1; TWO_PROJECT=1; shift 2 ;; + -*) + echo "unknown flag: $1" >&2; usage ;; + *) + CONN_ARGS+=("$1"); shift ;; + esac +done + +case "$STAGE" in + publisher|consumer|roundtrip) SUB_SCRIPT=guaranteed_subscriber.py; PUB_SCRIPT=guaranteed_publisher.py ;; + direct) SUB_SCRIPT=direct_subscriber.py; PUB_SCRIPT=direct_publisher.py ;; + direct-request-reply) SUB_SCRIPT=direct_replier.py; PUB_SCRIPT=direct_requestor_blocking.py ;; + app) ;; + guaranteed-request-reply) + echo "the Solace Python API has no guaranteed request-reply, so there is no $STAGE stage" >&2; usage ;; + *) echo "unknown stage: $STAGE" >&2; usage ;; +esac + +if [ "$STAGE" = "app" ]; then + { [ "$TWO_PROJECT" -eq 0 ] && [ "${#CONN_ARGS[@]}" -eq 0 ]; } || { + echo "the app stage takes no arguments; the app reads its own config" >&2; usage; } +elif [ "$TWO_PROJECT" -eq 1 ]; then + { [ "$SUB_SET" -eq 1 ] && [ "$PUB_SET" -eq 1 ]; } || { + echo "separate-projects mode needs BOTH role directories" >&2; usage; } + [ "${#CONN_ARGS[@]}" -eq 0 ] || { + echo "separate-projects mode takes no connection arguments; each project reads its own config.json" >&2; usage; } + # Resolve to absolute paths so each role runs from INSIDE its own project + # (its config.json is read relative to that directory) whatever the cwd. + SUB_DIR="$(cd "$SUB_DIR" && pwd)" + PUB_DIR="$(cd "$PUB_DIR" && pwd)" +else + case "${#CONN_ARGS[@]}" in + 0) [ -f config.json ] || { + echo "no connection details: pass [pass], or create config.json from config.example.json" >&2; usage; } ;; + 3|4) [ ! -f config.json ] || { + echo "config.json exists and the scripts prefer it over connection arguments; omit the arguments, or remove config.json" >&2; usage; } ;; + *) echo "connection arguments are [pass]" >&2; usage ;; + esac +fi + +# The request-reply leaf generates the blocking requestor by default and +# direct_requestor.py on the non-blocking variant; use whichever the project has. +if [ "$STAGE" = "direct-request-reply" ] && [ ! -f "$PUB_DIR/$PUB_SCRIPT" ] && [ -f "$PUB_DIR/direct_requestor.py" ]; then + PUB_SCRIPT=direct_requestor.py +fi + +# Each project runs with its own virtual environment's interpreter, never the +# system python3 (implement-mode.md Step 3). +venv_python() { + if [ -x "$1/.venv/bin/python" ]; then + echo "$1/.venv/bin/python" + elif [ -x "$1/.venv/Scripts/python.exe" ]; then + echo "$1/.venv/Scripts/python.exe" + fi +} + +if [ "$STAGE" != "app" ]; then + for pair in "$SUB_DIR:$SUB_SCRIPT" "$PUB_DIR:$PUB_SCRIPT"; do + d="${pair%:*}"; s="${pair##*:}" + [ -f "$d/$s" ] || { echo "no $s found in: $d (implement-mode.md Step 4 fixes the script names)" >&2; exit 1; } + [ -n "$(venv_python "$d")" ] || { + echo "no virtual environment in: $d (create it: python3 -m venv .venv && .venv/bin/pip install -r requirements.txt)" >&2; exit 1; } + done + SUB_PY="$(venv_python "$SUB_DIR")" + PUB_PY="$(venv_python "$PUB_DIR")" +fi + +# ── Preflight conformance warnings (warn-only; never changes the exit contract) ── +# Reports the generation-conformance misses that the verification checklist's +# "Generation conformance" group records: a missing tailored checklist, scripts +# without the AI-assisted disclaimer header, and a solace-pubsubplus pin that +# drifted from the release PyPI reports now. +preflight() { + local d="$1" f missing="" pin rel + if [ ! -f "$d/solace-verification-checklist.md" ]; then + echo "PREFLIGHT WARN: $d/solace-verification-checklist.md is missing (every generation must emit it; implement-mode.md Step 4)" >&2 + fi + for f in "$d"/*.py; do + [ -f "$f" ] || continue + head -n 1 "$f" | grep -qxF '# AI-assisted code. Review before production use.' || missing="$missing$f"$'\n' + done + if [ -n "$missing" ]; then + echo "PREFLIGHT WARN: generated scripts missing the AI-assisted disclaimer header:" >&2 + printf '%s' "$missing" >&2 + fi + if [ -f "$d/requirements.txt" ]; then + pin=$(grep -oE '^solace-pubsubplus==[^[:space:]]+' "$d/requirements.txt" 2>/dev/null | sed 's/^solace-pubsubplus==//' | head -1) || true + rel=$(curl -s --max-time 5 https://pypi.org/pypi/solace-pubsubplus/json 2>/dev/null \ + | python3 -c 'import json, sys; print(json.load(sys.stdin)["info"]["version"])' 2>/dev/null) || true + if [ -z "${pin:-}" ]; then + echo "PREFLIGHT WARN: $d/requirements.txt does not pin solace-pubsubplus== (implement-mode.md Step 3)" >&2 + elif [ -n "${rel:-}" ] && [ "$pin" != "$rel" ]; then + echo "PREFLIGHT WARN: requirements.txt pins solace-pubsubplus $pin in $d, but PyPI reports $rel as the latest release" >&2 + fi + fi +} +if [ "$TWO_PROJECT" -eq 1 ]; then + preflight "$SUB_DIR" + preflight "$PUB_DIR" +else + preflight "." +fi + +# ── Live-run heads-up (SKILL.md Invariant 6): name the target and the side effects ─ +# The host comes from the connection arguments, else from the long-running role's +# config.json (the app stage's own config is not read). +config_host() { + [ -f "$1/config.json" ] || return 0 + grep -oE '"solace\.messaging\.transport\.host"[[:space:]]*:[[:space:]]*"[^"]*"' "$1/config.json" 2>/dev/null \ + | head -1 | sed -E 's/.*:[[:space:]]*"([^"]*)"$/\1/' || true +} +TARGET_HOST="${CONN_ARGS[0]:-}" +if [ -z "$TARGET_HOST" ] && [ "$STAGE" != "app" ]; then + TARGET_HOST="$(config_host "$SUB_DIR")" +fi +echo "── Live-run heads-up: stage '$STAGE' starts processes that connect to broker ${TARGET_HOST:-}; broker-side effects can include provisioned durable queues, opened connections, and published messages ──" + +# ── Environment-failure signatures (doc-traceable API symbols) ────────────────── +# The API's named error classes and the SOLCLIENT_SUBCODE_* names it logs when the +# broker or the transport rejects the session. Observed live with solace-pubsubplus: +# wrong password AuthenticationError, SOLCLIENT_SUBCODE_LOGIN_FAILURE +# wrong VPN SOLCLIENT_SUBCODE_MSG_VPN_NOT_ALLOWED +# unresolvable host ServiceUnreachableError +# unroutable host Session connect timeout (SOLCLIENT_SUBCODE_TIMEOUT, also used by non-connect timeouts) +# refused port SOLCLIENT_SUBCODE_COMMUNICATION_ERROR +# no trust store SOLCLIENT_SUBCODE_FAILED_LOADING_TRUSTSTORE +# wrong CA SOLCLIENT_SUBCODE_UNTRUSTED_CERTIFICATE +# AuthorizationError, CLIENT_USERNAME_IS_SHUTDOWN, and INVALID_VIRTUAL_ADDRESS are +# the remaining broker rejections of a session. A match means the broker, +# credentials, or trust store are wrong, NOT the code, so exit 2 bypasses the fix loop. +ENV_SIGNATURES='AuthenticationError|AuthorizationError|ServiceUnreachableError|SOLCLIENT_SUBCODE_LOGIN_FAILURE|SOLCLIENT_SUBCODE_MSG_VPN_NOT_ALLOWED|SOLCLIENT_SUBCODE_CLIENT_USERNAME_IS_SHUTDOWN|SOLCLIENT_SUBCODE_INVALID_VIRTUAL_ADDRESS|SOLCLIENT_SUBCODE_COMMUNICATION_ERROR|SOLCLIENT_SUBCODE_FAILED_LOADING_TRUSTSTORE|SOLCLIENT_SUBCODE_UNTRUSTED_CERTIFICATE|Session connect timeout' + +# Every baked sample's signal handler prints this line on SIGINT. Seeing it in the +# long-running role's log is the observable proof that the graceful shutdown ran. +SHUTDOWN_LINE='Shutdown signal received' + +# The baked guaranteed subscriber logs this line and exits when receiver.start() +# fails to provision or bind the queue, which is what a client without the +# endpoint-create permission sees. The code is not the problem: exit 2. +ENDPOINT_DENIED='Could not bind to queue' + +TIMEOUT_S=30 # marker-watch deadline +PUBLISH_WAIT_S=30 # foreground-role deadline before escalating to a code failure +SHUTDOWN_WAIT_S=15 # bounded post-SIGINT wait before escalating to SIGTERM/SIGKILL + +# A long-running role that is still alive at the readiness deadline and never +# printed VERIFY: CONNECTED is still connecting: an unreachable host or a blocked +# port, where the connect attempt (and its retries) outlasts TIMEOUT_S without +# logging an error. That is the environment, not the code, so exit 2. +classify_stalled_connect() { # + if [ "$status" -ne 2 ] && [ "$3" -eq 0 ] && kill -0 "$1" 2>/dev/null && ! grep -qF "VERIFY: CONNECTED" "$2"; then + status=2 + CONNECT_STALLED=1 + fi +} + +# Print the count of in once it has stopped changing for 2 seconds +# (bounded by TIMEOUT_S), so messages an earlier run left on a durable queue, which +# arrive just after the bind, are all counted before the publisher starts. +settled_count() { # + local n m settle_deadline=$(( SECONDS + TIMEOUT_S )) + n=$(grep -cF "$2" "$1" || true) + while [ "$SECONDS" -lt "$settle_deadline" ]; do + sleep 2 + m=$(grep -cF "$2" "$1" || true) + [ "$m" -eq "$n" ] && break + n=$m + done + echo "$n" +} + +# ── 1. py_compile first (a syntax error is a code failure, exit 1) ────────────── +compile_project() { + local d="$1" py="$2" files=() + local f + for f in "$d"/*.py; do [ -f "$f" ] && files+=("$f"); done + [ "${#files[@]}" -gt 0 ] || return 0 + echo "── Checking $d with py_compile ──" + if ! "$py" -m py_compile "${files[@]}"; then + echo "py_compile failed in $d (code failure, exit 1)" >&2 + exit 1 + fi +} +if [ "$STAGE" = "app" ]; then + if [ -n "$(venv_python .)" ]; then + compile_project . "$(venv_python .)" + else + echo "── No .venv in the working directory; skipping py_compile (the app stage runs through its own START_CMD) ──" + fi +else + compile_project "$SUB_DIR" "$SUB_PY" + [ "$PUB_DIR" = "$SUB_DIR" ] || compile_project "$PUB_DIR" "$PUB_PY" +fi + +# ── Per-process logs: each role writes its own log, because both print ───────── +# VERIFY: CONNECTED and a merged log could not say which process reached a marker. +SUB_LOG="$(mktemp)" +PUB_LOG="$(mktemp)" + +# If the script itself dies (agent-harness timeout, Ctrl-C, CI cancellation, or a +# set -e exit), the backgrounded long-running role would keep running detached and +# hold its queue. The EXIT trap stops both roles and removes the logs. SIGKILL of +# this script is the one residual: `set -m` puts each child in its own process +# group, so a hard kill of this script's group cannot reach them. +cleanup() { + if [ -n "${SUB_PID:-}" ]; then kill -TERM -- "-$SUB_PID" 2>/dev/null || true; fi + if [ -n "${PUB_PID:-}" ]; then kill -TERM -- "-$PUB_PID" 2>/dev/null || true; fi + rm -f "${SUB_LOG:-}" "${PUB_LOG:-}" +} +trap cleanup EXIT + +status=1 # default to code failure until a stage milestone is confirmed +DIRECT_ZERO_RECEIPT=0 # set when the at-most-once burst was lost after a confirmed subscription +RR_NO_REPLY=0 # set when the replier subscribed but no reply came back +CONNECT_STALLED=0 # set when the long-running role was still connecting at the readiness deadline + +# ── Launchers ────────────────────────────────────────────────────────────────── +# `set -m` starts each role in its own process group, so the group-directed kills +# below reach the whole role (and a `bash -c` wrapper's child on the app stage). +# In separate-projects mode each role runs from inside its own project, so it reads +# that project's config.json; in single-project mode the connection arguments, when +# given, pass through as the scripts' command-line fallback. +start_role() { #