EBP-3107: Add Python verify.sh, checklist template, and smoke fixture

.github/workflows/ci.yml

@@ -6,7 +6,8 @@ name: ci
 #                  + marketplace.json/plugins/ sync + version bumps on release PRs
 #                  + post-merge version backstop on pushes to main
 #   link-check:    CI-02: the tools/check-links.sh sweep
-#   compile-check: EVAL-02 compile the baked JCSMP samples verbatim
+#   jcsmp-sample-check:  EVAL-02 compile the baked JCSMP samples verbatim
+#   python-sample-check: check the baked Python samples against the latest solace-pubsubplus
 #   trigger-evals: lightweight skill-triggering check; disabled until CI may hold an API key
 on:
   # Deliberately no `edited` type: running on title/body edits either re-runs
@@ -50,7 +51,7 @@ jobs:
   # missing license are warnings only. Also validate the Claude
   # plugin/marketplace manifests.
   validate:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-26.04
     timeout-minutes: 10
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
@@ -99,7 +100,7 @@ jobs:
 
   # CI-02: sweep the plugins/ tree for broken and soft-404 documentation links.
   link-check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-26.04
     timeout-minutes: 15
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
@@ -108,8 +109,8 @@ jobs:
 
   # EVAL-02: compile the baked JCSMP samples verbatim against the live-resolved
   # sol-jcsmp. Pure compile gate, no API budget and no credential.
-  compile-check:
-    runs-on: ubuntu-latest
+  jcsmp-sample-check:
+    runs-on: ubuntu-26.04
     timeout-minutes: 25
     steps:
       - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
@@ -118,8 +119,23 @@ jobs:
           distribution: temurin
           java-version: 11
           cache: maven
-      - name: Compile baked samples (EVAL-02 hard gate)
-        run: ./plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/compile.sh
+      - name: Check baked JCSMP samples (EVAL-02 hard gate)
+        run: ./plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/check-samples.sh
+
+  # Check the baked Python samples against the latest solace-pubsubplus from PyPI:
+  # py_compile, then import each sample and build its API objects against a
+  # resolvable host without a broker. Python has no compile step, so this is the
+  # Python counterpart of jcsmp-sample-check. No API budget and no credential.
+  python-sample-check:
+    runs-on: ubuntu-26.04
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
+      - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: '3.11'
+      - name: Check baked Python samples
+        run: ./plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check-samples.sh
 
   # Lightweight skill-triggering check. For every case in each plugin's
   # evals/trigger-evals.json, run the prompt with only that plugin loaded and
@@ -138,7 +154,7 @@ jobs:
   # re-enable the job once a key is allowed.
   trigger-evals:
     if: false
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-26.04
     timeout-minutes: 120                  # per model leg; 57 cases x 3 runs (~50 min observed)
     strategy:
       fail-fast: false                    # each model is an independent check

CLAUDE.md

@@ -16,7 +16,7 @@ This repository is a Claude Code plugin marketplace (`solace-agent-plugins`) of
 | `tools/` | CI check scripts, the two eval runners, and `output-evals.schema.json`. |
 | `.github/workflows/ci.yml` | The only CI workflow. |
 
-Nothing in the repository is generated or vendored. The Eclipse files under `references/jcsmp/evals/compile-fixture/` are IDE metadata for the compile fixture; leave them alone.
+Nothing in the repository is generated or vendored. The Eclipse files under `references/jcsmp/evals/sample-check/` are IDE metadata for the JCSMP sample check; leave them alone.
 
 ## Commands
 
@@ -27,7 +27,8 @@ gh skill publish --dry-run                       # skills against the Agent Skil
 claude plugin validate . --strict                # marketplace and plugin manifests
 ./tools/check-plugin-versions.sh --sync-only     # marketplace.json matches plugins/
 ./tools/check-links.sh plugins                   # broken and soft-404 doc links
-./plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/compile.sh   # compile the baked JCSMP samples
+./plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/check-samples.sh    # check the baked JCSMP samples (compile and config smoke)
+./plugins/solace-messaging-skills/skills/solace-application-development/references/python/evals/sample-check/check-samples.sh   # check the baked Python samples (python3 and PyPI access)
 claude --plugin-dir plugins/solace-messaging-skills   # load the plugin from the clone
 ```
 
@@ -47,7 +48,7 @@ The trigger runner has no single-case mode; set `TRIGGER_EVAL_RUNS=1` for a quic
 - Link documentation to the live `docs.solace.com` `.md` URL. Do not copy doc content into a skill. Every link under `plugins/` must pass `check-links.sh`.
 - A new skill needs cases in both `trigger-evals.json` (positive and near-miss negative prompts) and `output-evals.json` (validate the latter against `tools/output-evals.schema.json`).
 - A new, removed, or renamed plugin directory needs the matching `marketplace.json` change in the same pull request.
-- A new baked JCSMP sample must be added to the sample roster in `compile.sh`, or the compile check does not cover it.
+- A new baked JCSMP sample must be added to the sample roster in `jcsmp/evals/sample-check/check-samples.sh`, or the sample check does not cover it. A new baked Python sample goes into the roster in `python/evals/sample-check/check-samples.sh`; `check_samples.py` picks it up from there.
 - Do not put exact counts (of cases, skills, or graders) in READMEs or skill docs. They go stale.
 
 ## Gotchas / do not

plugins/solace-messaging-skills/evals/README.md

@@ -49,9 +49,10 @@ Each `turnN.jsonl` in the work directory is the raw `--output-format stream-json
 - The `claude` CLI (Claude Code) on your PATH.
 - `jq` and `curl` on your PATH.
 - `mvn` (Apache Maven) with a JDK 11 or newer on your PATH. Only the cases that compile generated code need it, and the runner checks for it only when such a case is selected.
+- `python3` with the `venv` module and `pip`, for the cases that generate Solace Python projects. A generated Python project installs `solace-pubsubplus` into its own virtual environment, and its `verify.sh` runs that environment's interpreter.
 - An exported credential, either `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN`. An existing `claude` login does not satisfy this: the runner uses a throwaway config directory so that only the plugin under test is loaded, and that directory has no login. Claude seat holders can mint a token with `claude setup-token`.
 - `timeout` from GNU coreutils (or Homebrew's `gtimeout`) on your PATH, to cap each `mvn compile` and `verify.sh` run at 600 seconds. Optional: without it those steps run unbounded, and the runner prints a warning. On macOS, install it with `brew install coreutils`.
-- Network access to `repo1.maven.org` (Maven Central metadata and dependencies) and `docs.solace.com` (the skills fetch documentation pages).
+- Network access to `repo1.maven.org` (Maven Central metadata and dependencies), `pypi.org` and `files.pythonhosted.org` (the `solace-pubsubplus` release metadata and package), and `docs.solace.com` (the skills fetch documentation pages).
 - Optional: the four `OUTPUT_EVAL_BROKER_*` variables for the live round trip against a real broker (see below).
 - On macOS, `caffeinate` to keep the machine awake for a full leg.
 - On Windows, Git Bash. The repository's `.gitattributes` checks shell scripts out with LF endings, which bash and the byte-identity grader on `verify.sh` both need. A checkout made with `core.autocrlf=true` before that file existed keeps its CRLF copies, and git reports the tree clean because it treats them as unchanged, so `git pull` and `git checkout -- .` do not rewrite them. Re-clone, or delete the tracked scripts and check them out again: `git ls-files -z '*.sh' | xargs -0 rm -f && git checkout -- .`.

plugins/solace-messaging-skills/skills/solace-application-development/SKILL.md

@@ -44,12 +44,12 @@ Non-negotiable rules that apply across every API. Apply all.
 2. **WebFetch-on-demand doc grounding**: This skill bundles no documentation. When a step needs documentation content to ground a generation or design decision, WebFetch the live canonical page on demand (its `docs.solace.com` `.md` URL, the same link the reference files carry), then quote or summarize the fetched page. Do not answer from memory and do not paraphrase guidance the page does not contain. Do not restate doc content in the skill; link the live `.md` URL. Never substitute another channel for that WebFetch: grounding taken from a docs chatbot, a search tool, or decompiling a jar does not satisfy this invariant, and a page never fetched in the session must never be cited as grounding. A short list of references stays live for the same reason it always has (the JCSMP Javadoc and the Python API reference, the JCSMP and Python API Release Notes, Maven Central and PyPI, the tutorials, and the GitHub samples) because those have no `docs.solace.com` `.md` form. Each API's entry file lists its own live exceptions.
 3. **No hardcoded versions**: Never pin a library version anywhere in skill content. The generated build resolves the latest release of every dependency at generation time from the API's AUTHORITATIVE registry: the Maven repository metadata for JCSMP, the PyPI JSON `info.version` for Python. Each API's entry file names the exact coordinate and carries the exact lookup command. Never read a JCSMP version from the legacy `search.maven.org/solrsearch` index: it lags the repository metadata and reports stale versions. Security floors (a minimum version below which known CVEs live) are enforced as checks on the resolved result, never as pins.
 4. **AI-assisted disclaimer header**: Every source file generated under this skill starts with this exact line: `AI-assisted code. Review before production use.` Directly below it: `See the verification checklist: solace-verification-checklist.md`. This applies to EVERY generated source file, including custom builds, web apps, and embedded shapes that adapt the reference samples outside the canonical generator. Build files (the pom, `requirements.txt`) and other non-source artifacts are exempt; the bundled reference samples themselves carry no header.
-5. **Verification artifacts, always**: Any session that generates Solace code from this skill writes a tailored `solace-verification-checklist.md` beside the code as generation output, copies the API's own bundled verify script (plus its generated `verify-hooks.sh`) into the project, and ends in a real verify run or the compile-only fallback with the exact handed-back commands. The API's entry file names its verify script. An API that does not bundle a verify script yet (Python, until its `verify.sh` lands) ends in the compile-only fallback every time, and its implement-mode file defines that fallback; never claim a verified run without the script. The `VERIFY:` markers are non-optional in generated Solace messaging code, whatever the app shape and whatever the API. An improvised check (for example curl against the app's own HTTP API) may TRIGGER traffic but never renders the verdict; the verdict comes from the markers and the verify exit code.
+5. **Verification artifacts, always**: Any session that generates Solace code from this skill writes a tailored `solace-verification-checklist.md` beside the code as generation output, copies the API's own bundled verify script (plus its generated `verify-hooks.sh`) into the project, and ends in a real verify run or the compile-only fallback with the exact handed-back commands. The API's entry file names its verify script. Never claim a verified run without the script. The `VERIFY:` markers are non-optional in generated Solace messaging code, whatever the app shape and whatever the API. An improvised check (for example curl against the app's own HTTP API) may TRIGGER traffic but never renders the verdict; the verdict comes from the markers and the verify exit code.
 6. **Live-broker heads-up**: Immediately before starting ANY process that will connect to a live broker — verify.sh, `java -jar`, a `mvn` run, a `python` run, a run script, anything — print one line that names the broker host and states the actual broker-side effects of this run (the queues it provisions, the connections it opens, the messages it publishes), scaled to the real app. Environment discovery is never consent and never an answer: a running local broker, an existing config file, or found credentials are facts to report, not answers to consume; report them, then still ask which broker the developer wants to target.
 
 ## Reference Files (read on-demand only)
 
 - `references/jcsmp.md`: the JCSMP entry file (mode detection, the JCSMP coordinate, and the live JCSMP doc links); read this for any JCSMP request.
-- `references/jcsmp/`: all JCSMP API content (mode files, the reference samples, the verify script, and the compile fixture), reached through `references/jcsmp.md`.
+- `references/jcsmp/`: all JCSMP API content (mode files, the reference samples, the verify script, and the sample check), reached through `references/jcsmp.md`.
 - `references/python.md`: the Python entry file (mode detection, the `solace-pubsubplus` coordinate, and the live Python doc links); read this for any Solace Python API request.
-- `references/python/`: all Python API content (mode files and the reference samples), reached through `references/python.md`.
+- `references/python/`: all Python API content (mode files, the reference samples, the verify script, and the sample check), reached through `references/python.md`.

plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/ConfigSmokeTest.java → plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/ConfigSmokeTest.java

@@ -1,5 +1,5 @@
 /*
- * Runtime smoke test for SolaceConnectionConfig, driven by compile.sh.
+ * Runtime smoke test for SolaceConnectionConfig, driven by check-samples.sh.
  *
  * NOT a reference sample: compiling the helper never executes its config.json
  * parser or CLI-args fallback, so this tiny driver is compiled against the
@@ -9,7 +9,7 @@
  * value, blank CLI arg).
  *
  * One case per JVM invocation: the parser reads a CWD-relative config.json and
- * the fail-fast paths throw/exit, so compile.sh drives each case from its own
+ * the fail-fast paths throw/exit, so check-samples.sh drives each case from its own
  * working directory and checks the exit code. The case name is argv[0]; any
  * connection arguments the case needs are built here, not taken from argv.
  */

plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/compile.sh → plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/check-samples.sh

@@ -23,7 +23,7 @@ set -euo pipefail
 # Resolve this script's own directory so sample/pom paths are independent of CWD.
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 POM="${SCRIPT_DIR}/pom.xml"
-# references/jcsmp/ lives two levels up from compile-fixture/ (evals/ -> jcsmp/).
+# references/jcsmp/ lives two levels up from sample-check/ (evals/ -> jcsmp/).
 REFERENCES_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
 # Per-sample source paths. Each new baked sample a later per-pattern plan adds is
 # a ONE-LINE append here, mirroring this shape:

plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/compile-fixture/pom.xml → plugins/solace-messaging-skills/skills/solace-application-development/references/jcsmp/evals/sample-check/pom.xml

@@ -5,13 +5,13 @@
   <modelVersion>4.0.0</modelVersion>
 
   <!--
-    EVAL-02 compile-fixture template. Compiles the two baked JCSMP reference
+    EVAL-02 sample-check template. Compiles the two baked JCSMP reference
     samples (GuaranteedPublisher / GuaranteedSubscriber) VERBATIM against the
     latest sol-jcsmp resolved live from Maven Central. This pom only compiles;
     it never runs the samples, so it declares no run/exec plugin.
 
     The sol-jcsmp version is intentionally version-free here (Invariant 3): the
-    property default below is a NON-NUMERIC placeholder token that compile.sh
+    property default below is a NON-NUMERIC placeholder token that check-samples.sh
     overrides at run time with the live <release> from Maven Central metadata.
     No sol-jcsmp version number is committed anywhere. The log4j2 version below
     is FIXTURE-ONLY tooling (a concrete number so this CI fixture compiles
@@ -21,14 +21,14 @@
   -->
 
   <groupId>com.solace.samples</groupId>
-  <artifactId>jcsmp-compile-fixture</artifactId>
+  <artifactId>jcsmp-sample-check</artifactId>
   <version>0.0.1-SNAPSHOT</version>
   <packaging>jar</packaging>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <maven.compiler.release>11</maven.compiler.release>
-    <!-- Overridden by compile.sh via -Dsol.jcsmp.version=<live release>.
+    <!-- Overridden by check-samples.sh via -Dsol.jcsmp.version=<live release>.
          The default is a placeholder token, never a committed version number. -->
     <sol.jcsmp.version>RESOLVED_AT_GENERATION_TIME</sol.jcsmp.version>
   </properties>