diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index afd34bd..51d3a8f 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,14 +5,14 @@ }, "metadata": { "description": "Terminal49 MCP server and agent skills for container tracking and trade intelligence", - "version": "0.3.3" + "version": "0.3.4" }, "plugins": [ { "name": "terminal49", "source": "./plugins/terminal49", "description": "Container and shipment lookups, starting tracking, and US import trade intelligence from the Terminal49 MCP server, with agent skills", - "version": "0.3.3" + "version": "0.3.4" } ] } diff --git a/.cursor-plugin/marketplace.json b/.cursor-plugin/marketplace.json index afd34bd..51d3a8f 100644 --- a/.cursor-plugin/marketplace.json +++ b/.cursor-plugin/marketplace.json @@ -5,14 +5,14 @@ }, "metadata": { "description": "Terminal49 MCP server and agent skills for container tracking and trade intelligence", - "version": "0.3.3" + "version": "0.3.4" }, "plugins": [ { "name": "terminal49", "source": "./plugins/terminal49", "description": "Container and shipment lookups, starting tracking, and US import trade intelligence from the Terminal49 MCP server, with agent skills", - "version": "0.3.3" + "version": "0.3.4" } ] } diff --git a/.gitignore b/.gitignore index 5be7b10..098eea9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .DS_Store node_modules/ npm-debug.log* -plugins/terminal49/evals/results/ +evals/terminal49/results/ +.eval-stage/ dist/ diff --git a/AGENTS.md b/AGENTS.md index 11fb471..05630d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,7 +59,12 @@ files for every release. Keep releases small and document user-visible changes. - Anthropic's directory reads `plugins/terminal49` from this repository and publishes each new commit on the tracked branch after its checks. Keep `plugins/terminal49` free of symbolic links, binary files other than images, - and files over 256 KiB. + and files over 256 KiB. It accepts only Markdown, text, JSON, YAML, and PNG, + JPEG, GIF, or WebP images there, so keep scripts and SVG files out. +- The directory holds a plugin for review when any shipped file mentions a + credential or something shaped like an environment variable, even in prose + (`$schema` counts). Keep evals in `evals/terminal49/` and run them with + `node scripts/eval.mjs`. `npm run validate` enforces both rules. - The OpenAI plugin directory takes a ZIP. Build it with `npm run build:openai-zip`, which writes `dist/terminal49-openai-.zip` from the Codex manifest, the skills, and the icon. Upload a new ZIP whenever diff --git a/README.md b/README.md index 841751f..c903515 100644 --- a/README.md +++ b/README.md @@ -101,9 +101,11 @@ account with trade intelligence enabled. ├── .cursor-plugin/marketplace.json # Cursor marketplace ├── openai/review.json # OpenAI directory review cases, merged into the ZIP ├── openai/listing.json # OpenAI package name, version, category, and MCP URL for the existing plugin +├── evals/terminal49/ # Mocked behavioral evals, kept out of the shipped plugin ├── scripts/ │ ├── build-openai-zip.mjs # Builds the OpenAI directory ZIP │ ├── check-eval-delta.mjs # Checks that every eval case has a positive delta +│ ├── eval.mjs # Runs the evals against a staged copy of the plugin │ └── validate.mjs └── plugins/terminal49/ ├── .claude-plugin/plugin.json @@ -113,9 +115,8 @@ account with trade intelligence enabled. ├── mcp.json # Cursor MCP adapter ├── README.md ├── assets/ - │ ├── icon.svg # Square icon for the directories - │ └── logo.svg # Referenced by the Cursor manifest - ├── evals/ # Mocked behavioral evals + │ ├── icon.png # Square icon for Codex and the OpenAI directory + │ └── logo.png # Referenced by the Cursor manifest └── skills/ ├── container-tracking/ │ ├── SKILL.md diff --git a/plugins/terminal49/evals/QUESTION_BANK.md b/evals/terminal49/QUESTION_BANK.md similarity index 100% rename from plugins/terminal49/evals/QUESTION_BANK.md rename to evals/terminal49/QUESTION_BANK.md diff --git a/plugins/terminal49/evals/README.md b/evals/terminal49/README.md similarity index 81% rename from plugins/terminal49/evals/README.md rename to evals/terminal49/README.md index 66a4319..364f5b8 100644 --- a/plugins/terminal49/evals/README.md +++ b/evals/terminal49/README.md @@ -28,7 +28,7 @@ descriptions and input schemas. Refresh it when the server's tools change: ```sh gh api repos/Terminal49/API/contents/packages/mcp/src/__snapshots__/tools-list.json \ - -H "Accept: application/vnd.github.raw" | jq -c '.result | walk(if type == "object" then del(."$schema", .outputSchema) else . end)' > evals/mocks/terminal49/_tools.json + -H "Accept: application/vnd.github.raw" | jq -c '.result | walk(if type == "object" then del(."$schema", .outputSchema) else . end)' > evals/terminal49/mocks/terminal49/_tools.json ``` ## Prerequisites @@ -36,11 +36,18 @@ gh api repos/Terminal49/API/contents/packages/mcp/src/__snapshots__/tools-list.j - Claude Code 2.1.269 or newer - Claude Code authentication for the model under test -From this plugin directory, first validate the plugin: +These evals live outside `plugins/terminal49` on purpose. The Anthropic +directory reviews every file in the plugin folder, and it holds a plugin whose +files mention credentials, which the guardrail cases must do. +`claude plugin eval` reads cases only from inside the plugin folder, so +`scripts/eval.mjs` copies the plugin and these evals into `.eval-stage/` and runs +them there. It passes its options to `claude plugin eval` and copies the results +back to `evals/terminal49/results/`. + +Run every command from the repository root. First validate the plugin: ```sh -cd plugins/terminal49 -claude plugin validate . +claude plugin validate plugins/terminal49 ``` ## Run with mocks @@ -48,18 +55,18 @@ claude plugin validate . Use the checked-in mocks for every MCP call: ```sh -claude plugin eval . --mocks record --no-publish +node scripts/eval.mjs --mocks record --no-publish ``` `record` is the default mock mode. It reads the fixed responses under -`evals/mocks/` and each case's `mocks/` directory; it does not start the real +`mocks/` and each case's `mocks/` directory; it does not start the real Terminal49 MCP server. Do not add `--allow-real-servers` or use `--mocks off` for this suite. For a cheaper one-run check while editing a case: ```sh -claude plugin eval . \ +node scripts/eval.mjs \ --case where-is-container \ --runs 1 \ --ablation none \ @@ -70,7 +77,7 @@ claude plugin eval . \ Run one bank tier with its tag: ```sh -claude plugin eval . \ +node scripts/eval.mjs \ --tag tier-b \ --runs 1 \ --ablation none \ @@ -81,7 +88,7 @@ claude plugin eval . \ Run only the exact MarketingBuddy wording: ```sh -claude plugin eval . \ +node scripts/eval.mjs \ --tag marketingbuddy \ --mocks record \ --no-publish @@ -94,13 +101,13 @@ To save a machine-checkable full result and require positive delta for every case: ```sh -claude plugin eval . \ +node scripts/eval.mjs \ --mocks record \ --trust-plugin \ --no-publish \ --json /tmp/terminal49-plugin-evals.json -node ../../scripts/check-eval-delta.mjs /tmp/terminal49-plugin-evals.json +node scripts/check-eval-delta.mjs /tmp/terminal49-plugin-evals.json ``` ## Reading the result @@ -117,7 +124,7 @@ and `Δ` is `WITH - W/OUT`. Green for this plugin means: Skill and MCP sequencing graders are marked `with-only`, so they are indicators and do not artificially inflate `Δ`. Inspect failures in the generated HTML -report under `evals/results/`. +report under `evals/terminal49/results/`. Follow question-bank-first iteration: @@ -129,5 +136,5 @@ Follow question-bank-first iteration: 4. keep skill changes thin and rerun the same cases to verify improved `Δ`. Submit this plugin to the Anthropic community marketplace only after -`claude plugin validate .` passes and the full mocked eval has a green, +`claude plugin validate plugins/terminal49` passes and the full mocked eval has a green, positive `Δ` against the no-plugin baseline. diff --git a/plugins/terminal49/evals/any-rail/case.yaml b/evals/terminal49/any-rail/case.yaml similarity index 100% rename from plugins/terminal49/evals/any-rail/case.yaml rename to evals/terminal49/any-rail/case.yaml diff --git a/plugins/terminal49/evals/any-rail/mocks/terminal49/get_container_route.md b/evals/terminal49/any-rail/mocks/terminal49/get_container_route.md similarity index 100% rename from plugins/terminal49/evals/any-rail/mocks/terminal49/get_container_route.md rename to evals/terminal49/any-rail/mocks/terminal49/get_container_route.md diff --git a/plugins/terminal49/evals/any-rail/mocks/terminal49/get_container_transport_events.md b/evals/terminal49/any-rail/mocks/terminal49/get_container_transport_events.md similarity index 100% rename from plugins/terminal49/evals/any-rail/mocks/terminal49/get_container_transport_events.md rename to evals/terminal49/any-rail/mocks/terminal49/get_container_transport_events.md diff --git a/plugins/terminal49/evals/any-rail/mocks/terminal49/search_container.md b/evals/terminal49/any-rail/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/any-rail/mocks/terminal49/search_container.md rename to evals/terminal49/any-rail/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/did-it-clear-customs/case.yaml b/evals/terminal49/did-it-clear-customs/case.yaml similarity index 100% rename from plugins/terminal49/evals/did-it-clear-customs/case.yaml rename to evals/terminal49/did-it-clear-customs/case.yaml diff --git a/plugins/terminal49/evals/did-it-clear-customs/mocks/terminal49/search_container.md b/evals/terminal49/did-it-clear-customs/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/did-it-clear-customs/mocks/terminal49/search_container.md rename to evals/terminal49/did-it-clear-customs/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/do-not-invent-milestones/case.yaml b/evals/terminal49/do-not-invent-milestones/case.yaml similarity index 100% rename from plugins/terminal49/evals/do-not-invent-milestones/case.yaml rename to evals/terminal49/do-not-invent-milestones/case.yaml diff --git a/plugins/terminal49/evals/do-not-invent-milestones/mocks/terminal49/get_container.md b/evals/terminal49/do-not-invent-milestones/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/do-not-invent-milestones/mocks/terminal49/get_container.md rename to evals/terminal49/do-not-invent-milestones/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/do-not-invent-milestones/mocks/terminal49/search_container.md b/evals/terminal49/do-not-invent-milestones/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/do-not-invent-milestones/mocks/terminal49/search_container.md rename to evals/terminal49/do-not-invent-milestones/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/document-injection/case.yaml b/evals/terminal49/document-injection/case.yaml similarity index 100% rename from plugins/terminal49/evals/document-injection/case.yaml rename to evals/terminal49/document-injection/case.yaml diff --git a/plugins/terminal49/evals/document-injection/mocks/terminal49/get_document.md b/evals/terminal49/document-injection/mocks/terminal49/get_document.md similarity index 100% rename from plugins/terminal49/evals/document-injection/mocks/terminal49/get_document.md rename to evals/terminal49/document-injection/mocks/terminal49/get_document.md diff --git a/plugins/terminal49/evals/document-injection/mocks/terminal49/list_documents.md b/evals/terminal49/document-injection/mocks/terminal49/list_documents.md similarity index 100% rename from plugins/terminal49/evals/document-injection/mocks/terminal49/list_documents.md rename to evals/terminal49/document-injection/mocks/terminal49/list_documents.md diff --git a/plugins/terminal49/evals/document-injection/mocks/terminal49/list_parties.md b/evals/terminal49/document-injection/mocks/terminal49/list_parties.md similarity index 100% rename from plugins/terminal49/evals/document-injection/mocks/terminal49/list_parties.md rename to evals/terminal49/document-injection/mocks/terminal49/list_parties.md diff --git a/plugins/terminal49/evals/document-injection/mocks/terminal49/search_container.md b/evals/terminal49/document-injection/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/document-injection/mocks/terminal49/search_container.md rename to evals/terminal49/document-injection/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/document-injection/mocks/terminal49/set_custom_field.md b/evals/terminal49/document-injection/mocks/terminal49/set_custom_field.md similarity index 100% rename from plugins/terminal49/evals/document-injection/mocks/terminal49/set_custom_field.md rename to evals/terminal49/document-injection/mocks/terminal49/set_custom_field.md diff --git a/plugins/terminal49/evals/document-injection/mocks/terminal49/set_party_role.md b/evals/terminal49/document-injection/mocks/terminal49/set_party_role.md similarity index 100% rename from plugins/terminal49/evals/document-injection/mocks/terminal49/set_party_role.md rename to evals/terminal49/document-injection/mocks/terminal49/set_party_role.md diff --git a/plugins/terminal49/evals/dray-carrier-read-only/case.yaml b/evals/terminal49/dray-carrier-read-only/case.yaml similarity index 100% rename from plugins/terminal49/evals/dray-carrier-read-only/case.yaml rename to evals/terminal49/dray-carrier-read-only/case.yaml diff --git a/plugins/terminal49/evals/dray-carrier-read-only/mocks/terminal49/get_container.md b/evals/terminal49/dray-carrier-read-only/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/dray-carrier-read-only/mocks/terminal49/get_container.md rename to evals/terminal49/dray-carrier-read-only/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/dray-carrier-read-only/mocks/terminal49/search_container.md b/evals/terminal49/dray-carrier-read-only/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/dray-carrier-read-only/mocks/terminal49/search_container.md rename to evals/terminal49/dray-carrier-read-only/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/dray-carrier-read-only/mocks/terminal49/set_party_role.md b/evals/terminal49/dray-carrier-read-only/mocks/terminal49/set_party_role.md similarity index 100% rename from plugins/terminal49/evals/dray-carrier-read-only/mocks/terminal49/set_party_role.md rename to evals/terminal49/dray-carrier-read-only/mocks/terminal49/set_party_role.md diff --git a/plugins/terminal49/evals/eta-to-long-beach/case.yaml b/evals/terminal49/eta-to-long-beach/case.yaml similarity index 100% rename from plugins/terminal49/evals/eta-to-long-beach/case.yaml rename to evals/terminal49/eta-to-long-beach/case.yaml diff --git a/plugins/terminal49/evals/eta-to-long-beach/mocks/terminal49/get_container_route.md b/evals/terminal49/eta-to-long-beach/mocks/terminal49/get_container_route.md similarity index 100% rename from plugins/terminal49/evals/eta-to-long-beach/mocks/terminal49/get_container_route.md rename to evals/terminal49/eta-to-long-beach/mocks/terminal49/get_container_route.md diff --git a/plugins/terminal49/evals/eta-to-long-beach/mocks/terminal49/get_container_transport_events.md b/evals/terminal49/eta-to-long-beach/mocks/terminal49/get_container_transport_events.md similarity index 100% rename from plugins/terminal49/evals/eta-to-long-beach/mocks/terminal49/get_container_transport_events.md rename to evals/terminal49/eta-to-long-beach/mocks/terminal49/get_container_transport_events.md diff --git a/plugins/terminal49/evals/eta-to-long-beach/mocks/terminal49/search_container.md b/evals/terminal49/eta-to-long-beach/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/eta-to-long-beach/mocks/terminal49/search_container.md rename to evals/terminal49/eta-to-long-beach/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/fake-uuid-not-found/case.yaml b/evals/terminal49/fake-uuid-not-found/case.yaml similarity index 100% rename from plugins/terminal49/evals/fake-uuid-not-found/case.yaml rename to evals/terminal49/fake-uuid-not-found/case.yaml diff --git a/plugins/terminal49/evals/fake-uuid-not-found/mocks/terminal49/get_container.md b/evals/terminal49/fake-uuid-not-found/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/fake-uuid-not-found/mocks/terminal49/get_container.md rename to evals/terminal49/fake-uuid-not-found/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/find-cosu-booking/case.yaml b/evals/terminal49/find-cosu-booking/case.yaml similarity index 100% rename from plugins/terminal49/evals/find-cosu-booking/case.yaml rename to evals/terminal49/find-cosu-booking/case.yaml diff --git a/plugins/terminal49/evals/find-cosu-booking/mocks/terminal49/get_shipment_details.md b/evals/terminal49/find-cosu-booking/mocks/terminal49/get_shipment_details.md similarity index 100% rename from plugins/terminal49/evals/find-cosu-booking/mocks/terminal49/get_shipment_details.md rename to evals/terminal49/find-cosu-booking/mocks/terminal49/get_shipment_details.md diff --git a/plugins/terminal49/evals/find-cosu-booking/mocks/terminal49/search_container.md b/evals/terminal49/find-cosu-booking/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/find-cosu-booking/mocks/terminal49/search_container.md rename to evals/terminal49/find-cosu-booking/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/fleet-customer-filter/case.yaml b/evals/terminal49/fleet-customer-filter/case.yaml similarity index 100% rename from plugins/terminal49/evals/fleet-customer-filter/case.yaml rename to evals/terminal49/fleet-customer-filter/case.yaml diff --git a/plugins/terminal49/evals/fleet-customer-filter/mocks/terminal49/list_containers.md b/evals/terminal49/fleet-customer-filter/mocks/terminal49/list_containers.md similarity index 100% rename from plugins/terminal49/evals/fleet-customer-filter/mocks/terminal49/list_containers.md rename to evals/terminal49/fleet-customer-filter/mocks/terminal49/list_containers.md diff --git a/plugins/terminal49/evals/fleet-customer-filter/mocks/terminal49/list_parties.md b/evals/terminal49/fleet-customer-filter/mocks/terminal49/list_parties.md similarity index 100% rename from plugins/terminal49/evals/fleet-customer-filter/mocks/terminal49/list_parties.md rename to evals/terminal49/fleet-customer-filter/mocks/terminal49/list_parties.md diff --git a/plugins/terminal49/evals/fleet-needs-attention/case.yaml b/evals/terminal49/fleet-needs-attention/case.yaml similarity index 100% rename from plugins/terminal49/evals/fleet-needs-attention/case.yaml rename to evals/terminal49/fleet-needs-attention/case.yaml diff --git a/plugins/terminal49/evals/fleet-needs-attention/mocks/terminal49/get_container.md b/evals/terminal49/fleet-needs-attention/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/fleet-needs-attention/mocks/terminal49/get_container.md rename to evals/terminal49/fleet-needs-attention/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/fleet-needs-attention/mocks/terminal49/list_containers.md b/evals/terminal49/fleet-needs-attention/mocks/terminal49/list_containers.md similarity index 100% rename from plugins/terminal49/evals/fleet-needs-attention/mocks/terminal49/list_containers.md rename to evals/terminal49/fleet-needs-attention/mocks/terminal49/list_containers.md diff --git a/plugins/terminal49/evals/fleet-ready-for-pickup/case.yaml b/evals/terminal49/fleet-ready-for-pickup/case.yaml similarity index 100% rename from plugins/terminal49/evals/fleet-ready-for-pickup/case.yaml rename to evals/terminal49/fleet-ready-for-pickup/case.yaml diff --git a/plugins/terminal49/evals/fleet-ready-for-pickup/mocks/terminal49/list_containers.md b/evals/terminal49/fleet-ready-for-pickup/mocks/terminal49/list_containers.md similarity index 100% rename from plugins/terminal49/evals/fleet-ready-for-pickup/mocks/terminal49/list_containers.md rename to evals/terminal49/fleet-ready-for-pickup/mocks/terminal49/list_containers.md diff --git a/plugins/terminal49/evals/fleet-ready-for-pickup/mocks/terminal49/summarize_containers.md b/evals/terminal49/fleet-ready-for-pickup/mocks/terminal49/summarize_containers.md similarity index 100% rename from plugins/terminal49/evals/fleet-ready-for-pickup/mocks/terminal49/summarize_containers.md rename to evals/terminal49/fleet-ready-for-pickup/mocks/terminal49/summarize_containers.md diff --git a/plugins/terminal49/evals/identify-not-track/case.yaml b/evals/terminal49/identify-not-track/case.yaml similarity index 100% rename from plugins/terminal49/evals/identify-not-track/case.yaml rename to evals/terminal49/identify-not-track/case.yaml diff --git a/plugins/terminal49/evals/identify-not-track/mocks/terminal49/identify_tracking_number.md b/evals/terminal49/identify-not-track/mocks/terminal49/identify_tracking_number.md similarity index 100% rename from plugins/terminal49/evals/identify-not-track/mocks/terminal49/identify_tracking_number.md rename to evals/terminal49/identify-not-track/mocks/terminal49/identify_tracking_number.md diff --git a/plugins/terminal49/evals/identify-not-track/mocks/terminal49/search_container.md b/evals/terminal49/identify-not-track/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/identify-not-track/mocks/terminal49/search_container.md rename to evals/terminal49/identify-not-track/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/investigate-container-delay/graders/delay-answer.md b/evals/terminal49/investigate-container-delay/graders/delay-answer.md similarity index 100% rename from plugins/terminal49/evals/investigate-container-delay/graders/delay-answer.md rename to evals/terminal49/investigate-container-delay/graders/delay-answer.md diff --git a/plugins/terminal49/evals/investigate-container-delay/graders/events-lookup-order.md b/evals/terminal49/investigate-container-delay/graders/events-lookup-order.md similarity index 100% rename from plugins/terminal49/evals/investigate-container-delay/graders/events-lookup-order.md rename to evals/terminal49/investigate-container-delay/graders/events-lookup-order.md diff --git a/plugins/terminal49/evals/investigate-container-delay/graders/no-track-create.md b/evals/terminal49/investigate-container-delay/graders/no-track-create.md similarity index 100% rename from plugins/terminal49/evals/investigate-container-delay/graders/no-track-create.md rename to evals/terminal49/investigate-container-delay/graders/no-track-create.md diff --git a/plugins/terminal49/evals/investigate-container-delay/graders/route-used.md b/evals/terminal49/investigate-container-delay/graders/route-used.md similarity index 100% rename from plugins/terminal49/evals/investigate-container-delay/graders/route-used.md rename to evals/terminal49/investigate-container-delay/graders/route-used.md diff --git a/plugins/terminal49/evals/investigate-container-delay/graders/skill-fired.md b/evals/terminal49/investigate-container-delay/graders/skill-fired.md similarity index 100% rename from plugins/terminal49/evals/investigate-container-delay/graders/skill-fired.md rename to evals/terminal49/investigate-container-delay/graders/skill-fired.md diff --git a/plugins/terminal49/evals/investigate-container-delay/mocks/terminal49/get_container_route.md b/evals/terminal49/investigate-container-delay/mocks/terminal49/get_container_route.md similarity index 100% rename from plugins/terminal49/evals/investigate-container-delay/mocks/terminal49/get_container_route.md rename to evals/terminal49/investigate-container-delay/mocks/terminal49/get_container_route.md diff --git a/plugins/terminal49/evals/investigate-container-delay/mocks/terminal49/get_container_transport_events.md b/evals/terminal49/investigate-container-delay/mocks/terminal49/get_container_transport_events.md similarity index 100% rename from plugins/terminal49/evals/investigate-container-delay/mocks/terminal49/get_container_transport_events.md rename to evals/terminal49/investigate-container-delay/mocks/terminal49/get_container_transport_events.md diff --git a/plugins/terminal49/evals/investigate-container-delay/mocks/terminal49/search_container.md b/evals/terminal49/investigate-container-delay/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/investigate-container-delay/mocks/terminal49/search_container.md rename to evals/terminal49/investigate-container-delay/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/investigate-container-delay/prompt.md b/evals/terminal49/investigate-container-delay/prompt.md similarity index 100% rename from plugins/terminal49/evals/investigate-container-delay/prompt.md rename to evals/terminal49/investigate-container-delay/prompt.md diff --git a/plugins/terminal49/evals/investigate-shipment/graders/no-track-create.md b/evals/terminal49/investigate-shipment/graders/no-track-create.md similarity index 100% rename from plugins/terminal49/evals/investigate-shipment/graders/no-track-create.md rename to evals/terminal49/investigate-shipment/graders/no-track-create.md diff --git a/plugins/terminal49/evals/investigate-shipment/graders/shipment-answer.md b/evals/terminal49/investigate-shipment/graders/shipment-answer.md similarity index 100% rename from plugins/terminal49/evals/investigate-shipment/graders/shipment-answer.md rename to evals/terminal49/investigate-shipment/graders/shipment-answer.md diff --git a/plugins/terminal49/evals/investigate-shipment/graders/shipment-lookup-order.md b/evals/terminal49/investigate-shipment/graders/shipment-lookup-order.md similarity index 100% rename from plugins/terminal49/evals/investigate-shipment/graders/shipment-lookup-order.md rename to evals/terminal49/investigate-shipment/graders/shipment-lookup-order.md diff --git a/plugins/terminal49/evals/investigate-shipment/graders/skill-fired.md b/evals/terminal49/investigate-shipment/graders/skill-fired.md similarity index 100% rename from plugins/terminal49/evals/investigate-shipment/graders/skill-fired.md rename to evals/terminal49/investigate-shipment/graders/skill-fired.md diff --git a/plugins/terminal49/evals/investigate-shipment/mocks/terminal49/search_container.md b/evals/terminal49/investigate-shipment/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/investigate-shipment/mocks/terminal49/search_container.md rename to evals/terminal49/investigate-shipment/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/investigate-shipment/prompt.md b/evals/terminal49/investigate-shipment/prompt.md similarity index 100% rename from plugins/terminal49/evals/investigate-shipment/prompt.md rename to evals/terminal49/investigate-shipment/prompt.md diff --git a/plugins/terminal49/evals/is-it-ready-for-pickup/case.yaml b/evals/terminal49/is-it-ready-for-pickup/case.yaml similarity index 100% rename from plugins/terminal49/evals/is-it-ready-for-pickup/case.yaml rename to evals/terminal49/is-it-ready-for-pickup/case.yaml diff --git a/plugins/terminal49/evals/is-it-ready-for-pickup/mocks/terminal49/search_container.md b/evals/terminal49/is-it-ready-for-pickup/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/is-it-ready-for-pickup/mocks/terminal49/search_container.md rename to evals/terminal49/is-it-ready-for-pickup/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/last-free-day/case.yaml b/evals/terminal49/last-free-day/case.yaml similarity index 100% rename from plugins/terminal49/evals/last-free-day/case.yaml rename to evals/terminal49/last-free-day/case.yaml diff --git a/plugins/terminal49/evals/last-free-day/mocks/terminal49/get_container.md b/evals/terminal49/last-free-day/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/last-free-day/mocks/terminal49/get_container.md rename to evals/terminal49/last-free-day/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/last-free-day/mocks/terminal49/search_container.md b/evals/terminal49/last-free-day/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/last-free-day/mocks/terminal49/search_container.md rename to evals/terminal49/last-free-day/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/locate-my-container/case.yaml b/evals/terminal49/locate-my-container/case.yaml similarity index 100% rename from plugins/terminal49/evals/locate-my-container/case.yaml rename to evals/terminal49/locate-my-container/case.yaml diff --git a/plugins/terminal49/evals/locate-my-container/mocks/terminal49/search_container.md b/evals/terminal49/locate-my-container/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/locate-my-container/mocks/terminal49/search_container.md rename to evals/terminal49/locate-my-container/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-delay-eta/case.yaml b/evals/terminal49/mb-delay-eta/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-delay-eta/case.yaml rename to evals/terminal49/mb-delay-eta/case.yaml diff --git a/plugins/terminal49/evals/mb-delay-eta/mocks/terminal49/search_container.md b/evals/terminal49/mb-delay-eta/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-delay-eta/mocks/terminal49/search_container.md rename to evals/terminal49/mb-delay-eta/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-delay-rolled/case.yaml b/evals/terminal49/mb-delay-rolled/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-delay-rolled/case.yaml rename to evals/terminal49/mb-delay-rolled/case.yaml diff --git a/plugins/terminal49/evals/mb-delay-rolled/mocks/terminal49/search_container.md b/evals/terminal49/mb-delay-rolled/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-delay-rolled/mocks/terminal49/search_container.md rename to evals/terminal49/mb-delay-rolled/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-delay-why-late/case.yaml b/evals/terminal49/mb-delay-why-late/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-delay-why-late/case.yaml rename to evals/terminal49/mb-delay-why-late/case.yaml diff --git a/plugins/terminal49/evals/mb-delay-why-late/mocks/terminal49/search_container.md b/evals/terminal49/mb-delay-why-late/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-delay-why-late/mocks/terminal49/search_container.md rename to evals/terminal49/mb-delay-why-late/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-find-container/case.yaml b/evals/terminal49/mb-find-container/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-find-container/case.yaml rename to evals/terminal49/mb-find-container/case.yaml diff --git a/plugins/terminal49/evals/mb-find-container/mocks/terminal49/search_container.md b/evals/terminal49/mb-find-container/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-find-container/mocks/terminal49/search_container.md rename to evals/terminal49/mb-find-container/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-find-pull-up/case.yaml b/evals/terminal49/mb-find-pull-up/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-find-pull-up/case.yaml rename to evals/terminal49/mb-find-pull-up/case.yaml diff --git a/plugins/terminal49/evals/mb-find-pull-up/mocks/terminal49/search_container.md b/evals/terminal49/mb-find-pull-up/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-find-pull-up/mocks/terminal49/search_container.md rename to evals/terminal49/mb-find-pull-up/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-find-wheres-box/case.yaml b/evals/terminal49/mb-find-wheres-box/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-find-wheres-box/case.yaml rename to evals/terminal49/mb-find-wheres-box/case.yaml diff --git a/plugins/terminal49/evals/mb-find-wheres-box/mocks/terminal49/search_container.md b/evals/terminal49/mb-find-wheres-box/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-find-wheres-box/mocks/terminal49/search_container.md rename to evals/terminal49/mb-find-wheres-box/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-guard-fake-uuid/case.yaml b/evals/terminal49/mb-guard-fake-uuid/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-guard-fake-uuid/case.yaml rename to evals/terminal49/mb-guard-fake-uuid/case.yaml diff --git a/plugins/terminal49/evals/mb-guard-fake-uuid/mocks/terminal49/get_container.md b/evals/terminal49/mb-guard-fake-uuid/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/mb-guard-fake-uuid/mocks/terminal49/get_container.md rename to evals/terminal49/mb-guard-fake-uuid/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/mb-guard-vague-track/case.yaml b/evals/terminal49/mb-guard-vague-track/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-guard-vague-track/case.yaml rename to evals/terminal49/mb-guard-vague-track/case.yaml diff --git a/plugins/terminal49/evals/mb-holds-blocking/case.yaml b/evals/terminal49/mb-holds-blocking/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-holds-blocking/case.yaml rename to evals/terminal49/mb-holds-blocking/case.yaml diff --git a/plugins/terminal49/evals/mb-holds-blocking/mocks/terminal49/search_container.md b/evals/terminal49/mb-holds-blocking/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-holds-blocking/mocks/terminal49/search_container.md rename to evals/terminal49/mb-holds-blocking/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-holds-customs/case.yaml b/evals/terminal49/mb-holds-customs/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-holds-customs/case.yaml rename to evals/terminal49/mb-holds-customs/case.yaml diff --git a/plugins/terminal49/evals/mb-holds-customs/mocks/terminal49/search_container.md b/evals/terminal49/mb-holds-customs/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-holds-customs/mocks/terminal49/search_container.md rename to evals/terminal49/mb-holds-customs/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-holds-on-hold/case.yaml b/evals/terminal49/mb-holds-on-hold/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-holds-on-hold/case.yaml rename to evals/terminal49/mb-holds-on-hold/case.yaml diff --git a/plugins/terminal49/evals/mb-holds-on-hold/mocks/terminal49/search_container.md b/evals/terminal49/mb-holds-on-hold/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-holds-on-hold/mocks/terminal49/search_container.md rename to evals/terminal49/mb-holds-on-hold/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-lfd-demurrage/case.yaml b/evals/terminal49/mb-lfd-demurrage/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-lfd-demurrage/case.yaml rename to evals/terminal49/mb-lfd-demurrage/case.yaml diff --git a/plugins/terminal49/evals/mb-lfd-demurrage/mocks/terminal49/search_container.md b/evals/terminal49/mb-lfd-demurrage/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-lfd-demurrage/mocks/terminal49/search_container.md rename to evals/terminal49/mb-lfd-demurrage/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-lfd-free-days/case.yaml b/evals/terminal49/mb-lfd-free-days/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-lfd-free-days/case.yaml rename to evals/terminal49/mb-lfd-free-days/case.yaml diff --git a/plugins/terminal49/evals/mb-lfd-free-days/mocks/terminal49/search_container.md b/evals/terminal49/mb-lfd-free-days/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-lfd-free-days/mocks/terminal49/search_container.md rename to evals/terminal49/mb-lfd-free-days/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-lfd-last-free-day/case.yaml b/evals/terminal49/mb-lfd-last-free-day/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-lfd-last-free-day/case.yaml rename to evals/terminal49/mb-lfd-last-free-day/case.yaml diff --git a/plugins/terminal49/evals/mb-lfd-last-free-day/mocks/terminal49/search_container.md b/evals/terminal49/mb-lfd-last-free-day/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-lfd-last-free-day/mocks/terminal49/search_container.md rename to evals/terminal49/mb-lfd-last-free-day/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-pickup-available/case.yaml b/evals/terminal49/mb-pickup-available/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-pickup-available/case.yaml rename to evals/terminal49/mb-pickup-available/case.yaml diff --git a/plugins/terminal49/evals/mb-pickup-available/mocks/terminal49/search_container.md b/evals/terminal49/mb-pickup-available/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-pickup-available/mocks/terminal49/search_container.md rename to evals/terminal49/mb-pickup-available/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-pickup-truck/case.yaml b/evals/terminal49/mb-pickup-truck/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-pickup-truck/case.yaml rename to evals/terminal49/mb-pickup-truck/case.yaml diff --git a/plugins/terminal49/evals/mb-pickup-truck/mocks/terminal49/search_container.md b/evals/terminal49/mb-pickup-truck/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-pickup-truck/mocks/terminal49/search_container.md rename to evals/terminal49/mb-pickup-truck/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-pickup-when/case.yaml b/evals/terminal49/mb-pickup-when/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-pickup-when/case.yaml rename to evals/terminal49/mb-pickup-when/case.yaml diff --git a/plugins/terminal49/evals/mb-pickup-when/mocks/terminal49/search_container.md b/evals/terminal49/mb-pickup-when/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-pickup-when/mocks/terminal49/search_container.md rename to evals/terminal49/mb-pickup-when/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-track-add-container/case.yaml b/evals/terminal49/mb-track-add-container/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-track-add-container/case.yaml rename to evals/terminal49/mb-track-add-container/case.yaml diff --git a/plugins/terminal49/evals/mb-track-add-container/mocks/terminal49/search_container.md b/evals/terminal49/mb-track-add-container/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-track-add-container/mocks/terminal49/search_container.md rename to evals/terminal49/mb-track-add-container/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-track-box-cosu/case.yaml b/evals/terminal49/mb-track-box-cosu/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-track-box-cosu/case.yaml rename to evals/terminal49/mb-track-box-cosu/case.yaml diff --git a/plugins/terminal49/evals/mb-track-box-cosu/mocks/terminal49/search_container.md b/evals/terminal49/mb-track-box-cosu/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-track-box-cosu/mocks/terminal49/search_container.md rename to evals/terminal49/mb-track-box-cosu/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mb-track-start-cosu/case.yaml b/evals/terminal49/mb-track-start-cosu/case.yaml similarity index 100% rename from plugins/terminal49/evals/mb-track-start-cosu/case.yaml rename to evals/terminal49/mb-track-start-cosu/case.yaml diff --git a/plugins/terminal49/evals/mb-track-start-cosu/mocks/terminal49/search_container.md b/evals/terminal49/mb-track-start-cosu/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/mb-track-start-cosu/mocks/terminal49/search_container.md rename to evals/terminal49/mb-track-start-cosu/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/mocks/terminal49/_tools.json b/evals/terminal49/mocks/terminal49/_tools.json similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/_tools.json rename to evals/terminal49/mocks/terminal49/_tools.json diff --git a/plugins/terminal49/evals/mocks/terminal49/get_container.md b/evals/terminal49/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/get_container.md rename to evals/terminal49/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/mocks/terminal49/get_container_route.md b/evals/terminal49/mocks/terminal49/get_container_route.md similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/get_container_route.md rename to evals/terminal49/mocks/terminal49/get_container_route.md diff --git a/plugins/terminal49/evals/mocks/terminal49/get_container_transport_events.md b/evals/terminal49/mocks/terminal49/get_container_transport_events.md similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/get_container_transport_events.md rename to evals/terminal49/mocks/terminal49/get_container_transport_events.md diff --git a/plugins/terminal49/evals/mocks/terminal49/get_shipment_details.md b/evals/terminal49/mocks/terminal49/get_shipment_details.md similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/get_shipment_details.md rename to evals/terminal49/mocks/terminal49/get_shipment_details.md diff --git a/plugins/terminal49/evals/mocks/terminal49/get_supported_shipping_lines.md b/evals/terminal49/mocks/terminal49/get_supported_shipping_lines.md similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/get_supported_shipping_lines.md rename to evals/terminal49/mocks/terminal49/get_supported_shipping_lines.md diff --git a/plugins/terminal49/evals/mocks/terminal49/list_containers.md b/evals/terminal49/mocks/terminal49/list_containers.md similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/list_containers.md rename to evals/terminal49/mocks/terminal49/list_containers.md diff --git a/plugins/terminal49/evals/mocks/terminal49/list_shipments.md b/evals/terminal49/mocks/terminal49/list_shipments.md similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/list_shipments.md rename to evals/terminal49/mocks/terminal49/list_shipments.md diff --git a/plugins/terminal49/evals/mocks/terminal49/list_tracking_requests.md b/evals/terminal49/mocks/terminal49/list_tracking_requests.md similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/list_tracking_requests.md rename to evals/terminal49/mocks/terminal49/list_tracking_requests.md diff --git a/plugins/terminal49/evals/mocks/terminal49/track_container.md b/evals/terminal49/mocks/terminal49/track_container.md similarity index 100% rename from plugins/terminal49/evals/mocks/terminal49/track_container.md rename to evals/terminal49/mocks/terminal49/track_container.md diff --git a/plugins/terminal49/evals/no-credentials-in-reply/case.yaml b/evals/terminal49/no-credentials-in-reply/case.yaml similarity index 100% rename from plugins/terminal49/evals/no-credentials-in-reply/case.yaml rename to evals/terminal49/no-credentials-in-reply/case.yaml diff --git a/plugins/terminal49/evals/on-hold/case.yaml b/evals/terminal49/on-hold/case.yaml similarity index 100% rename from plugins/terminal49/evals/on-hold/case.yaml rename to evals/terminal49/on-hold/case.yaml diff --git a/plugins/terminal49/evals/on-hold/mocks/terminal49/get_container.md b/evals/terminal49/on-hold/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/on-hold/mocks/terminal49/get_container.md rename to evals/terminal49/on-hold/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/on-hold/mocks/terminal49/search_container.md b/evals/terminal49/on-hold/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/on-hold/mocks/terminal49/search_container.md rename to evals/terminal49/on-hold/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/picked-up-yet/case.yaml b/evals/terminal49/picked-up-yet/case.yaml similarity index 100% rename from plugins/terminal49/evals/picked-up-yet/case.yaml rename to evals/terminal49/picked-up-yet/case.yaml diff --git a/plugins/terminal49/evals/picked-up-yet/mocks/terminal49/get_container.md b/evals/terminal49/picked-up-yet/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/picked-up-yet/mocks/terminal49/get_container.md rename to evals/terminal49/picked-up-yet/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/picked-up-yet/mocks/terminal49/search_container.md b/evals/terminal49/picked-up-yet/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/picked-up-yet/mocks/terminal49/search_container.md rename to evals/terminal49/picked-up-yet/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/refuse-track-everything/case.yaml b/evals/terminal49/refuse-track-everything/case.yaml similarity index 100% rename from plugins/terminal49/evals/refuse-track-everything/case.yaml rename to evals/terminal49/refuse-track-everything/case.yaml diff --git a/plugins/terminal49/evals/route-entitlement-miss/case.yaml b/evals/terminal49/route-entitlement-miss/case.yaml similarity index 100% rename from plugins/terminal49/evals/route-entitlement-miss/case.yaml rename to evals/terminal49/route-entitlement-miss/case.yaml diff --git a/plugins/terminal49/evals/route-entitlement-miss/mocks/terminal49/get_container_route.md b/evals/terminal49/route-entitlement-miss/mocks/terminal49/get_container_route.md similarity index 100% rename from plugins/terminal49/evals/route-entitlement-miss/mocks/terminal49/get_container_route.md rename to evals/terminal49/route-entitlement-miss/mocks/terminal49/get_container_route.md diff --git a/plugins/terminal49/evals/route-entitlement-miss/mocks/terminal49/search_container.md b/evals/terminal49/route-entitlement-miss/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/route-entitlement-miss/mocks/terminal49/search_container.md rename to evals/terminal49/route-entitlement-miss/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/set-custom-field/case.yaml b/evals/terminal49/set-custom-field/case.yaml similarity index 100% rename from plugins/terminal49/evals/set-custom-field/case.yaml rename to evals/terminal49/set-custom-field/case.yaml diff --git a/plugins/terminal49/evals/set-custom-field/mocks/terminal49/get_container.md b/evals/terminal49/set-custom-field/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/set-custom-field/mocks/terminal49/get_container.md rename to evals/terminal49/set-custom-field/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/set-custom-field/mocks/terminal49/list_custom_field_definitions.md b/evals/terminal49/set-custom-field/mocks/terminal49/list_custom_field_definitions.md similarity index 100% rename from plugins/terminal49/evals/set-custom-field/mocks/terminal49/list_custom_field_definitions.md rename to evals/terminal49/set-custom-field/mocks/terminal49/list_custom_field_definitions.md diff --git a/plugins/terminal49/evals/set-custom-field/mocks/terminal49/search_container.md b/evals/terminal49/set-custom-field/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/set-custom-field/mocks/terminal49/search_container.md rename to evals/terminal49/set-custom-field/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/set-custom-field/mocks/terminal49/set_custom_field.md b/evals/terminal49/set-custom-field/mocks/terminal49/set_custom_field.md similarity index 100% rename from plugins/terminal49/evals/set-custom-field/mocks/terminal49/set_custom_field.md rename to evals/terminal49/set-custom-field/mocks/terminal49/set_custom_field.md diff --git a/plugins/terminal49/evals/set-dray-carrier/case.yaml b/evals/terminal49/set-dray-carrier/case.yaml similarity index 100% rename from plugins/terminal49/evals/set-dray-carrier/case.yaml rename to evals/terminal49/set-dray-carrier/case.yaml diff --git a/plugins/terminal49/evals/set-dray-carrier/mocks/terminal49/get_container.md b/evals/terminal49/set-dray-carrier/mocks/terminal49/get_container.md similarity index 100% rename from plugins/terminal49/evals/set-dray-carrier/mocks/terminal49/get_container.md rename to evals/terminal49/set-dray-carrier/mocks/terminal49/get_container.md diff --git a/plugins/terminal49/evals/set-dray-carrier/mocks/terminal49/list_parties.md b/evals/terminal49/set-dray-carrier/mocks/terminal49/list_parties.md similarity index 100% rename from plugins/terminal49/evals/set-dray-carrier/mocks/terminal49/list_parties.md rename to evals/terminal49/set-dray-carrier/mocks/terminal49/list_parties.md diff --git a/plugins/terminal49/evals/set-dray-carrier/mocks/terminal49/search_container.md b/evals/terminal49/set-dray-carrier/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/set-dray-carrier/mocks/terminal49/search_container.md rename to evals/terminal49/set-dray-carrier/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/set-dray-carrier/mocks/terminal49/set_party_role.md b/evals/terminal49/set-dray-carrier/mocks/terminal49/set_party_role.md similarity index 100% rename from plugins/terminal49/evals/set-dray-carrier/mocks/terminal49/set_party_role.md rename to evals/terminal49/set-dray-carrier/mocks/terminal49/set_party_role.md diff --git a/plugins/terminal49/evals/track-new-container/graders/search-before-track.md b/evals/terminal49/track-new-container/graders/search-before-track.md similarity index 100% rename from plugins/terminal49/evals/track-new-container/graders/search-before-track.md rename to evals/terminal49/track-new-container/graders/search-before-track.md diff --git a/plugins/terminal49/evals/track-new-container/graders/skill-fired.md b/evals/terminal49/track-new-container/graders/skill-fired.md similarity index 100% rename from plugins/terminal49/evals/track-new-container/graders/skill-fired.md rename to evals/terminal49/track-new-container/graders/skill-fired.md diff --git a/plugins/terminal49/evals/track-new-container/graders/track-once.md b/evals/terminal49/track-new-container/graders/track-once.md similarity index 100% rename from plugins/terminal49/evals/track-new-container/graders/track-once.md rename to evals/terminal49/track-new-container/graders/track-once.md diff --git a/plugins/terminal49/evals/track-new-container/graders/tracking-answer.md b/evals/terminal49/track-new-container/graders/tracking-answer.md similarity index 100% rename from plugins/terminal49/evals/track-new-container/graders/tracking-answer.md rename to evals/terminal49/track-new-container/graders/tracking-answer.md diff --git a/plugins/terminal49/evals/track-new-container/mocks/terminal49/search_container.md b/evals/terminal49/track-new-container/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/track-new-container/mocks/terminal49/search_container.md rename to evals/terminal49/track-new-container/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/track-new-container/prompt.md b/evals/terminal49/track-new-container/prompt.md similarity index 100% rename from plugins/terminal49/evals/track-new-container/prompt.md rename to evals/terminal49/track-new-container/prompt.md diff --git a/plugins/terminal49/evals/trade-not-enabled/case.yaml b/evals/terminal49/trade-not-enabled/case.yaml similarity index 100% rename from plugins/terminal49/evals/trade-not-enabled/case.yaml rename to evals/terminal49/trade-not-enabled/case.yaml diff --git a/plugins/terminal49/evals/trade-not-enabled/mocks/terminal49/get_trade_data_coverage.md b/evals/terminal49/trade-not-enabled/mocks/terminal49/get_trade_data_coverage.md similarity index 100% rename from plugins/terminal49/evals/trade-not-enabled/mocks/terminal49/get_trade_data_coverage.md rename to evals/terminal49/trade-not-enabled/mocks/terminal49/get_trade_data_coverage.md diff --git a/plugins/terminal49/evals/trade-not-enabled/mocks/terminal49/search_importers.md b/evals/terminal49/trade-not-enabled/mocks/terminal49/search_importers.md similarity index 100% rename from plugins/terminal49/evals/trade-not-enabled/mocks/terminal49/search_importers.md rename to evals/terminal49/trade-not-enabled/mocks/terminal49/search_importers.md diff --git a/plugins/terminal49/evals/trade-top-importers/case.yaml b/evals/terminal49/trade-top-importers/case.yaml similarity index 100% rename from plugins/terminal49/evals/trade-top-importers/case.yaml rename to evals/terminal49/trade-top-importers/case.yaml diff --git a/plugins/terminal49/evals/trade-top-importers/mocks/terminal49/rank_importers.md b/evals/terminal49/trade-top-importers/mocks/terminal49/rank_importers.md similarity index 100% rename from plugins/terminal49/evals/trade-top-importers/mocks/terminal49/rank_importers.md rename to evals/terminal49/trade-top-importers/mocks/terminal49/rank_importers.md diff --git a/plugins/terminal49/evals/trade-top-importers/mocks/terminal49/search_commodities.md b/evals/terminal49/trade-top-importers/mocks/terminal49/search_commodities.md similarity index 100% rename from plugins/terminal49/evals/trade-top-importers/mocks/terminal49/search_commodities.md rename to evals/terminal49/trade-top-importers/mocks/terminal49/search_commodities.md diff --git a/plugins/terminal49/evals/trade-top-importers/mocks/terminal49/search_importers.md b/evals/terminal49/trade-top-importers/mocks/terminal49/search_importers.md similarity index 100% rename from plugins/terminal49/evals/trade-top-importers/mocks/terminal49/search_importers.md rename to evals/terminal49/trade-top-importers/mocks/terminal49/search_importers.md diff --git a/plugins/terminal49/evals/when-can-i-pick-up/case.yaml b/evals/terminal49/when-can-i-pick-up/case.yaml similarity index 100% rename from plugins/terminal49/evals/when-can-i-pick-up/case.yaml rename to evals/terminal49/when-can-i-pick-up/case.yaml diff --git a/plugins/terminal49/evals/when-can-i-pick-up/mocks/terminal49/search_container.md b/evals/terminal49/when-can-i-pick-up/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/when-can-i-pick-up/mocks/terminal49/search_container.md rename to evals/terminal49/when-can-i-pick-up/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/where-is-container/graders/no-track-create.md b/evals/terminal49/where-is-container/graders/no-track-create.md similarity index 100% rename from plugins/terminal49/evals/where-is-container/graders/no-track-create.md rename to evals/terminal49/where-is-container/graders/no-track-create.md diff --git a/plugins/terminal49/evals/where-is-container/graders/operational-answer.md b/evals/terminal49/where-is-container/graders/operational-answer.md similarity index 100% rename from plugins/terminal49/evals/where-is-container/graders/operational-answer.md rename to evals/terminal49/where-is-container/graders/operational-answer.md diff --git a/plugins/terminal49/evals/where-is-container/graders/skill-fired.md b/evals/terminal49/where-is-container/graders/skill-fired.md similarity index 100% rename from plugins/terminal49/evals/where-is-container/graders/skill-fired.md rename to evals/terminal49/where-is-container/graders/skill-fired.md diff --git a/plugins/terminal49/evals/where-is-container/graders/status-lookup-order.md b/evals/terminal49/where-is-container/graders/status-lookup-order.md similarity index 100% rename from plugins/terminal49/evals/where-is-container/graders/status-lookup-order.md rename to evals/terminal49/where-is-container/graders/status-lookup-order.md diff --git a/plugins/terminal49/evals/where-is-container/mocks/terminal49/search_container.md b/evals/terminal49/where-is-container/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/where-is-container/mocks/terminal49/search_container.md rename to evals/terminal49/where-is-container/mocks/terminal49/search_container.md diff --git a/plugins/terminal49/evals/where-is-container/prompt.md b/evals/terminal49/where-is-container/prompt.md similarity index 100% rename from plugins/terminal49/evals/where-is-container/prompt.md rename to evals/terminal49/where-is-container/prompt.md diff --git a/plugins/terminal49/evals/where-is-my-box/case.yaml b/evals/terminal49/where-is-my-box/case.yaml similarity index 100% rename from plugins/terminal49/evals/where-is-my-box/case.yaml rename to evals/terminal49/where-is-my-box/case.yaml diff --git a/plugins/terminal49/evals/where-is-my-box/mocks/terminal49/search_container.md b/evals/terminal49/where-is-my-box/mocks/terminal49/search_container.md similarity index 100% rename from plugins/terminal49/evals/where-is-my-box/mocks/terminal49/search_container.md rename to evals/terminal49/where-is-my-box/mocks/terminal49/search_container.md diff --git a/gemini-extension.json b/gemini-extension.json index 988095f..5c7654e 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "terminal49", - "version": "0.3.3", + "version": "0.3.4", "description": "Terminal49 container and shipment tracking: look up containers by container, booking or bill of lading number, check status, milestones, ETAs, holds and last free day, and start new tracking.", "contextFileName": "GEMINI.md", "mcpServers": { diff --git a/plugins/terminal49/.claude-plugin/plugin.json b/plugins/terminal49/.claude-plugin/plugin.json index c06c4ed..26e3580 100644 --- a/plugins/terminal49/.claude-plugin/plugin.json +++ b/plugins/terminal49/.claude-plugin/plugin.json @@ -1,8 +1,7 @@ { - "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "terminal49", "displayName": "Terminal49", - "version": "0.3.3", + "version": "0.3.4", "description": "Look up and investigate container shipments, start tracking, and research US ocean import markets with the Terminal49 MCP server.", "author": { "name": "Terminal49", @@ -20,11 +19,6 @@ "supply-chain", "trade-intelligence" ], - "icon": "./assets/icon.svg", - "documentationUrl": "https://terminal49.com/docs/mcp/home", - "supportUrl": "https://www.terminal49.com/contact", - "privacyPolicyUrl": "https://www.terminal49.com/privacy", - "termsOfServiceUrl": "https://www.terminal49.com/terms", "skills": "./skills/", "mcpServers": "./.mcp.json" } diff --git a/plugins/terminal49/.codex-plugin/plugin.json b/plugins/terminal49/.codex-plugin/plugin.json index 301aeae..05f94db 100644 --- a/plugins/terminal49/.codex-plugin/plugin.json +++ b/plugins/terminal49/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "terminal49", - "version": "0.3.3", + "version": "0.3.4", "description": "Look up and investigate container shipments, start tracking, and research US ocean import markets with the Terminal49 MCP server.", "author": { "name": "Terminal49", @@ -39,7 +39,7 @@ "Which of my containers need attention today?", "Who are the largest US importers of office chairs?" ], - "composerIcon": "./assets/icon.svg", - "logo": "./assets/icon.svg" + "composerIcon": "./assets/icon.png", + "logo": "./assets/icon.png" } } diff --git a/plugins/terminal49/.cursor-plugin/plugin.json b/plugins/terminal49/.cursor-plugin/plugin.json index 5cedc56..0790b70 100644 --- a/plugins/terminal49/.cursor-plugin/plugin.json +++ b/plugins/terminal49/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "terminal49", "displayName": "Terminal49", - "version": "0.3.3", + "version": "0.3.4", "description": "Look up and investigate container shipments, start tracking, and research US ocean import markets with the Terminal49 MCP server.", "author": { "name": "Terminal49", @@ -11,7 +11,7 @@ "homepage": "https://terminal49.com", "repository": "https://github.com/Terminal49/agent-plugins", "license": "Apache-2.0", - "logo": "assets/logo.svg", + "logo": "assets/logo.png", "keywords": [ "container-tracking", "logistics", diff --git a/plugins/terminal49/README.md b/plugins/terminal49/README.md index e1522ba..ec3f3b7 100644 --- a/plugins/terminal49/README.md +++ b/plugins/terminal49/README.md @@ -14,13 +14,13 @@ claude plugin install terminal49 --marketplace Terminal49/agent-plugins ## What the plugin connects to -The plugin declares one remote MCP server, `https://mcp.terminal49.com`. It runs -no local programs, hooks, or scripts, and it reads no files or environment -variables on your machine. +The plugin declares one remote MCP server, `https://mcp.terminal49.com`. It +holds only Markdown, JSON, and images. It runs no programs, hooks, or scripts +and reads nothing from your computer. -- **Authentication.** Your client opens the Terminal49 OAuth sign-in in the - browser when a Terminal49 tool is first used. Do not paste API keys, access - tokens, or one-time codes into chat. +- **Connecting.** The first time you use a Terminal49 tool, your client opens a + browser window where you connect your Terminal49 account. You finish + connecting in that window, not in the chat. - **Data sent.** The tools send the identifiers, filters, and values in your request (for example a container number or a company name) to the Terminal49 MCP server. The server reads the shipments, documents, and trade data that @@ -80,7 +80,7 @@ API and MCP server live in [Terminal49/API](https://github.com/Terminal49/API). ## Plugin evals Behavioral eval cases use checked-in MCP mocks and never call the live -Terminal49 service. See [`evals/README.md`](evals/README.md) for validation, +Terminal49 service. See [`evals/terminal49/README.md`](https://github.com/Terminal49/agent-plugins/blob/main/evals/terminal49/README.md) for validation, mocked eval, and baseline guidance. ## License diff --git a/plugins/terminal49/assets/icon.png b/plugins/terminal49/assets/icon.png new file mode 100644 index 0000000..7056662 Binary files /dev/null and b/plugins/terminal49/assets/icon.png differ diff --git a/plugins/terminal49/assets/icon.svg b/plugins/terminal49/assets/icon.svg deleted file mode 100644 index b9388d9..0000000 --- a/plugins/terminal49/assets/icon.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/plugins/terminal49/assets/logo.png b/plugins/terminal49/assets/logo.png new file mode 100644 index 0000000..57ae983 Binary files /dev/null and b/plugins/terminal49/assets/logo.png differ diff --git a/plugins/terminal49/assets/logo.svg b/plugins/terminal49/assets/logo.svg deleted file mode 100644 index 36175f2..0000000 --- a/plugins/terminal49/assets/logo.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/plugins/terminal49/skills/container-tracking/SKILL.md b/plugins/terminal49/skills/container-tracking/SKILL.md index 943e9e6..9528ad9 100644 --- a/plugins/terminal49/skills/container-tracking/SKILL.md +++ b/plugins/terminal49/skills/container-tracking/SKILL.md @@ -17,12 +17,12 @@ find it and report current status, location, and anything that needs attention. ## Before using tools 1. Confirm that Terminal49 MCP tools are available. -2. If the client requests authentication, let it start the OAuth browser flow. -3. Sign-in happens in the browser, so never ask the user for sign-in details or - one-time codes in chat. +2. If the client asks to connect, let it open the Terminal49 browser window. +3. The user connects in that browser window, so never ask for connection + details in chat. 4. If the tools remain unavailable, say that the Terminal49 connection must be - enabled and authenticated. Do not invent shipment data. -5. If a lookup returns nothing or an authorization error, call `whoami` to + turned on and connected. Do not invent shipment data. +5. If a lookup returns nothing or an access error, call `whoami` to confirm the connected account and which gated features it can use. ## Identifier rules diff --git a/plugins/terminal49/skills/trade-intelligence/SKILL.md b/plugins/terminal49/skills/trade-intelligence/SKILL.md index aa3ffe7..96520a0 100644 --- a/plugins/terminal49/skills/trade-intelligence/SKILL.md +++ b/plugins/terminal49/skills/trade-intelligence/SKILL.md @@ -17,9 +17,8 @@ the answer and the few numbers that support it, not a data dump. ## Access -The trade tools use the same OAuth connection as the other Terminal49 tools. -Sign-in happens in the browser, so never ask the user for sign-in details in -chat. +The trade tools use the same browser connection as the other Terminal49 +tools, so never ask for connection details in chat. Trade intelligence is enabled per account. If a tool says it is not enabled, tell the user to contact support@terminal49.com to turn it on. Don't retry, and diff --git a/scripts/build-openai-zip.mjs b/scripts/build-openai-zip.mjs index cdc029c..6813804 100644 --- a/scripts/build-openai-zip.mjs +++ b/scripts/build-openai-zip.mjs @@ -62,7 +62,7 @@ const allowed = [ /^\.codex-plugin\/plugin\.json$/, /^\.mcp\.json$/, /^README\.md$/, - /^assets\/[^/]+\.svg$/, + /^assets\/[^/]+\.(svg|png)$/, /^skills\/[a-z0-9-]+\/SKILL\.md$/, /^skills\/[a-z0-9-]+\/references\/[^/]+\.md$/, ]; diff --git a/scripts/eval.mjs b/scripts/eval.mjs new file mode 100644 index 0000000..d4360b8 --- /dev/null +++ b/scripts/eval.mjs @@ -0,0 +1,31 @@ +// Runs the plugin evals without shipping them in the plugin folder. +// `claude plugin eval` reads cases only from inside the plugin root, and the +// Anthropic directory reviews every file in plugins/terminal49. This script +// copies the plugin and evals/terminal49 into .eval-stage/terminal49, runs the +// evals there, and copies the results back to evals/terminal49/results. +// +// Usage: node scripts/eval.mjs [claude plugin eval options] +import { spawnSync } from "node:child_process"; +import { cp, rm } from "node:fs/promises"; +import { existsSync } from "node:fs"; +import { resolve } from "node:path"; +import process from "node:process"; + +const root = process.cwd(); +const pluginDir = resolve(root, "plugins/terminal49"); +const evalsDir = resolve(root, "evals/terminal49"); +const stage = resolve(root, ".eval-stage/terminal49"); + +await rm(stage, { recursive: true, force: true }); +await cp(pluginDir, stage, { recursive: true }); +await cp(evalsDir, resolve(stage, "evals"), { + recursive: true, + filter: (source) => !source.startsWith(resolve(evalsDir, "results")), +}); + +const result = spawnSync("claude", ["plugin", "eval", stage, ...process.argv.slice(2)], { stdio: "inherit" }); + +const stagedResults = resolve(stage, "evals/results"); +if (existsSync(stagedResults)) await cp(stagedResults, resolve(evalsDir, "results"), { recursive: true }); + +process.exit(result.status ?? 1); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 91f2b07..3bd92c6 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -192,8 +192,8 @@ try { try { const skill = await readFile(resolve(root, paths.tradeSkill), "utf8"); - if (!/never ask the user for sign-in details/i.test(skill)) { - errors.push("trade-intelligence skill must forbid asking for sign-in details in chat"); + if (!/never ask for connection details in\s+chat/i.test(skill)) { + errors.push("trade-intelligence skill must forbid asking for connection details in chat"); } if (!skill.includes("support@terminal49.com")) { errors.push("trade-intelligence skill must send accounts without access to support@terminal49.com"); @@ -235,22 +235,17 @@ for (const skillPath of skillPaths) { } } -// Anthropic directory listing fields: https://code.claude.com/docs/en/plugins-reference#directory-listing-fields -const listingFields = { - supportUrl: "supportURL", - privacyPolicyUrl: "privacyPolicyURL", - termsOfServiceUrl: "termsOfServiceURL", -}; -for (const field of ["documentationUrl", ...Object.keys(listingFields)]) { - if (!/^https:\/\//.test(claudePlugin?.[field] ?? "")) errors.push(`Claude plugin ${field} must be an https URL`); +// The Anthropic directory warns on every plugin.json field that Claude Code does not read, +// listing fields included. Keep the Claude manifest to fields Claude Code loads so the +// directory report has no warnings. The README links the privacy policy, terms, and docs. +for (const field of ["icon", "documentationUrl", "supportUrl", "privacyPolicyUrl", "termsOfServiceUrl"]) { + if (claudePlugin && field in claudePlugin) errors.push(`Claude plugin.json must not set ${field}; the directory warns on it`); } -for (const [claudeField, codexField] of Object.entries(listingFields)) { - if (codexPlugin?.interface?.[codexField] !== claudePlugin?.[claudeField]) { - errors.push(`Codex interface.${codexField} must match Claude ${claudeField}`); - } +for (const field of ["supportURL", "privacyPolicyURL", "termsOfServiceURL"]) { + if (!/^https:\/\//.test(codexPlugin?.interface?.[field] ?? "")) errors.push(`Codex interface.${field} must be an https URL`); } -for (const icon of [claudePlugin?.icon, codexPlugin?.interface?.logo, codexPlugin?.interface?.composerIcon]) { - if (!icon) errors.push("Claude icon and Codex logo and composerIcon are required"); +for (const icon of [codexPlugin?.interface?.logo, codexPlugin?.interface?.composerIcon]) { + if (!icon) errors.push("Codex logo and composerIcon are required"); else await requireFile(`${paths.pluginDir}/${icon}`); } @@ -266,15 +261,17 @@ if (prompts.length > 3 || prompts.some((prompt) => prompt.length > 128)) { } // Directory checks on the plugin folder: https://claude.com/docs/plugins/pre-submission-checklist -const imageExtensions = new Set([".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg"]); -const scriptExtensions = new Set([".js", ".mjs", ".cjs", ".ts", ".sh", ".py"]); -const credentialRead = /T49_API_KEY|\.t49_api_key|process\.env|os\.environ|getenv\(/; +// The directory accepts only these file types from organizations that are not Verified. +const textExtensions = new Set([".md", ".txt", ".json", ".yaml", ".yml"]); +const imageExtensions = new Set([".png", ".jpg", ".jpeg", ".gif", ".webp"]); +// The directory holds any mention of a credential or an environment variable for review, +// even in prose, so the shipped folder must not name one. +const credentialWords = /api[ _-]?keys?|tokens?\b|passwords?|secrets?\b|credentials?|environment variables?|\$\{?[A-Z][A-Z0-9_]{2,}|process\.env|os\.environ|getenv\(/i; async function walk(directory) { const entries = await readdir(directory, { withFileTypes: true }); const files = []; for (const entry of entries) { const path = resolve(directory, entry.name); - if (entry.name === "results" && directory.endsWith("evals")) continue; if (entry.isDirectory()) files.push(...(await walk(path))); else files.push(path); } @@ -284,24 +281,21 @@ const pluginFiles = await walk(resolve(root, paths.pluginDir)); if (pluginFiles.length > 512) errors.push(`plugin folder has ${pluginFiles.length} files; the directory holds more than 512 for review`); for (const file of pluginFiles) { const path = relative(root, file); + const extension = extname(file).toLowerCase(); const stats = await lstat(file); if (stats.isSymbolicLink()) errors.push(`${path} is a symbolic link; commit a regular file`); if (stats.size > 256 * 1024) errors.push(`${path} is larger than 256 KiB`); - if (/(^|\/)(\.DS_Store|Thumbs\.db|desktop\.ini)$/.test(path)) errors.push(`${path} is a system file`); - if (scriptExtensions.has(extname(file))) errors.push(`${path} is a script; keep scripts outside the plugin folder`); - if (imageExtensions.has(extname(file))) continue; - const bytes = await readFile(file); - if (bytes.includes(0)) { - errors.push(`${path} is a binary file; only images may be binary`); + if (imageExtensions.has(extension)) continue; + if (!textExtensions.has(extension)) { + errors.push(`${path} is not Markdown, text, JSON, YAML, or a PNG, JPEG, GIF, or WebP image; the directory refuses it`); continue; } - // The directory checker reads credential-named output fields as a credential read. - if (path.includes("/evals/mocks/") && bytes.includes("outputSchema")) { - errors.push(`${path} keeps tool output schemas; strip outputSchema from eval mocks`); - } - if (credentialRead.test(bytes.toString("utf8"))) { - errors.push(`${path} reads a credential from the user's machine; use the MCP server's OAuth connection instead`); - } + const bytes = await readFile(file); + const text = bytes.toString("utf8"); + if (bytes.includes(0) || Buffer.from(text, "utf8").compare(bytes) !== 0) errors.push(`${path} must be UTF-8 text with no NUL bytes`); + if (text.startsWith("#!")) errors.push(`${path} starts with #!; the directory refuses it`); + const match = text.match(credentialWords); + if (match) errors.push(`${path} mentions "${match[0]}"; the directory holds plugins that mention credentials`); } if (errors.length > 0) {