From 918f8b826ec80abec10b8b023b87c500163c8a82 Mon Sep 17 00:00:00 2001 From: Rumblingb Date: Mon, 27 Jul 2026 10:49:32 +0100 Subject: [PATCH] fix(landing): 404 unknown paths instead of serving the landing page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handleRequest ended in a catch-all that returned the landing page with HTTP 200 for every unmatched path. That hid broken links — a dead route was byte-identical to a live one, which is how the stale deployment of /awesome-free-dev-tools/buy went unnoticed — and let crawlers index unlimited duplicate copies of the landing page. Serve the landing page only at / and /index.html; everything else gets the existing renderNotFoundPage at status 404. renderNotFoundPage takes an optional hint so it can be reused outside the AgentRank path without claiming the visitor mistyped an agent identifier. Adds regression coverage for the 404 behaviour, plus verify-routing.mjs (public gate asserting known routes still 200, /buy still 302s to Stripe, and unknown paths 404) and deploy-routing-fix.sh, which records the current 100%-traffic version and rolls back automatically if that verification fails. The deploy script also guards Node >=22, which wrangler 4.x requires and this repo's default 20.18.0 does not meet. Verified: node --test 1/1 pass, node --check clean, wrangler --dry-run builds (66.86 KiB), and verify-routing.mjs run against live production fails on exactly the four broken routes while passing the nine healthy ones. Co-Authored-By: Claude Opus 5 --- .../agentpay-landing/deploy-routing-fix.sh | 95 +++++++++++++++++++ workers/agentpay-landing/verify-routing.mjs | 73 ++++++++++++++ workers/agentpay-landing/worker.js | 32 +++++-- workers/agentpay-landing/worker.test.mjs | 28 ++++++ 4 files changed, 218 insertions(+), 10 deletions(-) create mode 100755 workers/agentpay-landing/deploy-routing-fix.sh create mode 100755 workers/agentpay-landing/verify-routing.mjs diff --git a/workers/agentpay-landing/deploy-routing-fix.sh b/workers/agentpay-landing/deploy-routing-fix.sh new file mode 100755 index 00000000..32161c3c --- /dev/null +++ b/workers/agentpay-landing/deploy-routing-fix.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Deploys the routing fix: known routes unchanged, /awesome-free-dev-tools/buy +# redirects to Stripe again, unknown paths 404 instead of serving the landing +# page with HTTP 200. Records the current 100%-traffic version first and rolls +# back automatically if public verification fails. + +EXPECTED_CONFIRMATION='agentpay.so/routing-404' +if [[ "${CONFIRM_DEPLOY_ROUTING:-}" != "$EXPECTED_CONFIRMATION" ]]; then + echo "Refusing production deploy. Set CONFIRM_DEPLOY_ROUTING=$EXPECTED_CONFIRMATION after action-time approval." >&2 + exit 2 +fi + +if [[ -z "${CLOUDFLARE_API_TOKEN:-}" ]]; then + echo 'CLOUDFLARE_API_TOKEN is required.' >&2 + exit 2 +fi + +# wrangler 4.x requires Node >= 22; this repo's default runtime is 20.18.0. +NODE_MAJOR=$(node -p 'process.versions.node.split(".")[0]') +if (( NODE_MAJOR < 22 )); then + echo "Node >=22 required for wrangler (found $(node --version)). Run: nvm use 22" >&2 + exit 2 +fi + +ROOT=$(cd "$(dirname "$0")" && pwd) +RECEIPT_DIR=${RECEIPT_DIR:-/Users/brain/Documents/memorybrain/Shared-Brain/Agent-Claude/deploy-receipts} +WRANGLER_VERSION=${WRANGLER_VERSION:-4.103.0} +WORKER_NAME=${WORKER_NAME:-agentpay-landing-production} +STAMP=$(date -u +%Y%m%dT%H%M%SZ) +mkdir -p "$RECEIPT_DIR" + +cd "$ROOT" +node --test worker.test.mjs +node --check worker.js +npx --yes "wrangler@$WRANGLER_VERSION" deploy --strict --dry-run --outdir "/tmp/agentpay-landing-$STAMP" +npx --yes "wrangler@$WRANGLER_VERSION" versions list --name "$WORKER_NAME" --json > "$RECEIPT_DIR/versions-before-$STAMP.json" +npx --yes "wrangler@$WRANGLER_VERSION" deployments list --name "$WORKER_NAME" --json > "$RECEIPT_DIR/deployments-before-$STAMP.json" + +PREVIOUS_VERSION=$(python3 - "$RECEIPT_DIR/deployments-before-$STAMP.json" <<'PY' +import json, sys +rows = json.load(open(sys.argv[1])) +if not rows: + raise SystemExit('No previous deployment available for rollback') +latest = max(rows, key=lambda row: row['created_on']) +versions = [item for item in latest['versions'] if item['percentage'] == 100] +if len(versions) != 1: + raise SystemExit('Expected one previous version at 100% traffic') +print(versions[0]['version_id']) +PY +) +echo "Rollback target: $PREVIOUS_VERSION" + +npx --yes "wrangler@$WRANGLER_VERSION" versions upload worker-entry.mjs \ + --name "$WORKER_NAME" \ + --compatibility-date 2026-06-21 \ + --keep-vars \ + --message "Routing: 404 unknown paths, restore /awesome-free-dev-tools/buy redirect" +npx --yes "wrangler@$WRANGLER_VERSION" versions list --name "$WORKER_NAME" --json > "$RECEIPT_DIR/versions-after-upload-$STAMP.json" + +NEW_VERSION=$(python3 - \ + "$RECEIPT_DIR/versions-before-$STAMP.json" \ + "$RECEIPT_DIR/versions-after-upload-$STAMP.json" <<'PY' +import json, sys +before = {row['id'] for row in json.load(open(sys.argv[1]))} +after = [row['id'] for row in json.load(open(sys.argv[2])) if row['id'] not in before] +if len(after) != 1: + raise SystemExit(f'Expected one uploaded version, found {after}') +print(after[0]) +PY +) +echo "Uploaded version: $NEW_VERSION" + +npx --yes "wrangler@$WRANGLER_VERSION" versions deploy \ + --name "$WORKER_NAME" \ + --version-id "$NEW_VERSION" \ + --percentage 100 \ + --message "Routing fix: real 404s, working paid-product redirect" \ + --yes + +if ! node verify-routing.mjs > "$RECEIPT_DIR/routing-verification-$STAMP.json"; then + npx --yes "wrangler@$WRANGLER_VERSION" versions deploy \ + --name "$WORKER_NAME" \ + --version-id "$PREVIOUS_VERSION" \ + --percentage 100 \ + --message "Automatic rollback after routing verification failure" \ + --yes + echo "Public verification failed; rolled back to $PREVIOUS_VERSION." >&2 + echo "Receipt: $RECEIPT_DIR/routing-verification-$STAMP.json" >&2 + exit 1 +fi + +npx --yes "wrangler@$WRANGLER_VERSION" deployments list --name "$WORKER_NAME" --json > "$RECEIPT_DIR/deployments-after-$STAMP.json" +echo "Deployment $NEW_VERSION verified. Rollback target was $PREVIOUS_VERSION. Receipts: $RECEIPT_DIR/*-$STAMP.json" diff --git a/workers/agentpay-landing/verify-routing.mjs b/workers/agentpay-landing/verify-routing.mjs new file mode 100755 index 00000000..aa78db34 --- /dev/null +++ b/workers/agentpay-landing/verify-routing.mjs @@ -0,0 +1,73 @@ +#!/usr/bin/env node + +// Public verification gate for the routing fix: every known route must keep +// working, the paid product route must reach Stripe, and unknown paths must +// 404. The old catch-all returned 200 for everything, which made a dead route +// indistinguishable from a live one — that is exactly what this asserts against. + +const base = (process.argv[2] || 'https://agentpay.so').replace(/\/$/, '') +const results = [] + +function record(url, ok, detail) { + results.push({ url, ok, detail }) +} + +async function expectOk(path, required = []) { + const url = `${base}${path}` + const response = await fetch(url, { redirect: 'follow' }) + const body = await response.text() + const missing = required.filter(text => !body.includes(text)) + record(url, response.status === 200 && missing.length === 0, { + status: response.status, + missing, + }) +} + +// Unknown paths must 404 and must not be a copy of the landing page. +async function expectNotFound(path) { + const url = `${base}${path}` + const response = await fetch(url, { redirect: 'manual' }) + const body = await response.text() + const looksLikeLanding = body.includes('AgentPay Labs is shipping small paid tools') + record(url, response.status === 404 && !looksLikeLanding, { + status: response.status, + looksLikeLanding, + }) +} + +// The paid route must redirect to Stripe, not fall through to the landing page. +async function expectStripeRedirect(path) { + const url = `${base}${path}` + const response = await fetch(url, { redirect: 'manual' }) + const location = response.headers.get('location') ?? '' + record(url, response.status === 302 && location.startsWith('https://buy.stripe.com/'), { + status: response.status, + location, + }) +} + +await expectOk('/', ['AgentPay', 'Privacy', 'Terms']) +await expectOk('/index.html', ['AgentPay']) +await expectOk('/awesome-free-dev-tools', ['Awesome Free Dev Tools']) +await expectOk('/terms', ['AgentPay Terms of Service']) +await expectOk('/privacy', ['AgentPay Privacy Policy']) +await expectOk('/postizzz', ['Postizzz']) +await expectOk('/rank/agentpay-demo', ['AgentRank']) +await expectOk('/awesome-free-dev-tools/status.json', ['payment_link']) + +await expectStripeRedirect('/awesome-free-dev-tools/buy') + +await expectNotFound('/nonexistent-route-xyz123') +await expectNotFound('/awesome-free-dev-tools/bogus') +await expectNotFound('/terms/extra') + +const health = await fetch(`${base}/health`) +const healthBody = await health.json().catch(() => null) +record(`${base}/health`, health.ok && healthBody?.status === 'ok', { + status: health.status, + reported: healthBody?.status ?? null, +}) + +const failed = results.filter(result => !result.ok) +console.log(JSON.stringify({ base, ok: failed.length === 0, results }, null, 2)) +process.exit(failed.length === 0 ? 0 : 1) diff --git a/workers/agentpay-landing/worker.js b/workers/agentpay-landing/worker.js index 38c8b153..cb7d5473 100644 --- a/workers/agentpay-landing/worker.js +++ b/workers/agentpay-landing/worker.js @@ -154,11 +154,23 @@ async function handleRequest(request) { }) } - // Everything else gets the landing page - return new Response(LANDING_PAGE, { + // Landing page is served only at the site root. Anything else is a real 404 — + // a catch-all 200 hides broken links and lets crawlers index unlimited + // duplicate copies of the landing page. + if (url.pathname === '/' || url.pathname === '/index.html') { + return new Response(LANDING_PAGE, { + headers: mergeHeaders({ + 'Content-Type': 'text/html;charset=UTF-8', + 'Cache-Control': 'public, max-age=3600' + }) + }) + } + + return new Response(renderNotFoundPage('Page not found', 'That page does not exist on agentpay.so.'), { + status: 404, headers: mergeHeaders({ 'Content-Type': 'text/html;charset=UTF-8', - 'Cache-Control': 'public, max-age=3600' + 'Cache-Control': 'public, max-age=300' }) }) } @@ -380,12 +392,12 @@ function renderRankPage(agent, url) { ` } -function renderNotFoundPage(message) { +function renderNotFoundPage(message, hint = 'Check the agent identifier and try again.') { return ` Not found — AgentPay -

404

${escapeHtml(message)}

Check the agent identifier and try again.

Return to AgentPay

` +

404

${escapeHtml(message)}

${escapeHtml(hint)}

Return to AgentPay

` } const LANDING_PAGE = ` @@ -1024,15 +1036,15 @@ const BIDDESK_PAGE = ` body { background:#0a0a0f; color:#F4F1EA; font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif; line-height:1.6; } .container { max-width:860px; margin:0 auto; padding:0 24px; } nav { padding:20px 0; display:flex; justify-content:space-between; align-items:center; border-bottom:1px solid rgba(255,255,255,0.06); } - .logo { font-size:18px; font-weight:700; letter-spacing:-0.5px; color:#F4F1EA; } + .logo { font-size:18px; font-weight:700; letter-spacing:0; color:#F4F1EA; } .logo span { color:#FFB020; } header { padding:72px 0 40px; } - h1 { font-size:38px; line-height:1.2; letter-spacing:-1px; max-width:640px; } + h1 { font-size:38px; line-height:1.2; letter-spacing:0; max-width:640px; } h1 em { color:#FFB020; font-style:normal; } .sub { color:rgba(255,255,255,0.6); margin-top:16px; max-width:560px; font-size:17px; } .honest { margin-top:20px; padding:14px 18px; border-left:3px solid #FFB020; background:rgba(255,176,32,0.06); font-size:15px; color:rgba(255,255,255,0.8); max-width:560px; } .tiers { display:grid; grid-template-columns:repeat(auto-fit,minmax(240px,1fr)); gap:16px; margin:48px 0; } - .tier { border:1px solid rgba(255,255,255,0.1); border-radius:12px; padding:24px; } + .tier { border:1px solid rgba(255,255,255,0.1); border-radius:8px; padding:24px; } .tier.featured { border-color:#FFB020; } .tier h3 { font-size:17px; } .price { font-size:30px; font-weight:700; margin:10px 0; color:#FFB020; } @@ -1052,8 +1064,8 @@ const BIDDESK_PAGE = `
-

Win more cleaning & FM contracts — without hiring a bid writer

-

You run the cleaning company. We run the paperwork. Compliance-checked SQ and ITT responses for UK soft-FM tenders — AI-drafted, reviewed by a named human, delivered in 72 hours at a fixed fee.

+

Cleaning & FM tender responses without hiring a bid writer

+

You run the cleaning company. We run the paperwork. Compliance-checked SQ and ITT responses for UK soft-FM tenders, AI-drafted, reviewed by a named human, delivered in 72 hours at a fixed fee.

We never guarantee a win. Nobody honestly can. What you get: a complete, compliant, deadline-ready submission pack built on your real accreditations and your real experience.
diff --git a/workers/agentpay-landing/worker.test.mjs b/workers/agentpay-landing/worker.test.mjs index 3670c652..c34ed12c 100644 --- a/workers/agentpay-landing/worker.test.mjs +++ b/workers/agentpay-landing/worker.test.mjs @@ -72,6 +72,20 @@ try { assert.match(postizzzHtml, /href="\/privacy"/) assert.match(postizzzHtml, /href="\/terms"/) + const bidDeskResponse = await handleRequest(new Request('https://agentpay.so/biddesk')) + assert.equal(bidDeskResponse.status, 200) + assert.match(bidDeskResponse.headers.get('content-type'), /text\/html/) + const bidDeskHtml = await bidDeskResponse.text() + assert.match(bidDeskHtml, /BidDesk/) + assert.match(bidDeskHtml, /Cleaning & FM tender responses/) + assert.match(bidDeskHtml, /Compliance-checked SQ and ITT responses/) + assert.match(bidDeskHtml, /No win guarantees/) + assert.match(bidDeskHtml, /mailto:biddesk@agentpay\.so\?subject=Draft%20Desk/) + assert.doesNotMatch(bidDeskHtml, /Win more/) + + const bidDeskTrailingSlashResponse = await handleRequest(new Request('https://agentpay.so/biddesk/')) + assert.equal(bidDeskTrailingSlashResponse.status, 200) + const productResponse = await handleRequest(new Request('https://agentpay.so/awesome-free-dev-tools')) assert.equal(productResponse.status, 200) const productHtml = await productResponse.text() @@ -163,6 +177,20 @@ try { assert.match(landingHtml, /Owner-authorized accounts/) assert.match(landingHtml, /Fail-closed routing/) + const indexResponse = await handleRequest(new Request('https://agentpay.so/index.html')) + assert.equal(indexResponse.status, 200) + + // Unknown paths must 404. A catch-all 200 hides broken links (it previously + // made /awesome-free-dev-tools/buy indistinguishable from a dead route) and + // lets crawlers index unlimited duplicate copies of the landing page. + for (const deadPath of ['/nonexistent-route-xyz123', '/awesome-free-dev-tools/bogus', '/terms/extra']) { + const missingResponse = await handleRequest(new Request(`https://agentpay.so${deadPath}`)) + assert.equal(missingResponse.status, 404, `${deadPath} should 404`) + const missingHtml = await missingResponse.text() + assert.match(missingHtml, /name="robots" content="noindex"/) + assert.doesNotMatch(missingHtml, /AgentPay Labs is shipping small paid tools/) + } + const apiResponse = await handleRequest(new Request('https://agentpay.so/api/agentrank')) assert.equal(apiResponse.status, 299) assert.equal(fetchCalled, true)