chore(gha): run the telemetry delivery e2e on every build

.github/workflows/build.yml

@@ -61,3 +61,8 @@ jobs:
         env:
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
           GOCACHE: ${{ steps.global-cache-dir-path.outputs.go }}
+      # Builds a throwaway bundle pointed at a local sink; the script unsets
+      # the job-level CHECKPOINT_DISABLE=1 for its own child processes so the
+      # metrics flow. Nothing leaves the runner; the packaged bundle is untouched.
+      - name: telemetry delivery e2e
+        run: tools/validate-sentry-e2e.sh

packages/@cdktn/commons/src/telemetry.test.ts

@@ -44,6 +44,8 @@ type MetricItem = {
   attributes: Record<string, { value: unknown; type: string }>;
 };
 
+// Mirrors recordEnvelope in tools/sentry-sink.mjs: an envelope-format change
+// is fixed in both.
 function parseMetricItems(envelopeBodies: string[]): MetricItem[] {
   const items: MetricItem[] = [];
   for (const body of envelopeBodies) {

packages/cdktn-cli/.gitignore

@@ -2,5 +2,6 @@
 .yalc
 yalc.lock
 bundle
+bundle-e2e
 build-config
 !ambient.d.ts

packages/cdktn-cli/build.ts

@@ -52,11 +52,15 @@ const nativeNodeModulesPlugin = {
   },
 };
 
+// tools/validate-sentry-e2e.sh builds a throwaway copy with a local-sink DSN
+// next to the shipped bundle, which it never touches.
+const outdir = process.env.CDKTN_BUNDLE_OUTDIR || "./bundle";
+
 const config: esbuild.BuildOptions = {
   entryPoints: ["src/bin/cdktn.ts", "src/bin/cmds/handlers.ts"],
   outbase: "src",
   bundle: true,
-  outdir: "./bundle",
+  outdir,
   format: "cjs",
   target: "node22",
   minify: enableWatch ? false : true,
@@ -104,7 +108,7 @@ const config: esbuild.BuildOptions = {
 (async () => {
   console.log("Building…");
   await esbuild.build(config);
-  fs.copySync("../@cdktn/cli-core/templates", "./bundle/templates");
+  fs.copySync("../@cdktn/cli-core/templates", `${outdir}/templates`);
 
   if (enableWatch) {
     const ctx = await esbuild.context(config);

packages/cdktn-cli/src/bin/cmds/helper/utilities.ts

@@ -27,9 +27,10 @@ export const projectRootPath = () => {
 
 // deferred require to keep cdktn-cli main entrypoint small (e.g. for fast shell completions)
 export const requireHandlers = () => {
-  // if file exists relative to this file return its file path
-  // otherwise return the file path relative to the project root
-  const filePath = path.join(__dirname, "..", "handlers.js");
+  // the bundle lays out bin/cdktn.js next to bin/cmds/handlers.js, so a
+  // bundle built elsewhere (tools/validate-sentry-e2e.sh) loads its own
+  // handlers; otherwise fall back to the shipped bundle at the project root
+  const filePath = path.join(__dirname, "cmds", "handlers.js");
   if (fs.existsSync(filePath)) {
     return localRequire(filePath);
   }

tools/sentry-sink.mjs

@@ -0,0 +1,96 @@
+#!/usr/bin/env node
+// Copyright (c) HashiCorp, Inc
+// SPDX-License-Identifier: MPL-2.0
+//
+// Minimal local Sentry "sink" for end-to-end validation of the cdktn-cli
+// telemetry pipeline. Accepts Sentry envelopes on POST /api/<project>/envelope/
+// and records every envelope item type (and, for trace_metric, the metric
+// names with their attribute keys and the command they were counted under).
+//
+// Usage: node tools/sentry-sink.mjs [port=0]
+//   port 0 picks a free port; the chosen one is printed on the first line
+//   GET /__items  -> JSON array of recorded items
+//   GET /__raw    -> every decoded envelope body, concatenated
+//   GET /__reset  -> clears recorded items and bodies
+import * as http from "node:http";
+import * as zlib from "node:zlib";
+
+const port = Number(process.argv[2] ?? 0);
+const items = [];
+const bodies = [];
+
+// Mirrors parseMetricItems in packages/@cdktn/commons/src/telemetry.test.ts:
+// an envelope-format change is fixed in both.
+function recordEnvelope(body) {
+  bodies.push(body);
+  const lines = body.split("\n").filter(Boolean);
+  for (let i = 1; i < lines.length; i++) {
+    let header;
+    try {
+      header = JSON.parse(lines[i]);
+    } catch {
+      continue;
+    }
+    if (!header || typeof header.type !== "string") continue;
+    const item = { type: header.type };
+    if (header.type === "trace_metric" && lines[i + 1]) {
+      try {
+        item.metrics = JSON.parse(lines[i + 1]).items.map((m) => ({
+          name: m.name,
+          attributeKeys: Object.keys(m.attributes ?? {}),
+          command: m.attributes?.command?.value,
+        }));
+      } catch {
+        /* ignore malformed payloads */
+      }
+    }
+    items.push(item);
+    console.log(`[sentry-sink] ${JSON.stringify(item)}`);
+  }
+}
+
+const server = http.createServer((req, res) => {
+  if (req.method === "GET" && req.url === "/__items") {
+    res.setHeader("content-type", "application/json");
+    return res.end(JSON.stringify(items));
+  }
+  if (req.method === "GET" && req.url === "/__raw") {
+    res.setHeader("content-type", "text/plain");
+    return res.end(bodies.join("\n"));
+  }
+  if (req.method === "GET" && req.url === "/__reset") {
+    items.length = 0;
+    bodies.length = 0;
+    return res.end("ok");
+  }
+
+  const chunks = [];
+  req.on("data", (c) => chunks.push(c));
+  req.on("end", () => {
+    let body = Buffer.concat(chunks);
+    const encoding = req.headers["content-encoding"];
+    try {
+      if (encoding === "gzip") body = zlib.gunzipSync(body);
+      else if (encoding === "deflate") body = zlib.inflateSync(body);
+      else if (encoding === "br") body = zlib.brotliDecompressSync(body);
+    } catch {
+      /* fall through with the raw body */
+    }
+    console.log(
+      `[sentry-sink] ${req.method} ${req.url} (${body.length} bytes, encoding=${encoding ?? "none"})`,
+    );
+    // the SDK appends auth as a query string: /api/<p>/envelope/?sentry_key=…
+    if (req.method === "POST" && /\/envelope\/?(\?|$)/.test(req.url ?? "")) {
+      recordEnvelope(body.toString("utf8"));
+    }
+    res.statusCode = 200;
+    res.setHeader("content-type", "application/json");
+    res.end("{}");
+  });
+});
+
+server.listen(port, () => {
+  console.log(
+    `[sentry-sink] listening on http://localhost:${server.address().port}`,
+  );
+});

tools/validate-sentry-e2e.sh

@@ -0,0 +1,229 @@
+#!/usr/bin/env bash
+# Copyright (c) HashiCorp, Inc
+# SPDX-License-Identifier: MPL-2.0
+#
+# End-to-end check of cdktn-cli telemetry on the real esbuild bundle: builds a
+# throwaway copy of it with a local-sink DSN, runs convert (success), a failing
+# synth (error), a hand-written stack (per-stack metrics) and a crashing
+# command (entrypoint failure path), then asserts on what reached
+# tools/sentry-sink.mjs. The shipped bundle is never touched.
+set -euo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+OUTDIR="$ROOT/packages/cdktn-cli/bundle-e2e"
+CDKTN="$OUTDIR/bin/cdktn.js"
+SINK_LOG="$(mktemp)"
+SINK_PID=""
+
+cleanup() {
+  local status=$?
+  if [ -n "$SINK_PID" ]; then
+    kill "$SINK_PID" 2>/dev/null || true
+    wait "$SINK_PID" 2>/dev/null || true
+  fi
+  rm -rf "$OUTDIR" "$SINK_LOG"
+  exit "$status"
+}
+trap cleanup EXIT
+
+echo "==> starting sentry sink"
+node "$ROOT/tools/sentry-sink.mjs" "${1:-0}" >"$SINK_LOG" 2>&1 &
+SINK_PID=$!
+PORT=""
+for _ in $(seq 1 50); do
+  PORT="$(sed -n 's/.*listening on http:\/\/localhost:\([0-9]*\).*/\1/p' "$SINK_LOG" | head -1)"
+  [ -n "$PORT" ] && break
+  sleep 0.1
+done
+[ -n "$PORT" ] || { cat "$SINK_LOG" >&2; echo "FAIL: sink did not start" >&2; exit 1; }
+DSN="http://cdktn@localhost:${PORT}/1"
+
+echo "==> building a scratch bundle with DSN $DSN baked in"
+(cd "$ROOT/packages/cdktn-cli" && pnpm run compile-build-config >/dev/null && SENTRY_DSN="$DSN" CDKTN_BUNDLE_OUTDIR="$OUTDIR" node build-config/build.js)
+
+# Polls the sink until every pattern was recorded or 10 s pass; a timeout
+# names what never arrived instead of leaving it to the assertions below.
+await_items() {
+  local i pattern missing
+  for i in $(seq 1 100); do
+    ITEMS="$(curl -sf "http://localhost:${PORT}/__items" || true)"
+    missing=""
+    for pattern in "$@"; do
+      echo "$ITEMS" | grep -q -- "$pattern" || missing="$missing $pattern"
+    done
+    [ -z "$missing" ] && return 0
+    sleep 0.1
+  done
+  echo "FAIL: timed out waiting for$missing" >&2
+  exit 1
+}
+
+WORK="$(mktemp -d)"
+pushd "$WORK" >/dev/null
+unset CHECKPOINT_DISABLE
+# Sentry env vars a user or CI may export; none of their values may be sent.
+export SENTRY_ENVIRONMENT="LEAK-ENV-SENTRY"
+export SENTRY_TRACE="0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-1"
+export SENTRY_BAGGAGE="sentry-environment=LEAK-BAGGAGE-SENTRY"
+printf '{ "language": "typescript", "app": "true", "projectId": "e2e-validation", "sendCrashReports": true, "sendUsageTelemetry": true }' > cdktf.json
+
+echo "==> SUCCESS trigger: cdktn convert"
+echo 'resource "null_resource" "x" {}' | node "$CDKTN" convert --language typescript >/dev/null
+
+echo "==> ERROR trigger: cdktn synth (failing app)"
+node "$CDKTN" synth --app "node -e 'process.exit(1)'" >/dev/null 2>&1 || true
+
+popd >/dev/null
+rm -rf "$WORK"
+
+STACK_WORK="$(mktemp -d)"
+pushd "$STACK_WORK" >/dev/null
+printf '{ "language": "typescript", "app": "node fake-app.js", "projectId": "e2e-validation", "sendCrashReports": true, "sendUsageTelemetry": true, "terraformProviders": ["aws@~>5.0"] }' > cdktf.json
+cat > fake-app.js <<'JS'
+const fs = require("fs");
+const path = require("path");
+const outdir = process.env.CDKTF_OUTDIR;
+const name = "E2E-SECRET-STACK-NAME";
+fs.mkdirSync(path.join(outdir, "stacks", name), { recursive: true });
+fs.writeFileSync(
+  path.join(outdir, "stacks", name, "cdk.tf.json"),
+  JSON.stringify({
+    "//": {
+      metadata: {
+        version: "0.0.0-e2e",
+        stackName: name,
+        backend: "local",
+        overrides: { aws_s3_bucket: ["tags"] },
+        imports: { aws_s3_bucket: ["e2e-secret-resource-id"] },
+      },
+      outputs: {},
+    },
+    terraform: {
+      required_providers: {
+        aws: { source: "aws", version: "~> 5.0" },
+        random: { source: "hashicorp/random", version: "3.6.0" },
+        vault: { source: "tfe.leak-host.example/leak-org/vault", version: "~> 3.0" },
+        local: { source: "./leak-provider" },
+      },
+    },
+  }),
+);
+fs.writeFileSync(
+  path.join(outdir, "manifest.json"),
+  JSON.stringify({
+    version: "0.0.0-e2e",
+    stacks: {
+      [name]: {
+        name,
+        constructPath: name,
+        workingDirectory: `stacks/${name}`,
+        synthesizedStackPath: `stacks/${name}/cdk.tf.json`,
+        stackMetadataPath: `stacks/${name}/metadata.json`,
+        annotations: [],
+        dependencies: [],
+      },
+    },
+  }),
+);
+JS
+
+echo "==> STACK trigger: cdktn synth (hand-written stack)"
+node "$CDKTN" synth --check-code-maker-output=false >/dev/null \
+  || { echo "FAIL: the stack trigger synth exited non-zero; an unrelated synth regression also fails here" >&2; exit 1; }
+
+popd >/dev/null
+rm -rf "$STACK_WORK"
+
+# A corrupt synthesized stack read with --skip-synth is an unexpected error
+# that reaches runCli's failure reporter: crash event + cli.command.error.
+CRASH_WORK="$(mktemp -d)"
+pushd "$CRASH_WORK" >/dev/null
+mkdir -p cdktf.out/stacks/broken
+printf '{ "language": "typescript", "app": "true", "projectId": "e2e-validation", "sendCrashReports": true, "sendUsageTelemetry": true }' > cdktf.json
+printf '{ "version": "0.0.0-e2e", "stacks": { "broken": { "name": "broken", "constructPath": "broken", "workingDirectory": "stacks/broken", "synthesizedStackPath": "stacks/broken/cdk.tf.json", "stackMetadataPath": "stacks/broken/metadata.json", "annotations": [], "dependencies": [] } } }' > cdktf.out/manifest.json
+printf '{ not json' > cdktf.out/stacks/broken/cdk.tf.json
+
+echo "==> CRASH trigger: cdktn output --skip-synth (corrupt cdk.tf.json)"
+CRASH_OUTPUT="$(node "$CDKTN" output --skip-synth 2>&1 || true)"
+
+popd >/dev/null
+rm -rf "$CRASH_WORK"
+
+await_items 'cli.command.invoked' 'cli.command.completed' 'cli.command.error' '"cli.stack.provider"' '{"type":"event"}'
+RAW="$(curl -sf "http://localhost:${PORT}/__raw")"
+echo "==> sink recorded: $ITEMS"
+
+fail() { echo "FAIL: $1" >&2; exit 1; }
+
+echo "$ITEMS" | grep -q '"trace_metric"' \
+  || fail "no trace_metric envelope reached the sink — the success-path flush is missing/broken"
+echo "$ITEMS" | grep -q '"binary"' \
+  || fail "binary attribute missing on the command metrics"
+
+# Every run counts once as invoked at start, then once as completed or error:
+# the four triggers above, sorted as "<metric> <command>".
+RUNS="$(node -e '
+  const runs = [];
+  for (const item of JSON.parse(process.argv[1])) {
+    for (const metric of item.metrics ?? []) {
+      if (metric.name.startsWith("cli.command.")) {
+        runs.push(`${metric.name} ${metric.command}`);
+      }
+    }
+  }
+  console.log(runs.sort().join("\n"));
+' "$ITEMS")"
+EXPECTED_RUNS="$(printf '%s\n' \
+  'cli.command.completed convert' \
+  'cli.command.completed synth' \
+  'cli.command.error output' \
+  'cli.command.error synth' \
+  'cli.command.invoked convert' \
+  'cli.command.invoked output' \
+  'cli.command.invoked synth' \
+  'cli.command.invoked synth')"
+[ "$RUNS" = "$EXPECTED_RUNS" ] \
+  || fail "run metrics differ from one invoked per run plus one completed or error; got:
+$RUNS"
+echo "$ITEMS" | grep -q '"cli.stack"' \
+  || fail "cli.stack metric missing (stack trigger)"
+echo "$ITEMS" | grep -q '"cli.stack.provider"' \
+  || fail "cli.stack.provider metric missing (stack trigger)"
+echo "$ITEMS" | grep -q '"cli.stack.override"' \
+  || fail "cli.stack.override metric missing (stack trigger)"
+for key in binding library_version override_count import_count resource_type; do
+  echo "$ITEMS" | grep -q "\"$key\"" \
+    || fail "$key attribute missing on the stack metrics"
+done
+echo "$RAW" | grep -q 'hashicorp/random' \
+  || fail "normalized provider source missing from the stack metrics"
+echo "$RAW" | grep -q 'private-registry' \
+  || fail "private-registry provider was not reduced to its kind"
+# matched as key/value adjacency inside the metric item, so a "production"
+# elsewhere in the envelope cannot satisfy the check
+echo "$RAW" | grep -q '"sentry.environment":{"value":"production"' \
+  || fail "sentry.environment is not the fixed production value"
+for secret in E2E-SECRET-STACK-NAME e2e-secret-resource-id leak-host.example leak-org leak-provider LEAK-ENV-SENTRY LEAK-BAGGAGE-SENTRY 0af7651916cd43dd8448eb211c80319c; do
+  if echo "$RAW" | grep -q "$secret"; then
+    fail "$secret reached the sink: stack names, resource ids, provider hosts/paths and SENTRY_* env values must never be sent"
+  fi
+done
+echo "$ITEMS" | grep -q '"error_type"' \
+  || fail "error_type attribute missing on cli.command.error"
+echo "$RAW" | grep -q '"error_type":{"value":"unexpected"' \
+  || fail "the crash trigger was not counted as an unexpected cli.command.error"
+echo "$RAW" | grep -q '"converted_lines"' \
+  || fail "the convert scalars did not ride on cli.command.completed"
+echo "$ITEMS" | grep -q '{"type":"event"}' \
+  || fail "no crash event reached the sink from the entrypoint failure path"
+echo "$CRASH_OUTPUT" | grep -q '^Debug Information:' \
+  || fail "the crash trigger did not reach the debug information block"
+if echo "$CRASH_OUTPUT" | grep -q 'ERR_UNHANDLED_REJECTION\|PromiseRejectionHandledWarning'; then
+  fail "the crash trigger orphaned a rejection"
+fi
+
+# A cheap bundle scan; the sink assertions above are what prove the transport.
+HASHICORP_REFS="$(grep -c "checkpoint-api.hashicorp.com" "$CDKTN" || true)"
+[ "$HASHICORP_REFS" = "0" ] || fail "bundle still references checkpoint-api.hashicorp.com ($HASHICORP_REFS hits)"
+
+echo "PASS: success-path, error-path, per-stack and entrypoint-failure telemetry delivered to the local sink"