From fc4ecbf1d0d2a0e10d8c1f50f762458c34a0882c Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 8 Aug 2026 18:32:43 +0000 Subject: [PATCH 1/3] feat(roadmap): define launch success metrics baseline + 90-day targets Adds data/launch-metrics.json: a hand-curated baseline snapshot for 5 launch signals (GitHub stars, watchers, forks, good-first-issue conversion, inbound CNCF links) with a 90-day post-launch target for each, gated by a new validate-launch-metrics.mjs validator (wired into CI and the unit/smoke test suites). ROADMAP.md Phase 3 gets a 'Launch success metrics' section summarizing the table and linking back to the data file so it stays the single source of truth. Addresses #100 (no launch success metrics defined before Phase 1 closes). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/deploy-gh-pages.yml | 3 + ROADMAP.md | 71 ++++++++++++---- data/launch-metrics.json | 52 ++++++++++++ package.json | 1 + scripts/validate-launch-metrics.mjs | 79 ++++++++++++++++++ tests/validate-launch-metrics.test.mjs | 111 +++++++++++++++++++++++++ tests/validators-smoke.test.mjs | 1 + 7 files changed, 300 insertions(+), 18 deletions(-) create mode 100644 data/launch-metrics.json create mode 100644 scripts/validate-launch-metrics.mjs create mode 100644 tests/validate-launch-metrics.test.mjs diff --git a/.github/workflows/deploy-gh-pages.yml b/.github/workflows/deploy-gh-pages.yml index ce827d07..f3a277b1 100644 --- a/.github/workflows/deploy-gh-pages.yml +++ b/.github/workflows/deploy-gh-pages.yml @@ -46,6 +46,9 @@ jobs: - name: Validate awards data run: npm run validate:awards + - name: Validate launch metrics + run: npm run validate:launch-metrics + # Build for the GitHub Pages project URL. When the custom domain in # static/CNAME is verified and active, update these values to # SITE_URL=https://endusers.cncf.io and BASE_URL=/ so canonical and diff --git a/ROADMAP.md b/ROADMAP.md index 6a99572a..7851f4cb 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,21 +1,25 @@ # Roadmap: endusers.cncf.io -This roadmap describes the planned evolution of the CNCF End User Community site. -It is a living document; update it as priorities change. Tracking issue: #45. +This roadmap describes the planned evolution of the CNCF End User Community +site. It is a living document; update it as priorities change. Tracking issue: +#45. ## Vision -Be the trusted home of the CNCF End User Community: the place where organizations -running cloud native in production find reference architectures, metrics, events, -and pathways to participate. +Be the trusted home of the CNCF End User Community: the place where +organizations running cloud native in production find reference architectures, +metrics, events, and pathways to participate. ## Guiding principles -- **Audience first**: content serves end users (adopters), not project contributors. +- **Audience first**: content serves end users (adopters), not project + contributors. - **Data over prose**: metrics, awards, and architectures are generated from - authoritative sources (cncf/landscape, cncf/architecture, cncf/tab), never hand-edited. + authoritative sources (cncf/landscape, cncf/architecture, cncf/tab), never + hand-edited. - **Verified facts**: every CNCF fact is backed by an authoritative source link. -- **Green pipeline**: the site must always deploy; a broken deploy blocks everything. +- **Green pipeline**: the site must always deploy; a broken deploy blocks + everything. ## Phase 0 — Foundation (current) @@ -23,18 +27,24 @@ Goal: a reliable, trustworthy site skeleton. - [ ] Restore a green deploy pipeline (blocked: see issue #44) - [x] Add a LICENSE (PR #40, issue #32) -- [ ] Pin GitHub Actions by SHA and verify downloaded installers (issues #38, #39; PRs #41–#43) -- [ ] Shared validation utilities and test coverage for data pipelines (issue #36, PR #37) +- [ ] Pin GitHub Actions by SHA and verify downloaded installers (issues #38, + #39; PRs #41–#43) +- [ ] Shared validation utilities and test coverage for data pipelines (issue + #36, PR #37) - [ ] This roadmap, plus GitHub milestones matching these phases (issue #45) ## Phase 1 — Content completeness Goal: every pillar section is accurate, current, and self-maintaining. -- [ ] Architectures: automated import from cncf/architecture stays in sync (scheduled workflow exists; add freshness indicator on the page) -- [ ] Metrics: scheduled refresh of data/metrics.json with validation gating the build -- [ ] Awards: complete historical winner list, each entry verified against its cncf.io announcement -- [ ] Community: current TAB membership, End User Groups, and engagement pathways +- [ ] Architectures: automated import from cncf/architecture stays in sync + (scheduled workflow exists; add freshness indicator on the page) +- [ ] Metrics: scheduled refresh of data/metrics.json with validation gating the + build +- [ ] Awards: complete historical winner list, each entry verified against its + cncf.io announcement +- [ ] Community: current TAB membership, End User Groups, and engagement + pathways - [ ] Events: upcoming end-user events at KubeCon + CloudNativeCon - [ ] Blog: establish a publishing cadence beyond the welcome post @@ -43,7 +53,8 @@ Goal: every pillar section is accurate, current, and self-maintaining. Goal: the project can outlive any single maintainer. - [ ] MAINTAINERS.md with an explicit process for adding maintainers (issue #47) -- [ ] Governance note describing review/merge expectations, including agent-automation policy +- [ ] Governance note describing review/merge expectations, including + agent-automation policy - [ ] Good-first-issue curation to recruit human contributors - [ ] Decide the long-term home of the site (issue #46): transfer to a CNCF org, remain a personal staging site, or merge into an existing CNCF property @@ -53,9 +64,33 @@ Goal: the project can outlive any single maintainer. Goal: endusers.cncf.io becomes the authoritative end-user destination. - [ ] DNS cutover to endusers.cncf.io (depends on Phase 2 ownership decision) -- [ ] Cross-linking with contribute.cncf.io and cncf.io (clear audience boundaries) +- [ ] Cross-linking with contribute.cncf.io and cncf.io (clear audience + boundaries) - [ ] Public feedback loop: content-issue templates and a visible changelog +## Launch success metrics (tracking: #100) + +Goal: launch is measurable, not just announced. `data/launch-metrics.json` holds +the baseline snapshot and 90-day post-launch target for each signal (validated +by `npm run validate:launch-metrics`); update it by hand when a signal is +re-measured rather than editing this table. + +| Signal | Baseline (2026-08-08) | 90-day post-launch target | +| ---------------------------------------------------------- | --------------------------------- | ------------------------- | +| GitHub stars | 0 | 50 | +| GitHub watchers | 0 | 15 | +| GitHub forks | 1 (maintainer's own working fork) | 8 | +| Good-first-issues claimed or closed by a non-maintainer | 0 | 3 | +| Inbound links from cncf.io / contribute.cncf.io properties | 0 | 1 | + +- [ ] Re-measure each signal at the 90-day mark and record the outcome in + `data/launch-metrics.json` and this table. +- [ ] Add site analytics (e.g. a privacy-respecting tool such as Plausible or + GoatCounter) so inbound-traffic signals can move beyond a manual check. + +Related: #90 (launch strategy), #98 (good-first-issue curation), #104 (launch +window). + ## Non-goals - Contributor-facing documentation (belongs on contribute.cncf.io) @@ -64,7 +99,7 @@ Goal: endusers.cncf.io becomes the authoritative end-user destination. ## How this roadmap is maintained -- Strategic gaps and reprioritizations are filed as GitHub issues by the strategist - agent and reviewed by the maintainer. +- Strategic gaps and reprioritizations are filed as GitHub issues by the + strategist agent and reviewed by the maintainer. - Each phase maps to a GitHub milestone; issues and PRs are tagged accordingly. - This document is updated by planning PRs, not ad hoc edits. diff --git a/data/launch-metrics.json b/data/launch-metrics.json new file mode 100644 index 00000000..a326a768 --- /dev/null +++ b/data/launch-metrics.json @@ -0,0 +1,52 @@ +{ + "generated": false, + "note": "Hand-curated baseline snapshot for the site launch (issue #100). Unlike data/metrics.json, this is not refreshed by an automated pipeline — update it manually when a signal is re-measured, recording the new snapshot date.", + "baselineSnapshotAt": "2026-08-08", + "signals": [ + { + "id": "github-stars", + "label": "GitHub stars", + "source": "https://github.com/castrojo/endusers", + "baseline": 0, + "target90Day": 50, + "targetWindow": "90 days post-launch", + "note": "Repo-level engagement signal; cheapest to measure, easiest to game, tracked alongside the others rather than alone." + }, + { + "id": "github-watchers", + "label": "GitHub watchers", + "source": "https://github.com/castrojo/endusers", + "baseline": 0, + "target90Day": 15, + "targetWindow": "90 days post-launch", + "note": "Proxy for people who want ongoing visibility into the project, not just a one-time star." + }, + { + "id": "github-forks", + "label": "GitHub forks", + "source": "https://github.com/castrojo/endusers", + "baseline": 1, + "target90Day": 8, + "targetWindow": "90 days post-launch", + "note": "Baseline of 1 is the maintainer's own working fork, not an external contributor." + }, + { + "id": "good-first-issue-conversion", + "label": "Good-first-issues claimed or closed by a non-maintainer", + "source": "https://github.com/castrojo/endusers/labels/good%20first%20issue", + "baseline": 0, + "target90Day": 3, + "targetWindow": "90 days post-launch", + "note": "Directly measures whether curated on-ramp issues (issue #98) convert visitors into contributors." + }, + { + "id": "inbound-links-cncf", + "label": "Inbound links from cncf.io / contribute.cncf.io properties", + "source": "manual check via site search or referrer data once analytics exists", + "baseline": 0, + "target90Day": 1, + "targetWindow": "90 days post-launch", + "note": "At least one authoritative CNCF property should link to this site by the 90-day mark; there is no analytics pipeline yet to measure traffic directly, so this is checked manually until one is added." + } + ] +} diff --git a/package.json b/package.json index 93c9c3e6..8d412127 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "validate:architecture-assets": "node scripts/validate-architecture-assets.mjs", "validate:button-contrast": "node scripts/validate-button-contrast.mjs", "validate:awards": "node scripts/validate-awards.mjs", + "validate:launch-metrics": "node scripts/validate-launch-metrics.mjs", "docusaurus": "docusaurus", "fix:format": "npm run _check:format -- --write", "fix": "npm run seq -- $(npm -s run _list:fix:*)", diff --git a/scripts/validate-launch-metrics.mjs b/scripts/validate-launch-metrics.mjs new file mode 100644 index 00000000..18806ebd --- /dev/null +++ b/scripts/validate-launch-metrics.mjs @@ -0,0 +1,79 @@ +#!/usr/bin/env node +import { readFileSync } from 'node:fs'; +import { reportAndExit } from './lib/validate-utils.mjs'; + +const data = JSON.parse( + readFileSync(new URL('../data/launch-metrics.json', import.meta.url)), +); +const errors = []; + +if (Number.isNaN(Date.parse(data.baselineSnapshotAt))) { + errors.push({ + path: 'launch-metrics.json', + severity: 'error', + message: 'baselineSnapshotAt must be a valid date', + }); +} +if ( + !Array.isArray(data.signals) || + data.signals.length < 3 || + data.signals.length > 5 +) { + errors.push({ + path: 'launch-metrics.json', + severity: 'error', + message: 'signals must contain between 3 and 5 entries', + }); +} + +const ids = new Set(); +for (const signal of data.signals || []) { + if (!signal.id || ids.has(signal.id)) { + errors.push({ + path: signal.id || '(missing id)', + severity: 'error', + message: 'duplicate or missing signal id', + }); + } + ids.add(signal.id); + if (!signal.label) + errors.push({ + path: signal.id, + severity: 'error', + message: 'missing label', + }); + if (!signal.source) + errors.push({ + path: signal.id, + severity: 'error', + message: 'missing source', + }); + if (signal.baseline === undefined || signal.baseline === null) { + errors.push({ + path: signal.id, + severity: 'error', + message: 'missing baseline value', + }); + } + if (signal.target90Day === undefined || signal.target90Day === null) { + errors.push({ + path: signal.id, + severity: 'error', + message: 'missing target90Day value', + }); + } + if ( + typeof signal.baseline === 'number' && + typeof signal.target90Day === 'number' && + signal.target90Day < signal.baseline + ) { + errors.push({ + path: signal.id, + severity: 'warn', + message: 'target90Day is lower than baseline', + }); + } +} + +reportAndExit(errors, 'launch metrics'); +console.log(`Validated ${data.signals.length} launch success signals`); diff --git a/tests/validate-launch-metrics.test.mjs b/tests/validate-launch-metrics.test.mjs new file mode 100644 index 00000000..398fe2fc --- /dev/null +++ b/tests/validate-launch-metrics.test.mjs @@ -0,0 +1,111 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { runScriptWithFixtures } from './helpers.mjs'; + +const SCRIPT = 'validate-launch-metrics.mjs'; + +const validSignal = { + id: 'github-stars', + label: 'GitHub stars', + source: 'https://github.com/castrojo/endusers', + baseline: 0, + target90Day: 50, + targetWindow: '90 days post-launch', +}; + +function launchMetricsFixture(overrides = {}) { + return { + 'data/launch-metrics.json': JSON.stringify({ + baselineSnapshotAt: '2026-08-08', + signals: [ + validSignal, + { ...validSignal, id: 'github-watchers', label: 'GitHub watchers' }, + { ...validSignal, id: 'github-forks', label: 'GitHub forks' }, + ], + ...overrides, + }), + }; +} + +test('accepts a valid launch metrics file', () => { + const result = runScriptWithFixtures(SCRIPT, launchMetricsFixture()); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /Validated 3 launch success signals/); +}); + +test('rejects an invalid baselineSnapshotAt date', () => { + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ baselineSnapshotAt: 'not-a-date' }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /valid date/); +}); + +test('rejects fewer than 3 signals', () => { + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ signals: [validSignal] }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /between 3 and 5/); +}); + +test('rejects more than 5 signals', () => { + const signals = Array.from({ length: 6 }, (_, i) => ({ + ...validSignal, + id: `signal-${i}`, + })); + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ signals }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /between 3 and 5/); +}); + +test('rejects duplicate signal ids', () => { + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ + signals: [ + validSignal, + { ...validSignal }, + { ...validSignal, id: 'github-forks' }, + ], + }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /duplicate or missing signal id/); +}); + +test('rejects a signal missing target90Day', () => { + const { target90Day, ...withoutTarget } = validSignal; + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ + signals: [ + withoutTarget, + { ...validSignal, id: 'github-watchers' }, + { ...validSignal, id: 'github-forks' }, + ], + }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /missing target90Day/); +}); + +test('warns when target90Day is lower than baseline', () => { + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ + signals: [ + { ...validSignal, baseline: 10, target90Day: 5 }, + { ...validSignal, id: 'github-watchers' }, + { ...validSignal, id: 'github-forks' }, + ], + }), + ); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stderr, /lower than baseline/); +}); diff --git a/tests/validators-smoke.test.mjs b/tests/validators-smoke.test.mjs index adeb6c15..75df9327 100644 --- a/tests/validators-smoke.test.mjs +++ b/tests/validators-smoke.test.mjs @@ -15,6 +15,7 @@ const READ_ONLY_VALIDATORS = [ 'validate-architectures.mjs', 'validate-architecture-assets.mjs', 'validate-button-contrast.mjs', + 'validate-launch-metrics.mjs', ]; for (const script of READ_ONLY_VALIDATORS) { From 12be0fa91f3fc082332b952db32145d9b6f4d797 Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Wed, 16 Sep 2026 17:55:51 +0000 Subject: [PATCH 2/3] Address review: pre-launch checkpoint framing, canonical URLs, strict number validation, reproducible inbound-link procedure, JSON-driven table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Renamed the baseline date field to preLaunchCheckpointAt / checkpointLabel ('W-6 pre-launch checkpoint') so the snapshot is explicitly tied to the launch plan's pre-launch measurement point rather than an arbitrary discovery date. - Replaced castrojo/endusers source URLs with the canonical cncf/endusers ones; the pre-transfer URLs are preserved in a new historicalSource field per signal so the evidence trail for the original measurement isn't lost. validate-launch-metrics.mjs now hard-fails if a source field still points at castrojo/endusers. - baseline/target90Day are now validated as finite, non-negative integers (previously any truthy value passed). - Added a single documented inboundLinkMeasurementProcedure (a specific, re-runnable GitHub code-search query) so the 90-day re-measurement uses the same method as the baseline instead of an undefined 'manual check'. - Made data/launch-metrics.json the actual source of truth for the ROADMAP.md table: scripts/sync-launch-metrics-table.mjs generates the table between markers, and validate-launch-metrics.mjs now fails the build if the committed table has drifted from the JSON (normalizing away Prettier's column padding so re-formatting doesn't false-positive as drift). - Extended tests/validate-launch-metrics.test.mjs to cover all of the above (14 tests); full suite (68 tests), npm run build, and npx prettier --check all pass. NOTE: I could not rebase this branch onto current main or push any .github/workflows/* change in this environment — my credentials lack the GitHub 'workflow' OAuth scope, and this fork predates main's .github/workflows/ci.yml, so even a no-op rebase would count as introducing that file from this token's perspective. This PR's own ask doesn't require a workflow-file change (validate:launch-metrics and the new sync:launch-metrics-table script are already wired via package.json and deploy-gh-pages.yml per the original PR), so this only affects the 'rebase onto main to resolve the conflict' part of the review — that still needs to happen by hand or with workflow-scope credentials. Addresses PR #143 review feedback. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ROADMAP.md | 33 ++++--- data/launch-metrics.json | 22 +++-- package.json | 1 + scripts/lib/launch-metrics-table.mjs | 34 +++++++ scripts/sync-launch-metrics-table.mjs | 32 +++++++ scripts/validate-launch-metrics.mjs | 86 +++++++++++++++-- tests/validate-launch-metrics.test.mjs | 126 ++++++++++++++++++++++--- 7 files changed, 291 insertions(+), 43 deletions(-) create mode 100644 scripts/lib/launch-metrics-table.mjs create mode 100644 scripts/sync-launch-metrics-table.mjs diff --git a/ROADMAP.md b/ROADMAP.md index 7851f4cb..22268be7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -70,21 +70,28 @@ Goal: endusers.cncf.io becomes the authoritative end-user destination. ## Launch success metrics (tracking: #100) -Goal: launch is measurable, not just announced. `data/launch-metrics.json` holds -the baseline snapshot and 90-day post-launch target for each signal (validated -by `npm run validate:launch-metrics`); update it by hand when a signal is -re-measured rather than editing this table. - -| Signal | Baseline (2026-08-08) | 90-day post-launch target | -| ---------------------------------------------------------- | --------------------------------- | ------------------------- | -| GitHub stars | 0 | 50 | -| GitHub watchers | 0 | 15 | -| GitHub forks | 1 (maintainer's own working fork) | 8 | -| Good-first-issues claimed or closed by a non-maintainer | 0 | 3 | -| Inbound links from cncf.io / contribute.cncf.io properties | 0 | 1 | +Goal: launch is measurable, not just announced. `data/launch-metrics.json` is +the source of truth for the baseline (captured at the W-6 pre-launch checkpoint) +and 90-day post-launch target for each signal. The table below is **generated +from that file** — do not hand-edit it. Run `npm run sync:launch-metrics-table` +after changing the JSON, and `npm run validate:launch-metrics` fails the build +if the two drift apart. + + + +| Signal | Baseline (W-6 pre-launch checkpoint, 2026-08-08) | 90-day post-launch target | +| ---------------------------------------------------------- | ------------------------------------------------ | ------------------------- | +| GitHub stars | 0 | 50 | +| GitHub watchers | 0 | 15 | +| GitHub forks | 1 | 8 | +| Good-first-issues claimed or closed by a non-maintainer | 0 | 3 | +| Inbound links from cncf.io / contribute.cncf.io properties | 0 | 1 | + + - [ ] Re-measure each signal at the 90-day mark and record the outcome in - `data/launch-metrics.json` and this table. + `data/launch-metrics.json`, then re-run + `npm run sync:launch-metrics-table`. - [ ] Add site analytics (e.g. a privacy-respecting tool such as Plausible or GoatCounter) so inbound-traffic signals can move beyond a manual check. diff --git a/data/launch-metrics.json b/data/launch-metrics.json index a326a768..e9593acb 100644 --- a/data/launch-metrics.json +++ b/data/launch-metrics.json @@ -1,12 +1,15 @@ { "generated": false, - "note": "Hand-curated baseline snapshot for the site launch (issue #100). Unlike data/metrics.json, this is not refreshed by an automated pipeline — update it manually when a signal is re-measured, recording the new snapshot date.", - "baselineSnapshotAt": "2026-08-08", + "note": "Hand-curated baseline snapshot for the site launch (issue #100). Unlike data/metrics.json, this is not refreshed by an automated pipeline — update it manually when a signal is re-measured, recording the new snapshot date. The ROADMAP.md launch-metrics table is generated FROM this file: run `npm run sync:launch-metrics-table` after any edit here, and `npm run validate:launch-metrics` fails if the two drift out of sync.", + "preLaunchCheckpointAt": "2026-08-08", + "checkpointLabel": "W-6 pre-launch checkpoint", + "inboundLinkMeasurementProcedure": "GitHub code search for the canonical repo URL restricted to the cncf.io/contribute.cncf.io properties' source repos: `gh search code \"cncf/endusers\" --owner cncf --repo cncf/foundation --repo cncf/contribute.cncf.io`. Re-run the identical query at the 90-day mark and record the resulting count as the new value; this keeps the measurement reproducible instead of an ad hoc manual search.", "signals": [ { "id": "github-stars", "label": "GitHub stars", - "source": "https://github.com/castrojo/endusers", + "source": "https://github.com/cncf/endusers", + "historicalSource": "https://github.com/castrojo/endusers (repo lived here at snapshot time, since transferred to cncf/endusers)", "baseline": 0, "target90Day": 50, "targetWindow": "90 days post-launch", @@ -15,7 +18,8 @@ { "id": "github-watchers", "label": "GitHub watchers", - "source": "https://github.com/castrojo/endusers", + "source": "https://github.com/cncf/endusers", + "historicalSource": "https://github.com/castrojo/endusers (repo lived here at snapshot time, since transferred to cncf/endusers)", "baseline": 0, "target90Day": 15, "targetWindow": "90 days post-launch", @@ -24,7 +28,8 @@ { "id": "github-forks", "label": "GitHub forks", - "source": "https://github.com/castrojo/endusers", + "source": "https://github.com/cncf/endusers", + "historicalSource": "https://github.com/castrojo/endusers (repo lived here at snapshot time, since transferred to cncf/endusers)", "baseline": 1, "target90Day": 8, "targetWindow": "90 days post-launch", @@ -33,7 +38,8 @@ { "id": "good-first-issue-conversion", "label": "Good-first-issues claimed or closed by a non-maintainer", - "source": "https://github.com/castrojo/endusers/labels/good%20first%20issue", + "source": "https://github.com/cncf/endusers/labels/good%20first%20issue", + "historicalSource": "https://github.com/castrojo/endusers/labels/good%20first%20issue (repo lived here at snapshot time, since transferred to cncf/endusers)", "baseline": 0, "target90Day": 3, "targetWindow": "90 days post-launch", @@ -42,11 +48,11 @@ { "id": "inbound-links-cncf", "label": "Inbound links from cncf.io / contribute.cncf.io properties", - "source": "manual check via site search or referrer data once analytics exists", + "source": "see inboundLinkMeasurementProcedure above", "baseline": 0, "target90Day": 1, "targetWindow": "90 days post-launch", - "note": "At least one authoritative CNCF property should link to this site by the 90-day mark; there is no analytics pipeline yet to measure traffic directly, so this is checked manually until one is added." + "note": "At least one authoritative CNCF property should link to this site by the 90-day mark; there is no analytics pipeline yet to measure traffic directly, so this uses the reproducible code-search procedure above until one is added." } ] } diff --git a/package.json b/package.json index 8d412127..61b76112 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "validate:button-contrast": "node scripts/validate-button-contrast.mjs", "validate:awards": "node scripts/validate-awards.mjs", "validate:launch-metrics": "node scripts/validate-launch-metrics.mjs", + "sync:launch-metrics-table": "node scripts/sync-launch-metrics-table.mjs", "docusaurus": "docusaurus", "fix:format": "npm run _check:format -- --write", "fix": "npm run seq -- $(npm -s run _list:fix:*)", diff --git a/scripts/lib/launch-metrics-table.mjs b/scripts/lib/launch-metrics-table.mjs new file mode 100644 index 00000000..396da6c3 --- /dev/null +++ b/scripts/lib/launch-metrics-table.mjs @@ -0,0 +1,34 @@ +// Shared table-generation logic used by both +// scripts/sync-launch-metrics-table.mjs (writes ROADMAP.md) and +// scripts/validate-launch-metrics.mjs (checks ROADMAP.md hasn't drifted +// from data/launch-metrics.json without being re-synced). +export const START = ''; +export const END = ''; + +export function buildTable(data) { + const checkpointLabel = data.checkpointLabel || 'baseline'; + const checkpointAt = data.preLaunchCheckpointAt || data.baselineSnapshotAt; + const header = `| Signal | Baseline (${checkpointLabel}, ${checkpointAt}) | 90-day post-launch target |`; + const divider = '| --- | --- | --- |'; + const rows = (data.signals || []).map( + (s) => `| ${s.label} | ${s.baseline} | ${s.target90Day} |`, + ); + return [header, divider, ...rows].join('\n'); +} + +// Prettier re-pads table column widths for readability, so a byte-for-byte +// comparison between the generated table and what's committed would false- +// positive on drift after every `npx prettier --write`. Normalize by +// collapsing whitespace within each cell before comparing. +export function normalizeTable(markdown) { + return markdown + .split('\n') + .filter((line) => !/^\|[\s-]*\|[\s-:]*\|[\s-:]*\|$/.test(line.trim())) + .map((line) => + line + .split('|') + .map((cell) => cell.trim()) + .join('|'), + ) + .join('\n'); +} diff --git a/scripts/sync-launch-metrics-table.mjs b/scripts/sync-launch-metrics-table.mjs new file mode 100644 index 00000000..586fe2db --- /dev/null +++ b/scripts/sync-launch-metrics-table.mjs @@ -0,0 +1,32 @@ +#!/usr/bin/env node +// Regenerates the launch-metrics table in ROADMAP.md from +// data/launch-metrics.json, which is the actual source of truth (issue +// #100 review feedback: don't let a hand-maintained table drift from the +// validated JSON). Run this after any edit to the JSON; CI verifies via +// validate-launch-metrics.mjs --check that the two haven't drifted apart. +import { readFileSync, writeFileSync } from 'node:fs'; +import { buildTable, START, END } from './lib/launch-metrics-table.mjs'; + +const roadmapPath = new URL('../ROADMAP.md', import.meta.url); +const data = JSON.parse( + readFileSync(new URL('../data/launch-metrics.json', import.meta.url)), +); +const roadmap = readFileSync(roadmapPath, 'utf8'); + +const startIdx = roadmap.indexOf(START); +const endIdx = roadmap.indexOf(END); +if (startIdx === -1 || endIdx === -1 || endIdx < startIdx) { + console.error( + `Could not find ${START} / ${END} markers in ROADMAP.md — cannot sync table.`, + ); + process.exit(1); +} + +const before = roadmap.slice(0, startIdx + START.length); +const after = roadmap.slice(endIdx); +const updated = `${before}\n\n${buildTable(data)}\n\n${after}`; + +writeFileSync(roadmapPath, updated); +console.log( + 'Synced ROADMAP.md launch-metrics table from data/launch-metrics.json', +); diff --git a/scripts/validate-launch-metrics.mjs b/scripts/validate-launch-metrics.mjs index 18806ebd..86f94e56 100644 --- a/scripts/validate-launch-metrics.mjs +++ b/scripts/validate-launch-metrics.mjs @@ -1,17 +1,33 @@ #!/usr/bin/env node import { readFileSync } from 'node:fs'; import { reportAndExit } from './lib/validate-utils.mjs'; +import { + buildTable, + normalizeTable, + START, + END, +} from './lib/launch-metrics-table.mjs'; const data = JSON.parse( readFileSync(new URL('../data/launch-metrics.json', import.meta.url)), ); const errors = []; -if (Number.isNaN(Date.parse(data.baselineSnapshotAt))) { +if (Number.isNaN(Date.parse(data.preLaunchCheckpointAt))) { errors.push({ path: 'launch-metrics.json', severity: 'error', - message: 'baselineSnapshotAt must be a valid date', + message: 'preLaunchCheckpointAt must be a valid date', + }); +} +if (!data.inboundLinkMeasurementProcedure) { + errors.push({ + path: 'launch-metrics.json', + severity: 'error', + message: + 'inboundLinkMeasurementProcedure must describe a single, reproducible ' + + 'procedure for measuring inbound links, so the 90-day re-measurement ' + + 'uses the same method as the baseline', }); } if ( @@ -26,6 +42,10 @@ if ( }); } +function isFiniteNonNegativeInt(value) { + return Number.isInteger(value) && Number.isFinite(value) && value >= 0; +} + const ids = new Set(); for (const signal of data.signals || []) { if (!signal.id || ids.has(signal.id)) { @@ -48,23 +68,35 @@ for (const signal of data.signals || []) { severity: 'error', message: 'missing source', }); - if (signal.baseline === undefined || signal.baseline === null) { + if ( + /castrojo\/endusers/.test(signal.source) || + /castrojo\/endusers/.test(signal.label) + ) { errors.push({ path: signal.id, severity: 'error', - message: 'missing baseline value', + message: + 'source must use the canonical cncf/endusers URL, not the pre-transfer ' + + 'castrojo/endusers fork (put historical evidence in historicalSource instead)', }); } - if (signal.target90Day === undefined || signal.target90Day === null) { + if (!isFiniteNonNegativeInt(signal.baseline)) { errors.push({ path: signal.id, severity: 'error', - message: 'missing target90Day value', + message: 'baseline must be a finite, non-negative integer', + }); + } + if (!isFiniteNonNegativeInt(signal.target90Day)) { + errors.push({ + path: signal.id, + severity: 'error', + message: 'target90Day must be a finite, non-negative integer', }); } if ( - typeof signal.baseline === 'number' && - typeof signal.target90Day === 'number' && + isFiniteNonNegativeInt(signal.baseline) && + isFiniteNonNegativeInt(signal.target90Day) && signal.target90Day < signal.baseline ) { errors.push({ @@ -75,5 +107,43 @@ for (const signal of data.signals || []) { } } +// The ROADMAP.md table must be generated from this JSON (see +// scripts/sync-launch-metrics-table.mjs), not hand-edited — otherwise the +// two silently drift apart. Recompute what the table SHOULD be and diff it +// against what's actually committed between the markers. +try { + const roadmap = readFileSync( + new URL('../ROADMAP.md', import.meta.url), + 'utf8', + ); + const startIdx = roadmap.indexOf(START); + const endIdx = roadmap.indexOf(END); + if (startIdx === -1 || endIdx === -1 || endIdx < startIdx) { + errors.push({ + path: 'ROADMAP.md', + severity: 'error', + message: `missing ${START} / ${END} markers around the launch-metrics table`, + }); + } else { + const actual = roadmap.slice(startIdx + START.length, endIdx).trim(); + const expected = buildTable(data).trim(); + if (normalizeTable(actual) !== normalizeTable(expected)) { + errors.push({ + path: 'ROADMAP.md', + severity: 'error', + message: + 'launch-metrics table is out of sync with data/launch-metrics.json — ' + + 'run `npm run sync:launch-metrics-table` and commit the result', + }); + } + } +} catch (error) { + errors.push({ + path: 'ROADMAP.md', + severity: 'error', + message: `could not read ROADMAP.md to verify table sync: ${error.message}`, + }); +} + reportAndExit(errors, 'launch metrics'); console.log(`Validated ${data.signals.length} launch success signals`); diff --git a/tests/validate-launch-metrics.test.mjs b/tests/validate-launch-metrics.test.mjs index 398fe2fc..ec443488 100644 --- a/tests/validate-launch-metrics.test.mjs +++ b/tests/validate-launch-metrics.test.mjs @@ -1,47 +1,66 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { runScriptWithFixtures } from './helpers.mjs'; +import { + buildTable, + START, + END, +} from '../scripts/lib/launch-metrics-table.mjs'; const SCRIPT = 'validate-launch-metrics.mjs'; const validSignal = { id: 'github-stars', label: 'GitHub stars', - source: 'https://github.com/castrojo/endusers', + source: 'https://github.com/cncf/endusers', baseline: 0, target90Day: 50, targetWindow: '90 days post-launch', }; function launchMetricsFixture(overrides = {}) { + const data = { + preLaunchCheckpointAt: '2026-08-08', + checkpointLabel: 'W-6 pre-launch checkpoint', + inboundLinkMeasurementProcedure: + 'run the same GitHub code search query each time', + signals: [ + validSignal, + { ...validSignal, id: 'github-watchers', label: 'GitHub watchers' }, + { ...validSignal, id: 'github-forks', label: 'GitHub forks' }, + ], + ...overrides, + }; return { - 'data/launch-metrics.json': JSON.stringify({ - baselineSnapshotAt: '2026-08-08', - signals: [ - validSignal, - { ...validSignal, id: 'github-watchers', label: 'GitHub watchers' }, - { ...validSignal, id: 'github-forks', label: 'GitHub forks' }, - ], - ...overrides, - }), + 'data/launch-metrics.json': JSON.stringify(data), + 'ROADMAP.md': `# Roadmap\n\n${START}\n\n${buildTable(data)}\n\n${END}\n`, }; } -test('accepts a valid launch metrics file', () => { +test('accepts a valid launch metrics file with a synced table', () => { const result = runScriptWithFixtures(SCRIPT, launchMetricsFixture()); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /Validated 3 launch success signals/); }); -test('rejects an invalid baselineSnapshotAt date', () => { +test('rejects an invalid preLaunchCheckpointAt date', () => { const result = runScriptWithFixtures( SCRIPT, - launchMetricsFixture({ baselineSnapshotAt: 'not-a-date' }), + launchMetricsFixture({ preLaunchCheckpointAt: 'not-a-date' }), ); assert.equal(result.status, 1); assert.match(result.stderr, /valid date/); }); +test('rejects a missing inboundLinkMeasurementProcedure', () => { + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ inboundLinkMeasurementProcedure: undefined }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /reproducible/); +}); + test('rejects fewer than 3 signals', () => { const result = runScriptWithFixtures( SCRIPT, @@ -92,7 +111,70 @@ test('rejects a signal missing target90Day', () => { }), ); assert.equal(result.status, 1); - assert.match(result.stderr, /missing target90Day/); + assert.match( + result.stderr, + /target90Day must be a finite, non-negative integer/, + ); +}); + +test('rejects a non-integer baseline', () => { + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ + signals: [ + { ...validSignal, baseline: 1.5 }, + { ...validSignal, id: 'github-watchers' }, + { ...validSignal, id: 'github-forks' }, + ], + }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /finite, non-negative integer/); +}); + +test('rejects a negative target90Day', () => { + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ + signals: [ + { ...validSignal, target90Day: -1 }, + { ...validSignal, id: 'github-watchers' }, + { ...validSignal, id: 'github-forks' }, + ], + }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /finite, non-negative integer/); +}); + +test('rejects a non-finite baseline (Infinity)', () => { + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ + signals: [ + { ...validSignal, baseline: Infinity }, + { ...validSignal, id: 'github-watchers' }, + { ...validSignal, id: 'github-forks' }, + ], + }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /finite, non-negative integer/); +}); + +test('rejects a source pointing at the pre-transfer castrojo/endusers fork', () => { + const result = runScriptWithFixtures( + SCRIPT, + launchMetricsFixture({ + signals: [ + { ...validSignal, source: 'https://github.com/castrojo/endusers' }, + { ...validSignal, id: 'github-watchers' }, + { ...validSignal, id: 'github-forks' }, + ], + }), + ); + assert.equal(result.status, 1); + assert.match(result.stderr, /canonical cncf\/endusers URL/); }); test('warns when target90Day is lower than baseline', () => { @@ -109,3 +191,19 @@ test('warns when target90Day is lower than baseline', () => { assert.equal(result.status, 0, result.stderr); assert.match(result.stderr, /lower than baseline/); }); + +test('rejects a ROADMAP.md table that has drifted from the JSON', () => { + const fixtures = launchMetricsFixture(); + fixtures['ROADMAP.md'] = fixtures['ROADMAP.md'].replace('50', '999'); + const result = runScriptWithFixtures(SCRIPT, fixtures); + assert.equal(result.status, 1); + assert.match(result.stderr, /out of sync/); +}); + +test('rejects a ROADMAP.md missing the table markers', () => { + const fixtures = launchMetricsFixture(); + fixtures['ROADMAP.md'] = '# Roadmap\n\nno markers here\n'; + const result = runScriptWithFixtures(SCRIPT, fixtures); + assert.equal(result.status, 1); + assert.match(result.stderr, /missing.*markers/); +}); From d02795c0589adc6b075a40b5d69d968e391dc190 Mon Sep 17 00:00:00 2001 From: mrbobbytables Date: Wed, 16 Sep 2026 20:01:06 +0000 Subject: [PATCH 3/3] fix: de-linkify historicalSource pre-transfer references Addresses scanner re-verification on #143: historicalSource fields still embedded live https://github.com/castrojo/endusers URLs. Since that account could be renamed/reused, a rendered link risks link rot and misattribution even though the field is explicitly historical evidence, not a live source (the validator only enforces canonical cncf/endusers on the 'source' field). Rewrites historicalSource to a plain (non-URL) repo-path reference that can't be mistaken for a live/authoritative link, while still preserving the pre-transfer evidence trail. Signed-off-by: mrbobbytables --- data/launch-metrics.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/data/launch-metrics.json b/data/launch-metrics.json index e9593acb..7267ac77 100644 --- a/data/launch-metrics.json +++ b/data/launch-metrics.json @@ -9,7 +9,7 @@ "id": "github-stars", "label": "GitHub stars", "source": "https://github.com/cncf/endusers", - "historicalSource": "https://github.com/castrojo/endusers (repo lived here at snapshot time, since transferred to cncf/endusers)", + "historicalSource": "castrojo/endusers (pre-transfer GitHub account/repo path, not a live URL; account may be renamed/reused, so this is deliberately not rendered as a link; canonical repo is https://github.com/cncf/endusers)", "baseline": 0, "target90Day": 50, "targetWindow": "90 days post-launch", @@ -19,7 +19,7 @@ "id": "github-watchers", "label": "GitHub watchers", "source": "https://github.com/cncf/endusers", - "historicalSource": "https://github.com/castrojo/endusers (repo lived here at snapshot time, since transferred to cncf/endusers)", + "historicalSource": "castrojo/endusers (pre-transfer GitHub account/repo path, not a live URL; account may be renamed/reused, so this is deliberately not rendered as a link; canonical repo is https://github.com/cncf/endusers)", "baseline": 0, "target90Day": 15, "targetWindow": "90 days post-launch", @@ -29,7 +29,7 @@ "id": "github-forks", "label": "GitHub forks", "source": "https://github.com/cncf/endusers", - "historicalSource": "https://github.com/castrojo/endusers (repo lived here at snapshot time, since transferred to cncf/endusers)", + "historicalSource": "castrojo/endusers (pre-transfer GitHub account/repo path, not a live URL; account may be renamed/reused, so this is deliberately not rendered as a link; canonical repo is https://github.com/cncf/endusers)", "baseline": 1, "target90Day": 8, "targetWindow": "90 days post-launch", @@ -39,7 +39,7 @@ "id": "good-first-issue-conversion", "label": "Good-first-issues claimed or closed by a non-maintainer", "source": "https://github.com/cncf/endusers/labels/good%20first%20issue", - "historicalSource": "https://github.com/castrojo/endusers/labels/good%20first%20issue (repo lived here at snapshot time, since transferred to cncf/endusers)", + "historicalSource": "castrojo/endusers/labels/good%20first%20issue (pre-transfer GitHub account/repo path, not a live URL; account may be renamed/reused, so this is deliberately not rendered as a link; canonical repo is https://github.com/cncf/endusers)", "baseline": 0, "target90Day": 3, "targetWindow": "90 days post-launch",