diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml new file mode 100644 index 0000000000..1234ce25e1 --- /dev/null +++ b/.github/workflows/indexnow.yml @@ -0,0 +1,185 @@ +name: IndexNow + +# Tells IndexNow search engines (Bing, Yandex, Naver, Seznam, Yep, Amazon) about aspire.dev URLs. +# +# - Daily: submits URLs that are new or removed in the sitemap since the previous run, then the next +# window of a slow rotation through the whole sitemap. +# - Release: when a release/X.Y pull request is merged into main, waits for that release's "What's +# new" page to be live (the docs slot swap) and then starts a full-sitemap burst that the daily +# runs finish. The sitemap has no lastmod, so the burst sends the English pages first (where the +# new and updated docs are), then the generated API reference, then the localized pages. +# - Manual: run it from the Actions tab (mode, limit, release, dry run). A flush restarts the +# full-sitemap burst, the same way a release does. +# +# All logic lives in src/frontend/scripts/indexnow.ts so the same flow can be reproduced locally: +# cd src/frontend && pnpm indexnow --state-dir /tmp/indexnow --dry-run +# The budgets and the key are in src/frontend/config/indexnow.json, and the key file is served from +# src/frontend/public/.txt. IndexNow keys are public by design: the key file has to be +# downloadable so search engines can verify ownership. To rotate the key, add the new key file and +# update config/indexnow.json in the same change, and delete the old key file once it has deployed. +# +# What has already been submitted is kept in the "indexnow-state" artifact (refreshed by every run). +# Losing it only causes a cold start that re-seeds the known URL list without submitting everything. + +on: + schedule: + # Once a day at 06:43 UTC (the fields are minute, hour, day of month, month and day of week). + # Each run submits the new and removed URLs, then the next window of the rotation through the + # whole sitemap. The window size is in config/indexnow.json: at 1,000 URLs a run, a sitemap of + # about 32,000 URLs takes about a month to cover. If GitHub skips a day, the rotation only + # slips by a day, because its position is saved in the state artifact. The odd minute avoids + # the top of the hour, when GitHub delays scheduled runs the most. Scheduled runs use the + # workflow on the default branch, so this starts once the pull request is merged. + - cron: "43 6 * * *" + pull_request: + types: [closed] + branches: [main] + workflow_dispatch: + inputs: + mode: + description: "daily: new and removed URLs plus the next rotation window. flush: restart the full-sitemap cycle at the release budget." + type: choice + options: + - daily + - flush + default: daily + limit: + description: "Number of URLs to submit in this run (1-10000). Leave empty to use the configured budget." + type: string + required: false + release: + description: "Release such as 13.6. Waits for that release's What's new page to be live before submitting." + type: string + required: false + dry_run: + description: "Only print what would be submitted. Nothing is sent and the state is not saved." + type: boolean + default: false + allow_shrink: + description: "Continue even if the sitemap lists more than 20% fewer URLs than the last run." + type: boolean + default: false + +permissions: {} + +jobs: + submit: + name: Submit URLs to IndexNow + # Pull requests only matter when they are merged release pull requests from this repository. The + # script checks the exact branch format again. + if: >- + ${{ github.repository == 'microsoft/aspire.dev' && + (github.event_name != 'pull_request' || + (github.event.pull_request.merged == true && + github.event.pull_request.head.repo.full_name == github.repository && + startsWith(github.event.pull_request.head.ref, 'release/'))) }} + runs-on: ubuntu-latest + timeout-minutes: 75 + # The group is on the job, not the workflow, so the many pull requests that are closed without + # being a release merge (their job is skipped above) never join it. A workflow-level group would + # let each of them replace a run that is waiting its turn. "queue: max" lets the daily run, a + # release run and a manual run wait in line instead of replacing each other. + concurrency: + group: indexnow + cancel-in-progress: false + queue: max + # The workflow-level "permissions: {}" gives every job no access by default. A job's own block + # replaces that default instead of adding to it, so this job gets exactly these two: contents to + # check out the repository, and actions to list and download the state artifact. Uploading the + # artifact needs no token permission. + permissions: + contents: read + actions: read + defaults: + run: + working-directory: src/frontend + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # A release merge runs the merged script from main, not the pull request's merge ref. + ref: ${{ github.event_name == 'pull_request' && github.event.repository.default_branch || github.ref }} + fetch-depth: 1 + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24.x" + cache: pnpm + cache-dependency-path: src/frontend/pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + + - name: Restore state from the previous run + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + state_dir="$RUNNER_TEMP/indexnow-state" + mkdir -p "$state_dir" + + # `gh run download` lists every artifact in the repository and fails with HTTP 500 here, so + # ask for this artifact by name and take the newest one that has not expired. Runs from + # forks are listed in this repository too and could upload an artifact with this name, so + # only artifacts from runs of this repository are trusted. The script validates the state + # again when it reads it. + artifact_id="" + for attempt in 1 2 3; do + if artifact_id=$(gh api "repos/$GH_REPO/actions/artifacts?name=indexnow-state&per_page=100" \ + --jq '[.artifacts[] | select(.expired == false and .workflow_run.head_repository_id == .workflow_run.repository_id)] | sort_by(.created_at) | last | .id // empty'); then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::Could not list the indexnow-state artifacts." + exit 1 + fi + sleep $((attempt * 10)) + done + + if [ -z "$artifact_id" ]; then + echo "No previous state found; this run starts cold." + exit 0 + fi + + echo "Restoring artifact $artifact_id." + gh api "repos/$GH_REPO/actions/artifacts/$artifact_id/zip" > "$RUNNER_TEMP/indexnow-state.zip" + unzip -q -o "$RUNNER_TEMP/indexnow-state.zip" -d "$state_dir" + + - name: Submit URLs + id: submit + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_MODE: ${{ inputs.mode }} + INPUT_LIMIT: ${{ inputs.limit }} + INPUT_RELEASE: ${{ inputs.release }} + INPUT_DRY_RUN: ${{ inputs.dry_run }} + INPUT_ALLOW_SHRINK: ${{ inputs.allow_shrink }} + shell: bash + run: | + set -euo pipefail + args=(--state-dir "$RUNNER_TEMP/indexnow-state" --trigger "$EVENT_NAME") + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + args+=(--mode "$INPUT_MODE") + [ -z "$INPUT_LIMIT" ] || args+=(--limit "$INPUT_LIMIT") + [ -z "$INPUT_RELEASE" ] || args+=(--release "$INPUT_RELEASE") + [ "$INPUT_DRY_RUN" != "true" ] || args+=(--dry-run) + [ "$INPUT_ALLOW_SHRINK" != "true" ] || args+=(--allow-shrink) + fi + pnpm indexnow "${args[@]}" + + # Runs after a failure too, because a failed cold start or flush still saves what it started. + - name: Save state for the next run + if: ${{ !cancelled() && steps.submit.outputs.state_changed == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: indexnow-state + path: ${{ runner.temp }}/indexnow-state + retention-days: 90 + if-no-files-found: error diff --git a/src/frontend/config/indexnow.json b/src/frontend/config/indexnow.json new file mode 100644 index 0000000000..b01b4e8e7d --- /dev/null +++ b/src/frontend/config/indexnow.json @@ -0,0 +1,10 @@ +{ + "host": "aspire.dev", + "key": "c5f794ab28dc0cb92f48bd970500daa3", + "endpoint": "https://api.indexnow.org/indexnow", + "sitemapIndex": "https://aspire.dev/sitemap-index.xml", + "budgets": { + "release": 10000, + "rotation": 1000 + } +} diff --git a/src/frontend/package.json b/src/frontend/package.json index 4dec81b28f..fc2fe3f489 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -60,6 +60,7 @@ "update:github-stats": "tsx ./scripts/update-github-stats.ts", "update:release-contributors": "tsx ./scripts/update-release-contributors.ts", "update:samples": "tsx ./scripts/update-samples.ts", + "indexnow": "tsx ./scripts/indexnow.ts", "normalize:api-data": "tsx ./scripts/normalize-generated-api-data.ts", "validate:api-data": "tsx ./scripts/validate-generated-api-data.ts" }, diff --git a/src/frontend/public/c5f794ab28dc0cb92f48bd970500daa3.txt b/src/frontend/public/c5f794ab28dc0cb92f48bd970500daa3.txt new file mode 100644 index 0000000000..2b4995db98 --- /dev/null +++ b/src/frontend/public/c5f794ab28dc0cb92f48bd970500daa3.txt @@ -0,0 +1 @@ +c5f794ab28dc0cb92f48bd970500daa3 \ No newline at end of file diff --git a/src/frontend/scripts/indexnow.ts b/src/frontend/scripts/indexnow.ts new file mode 100644 index 0000000000..1d64963a88 --- /dev/null +++ b/src/frontend/scripts/indexnow.ts @@ -0,0 +1,809 @@ +/** + * Tells IndexNow-participating search engines (Bing, Yandex, Naver, Seznam, Yep, Amazon) about + * aspire.dev URLs. See https://www.indexnow.org/documentation. + * + * Every run reads the live sitemap, compares it with the URL list saved by the previous run and + * sends one request (IndexNow takes at most 10,000 URLs in one) with, in this order: + * 1. URLs that left the sitemap (so engines drop them) and URLs that are new in it, + * 2. the next window of a rotation that walks the whole sitemap: English pages first, then the + * API reference, then localized pages. + * + * The rotation is a cursor over that order, so it survives pages being added or removed. A release + * is a "flush": it starts the rotation again from the top at the larger release budget, and the + * daily runs finish that pass before the small rotation budget applies again. IndexNow does not + * publish its rate limits, so both budgets are in config/indexnow.json. A request that fails (a + * 429 included) leaves the progress alone, so the next run simply tries again; only a cold start + * still seeds the known list and a flush still starts its pass. A page that is new while a pass + * is running can be sent twice: when it appears and when the rotation reaches it. + * + * The sitemap has no `lastmod`, so the script cannot tell an updated page from an unchanged one. + * A release flush sends the English pages first, which is where the new and updated docs are, and + * the generated API reference after them. + * + * Besides config/indexnow.json, the script assumes that: + * - pages under /reference/api/ are the generated API reference, and the locale folders listed + * in config/locales.ts hold the translations; both are sent after the English pages, + * - release X.Y is merged from a release/X.Y branch and publishes /whats-new/aspire-X-Y/ (or + * aspire-X for X.0), which is how a run knows that the new docs are live. + * + * The state (state.json) lives in --state-dir. The workflow keeps it between runs as an Actions + * artifact; losing it only causes a cold start, which seeds the known URL list without submitting + * every URL as new. + * + * Usage (from src/frontend): + * pnpm indexnow --state-dir [--mode daily|flush] [--limit ] [--release ] + * [--trigger ] [--dry-run] [--allow-shrink] + */ +import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; + +import { locales } from '../config/locales'; +import { fetchWithProxy } from './fetch-with-proxy'; + +export type FetchLike = typeof fetchWithProxy; +export type RunMode = 'daily' | 'flush'; + +export interface IndexNowConfig { + host: string; + key: string; + endpoint: string; + sitemapIndex: string; + budgets: { release: number; rotation: number }; +} + +/** A pass over the sitemap. `cursor` is the last URL covered, in the order of `compareUrls`. */ +export interface Cycle { + kind: 'release' | 'rotation'; + cursor: string | null; +} + +export interface IndexNowState { + host: string; + updatedAt: string; + /** The pass in progress, or null when the last one finished and the next run starts a new one. */ + cycle: Cycle | null; + /** The sitemap URLs that were submitted or are left to the pass in progress. */ + known: string[]; +} + +/** What a run needs from the outside world. Tests replace it. */ +export interface Deps { + fetch: FetchLike; + sleep: (ms: number) => Promise; + now: () => Date; +} + +const CONFIG_PATH = fileURLToPath(new URL('../config/indexnow.json', import.meta.url)); +const STATE_FILE = 'state.json'; +const SCHEMA_VERSION = 1; +const USER_AGENT = 'aspire.dev-indexnow (+https://github.com/microsoft/aspire.dev)'; +/** The generated API reference, which is sent after the English pages. */ +const API_REFERENCE_PATH = '/reference/api/'; + +/** IndexNow takes at most 10,000 URLs per request, and a run sends a single request. */ +export const MAX_URLS_PER_RUN = 10_000; +/** A sitemap with fewer URLs than this share of the last known list is treated as broken. */ +export const SHRINK_THRESHOLD = 0.8; +/** Waits before the retries of a request that failed with a 5xx or a network error. */ +const RETRY_DELAYS_MS = [5_000, 30_000, 120_000]; +const GATE_INTERVAL_MS = 60_000; +const GATE_TIMEOUT_MS = 45 * 60_000; + +export const LOCALE_CODES: ReadonlySet = new Set( + Object.keys(locales).filter((code) => code !== 'root') +); + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function annotate(level: 'warning' | 'error', message: string): string { + return process.env.GITHUB_ACTIONS === 'true' + ? `::${level}::${message.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A')}` + : `${level}: ${message}`; +} + +const log = { + info: (message: string) => console.log(message), + warn: (message: string) => console.warn(annotate('warning', message)), + error: (message: string) => console.error(annotate('error', message)), +}; + +export async function loadConfig(configPath = CONFIG_PATH): Promise { + return JSON.parse(await readFile(configPath, 'utf8')) as IndexNowConfig; +} + +export function keyLocation(config: Pick): string { + return `https://${config.host}/${config.key}.txt`; +} + +// --------------------------------------------------------------------------------------------- +// Sitemap +// --------------------------------------------------------------------------------------------- + +const XML_ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }; + +/** + * Reads the `` values of a sitemap or sitemap index. The `xhtml:link` alternates that Astro + * adds carry their URL in an attribute and are ignored: every localized page is also listed as a + * `` of its own. + */ +export function parseSitemap(xml: string): { kind: 'index' | 'urlset'; locations: string[] } { + const root = /<(sitemapindex|urlset)[\s>]/.exec(xml); + if (!root) throw new Error('Not a sitemap: expected a or element'); + const locations = [...xml.matchAll(/\s*([^<]*?)\s*<\/loc>/g)].map((match) => + match[1].replace(/&(amp|lt|gt|quot|apos);/g, (_, name: string) => XML_ENTITIES[name]) + ); + return { kind: root[1] === 'sitemapindex' ? 'index' : 'urlset', locations }; +} + +/** The https URL on `host` without a fragment, or null for anything else. */ +function canonicalUrl(value: string, host: string): string | null { + try { + const url = new URL(value); + if (url.protocol !== 'https:' || url.hostname !== host) return null; + if (url.port || url.username || url.password) return null; + url.hash = ''; + return url.href; + } catch { + return null; + } +} + +/** Every page URL of the sitemap (following a sitemap index one level deep), de-duplicated. */ +export async function fetchSitemapUrls( + indexUrl: string, + host: string, + fetch: FetchLike +): Promise { + const load = async (url: string) => { + const response = await fetch(url, { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) throw new Error(`GET ${url} failed: HTTP ${response.status}`); + return parseSitemap(await response.text()); + }; + + const root = await load(indexUrl); + let locations = root.locations; + if (root.kind === 'index') { + locations = []; + for (const child of root.locations) { + if (canonicalUrl(child, host) === null) { + throw new Error(`Refusing to read sitemap ${child}: it is not an https URL on ${host}`); + } + const sitemap = await load(child); + if (sitemap.kind !== 'urlset') throw new Error(`${child} is a nested sitemap index`); + for (const location of sitemap.locations) locations.push(location); + } + } + + const urls = new Set(); + let ignored = 0; + for (const location of locations) { + const url = canonicalUrl(location, host); + if (url) urls.add(url); + else ignored++; + } + if (ignored > 0) log.warn(`Ignored ${ignored} sitemap URLs that are not https URLs on ${host}.`); + return [...urls]; +} + +// --------------------------------------------------------------------------------------------- +// Ordering and planning +// --------------------------------------------------------------------------------------------- + +/** + * Priority of a page. Lower tiers are sent first: English pages (docs, integrations, What's new), + * then the API reference, then localized pages. + */ +export function tierOf(pathname: string): 0 | 1 | 2 { + if (LOCALE_CODES.has(pathname.split('/')[1] ?? '')) return 2; + return pathname.startsWith(API_REFERENCE_PATH) ? 1 : 0; +} + +interface Entry { + url: string; + tier: number; +} + +const toEntry = (url: string): Entry => ({ url, tier: tierOf(new URL(url).pathname) }); + +function compareEntries(a: Entry, b: Entry): number { + return a.tier - b.tier || (a.url < b.url ? -1 : a.url > b.url ? 1 : 0); +} + +/** Orders URLs by (tier, URL), the order the rotation cursor walks. */ +export function compareUrls(a: string, b: string): number { + return compareEntries(toEntry(a), toEntry(b)); +} + +export interface PlanInput { + /** Normalized, de-duplicated sitemap URLs. */ + current: readonly string[]; + state: IndexNowState | null; + mode: RunMode; + budgets: IndexNowConfig['budgets']; + /** Replaces the budget for this run. It is a hard cap: it bounds the removed and new URLs too. */ + limit?: number; + allowShrink?: boolean; +} + +export interface Plan { + coldStart: boolean; + mode: RunMode; + budget: number; + /** The pass this run works on (continued or newly started). */ + cycle: Cycle; + total: number; + /** Every URL that is in the sitemap but was not known. */ + newUrls: string[]; + /** Every known URL that left the sitemap. */ + removedUrls: string[]; + /** The removed URLs sent this run. */ + removed: string[]; + /** The new URLs sent ahead of the window. A flush leaves them to its window. */ + fresh: string[]; + /** The next URLs of the rotation. */ + window: string[]; + /** True when sending the window finishes the pass. */ + windowComplete: boolean; + /** How many sitemap URLs the pass has covered once the window is sent. */ + covered: number; +} + +/** + * Decides what to send. Pure: it does not touch the network or the disk. + * + * - A cold start (no state) seeds the known list and never treats the whole sitemap as new. + * - A flush starts a release pass from the top; otherwise the pass in progress continues, or a new + * rotation pass begins when the previous one has finished. + * - Removed and new URLs go first and count against the budget. A flush is the exception for new + * URLs: its pass covers the whole sitemap, so it leaves them to the window. + */ +export function planRun(input: PlanInput): Plan { + const { state, mode } = input; + const currentSet = new Set(input.current); + const total = currentSet.size; + if (total === 0) throw new Error('The sitemap lists no URLs; refusing to continue'); + + const previous = state?.known ?? []; + if (state && !input.allowShrink && total < previous.length * SHRINK_THRESHOLD) { + throw new Error( + `The sitemap lists ${total} URLs, more than ${Math.round((1 - SHRINK_THRESHOLD) * 100)}% ` + + `fewer than the ${previous.length} known from the last run. Refusing to continue in case ` + + 'the site was only partially deployed; rerun with --allow-shrink if the drop is intended.' + ); + } + + const entries = [...currentSet].map(toEntry).sort(compareEntries); + const knownSet = new Set(previous); + const newUrls = state ? entries.filter((e) => !knownSet.has(e.url)).map((e) => e.url) : []; + const removedUrls = previous + .filter((url) => !currentSet.has(url)) + .map(toEntry) + .sort(compareEntries) + .map((entry) => entry.url); + + const cycle: Cycle = + mode === 'flush' + ? { kind: 'release', cursor: null } + : (state?.cycle ?? { kind: 'rotation', cursor: null }); + const configured = cycle.kind === 'release' ? input.budgets.release : input.budgets.rotation; + const budget = Math.min(input.limit ?? configured, MAX_URLS_PER_RUN); + + const cap = Math.min(input.limit ?? MAX_URLS_PER_RUN, MAX_URLS_PER_RUN); + const removed = removedUrls.slice(0, cap); + const fresh = (mode === 'flush' ? [] : newUrls).slice(0, cap - removed.length); + const room = Math.max(0, budget - removed.length - fresh.length); + + let start = 0; + if (cycle.cursor !== null) { + const target = toEntry(cycle.cursor); + start = entries.findIndex((entry) => compareEntries(entry, target) > 0); + if (start === -1) start = entries.length; + } + const window = entries.slice(start, start + room).map((entry) => entry.url); + + return { + coldStart: state === null, + mode, + budget, + cycle, + total, + newUrls, + removedUrls, + removed, + fresh, + window, + windowComplete: start + window.length >= total, + covered: start + window.length, + }; +} + +export interface FinalizeOptions { + host: string; + /** The URLs the plan was computed from. */ + current: readonly string[]; + /** Whether IndexNow accepted the request. */ + submitted: boolean; + now: Date; +} + +/** + * Builds the state to remember after a run. URLs count as done only when the request was + * accepted: a new URL that was not sent stays new, a removed URL that was not sent stays known, and + * the cursor moves to the last URL of the window. The new URLs of a flush count as known right + * away, because the pass it starts reaches them. + */ +export function finalizeRun(plan: Plan, options: FinalizeOptions): IndexNowState { + const { host, current, submitted, now } = options; + const sent = new Set(submitted ? [...plan.removed, ...plan.fresh, ...plan.window] : []); + const known = new Set(current); + if (plan.mode !== 'flush') { + for (const url of plan.newUrls) if (!sent.has(url)) known.delete(url); + } + for (const url of plan.removedUrls) if (!sent.has(url)) known.add(url); + + const cursor = submitted ? (plan.window.at(-1) ?? plan.cycle.cursor) : plan.cycle.cursor; + return { + host, + updatedAt: now.toISOString(), + cycle: submitted && plan.windowComplete ? null : { ...plan.cycle, cursor }, + known: [...known].sort(), + }; +} + +// --------------------------------------------------------------------------------------------- +// State on disk +// --------------------------------------------------------------------------------------------- + +function isCycle(value: unknown): value is Cycle | null { + return ( + value === null || + (isRecord(value) && + (value.kind === 'release' || value.kind === 'rotation') && + (value.cursor === null || typeof value.cursor === 'string')) + ); +} + +/** Returns null for a missing state (a cold start) and for one that cannot be trusted. */ +export async function loadState(dir: string, host: string): Promise { + let raw: string; + try { + raw = await readFile(path.join(dir, STATE_FILE), 'utf8'); + } catch (error) { + if (isRecord(error) && error.code === 'ENOENT') return null; + throw error; + } + + try { + const file: unknown = JSON.parse(raw); + if (!isRecord(file) || file.schemaVersion !== SCHEMA_VERSION || file.host !== host) { + throw new Error('it was written for another schema version or host'); + } + const { updatedAt, cycle, known } = file; + // The state comes from an artifact, so anything that is not a page of this site is not trusted. + if ( + typeof updatedAt !== 'string' || + !isCycle(cycle) || + !Array.isArray(known) || + known.length === 0 || + !known.every((url) => typeof url === 'string' && canonicalUrl(url, host) === url) || + (typeof cycle?.cursor === 'string' && canonicalUrl(cycle.cursor, host) !== cycle.cursor) + ) { + throw new Error('it is not in the expected format'); + } + return { host, updatedAt, cycle, known: known as string[] }; + } catch (error) { + log.warn(`Ignoring unusable IndexNow state (${errorMessage(error)}); starting cold.`); + return null; + } +} + +export async function saveState(dir: string, state: IndexNowState): Promise { + await mkdir(dir, { recursive: true }); + const content = JSON.stringify({ schemaVersion: SCHEMA_VERSION, ...state }, null, 2); + await writeFile(path.join(dir, STATE_FILE), `${content}\n`, 'utf8'); +} + +// --------------------------------------------------------------------------------------------- +// Talking to IndexNow and the site +// --------------------------------------------------------------------------------------------- + +export type Submission = { ok: true } | { ok: false; status: number; detail: string }; + +/** + * Sends the URLs in a single request. A 5xx or a network error is retried after a back-off; any + * other answer, a 429 included, ends the attempt, and the next run tries again. + */ +export async function submitUrls( + config: IndexNowConfig, + urls: readonly string[], + deps: Pick +): Promise { + const body = JSON.stringify({ + host: config.host, + key: config.key, + keyLocation: keyLocation(config), + urlList: urls, + }); + + let status = 0; + let detail = ''; + for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) { + if (attempt > 0) { + const delay = RETRY_DELAYS_MS[attempt - 1]; + log.warn( + `IndexNow answered ${status || 'with a network error'}; retrying in ${delay / 1000}s.` + ); + await deps.sleep(delay); + } + try { + const response = await deps.fetch(config.endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': USER_AGENT }, + body, + signal: AbortSignal.timeout(60_000), + }); + status = response.status; + detail = (await response.text()).trim().slice(0, 300); + } catch (error) { + status = 0; + detail = errorMessage(error); + } + // 202 is what IndexNow answers while it is still verifying a new key. + if (status === 200 || status === 202) return { ok: true }; + if (status !== 0 && status < 500) break; + } + return { ok: false, status, detail }; +} + +/** IndexNow verifies ownership by fetching the key file, so it has to be live before submitting. */ +export async function keyFileProblem( + config: IndexNowConfig, + fetch: FetchLike +): Promise { + const url = keyLocation(config); + try { + const response = await fetch(url, { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) return `GET ${url} returned HTTP ${response.status}`; + return (await response.text()).trim() === config.key + ? null + : `${url} does not contain the configured key`; + } catch (error) { + return `GET ${url} failed: ${errorMessage(error)}`; + } +} + +/** The What's new page of a release: aspire-13-6 for 13.6, aspire-14 for 14.0. */ +export function whatsNewPath(version: string): string { + const match = /^(\d+)\.(\d+)$/.exec(version); + if (!match) throw new Error(`"${version}" is not a release version such as 13.6`); + const [major, minor] = [Number(match[1]), Number(match[2])]; + return minor === 0 ? `/whats-new/aspire-${major}/` : `/whats-new/aspire-${major}-${minor}/`; +} + +/** + * Waits for the release's What's new page to be served. The release merge normally lands right + * after the slot swap; this protects against sending URLs while the old build is still live, which + * would make engines crawl stale content. + */ +export async function waitForRelease( + host: string, + version: string, + deps: Deps, + { timeoutMs = GATE_TIMEOUT_MS, intervalMs = GATE_INTERVAL_MS } = {} +): Promise { + const url = `https://${host}${whatsNewPath(version)}`; + const deadline = deps.now().getTime() + timeoutMs; + for (;;) { + try { + const response = await deps.fetch(url, { + method: 'HEAD', + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(30_000), + }); + if (response.ok) return true; + } catch { + // Not reachable yet counts as not live. + } + if (deps.now().getTime() + intervalMs > deadline) return false; + log.info(`Release ${version} is not live yet; checking again in ${intervalMs / 1000}s.`); + await deps.sleep(intervalMs); + } +} + +/** + * The version of a merged release pull request: from this repository, merged, from a + * `release/X.Y` branch. Release pull requests are squash-merged, so Git ancestry cannot be used. + */ +export function releaseVersion(event: unknown): string | null { + const pull = isRecord(event) ? event.pull_request : undefined; + if (!isRecord(pull) || pull.merged !== true) return null; + const repository = (side: unknown) => + isRecord(side) && isRecord(side.repo) ? side.repo.full_name : undefined; + const ref = isRecord(pull.head) ? pull.head.ref : undefined; + if (typeof ref !== 'string' || typeof repository(pull.head) !== 'string') return null; + if (repository(pull.head) !== repository(pull.base)) return null; + return /^release\/(\d+\.\d+)$/.exec(ref)?.[1] ?? null; +} + +// --------------------------------------------------------------------------------------------- +// Run +// --------------------------------------------------------------------------------------------- + +export interface RunOptions extends Partial { + config: IndexNowConfig; + stateDir: string; + mode: RunMode; + trigger: string; + /** Wait for this release's What's new page before submitting. */ + release?: string; + limit?: number; + dryRun?: boolean; + allowShrink?: boolean; +} + +export interface RunResult { + outcome: 'completed' | 'dry-run' | 'skipped' | 'failed'; + /** URLs accepted by IndexNow, or planned in a dry run. */ + submitted: number; + stateChanged: boolean; + /** Markdown for the job summary page. */ + summary: string; + /** Set when the run failed after it had started submitting. */ + failure?: string; +} + +const count = (value: number) => value.toLocaleString('en-US'); + +function summarize(host: string, outcome: RunResult['outcome'], lines: (string | false)[]): string { + const bullets = lines.filter(Boolean).map((line) => `- ${line}`); + return [`## IndexNow submission for ${host}: ${outcome}`, '', ...bullets, ''].join('\n'); +} + +function skipped(host: string, note: string): RunResult { + const summary = summarize(host, 'skipped', [note]); + return { outcome: 'skipped', submitted: 0, stateChanged: false, summary }; +} + +function failureMessage(submission: { status: number; detail: string }, mode: RunMode): string { + const reason = + submission.status === 429 + ? 'IndexNow answered 429 (too many requests), so nothing was submitted. If this keeps happening, lower the budgets in config/indexnow.json.' + : `IndexNow rejected the submission with HTTP ${submission.status || 'a network error'}` + + `${submission.detail ? `: ${submission.detail}` : ''}. ` + + 'For 403 or 422, check that the key file is live and matches config/indexnow.json.'; + return mode === 'flush' + ? `${reason} The release pass was started, so the next run sends it from the top.` + : `${reason} The next run tries again.`; +} + +export async function run(options: RunOptions): Promise { + const { config, stateDir, mode, trigger, release, dryRun = false } = options; + const { host } = config; + const deps: Deps = { + fetch: options.fetch ?? fetchWithProxy, + sleep: options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), + now: options.now ?? (() => new Date()), + }; + + const state = await loadState(stateDir, host); + log.info( + state + ? `Loaded state from ${state.updatedAt}: ${state.known.length} known URLs.` + : 'No previous state; this is a cold start.' + ); + + const keyProblem = await keyFileProblem(config, deps.fetch); + if (keyProblem) { + if (dryRun) { + log.warn(`${keyProblem}. IndexNow would reject submissions until the key file is live.`); + } else if (state === null) { + // Before the first deployment that ships the key file there is nothing to submit yet. + const note = `The IndexNow key file is not live yet (${keyProblem}); skipping this run.`; + log.warn(note); + return skipped(host, note); + } else { + throw new Error(`The IndexNow key file is not reachable: ${keyProblem}`); + } + } + + if (release) { + // A dry run only checks once. + const gate = dryRun ? { timeoutMs: 0 } : undefined; + if (!(await waitForRelease(host, release, deps, gate))) { + if (!dryRun) { + throw new Error( + `The What's new page for release ${release} (${whatsNewPath(release)}) did not go live in ` + + `time. Once the new docs are live, run the workflow manually with mode "flush" and release "${release}".` + ); + } + log.warn(`The What's new page for release ${release} is not live yet.`); + } + } + + const urls = await fetchSitemapUrls(config.sitemapIndex, host, deps.fetch); + const plan = planRun({ + current: urls, + state, + mode, + budgets: config.budgets, + limit: options.limit, + allowShrink: options.allowShrink, + }); + const ordered = [...plan.removed, ...plan.fresh, ...plan.window]; + const facts = [ + `Sitemap: ${count(plan.total)} URLs, ${count(plan.newUrls.length)} new and ${count(plan.removedUrls.length)} removed since the last run`, + `Plan: ${count(ordered.length)} URLs (${count(plan.removed.length)} removed, ${count(plan.fresh.length)} new, ${count(plan.window.length)} from the ${plan.cycle.kind} pass) with a budget of ${count(plan.budget)}`, + ]; + for (const fact of facts) log.info(fact); + const describe = (outcome: RunResult['outcome'], covered: number, finished: boolean, note = '') => + summarize(host, outcome, [ + `Trigger \`${trigger}\`, mode \`${mode}\`${release ? `, release ${release}` : ''}`, + ...facts, + `Pass: ${plan.cycle.kind}, ${count(covered)} of ${count(plan.total)} URLs covered (${Math.floor((covered / plan.total) * 100)}%)${finished ? ', complete' : ''}`, + plan.coldStart && `Cold start: the known URL list ${dryRun ? 'would be' : 'was'} seeded`, + note, + ]); + if (dryRun) { + const note = 'Dry run: nothing was submitted and the state was not saved.'; + const summary = describe('dry-run', plan.covered, plan.windowComplete, note); + return { outcome: 'dry-run', submitted: ordered.length, stateChanged: false, summary }; + } + + const submission: Submission = + ordered.length === 0 ? { ok: true } : await submitUrls(config, ordered, deps); + + // A failed run leaves the state alone, except that a cold start still seeds the known list and a + // flush still starts the release pass: dropping that would lose the burst, which the next run + // then carries out. + const stateChanged = submission.ok || plan.coldStart || plan.mode === 'flush'; + if (stateChanged) { + const next = finalizeRun(plan, { + host, + current: urls, + submitted: submission.ok, + now: deps.now(), + }); + await saveState(stateDir, next); + } + + if (submission.ok) { + const summary = describe('completed', plan.covered, plan.windowComplete); + return { outcome: 'completed', submitted: ordered.length, stateChanged, summary }; + } + const failure = failureMessage(submission, plan.mode); + const summary = describe('failed', plan.covered - plan.window.length, false, failure); + return { outcome: 'failed', submitted: 0, stateChanged, summary, failure }; +} + +// --------------------------------------------------------------------------------------------- +// Command line +// --------------------------------------------------------------------------------------------- + +export interface CliOptions { + stateDir: string; + mode: RunMode; + limit?: number; + release?: string; + trigger: string; + dryRun: boolean; + allowShrink: boolean; +} + +export function parseCli(argv: readonly string[]): CliOptions { + const { values } = parseArgs({ + args: [...argv], + options: { + 'state-dir': { type: 'string' }, + mode: { type: 'string' }, + limit: { type: 'string' }, + release: { type: 'string' }, + trigger: { type: 'string' }, + 'dry-run': { type: 'boolean' }, + 'allow-shrink': { type: 'boolean' }, + }, + strict: true, + allowPositionals: false, + }); + + if (!values['state-dir']) throw new Error('--state-dir is required'); + const mode = values.mode ?? 'daily'; + if (mode !== 'daily' && mode !== 'flush') throw new Error('--mode must be "daily" or "flush"'); + + let limit: number | undefined; + if (values.limit !== undefined) { + limit = Number(values.limit); + if (!Number.isInteger(limit) || limit < 1 || limit > MAX_URLS_PER_RUN) { + throw new Error(`--limit must be an integer between 1 and ${MAX_URLS_PER_RUN}`); + } + } + if (values.release !== undefined && !/^\d+\.\d+$/.test(values.release)) { + throw new Error('--release must look like 13.6'); + } + + return { + stateDir: path.resolve(values['state-dir']), + mode, + limit, + release: values.release, + trigger: values.trigger || 'manual', + dryRun: values['dry-run'] ?? false, + allowShrink: values['allow-shrink'] ?? false, + }; +} + +/** Runs the CLI and returns the process exit code. */ +export async function main( + argv: readonly string[] = process.argv.slice(2), + env: NodeJS.ProcessEnv = process.env, + overrides: Partial = {} +): Promise { + let host = ''; + const publish = async (summary: string, stateChanged: boolean) => { + if (env.GITHUB_STEP_SUMMARY) await appendFile(env.GITHUB_STEP_SUMMARY, summary); + if (env.GITHUB_OUTPUT) await appendFile(env.GITHUB_OUTPUT, `state_changed=${stateChanged}\n`); + }; + + try { + const options = parseCli(argv); + const config = await loadConfig(); + host = config.host; + + let { mode, release } = options; + if (options.trigger === 'pull_request') { + if (!env.GITHUB_EVENT_PATH) throw new Error('GITHUB_EVENT_PATH is not set'); + const event: unknown = JSON.parse(await readFile(env.GITHUB_EVENT_PATH, 'utf8')); + const version = releaseVersion(event); + if (!version) { + const note = + 'This pull request is not a merged release/X.Y pull request; nothing to submit.'; + log.info(note); + await publish(skipped(host, note).summary, false); + return 0; + } + [mode, release] = ['flush', version]; + } + + const result = await run({ + ...overrides, + config, + stateDir: options.stateDir, + mode, + trigger: options.trigger, + release, + limit: options.limit, + dryRun: options.dryRun, + allowShrink: options.allowShrink, + }); + await publish(result.summary, result.stateChanged); + if (result.failure) { + log.error(result.failure); + return 1; + } + return 0; + } catch (error) { + const message = errorMessage(error); + log.error(message); + await publish(summarize(host, 'failed', [message]), false).catch(() => undefined); + return 1; + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + void main().then((code) => { + process.exitCode = code; + }); +} diff --git a/src/frontend/tests/unit/indexnow.vitest.test.ts b/src/frontend/tests/unit/indexnow.vitest.test.ts new file mode 100644 index 0000000000..b232bd838e --- /dev/null +++ b/src/frontend/tests/unit/indexnow.vitest.test.ts @@ -0,0 +1,1037 @@ +import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { Response } from 'node-fetch'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +import type { fetchWithProxy } from '../../scripts/fetch-with-proxy'; +import { + MAX_URLS_PER_RUN, + compareUrls, + fetchSitemapUrls, + finalizeRun, + keyFileProblem, + loadConfig, + loadState, + main, + parseCli, + parseSitemap, + planRun, + releaseVersion, + run, + saveState, + submitUrls, + tierOf, + waitForRelease, + whatsNewPath, + type IndexNowConfig, + type IndexNowState, + type RunOptions, +} from '../../scripts/indexnow'; + +const HOST = 'aspire.dev'; +const KEY = 'a1b2c3d4e5f60718293a4b5c6d7e8f90'; +const NOW = new Date('2026-10-13T06:43:00.000Z'); + +const config: IndexNowConfig = { + host: HOST, + key: KEY, + endpoint: 'https://api.indexnow.org/indexnow', + sitemapIndex: `https://${HOST}/sitemap-index.xml`, + budgets: { release: 5, rotation: 3 }, +}; + +const url = (pathname: string) => `https://${HOST}${pathname}`; +const urls = (paths: string[]) => paths.map(url); + +// Ten pages in the order they are sent: four English pages, three API pages, three localized pages. +const DOCS = ['/', '/app-host/', '/get-started/', '/whats-new/aspire-13-6/']; +const API = [ + '/reference/api/csharp/a/', + '/reference/api/csharp/b/', + '/reference/api/typescript/c/', +]; +const LOCALIZED = ['/de/', '/de/app-host/', '/ja/']; +const SMALL_SITE = [...DOCS, ...API, ...LOCALIZED]; + +function syntheticUrls(count: number, prefix = '/reference/api/csharp/page-'): string[] { + return Array.from({ length: count }, (_, index) => + url(`${prefix}${String(index).padStart(6, '0')}/`) + ); +} + +function makeState(overrides: Partial = {}): IndexNowState { + return { host: HOST, updatedAt: NOW.toISOString(), cycle: null, known: [], ...overrides }; +} + +/** A clock that only moves when the code under test sleeps. */ +function createClock() { + let current = NOW.getTime(); + return { + now: () => new Date(current), + sleep: vi.fn<(ms: number) => Promise>((ms) => { + current += ms; + return Promise.resolve(); + }), + }; +} + +interface PostCall { + host: string; + key: string; + keyLocation: string; + urlList: string[]; +} + +/** A stand-in for aspire.dev and the IndexNow endpoint. */ +function createSite(paths: string[], key = KEY) { + const site = { + paths: [...paths], + keyLive: true, + liveReleases: new Set(), + /** Statuses for the upcoming IndexNow POSTs; 200 when the queue is empty. */ + responses: [] as number[], + posts: [] as PostCall[], + fetch: vi.fn(), + }; + + const urlset = () => + '' + + site.paths + .map( + (pathname) => + `${url(pathname)}` + ) + .join('') + + ''; + + site.fetch.mockImplementation((input, init) => { + const target = new URL(String(input)); + if (target.origin === 'https://api.indexnow.org') { + site.posts.push(JSON.parse(init?.body as string) as PostCall); + return Promise.resolve(new Response('', { status: site.responses.shift() ?? 200 })); + } + if (target.pathname === `/${key}.txt`) { + return Promise.resolve( + site.keyLive ? new Response(key) : new Response('Not found', { status: 404 }) + ); + } + if (target.pathname === '/sitemap-index.xml') { + return Promise.resolve( + new Response( + `${url('/sitemap-0.xml')}` + ) + ); + } + if (target.pathname === '/sitemap-0.xml') return Promise.resolve(new Response(urlset())); + const release = /^\/whats-new\/aspire-(\d+)(?:-(\d+))?\/$/.exec(target.pathname); + if (release) { + const live = site.liveReleases.has(`${release[1]}.${release[2] ?? '0'}`); + return Promise.resolve(new Response('', { status: live ? 200 : 404 })); + } + return Promise.resolve(new Response('Not found', { status: 404 })); + }); + + return site; +} + +let stateDir: string; + +beforeEach(async () => { + stateDir = await mkdtemp(path.join(os.tmpdir(), 'indexnow-test-')); + for (const method of ['log', 'warn', 'error'] as const) { + vi.spyOn(console, method).mockImplementation(() => undefined); + } +}); + +afterEach(async () => { + vi.restoreAllMocks(); + await rm(stateDir, { recursive: true, force: true }); +}); + +describe('config', () => { + test('the committed config is valid and its key file is served from the site root', async () => { + const real = await loadConfig(); + expect(real.host).toBe('aspire.dev'); + expect(real.key).toMatch(/^[A-Za-z0-9-]{8,128}$/); + expect(new URL(real.endpoint).protocol).toBe('https:'); + expect(new URL(real.sitemapIndex)).toMatchObject({ protocol: 'https:', hostname: real.host }); + // One request carries a whole run, so a budget cannot exceed what a request may hold. + for (const budget of Object.values(real.budgets)) { + expect(Number.isInteger(budget)).toBe(true); + expect(budget).toBeGreaterThanOrEqual(1); + expect(budget).toBeLessThanOrEqual(MAX_URLS_PER_RUN); + } + + const keyFile = fileURLToPath(new URL(`../../public/${real.key}.txt`, import.meta.url)); + const contents = await readFile(keyFile, 'utf8'); + expect(contents.charCodeAt(0)).not.toBe(0xfeff); + expect(contents.trim()).toBe(real.key); + }); +}); + +describe('sitemap', () => { + const urlset = (...locations: string[]) => + `${locations.map((location) => `${location}`).join('')}`; + + test('reads values, decodes entities and ignores xhtml alternates', () => { + const parsed = parseSitemap( + '' + + 'https://aspire.dev/' + + ' https://aspire.dev/a?b=1&c=2 ' + + '' + ); + expect(parsed).toEqual({ + kind: 'urlset', + locations: ['https://aspire.dev/', 'https://aspire.dev/a?b=1&c=2'], + }); + }); + + test('recognizes a sitemap index and rejects documents that are not sitemaps', () => { + expect( + parseSitemap( + 'https://aspire.dev/s.xml' + ) + ).toEqual({ kind: 'index', locations: ['https://aspire.dev/s.xml'] }); + expect(() => parseSitemap('Not found')).toThrow(/not a sitemap/i); + }); + + test('follows a sitemap index to its child sitemaps', async () => { + const site = createSite(['/', '/a/']); + expect(await fetchSitemapUrls(config.sitemapIndex, HOST, site.fetch)).toEqual( + urls(['/', '/a/']) + ); + expect(site.fetch).toHaveBeenCalledTimes(2); + }); + + test('keeps https pages on the host, drops fragments and de-duplicates', async () => { + const fetch = vi + .fn() + .mockResolvedValue( + new Response( + urlset( + 'https://aspire.dev/a/', + 'https://aspire.dev/a/#section', + 'https://ASPIRE.dev/b/', + 'http://aspire.dev/c/', + 'https://aspire.dev:8443/d/', + 'https://example.com/f/', + 'not a url' + ) + ) + ); + expect(await fetchSitemapUrls(config.sitemapIndex, HOST, fetch)).toEqual(urls(['/a/', '/b/'])); + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('Ignored 4')); + }); + + test('refuses child sitemaps on another host, nested indexes and failed requests', async () => { + const index = (child: string) => + new Response(`${child}`); + const fetch = vi.fn(); + + fetch.mockResolvedValueOnce(index('https://evil.example/s.xml')); + await expect(fetchSitemapUrls(config.sitemapIndex, HOST, fetch)).rejects.toThrow( + /not an https URL on aspire\.dev/ + ); + + fetch.mockResolvedValueOnce(index(url('/a.xml'))).mockResolvedValueOnce(index(url('/b.xml'))); + await expect(fetchSitemapUrls(config.sitemapIndex, HOST, fetch)).rejects.toThrow(/nested/); + + fetch.mockResolvedValueOnce(new Response('oops', { status: 503 })); + await expect(fetchSitemapUrls(config.sitemapIndex, HOST, fetch)).rejects.toThrow(/HTTP 503/); + }); +}); + +describe('ordering', () => { + test('puts English pages first, then the API reference, then localized pages', () => { + expect(tierOf('/')).toBe(0); + expect(tierOf('/app-host/certificate-configuration/')).toBe(0); + expect(tierOf('/reference/cli/')).toBe(0); + expect(tierOf('/reference/api')).toBe(0); + expect(tierOf('/deployment/')).toBe(0); + expect(tierOf('/reference/api/csharp/aspire.hosting/')).toBe(1); + expect(tierOf('/zh-cn/whats-new/upgrade-aspire/')).toBe(2); + expect(tierOf('/pt-br/')).toBe(2); + expect(tierOf('/de/reference/api/csharp/x/')).toBe(2); + }); + + test('sorts by tier and then by URL', () => { + const shuffled = urls([...SMALL_SITE].reverse()); + expect(shuffled.sort(compareUrls)).toEqual(urls(SMALL_SITE)); + }); +}); + +describe('planning', () => { + const base = { + current: urls(SMALL_SITE), + state: null as IndexNowState | null, + mode: 'daily' as const, + budgets: config.budgets, + }; + const rotation = (cursor: string | null) => ({ kind: 'rotation' as const, cursor }); + const known = urls(SMALL_SITE); + + test('a cold start seeds the known list instead of treating every URL as new', () => { + const plan = planRun(base); + expect(plan).toMatchObject({ + coldStart: true, + newUrls: [], + removedUrls: [], + fresh: [], + windowComplete: false, + cycle: { kind: 'rotation', cursor: null }, + }); + expect(plan.window).toEqual(urls(['/', '/app-host/', '/get-started/'])); + }); + + test('a pass continues after its cursor, even when the cursor left the sitemap', () => { + for (const cursor of ['/get-started/', '/get-started/gone/']) { + const state = makeState({ known, cycle: rotation(url(cursor)) }); + const plan = planRun({ ...base, state }); + expect(plan.coldStart).toBe(false); + expect(plan.window).toEqual(urls([DOCS[3], API[0], API[1]])); + } + }); + + test('a flush starts a release pass from the top and leaves new URLs to its window', () => { + const created = url('/reference/api/csharp/ab/'); + const state = makeState({ known, cycle: rotation(url('/ja/')) }); + const plan = planRun({ ...base, current: [...known, created], state, mode: 'flush' }); + expect(plan.cycle).toEqual({ kind: 'release', cursor: null }); + expect(plan.budget).toBe(5); + expect(plan.newUrls).toEqual([created]); + expect(plan.fresh).toEqual([]); + expect(plan.window).toEqual(urls([...DOCS, API[0]])); + }); + + test('a release pass keeps its larger budget until it finishes', () => { + const state = makeState({ known, cycle: { kind: 'release', cursor: url('/get-started/') } }); + const plan = planRun({ ...base, state }); + expect(plan.budget).toBe(5); + expect(plan.window).toEqual(urls([DOCS[3], ...API, LOCALIZED[0]])); + }); + + test('removed and new URLs go first and count against the budget', () => { + const current = [...known.filter((entry) => entry !== url('/ja/')), url('/aaa/')]; + const state = makeState({ known, cycle: rotation(url('/get-started/')) }); + const plan = planRun({ ...base, current, state }); + expect(plan.removed).toEqual([url('/ja/')]); + expect(plan.fresh).toEqual([url('/aaa/')]); + expect(plan.window).toEqual([url(DOCS[3])]); + }); + + test('--limit replaces the budget and also caps the removed and new URLs', () => { + expect(planRun({ ...base, limit: 2 }).window).toHaveLength(2); + + const gone = [url(API[2]), url('/ja/')]; + const removing = planRun({ + ...base, + current: known.filter((entry) => !gone.includes(entry)), + state: makeState({ known }), + limit: 1, + }); + expect(removing.removedUrls).toEqual(gone); + expect(removing.removed).toEqual([gone[0]]); + expect(removing.window).toEqual([]); + + const created = urls(['/aaa/', '/aab/', '/aac/']); + const adding = planRun({ + ...base, + current: [...known, ...created], + state: makeState({ known }), + limit: 2, + }); + expect(adding.newUrls).toHaveLength(3); + expect(adding.fresh).toEqual(created.slice(0, 2)); + expect(adding.window).toEqual([]); + }); + + test('never plans more than one request, whatever the budget', () => { + const current = syntheticUrls(12_000, '/docs/page-'); + const budgets = { release: 50_000, rotation: 50_000 }; + const plan = planRun({ ...base, current, budgets, mode: 'flush' }); + expect(plan.budget).toBe(MAX_URLS_PER_RUN); + expect(plan.window).toHaveLength(MAX_URLS_PER_RUN); + }); + + test('refuses an empty sitemap', () => { + expect(() => planRun({ ...base, current: [] })).toThrow(/no URLs/); + }); + + test('refuses a sitemap that shrank by more than 20% unless allowed', () => { + const before = syntheticUrls(100); + const state = makeState({ known: before }); + expect(() => planRun({ ...base, current: before.slice(0, 79), state })).toThrow( + /--allow-shrink/ + ); + expect(() => planRun({ ...base, current: before.slice(0, 80), state })).not.toThrow(); + const allowed = planRun({ ...base, current: before.slice(0, 79), state, allowShrink: true }); + expect(allowed.removedUrls).toHaveLength(21); + }); +}); + +describe('finalizing a run', () => { + const known = urls(SMALL_SITE); + const base = { current: known, mode: 'daily' as const, budgets: config.budgets }; + const options = (current: string[], submitted = true) => ({ + host: HOST, + current, + submitted, + now: NOW, + }); + + test('remembers the whole sitemap and moves the cursor once the request was accepted', () => { + const plan = planRun({ ...base, state: null }); + const next = finalizeRun(plan, options(known)); + expect(next.known).toEqual([...known].sort()); + expect(next.cycle).toEqual({ kind: 'rotation', cursor: url('/get-started/') }); + expect(next.updatedAt).toBe(NOW.toISOString()); + }); + + test('finishes the pass when the window reaches the end of the sitemap', () => { + const state = makeState({ known, cycle: { kind: 'release', cursor: url(API[2]) } }); + const plan = planRun({ ...base, state }); + expect(plan.window).toEqual(urls(LOCALIZED)); + expect(plan.windowComplete).toBe(true); + expect(finalizeRun(plan, options(known)).cycle).toBeNull(); + expect(finalizeRun(plan, options(known, false)).cycle).toEqual(state.cycle); + }); + + test('keeps URLs that were not sent pending', () => { + const created = urls(['/aaa/', '/aab/']); + const current = [...known.filter((entry) => entry !== url('/ja/')), ...created]; + const plan = planRun({ ...base, current, state: makeState({ known }) }); + expect(plan.removed).toEqual([url('/ja/')]); + expect(plan.fresh).toEqual(created); + + const failed = finalizeRun(plan, options(current, false)); + expect(failed.known).toContain(url('/ja/')); + expect(failed.known).not.toContain(created[0]); + expect(failed.known).not.toContain(created[1]); + expect(failed.cycle).toEqual({ kind: 'rotation', cursor: null }); + + const accepted = finalizeRun(plan, options(current)); + expect(accepted.known).not.toContain(url('/ja/')); + expect(accepted.known).toEqual(expect.arrayContaining(created)); + }); + + test('keeps the new URLs beyond the per-run cap pending', () => { + const before = syntheticUrls(3); + const created = syntheticUrls(10_005, '/reference/api/csharp/new-'); + const current = [...before, ...created]; + const plan = planRun({ ...base, current, state: makeState({ known: before }) }); + expect(plan.fresh).toHaveLength(MAX_URLS_PER_RUN); + + const next = finalizeRun(plan, options(current)); + expect(current.filter((entry) => !next.known.includes(entry))).toEqual(created.slice(10_000)); + }); + + test('a flush counts its new URLs as known even when the request failed', () => { + const created = url('/reference/api/csharp/ab/'); + const current = [...known, created]; + const plan = planRun({ ...base, current, state: makeState({ known }), mode: 'flush' }); + const next = finalizeRun(plan, options(current, false)); + expect(next.known).toContain(created); + expect(next.cycle).toEqual({ kind: 'release', cursor: null }); + }); + + test('a new page ahead of the cursor is sent now and again when the pass reaches it', () => { + const created = url('/reference/api/csharp/ab/'); + const current = [...known, created]; + const state = makeState({ known, cycle: { kind: 'rotation', cursor: url('/get-started/') } }); + const plan = planRun({ ...base, current, state }); + expect(plan.fresh).toEqual([created]); + + const later = planRun({ ...base, current, state: finalizeRun(plan, options(current)) }); + expect(later.window).toContain(created); + }); + + test('a release burst over a sitemap shaped like aspire.dev takes four runs and sends each URL once', () => { + const docs = syntheticUrls(611, '/docs/page-'); + const api = syntheticUrls(24_207); + const localized = syntheticUrls(7_518, '/de/page-'); + const current = [...localized, ...api, ...docs]; + const budgets = { release: 10_000, rotation: 1_000 }; + + let state = makeState({ known: current }); + const sizes: number[] = []; + const seen = new Set(); + for (let day = 0; day < 10; day++) { + const plan = planRun({ + current, + state, + mode: day === 0 ? 'flush' : 'daily', + budgets, + }); + if (day === 0) expect(plan.window.slice(0, 611)).toEqual(docs); + for (const entry of plan.window) { + expect(seen.has(entry)).toBe(false); + seen.add(entry); + } + sizes.push(plan.window.length); + state = finalizeRun(plan, options(current)); + if (state.cycle === null) break; + } + expect(sizes).toEqual([10_000, 10_000, 10_000, 2_336]); + expect(seen.size).toBe(current.length); + + // The next run starts a slow rotation pass from the top. + const after = planRun({ current, state, mode: 'daily', budgets }); + expect(after.cycle.kind).toBe('rotation'); + expect(after.window).toHaveLength(1_000); + expect(after.window[0]).toBe(docs[0]); + }); +}); + +describe('state on disk', () => { + const valid = { + schemaVersion: 1, + host: HOST, + updatedAt: NOW.toISOString(), + cycle: null, + known: [url('/')], + }; + const stateFile = () => path.join(stateDir, 'state.json'); + + test('round-trips in a single file', async () => { + const state = makeState({ + cycle: { kind: 'release', cursor: url('/de/') }, + known: urls(SMALL_SITE), + }); + await saveState(stateDir, state); + expect(await readdir(stateDir)).toEqual(['state.json']); + expect(await loadState(stateDir, HOST)).toEqual(state); + }); + + test('a missing state is a cold start without a warning', async () => { + expect(await loadState(stateDir, HOST)).toBeNull(); + expect(console.warn).not.toHaveBeenCalled(); + }); + + test.each([ + ['invalid JSON', '{not json'], + ['another schema version', JSON.stringify({ ...valid, schemaVersion: 2 })], + ['another host', JSON.stringify({ ...valid, host: 'example.com' })], + ['an empty known list', JSON.stringify({ ...valid, known: [] })], + [ + 'URLs of another host', + JSON.stringify({ ...valid, known: [url('/'), 'https://evil.example/'] }), + ], + ['an insecure URL', JSON.stringify({ ...valid, known: ['http://aspire.dev/'] })], + [ + 'a cursor of another host', + JSON.stringify({ ...valid, cycle: { kind: 'release', cursor: 'https://evil.example/' } }), + ], + ['an empty cursor', JSON.stringify({ ...valid, cycle: { kind: 'release', cursor: '' } })], + ['a malformed pass', JSON.stringify({ ...valid, cycle: { kind: 'weekly', cursor: null } })], + ['a missing pass', JSON.stringify({ ...valid, cycle: undefined })], + ])('ignores a state with %s and starts cold', async (_name, content) => { + await writeFile(stateFile(), content); + expect(await loadState(stateDir, HOST)).toBeNull(); + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring unusable')); + }); +}); + +describe('submitting', () => { + function fetchWith(...responses: (Response | Error)[]) { + const fetch = vi.fn(); + for (const response of responses) { + if (response instanceof Error) fetch.mockRejectedValueOnce(response); + else fetch.mockResolvedValueOnce(response); + } + return fetch; + } + const batch = syntheticUrls(3); + + test('POSTs the documented payload in a single request', async () => { + const fetch = fetchWith(new Response('', { status: 200 })); + const clock = createClock(); + expect(await submitUrls(config, batch, { fetch, sleep: clock.sleep })).toEqual({ ok: true }); + + expect(fetch).toHaveBeenCalledTimes(1); + const [target, init] = fetch.mock.calls[0]; + expect(String(target)).toBe(config.endpoint); + expect(init?.method).toBe('POST'); + expect(init?.headers).toMatchObject({ 'Content-Type': 'application/json; charset=utf-8' }); + expect(JSON.parse(init?.body as string)).toEqual({ + host: HOST, + key: KEY, + keyLocation: `https://${HOST}/${KEY}.txt`, + urlList: batch, + }); + }); + + test('accepts 202, which IndexNow answers while it verifies a new key', async () => { + const fetch = fetchWith(new Response('', { status: 202 })); + expect(await submitUrls(config, batch, { fetch, sleep: createClock().sleep })).toEqual({ + ok: true, + }); + }); + + test('retries server errors with back-off, then gives up', async () => { + const fetch = fetchWith( + new Response('', { status: 503 }), + new Response('', { status: 503 }), + new Response('', { status: 503 }), + new Response('unavailable', { status: 503 }) + ); + const clock = createClock(); + expect(await submitUrls(config, batch, { fetch, sleep: clock.sleep })).toEqual({ + ok: false, + status: 503, + detail: 'unavailable', + }); + expect(clock.sleep.mock.calls.map(([ms]) => ms)).toEqual([5_000, 30_000, 120_000]); + }); + + test('succeeds when a retry gets through, and treats network errors like server errors', async () => { + const clock = createClock(); + const recovered = fetchWith( + new Response('', { status: 502 }), + new Response('', { status: 200 }) + ); + expect(await submitUrls(config, batch, { fetch: recovered, sleep: clock.sleep })).toEqual({ + ok: true, + }); + expect(clock.sleep).toHaveBeenCalledTimes(1); + + const boom = new Error('socket hang up'); + const offline = fetchWith(boom, boom, boom, boom); + expect(await submitUrls(config, batch, { fetch: offline, sleep: clock.sleep })).toEqual({ + ok: false, + status: 0, + detail: 'socket hang up', + }); + }); + + test.each([400, 403, 422, 429])('does not retry HTTP %s', async (status) => { + const fetch = fetchWith(new Response('nope', { status })); + const clock = createClock(); + expect(await submitUrls(config, batch, { fetch, sleep: clock.sleep })).toEqual({ + ok: false, + status, + detail: 'nope', + }); + expect(fetch).toHaveBeenCalledTimes(1); + expect(clock.sleep).not.toHaveBeenCalled(); + }); +}); + +describe('the live site', () => { + test('checks that the key file is served and contains the key', async () => { + const site = createSite([]); + expect(await keyFileProblem(config, site.fetch)).toBeNull(); + + site.keyLive = false; + expect(await keyFileProblem(config, site.fetch)).toBe( + `GET https://aspire.dev/${KEY}.txt returned HTTP 404` + ); + + const wrong = vi.fn().mockResolvedValue(new Response('another-key')); + expect(await keyFileProblem(config, wrong)).toMatch(/does not contain/); + const down = vi.fn().mockRejectedValue(new Error('offline')); + expect(await keyFileProblem(config, down)).toMatch(/offline/); + }); + + test("derives the What's new page of a release", () => { + expect(whatsNewPath('13.6')).toBe('/whats-new/aspire-13-6/'); + expect(whatsNewPath('9.5')).toBe('/whats-new/aspire-9-5/'); + expect(whatsNewPath('14.0')).toBe('/whats-new/aspire-14/'); + expect(() => whatsNewPath('13')).toThrow(); + expect(() => whatsNewPath('13.6.1')).toThrow(); + }); + + test('waits until the release page is served, and gives up when it never is', async () => { + const site = createSite([]); + const clock = createClock(); + let polls = 0; + clock.sleep.mockImplementation(() => { + if (++polls === 2) site.liveReleases.add('13.6'); + return Promise.resolve(); + }); + const deps = { fetch: site.fetch, sleep: clock.sleep, now: clock.now }; + expect( + await waitForRelease(HOST, '13.6', deps, { timeoutMs: 600_000, intervalMs: 60_000 }) + ).toBe(true); + expect(clock.sleep).toHaveBeenCalledTimes(2); + expect(site.fetch.mock.calls.every(([, init]) => init?.method === 'HEAD')).toBe(true); + + const idle = createClock(); + const start = idle.now().getTime(); + const never = { fetch: site.fetch, sleep: idle.sleep, now: idle.now }; + expect( + await waitForRelease(HOST, '14.0', never, { timeoutMs: 180_000, intervalMs: 60_000 }) + ).toBe(false); + expect(idle.now().getTime() - start).toBeLessThanOrEqual(180_000); + }); +}); + +describe('triggers', () => { + const pullRequest = ( + overrides: Record = {}, + head: Record = {} + ) => ({ + pull_request: { + merged: true, + title: 'Content for release: 13.6', + head: { ref: 'release/13.6', repo: { full_name: 'microsoft/aspire.dev' }, ...head }, + base: { ref: 'main', repo: { full_name: 'microsoft/aspire.dev' } }, + ...overrides, + }, + }); + + test('recognizes a merged release/X.Y pull request from this repository', () => { + expect(releaseVersion(pullRequest())).toBe('13.6'); + expect(releaseVersion(pullRequest({}, { ref: 'release/14.0' }))).toBe('14.0'); + }); + + test.each([ + ['one that was closed without merging', pullRequest({ merged: false })], + ['one from a fork', pullRequest({}, { repo: { full_name: 'someone/aspire.dev' } })], + ['one from another branch', pullRequest({}, { ref: 'someone/fix-typo' })], + ['one from release/X.Y.Z', pullRequest({}, { ref: 'release/13.6.1' })], + [ + 'one that only has a release title', + pullRequest({ title: 'Merge release/13.4 into main' }, { ref: 'joperezr/merge-13-4' }), + ], + ['a payload without a pull request', {}], + ['no payload', undefined], + ])('ignores %s', (_name, event) => { + expect(releaseVersion(event)).toBeNull(); + }); +}); + +describe('command line', () => { + test('reads the options and applies the defaults', () => { + expect(parseCli(['--state-dir', stateDir])).toEqual({ + stateDir, + mode: 'daily', + limit: undefined, + release: undefined, + trigger: 'manual', + dryRun: false, + allowShrink: false, + }); + expect( + parseCli([ + ...['--state-dir', stateDir, '--mode', 'flush', '--limit', '500', '--release', '13.6'], + ...['--trigger', 'workflow_dispatch', '--dry-run', '--allow-shrink'], + ]) + ).toEqual({ + stateDir, + mode: 'flush', + limit: 500, + release: '13.6', + trigger: 'workflow_dispatch', + dryRun: true, + allowShrink: true, + }); + }); + + test.each([ + ['no state directory', []], + ['an unknown mode', ['--state-dir', 'x', '--mode', 'weekly']], + ['a limit below 1', ['--state-dir', 'x', '--limit', '0']], + ['a limit above one request', ['--state-dir', 'x', '--limit', '10001']], + ['a limit that is not a number', ['--state-dir', 'x', '--limit', 'many']], + ['a release that is not X.Y', ['--state-dir', 'x', '--release', '13']], + ['an unknown flag', ['--state-dir', 'x', '--changed-files', 'f.jsonl']], + ])('rejects %s', (_name, argv) => { + expect(() => parseCli(argv)).toThrow(); + }); +}); + +describe('running end to end', () => { + let site: ReturnType; + let clock: ReturnType; + + beforeEach(() => { + site = createSite(SMALL_SITE); + clock = createClock(); + }); + + const runDay = (overrides: Partial = {}) => { + site.posts.length = 0; + return run({ + config, + stateDir, + mode: 'daily', + trigger: 'schedule', + fetch: site.fetch, + sleep: clock.sleep, + now: clock.now, + ...overrides, + }); + }; + const submitted = () => site.posts.flatMap((post) => post.urlList); + const savedState = () => loadState(stateDir, HOST); + const readState = () => readFile(path.join(stateDir, 'state.json'), 'utf8'); + + test('a cold start submits the first window and seeds the state', async () => { + const result = await runDay(); + expect(result).toMatchObject({ outcome: 'completed', submitted: 3, stateChanged: true }); + expect(result.summary).toContain('## IndexNow submission for aspire.dev: completed'); + expect(result.summary).toContain('Sitemap: 10 URLs'); + expect(result.summary).toContain('Pass: rotation, 3 of 10 URLs covered (30%)'); + expect(result.summary).toContain('Cold start'); + expect(submitted()).toEqual(urls(['/', '/app-host/', '/get-started/'])); + + const state = await savedState(); + expect(state?.known).toHaveLength(10); + expect(state?.cycle?.cursor).toBe(url('/get-started/')); + }); + + test('daily runs walk the sitemap without repeating a URL and then start over', async () => { + const days: string[][] = []; + for (let day = 0; day < 5; day++) { + await runDay(); + days.push(submitted()); + } + expect(days).toEqual([ + urls(['/', '/app-host/', '/get-started/']), + urls([DOCS[3], API[0], API[1]]), + urls([API[2], '/de/', '/de/app-host/']), + urls(['/ja/']), + urls(['/', '/app-host/', '/get-started/']), + ]); + }); + + test('removed and new URLs are sent first, and only a new page ahead of the cursor repeats', async () => { + await runDay(); + const days = [submitted()]; + + site.paths = [ + ...site.paths.filter((entry) => entry !== '/ja/'), + '/aaa/', + '/reference/api/csharp/ab/', + ]; + const second = await runDay(); + expect(second.summary).toContain('2 new and 1 removed'); + expect(submitted()).toEqual(urls(['/ja/', '/aaa/', '/reference/api/csharp/ab/'])); + days.push(submitted()); + + for (let day = 0; day < 3; day++) { + await runDay(); + days.push(submitted()); + } + const everything = days.flat(); + const seen = new Set(everything); + expect(seen).toEqual(new Set([...urls(site.paths), url('/ja/')])); + expect(everything).toHaveLength(seen.size + 1); + expect(everything.filter((entry) => entry === url('/reference/api/csharp/ab/'))).toHaveLength( + 2 + ); + }); + + test('a flush restarts from the top at the release budget and then returns to the rotation', async () => { + await runDay(); + const flush = await runDay({ mode: 'flush', release: undefined }); + expect(flush.summary).toContain('Pass: release'); + expect(submitted()).toEqual(urls([...DOCS, API[0]])); + + await runDay(); + expect(submitted()).toEqual(urls([API[1], API[2], ...LOCALIZED])); + expect((await savedState())?.cycle).toBeNull(); + + await runDay(); + expect(submitted()).toEqual(urls(['/', '/app-host/', '/get-started/'])); + }); + + test('skips a cold start while the key file is not live yet', async () => { + site.keyLive = false; + const result = await runDay(); + expect(result).toMatchObject({ outcome: 'skipped', submitted: 0, stateChanged: false }); + expect(site.posts).toHaveLength(0); + expect(await savedState()).toBeNull(); + }); + + test('fails when the key file disappears after the first successful run', async () => { + await runDay(); + site.keyLive = false; + await expect(runDay()).rejects.toThrow(/key file is not reachable/); + }); + + test('a dry run plans without submitting or saving, even while the key file is missing', async () => { + site.keyLive = false; + const result = await runDay({ dryRun: true }); + expect(result).toMatchObject({ outcome: 'dry-run', submitted: 3, stateChanged: false }); + expect(result.summary).toContain('Dry run'); + expect(site.posts).toHaveLength(0); + expect(await savedState()).toBeNull(); + }); + + test('waits for the release page before submitting', async () => { + clock.sleep.mockImplementationOnce(() => { + site.liveReleases.add('13.6'); + return Promise.resolve(); + }); + const result = await runDay({ mode: 'flush', release: '13.6' }); + expect(result.outcome).toBe('completed'); + expect(result.summary).toContain('release 13.6'); + expect(clock.sleep).toHaveBeenCalledTimes(1); + expect(site.posts).toHaveLength(1); + }); + + test('does not submit when the release page never goes live', async () => { + await expect(runDay({ mode: 'flush', release: '13.6' })).rejects.toThrow(/did not go live/); + expect(site.posts).toHaveLength(0); + expect(await savedState()).toBeNull(); + }); + + test.each([429, 422])( + 'a request answered with HTTP %s fails and leaves the state alone', + async (status) => { + await runDay(); + const before = await readState(); + site.responses = [status]; + const result = await runDay(); + expect(result).toMatchObject({ outcome: 'failed', submitted: 0, stateChanged: false }); + expect(result.failure).toContain(String(status)); + expect(result.failure).toContain('The next run tries again.'); + expect(await readState()).toBe(before); + + await runDay(); + expect(submitted()).toEqual(urls([DOCS[3], API[0], API[1]])); + } + ); + + test('a flush whose request failed still starts the release pass', async () => { + await runDay(); + site.responses = [429]; + const result = await runDay({ mode: 'flush' }); + expect(result).toMatchObject({ outcome: 'failed', stateChanged: true }); + expect(result.failure).toContain('release pass was started'); + expect((await savedState())?.cycle).toEqual({ kind: 'release', cursor: null }); + + await runDay(); + expect(submitted()).toEqual(urls([...DOCS, API[0]])); + }); + + test('a cold start whose request failed still seeds the state', async () => { + site.responses = [422]; + const result = await runDay(); + expect(result).toMatchObject({ outcome: 'failed', stateChanged: true }); + const state = await savedState(); + expect(state?.known).toHaveLength(10); + expect(state?.cycle?.cursor).toBeNull(); + }); + + test('refuses a sitemap that suddenly lost most of its pages and keeps the state', async () => { + await runDay(); + const before = await readState(); + site.paths = site.paths.slice(0, 5); + await expect(runDay()).rejects.toThrow(/--allow-shrink/); + expect(await readState()).toBe(before); + }); + + test('replaces a state that lists other sites instead of submitting what it lists', async () => { + await saveState( + stateDir, + makeState({ known: [url('/'), 'https://evil.example/spam/', 'http://aspire.dev/insecure/'] }) + ); + const result = await runDay(); + expect(result.summary).toContain('Cold start'); + expect(submitted().every((entry) => entry.startsWith('https://aspire.dev/'))).toBe(true); + expect((await savedState())?.known).toEqual([...urls(SMALL_SITE)].sort()); + }); + + test('sends a single request even when the budget is larger than a request can hold', async () => { + site = createSite( + Array.from({ length: 12_000 }, (_, index) => `/docs/page-${String(index).padStart(5, '0')}/`) + ); + const budgets = { release: 50_000, rotation: 50_000 }; + await runDay({ config: { ...config, budgets }, mode: 'flush' }); + expect(site.posts).toHaveLength(1); + expect(site.posts[0].urlList).toHaveLength(MAX_URLS_PER_RUN); + }); +}); + +describe('the command line entry point', () => { + async function runMain(argv: string[], event?: unknown) { + const files = { + GITHUB_EVENT_PATH: path.join(stateDir, 'event.json'), + GITHUB_STEP_SUMMARY: path.join(stateDir, 'summary.md'), + GITHUB_OUTPUT: path.join(stateDir, 'output.txt'), + }; + if (event !== undefined) await writeFile(files.GITHUB_EVENT_PATH, JSON.stringify(event)); + const real = await loadConfig(); + const site = createSite(SMALL_SITE, real.key); + const clock = createClock(); + const code = await main(['--state-dir', path.join(stateDir, 'state'), ...argv], files, { + fetch: site.fetch, + sleep: clock.sleep, + now: clock.now, + }); + const read = (file: string) => readFile(file, 'utf8').catch(() => ''); + return { + code, + site, + real, + summary: await read(files.GITHUB_STEP_SUMMARY), + output: await read(files.GITHUB_OUTPUT), + }; + } + + const pullRequest = (title: string, ref: string) => ({ + pull_request: { + merged: true, + title, + head: { ref, repo: { full_name: 'microsoft/aspire.dev' } }, + base: { ref: 'main', repo: { full_name: 'microsoft/aspire.dev' } }, + }, + }); + + test('skips a merged pull request that is not a release', async () => { + const { code, output, summary, site } = await runMain( + ['--trigger', 'pull_request'], + pullRequest('Fix a typo', 'someone/typo') + ); + expect(code).toBe(0); + expect(output).toBe('state_changed=false\n'); + expect(summary).toContain('skipped'); + expect(summary).toContain('not a merged release'); + expect(site.fetch).not.toHaveBeenCalled(); + }); + + test('a merged release pull request starts a burst once the release page is live', async () => { + const files = { GITHUB_EVENT_PATH: path.join(stateDir, 'event.json') }; + await writeFile( + files.GITHUB_EVENT_PATH, + JSON.stringify(pullRequest('Content for release: 13.6', 'release/13.6')) + ); + const real = await loadConfig(); + const site = createSite(SMALL_SITE, real.key); + site.liveReleases.add('13.6'); + const clock = createClock(); + const env = { ...files, GITHUB_OUTPUT: path.join(stateDir, 'output.txt') }; + const code = await main(['--state-dir', stateDir, '--trigger', 'pull_request'], env, { + fetch: site.fetch, + sleep: clock.sleep, + now: clock.now, + }); + + expect(code).toBe(0); + expect(await readFile(env.GITHUB_OUTPUT, 'utf8')).toBe('state_changed=true\n'); + expect(site.posts).toHaveLength(1); + expect(site.posts[0]).toMatchObject({ host: real.host, key: real.key }); + expect(site.posts[0].urlList).toEqual(urls(SMALL_SITE)); + expect((await loadState(stateDir, real.host))?.cycle).toBeNull(); + }); + + test('reports a failed run and returns a failing exit code', async () => { + const { code, output, summary } = await runMain(['--mode', 'flush', '--release', '13.6']); + expect(code).toBe(1); + expect(output).toBe('state_changed=false\n'); + expect(summary).toContain('failed'); + expect(summary).toContain('did not go live'); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining('did not go live')); + }); + + test('rejects invalid arguments', async () => { + const { code } = await runMain(['--mode', 'weekly']); + expect(code).toBe(1); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining('--mode')); + }); +});