From b04692172067d4c47bd7b1d1e84875ba2e5788d9 Mon Sep 17 00:00:00 2001 From: David Pine <7679720+IEvangelist@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:09:50 -0500 Subject: [PATCH 1/6] Add a daily IndexNow submission workflow Adds a scheduled workflow that tells IndexNow search engines about aspire.dev URLs. Each run reads the live sitemap, submits URLs that are new or removed since the previous run, then the next window of a slow rotation through the whole sitemap. When a release/X.Y pull request is merged into main, the workflow waits for that release's What's new page to be live and then starts a full-sitemap burst that the daily runs finish. State is kept in an Actions artifact and IndexNow's undisclosed rate limits are handled by configurable budgets and 429 handling. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/indexnow.yml | 166 ++ src/frontend/config/indexnow.json | 10 + src/frontend/package.json | 1 + .../c5f794ab28dc0cb92f48bd970500daa3.txt | 1 + src/frontend/scripts/indexnow.ts | 1420 +++++++++++++++ .../tests/unit/indexnow.vitest.test.ts | 1515 +++++++++++++++++ 6 files changed, 3113 insertions(+) create mode 100644 .github/workflows/indexnow.yml create mode 100644 src/frontend/config/indexnow.json create mode 100644 src/frontend/public/c5f794ab28dc0cb92f48bd970500daa3.txt create mode 100644 src/frontend/scripts/indexnow.ts create mode 100644 src/frontend/tests/unit/indexnow.vitest.test.ts diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml new file mode 100644 index 0000000000..6ebbad0c7b --- /dev/null +++ b/.github/workflows/indexnow.yml @@ -0,0 +1,166 @@ +name: IndexNow + +# Tells IndexNow search engines (Bing, Yandex, Naver, Seznam, Yep, Amazon) about aspire.dev URLs. +# +# - Daily: submits URLs that are new or removed in the sitemap since the previous run, then the next +# window of a slow rotation through the whole sitemap. +# - Release: when a release/X.Y pull request is merged into main, waits for that release's "What's +# new" page to be live (the docs slot swap) and then starts a full-sitemap burst that the daily +# runs finish. +# - Manual: run it from the Actions tab (mode, limit, release, dry run). +# +# All logic lives in src/frontend/scripts/indexnow.ts so the same flow can be reproduced locally: +# cd src/frontend && pnpm indexnow --state-dir /tmp/indexnow --dry-run +# The budgets and the key are in src/frontend/config/indexnow.json, and the key file is served from +# src/frontend/public/.txt. IndexNow keys are public by design: the key file has to be +# downloadable so search engines can verify ownership. To rotate the key, add the new key file and +# update config/indexnow.json in the same change, and delete the old key file once it has deployed. +# +# What has already been submitted is kept in the "indexnow-state" artifact (refreshed by every run). +# Losing it only causes a cold start that re-seeds the known URL list without submitting everything. + +on: + schedule: + - cron: "43 6 * * *" + pull_request: + types: [closed] + branches: [main] + workflow_dispatch: + inputs: + mode: + description: "daily: new and removed URLs plus the next rotation window. flush: restart the full-sitemap cycle at the release budget." + type: choice + options: + - daily + - flush + default: daily + limit: + description: "Number of URLs to submit in this run (1-50000). Leave empty to use the configured budget." + type: string + required: false + release: + description: "Release such as 13.6. Waits for that release's What's new page to be live before submitting." + type: string + required: false + dry_run: + description: "Only print what would be submitted. Nothing is sent and the state is not saved." + type: boolean + default: false + allow_shrink: + description: "Continue even if the sitemap lists more than 20% fewer URLs than the last run." + type: boolean + default: false + +permissions: {} + +concurrency: + group: indexnow + cancel-in-progress: false + +jobs: + submit: + name: Submit URLs to IndexNow + # Pull requests only matter when they are merged release pull requests from this repository. The + # script checks the exact branch or title format again. + if: >- + ${{ github.repository == 'microsoft/aspire.dev' && + (github.event_name != 'pull_request' || + (github.event.pull_request.merged == true && + github.event.pull_request.head.repo.full_name == github.repository && + (startsWith(github.event.pull_request.head.ref, 'release/') || + startsWith(github.event.pull_request.title, 'Merge release/')))) }} + runs-on: ubuntu-latest + timeout-minutes: 75 + permissions: + contents: read + actions: read + defaults: + run: + working-directory: src/frontend + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # A release merge runs the merged script from main, not the pull request's merge ref. + ref: ${{ github.event_name == 'pull_request' && github.event.repository.default_branch || github.ref }} + fetch-depth: 1 + persist-credentials: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: "24.x" + cache: pnpm + cache-dependency-path: src/frontend/pnpm-lock.yaml + + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + + - name: Restore state from the previous run + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + shell: bash + run: | + set -euo pipefail + state_dir="$RUNNER_TEMP/indexnow-state" + mkdir -p "$state_dir" + + # `gh run download` lists every artifact in the repository and fails with HTTP 500 here, so + # ask for this artifact by name and take the newest one that has not expired. + artifact_id="" + for attempt in 1 2 3; do + if artifact_id=$(gh api "repos/$GH_REPO/actions/artifacts?name=indexnow-state&per_page=100" \ + --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | last | .id // empty'); then + break + fi + if [ "$attempt" -eq 3 ]; then + echo "::error::Could not list the indexnow-state artifacts." + exit 1 + fi + sleep $((attempt * 10)) + done + + if [ -z "$artifact_id" ]; then + echo "No previous state found; this run starts cold." + exit 0 + fi + + echo "Restoring artifact $artifact_id." + gh api "repos/$GH_REPO/actions/artifacts/$artifact_id/zip" > "$RUNNER_TEMP/indexnow-state.zip" + unzip -q -o "$RUNNER_TEMP/indexnow-state.zip" -d "$state_dir" + + - name: Submit URLs + id: submit + env: + EVENT_NAME: ${{ github.event_name }} + INPUT_MODE: ${{ inputs.mode }} + INPUT_LIMIT: ${{ inputs.limit }} + INPUT_RELEASE: ${{ inputs.release }} + INPUT_DRY_RUN: ${{ inputs.dry_run }} + INPUT_ALLOW_SHRINK: ${{ inputs.allow_shrink }} + shell: bash + run: | + set -euo pipefail + args=(--state-dir "$RUNNER_TEMP/indexnow-state" --trigger "$EVENT_NAME") + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + args+=(--mode "$INPUT_MODE") + [ -z "$INPUT_LIMIT" ] || args+=(--limit "$INPUT_LIMIT") + [ -z "$INPUT_RELEASE" ] || args+=(--release "$INPUT_RELEASE") + [ "$INPUT_DRY_RUN" != "true" ] || args+=(--dry-run) + [ "$INPUT_ALLOW_SHRINK" != "true" ] || args+=(--allow-shrink) + fi + pnpm indexnow "${args[@]}" + + # Runs after a failure too, so URLs that were accepted before an error are not sent again. + - name: Save state for the next run + if: ${{ !cancelled() && steps.submit.outputs.state_changed == 'true' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: indexnow-state + path: ${{ runner.temp }}/indexnow-state + retention-days: 90 + if-no-files-found: error diff --git a/src/frontend/config/indexnow.json b/src/frontend/config/indexnow.json new file mode 100644 index 0000000000..b01b4e8e7d --- /dev/null +++ b/src/frontend/config/indexnow.json @@ -0,0 +1,10 @@ +{ + "host": "aspire.dev", + "key": "c5f794ab28dc0cb92f48bd970500daa3", + "endpoint": "https://api.indexnow.org/indexnow", + "sitemapIndex": "https://aspire.dev/sitemap-index.xml", + "budgets": { + "release": 10000, + "rotation": 1000 + } +} diff --git a/src/frontend/package.json b/src/frontend/package.json index 4dec81b28f..fc2fe3f489 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -60,6 +60,7 @@ "update:github-stats": "tsx ./scripts/update-github-stats.ts", "update:release-contributors": "tsx ./scripts/update-release-contributors.ts", "update:samples": "tsx ./scripts/update-samples.ts", + "indexnow": "tsx ./scripts/indexnow.ts", "normalize:api-data": "tsx ./scripts/normalize-generated-api-data.ts", "validate:api-data": "tsx ./scripts/validate-generated-api-data.ts" }, diff --git a/src/frontend/public/c5f794ab28dc0cb92f48bd970500daa3.txt b/src/frontend/public/c5f794ab28dc0cb92f48bd970500daa3.txt new file mode 100644 index 0000000000..2b4995db98 --- /dev/null +++ b/src/frontend/public/c5f794ab28dc0cb92f48bd970500daa3.txt @@ -0,0 +1 @@ +c5f794ab28dc0cb92f48bd970500daa3 \ No newline at end of file diff --git a/src/frontend/scripts/indexnow.ts b/src/frontend/scripts/indexnow.ts new file mode 100644 index 0000000000..207a578012 --- /dev/null +++ b/src/frontend/scripts/indexnow.ts @@ -0,0 +1,1420 @@ +/** + * Tells IndexNow-participating search engines (Bing, Yandex, Naver, Seznam, Yep, Amazon) about + * aspire.dev URLs. See https://www.indexnow.org/documentation. + * + * Every run reads the live sitemap, compares it with the URL list remembered from the previous run + * and submits, in priority order: + * 1. removed URLs (so engines drop them) and new URLs (so engines discover them), + * 2. the next window of the rotation cycle, which walks the whole sitemap in (tier, path) order. + * + * The rotation cycle is a cursor over that order, so it survives URLs being added or removed. A + * `flush` (a release merge or a manual dispatch) restarts the cycle from the top at the larger + * release budget; the daily runs then drain it until the whole sitemap has been covered, after + * which the slow rotation budget applies. IndexNow does not publish its rate limits, so both + * budgets live in config/indexnow.json and a 429 simply pauses the run until the next one. + * + * State (`state.json` and `known-urls.txt.gz`) lives in --state-dir. The workflow keeps it between + * runs as an Actions artifact; losing it only causes a cold start, which seeds the known URL list + * without submitting every URL as "new". + * + * Usage (from src/frontend): + * pnpm indexnow --state-dir [--mode daily|flush] [--limit ] [--release ] + * [--trigger ] [--dry-run] [--allow-shrink] + */ +import { appendFile, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; +import { gunzipSync, gzipSync } from 'node:zlib'; + +import { locales } from '../config/locales'; +import { fetchWithProxy } from './fetch-with-proxy'; + +export type FetchLike = typeof fetchWithProxy; +export type Sleep = (ms: number) => Promise; +export type RunMode = 'daily' | 'flush'; +export type CycleKind = 'release' | 'rotation'; + +export interface Logger { + info(message: string): void; + warn(message: string): void; + error(message: string): void; +} + +export interface IndexNowConfig { + host: string; + key: string; + endpoint: string; + sitemapIndex: string; + budgets: { release: number; rotation: number }; +} + +/** A pass over the sitemap. `cursor` is the last URL covered, in (tier, path) order. */ +export interface Cycle { + kind: CycleKind; + reason: string; + startedAt: string; + cursor: string | null; + /** URLs ahead of the cursor that were already submitted as new, so the window skips them. */ + skip: string[]; +} + +export interface RunRecord { + at: string; + trigger: string; + mode: RunMode; + coldStart: boolean; + newUrls: number; + removedUrls: number; + windowUrls: number; + submitted: number; + statuses: number[]; +} + +export interface IndexNowState { + host: string; + updatedAt: string; + /** The `git-commit-id` meta value served by the site when the state was written. */ + deployment: string; + /** `null` once the previous cycle has covered the whole sitemap. */ + cycle: Cycle | null; + lastRun: RunRecord | null; + /** Every sitemap URL as of the last run, sorted. */ + known: string[]; +} + +const CONFIG_PATH = fileURLToPath(new URL('../config/indexnow.json', import.meta.url)); +const STATE_FILE = 'state.json'; +const KNOWN_FILE = 'known-urls.txt.gz'; +const SCHEMA_VERSION = 1; +const KEY_PATTERN = /^[A-Za-z0-9-]{8,128}$/; +const USER_AGENT = 'aspire.dev-indexnow (+https://github.com/microsoft/aspire.dev)'; + +/** IndexNow accepts at most 10,000 URLs per POST. */ +export const MAX_URLS_PER_REQUEST = 10_000; +/** Cap on new URLs, and separately on removed URLs, submitted ahead of the window in one run. */ +const MAX_PRIORITY_URLS = 10_000; +export const MAX_LIMIT = 50_000; +/** A sitemap with fewer URLs than this share of the last known list is treated as broken. */ +export const SHRINK_THRESHOLD = 0.8; +const MAX_SITEMAP_CHARS = 100_000_000; +const MAX_CHILD_SITEMAPS = 50; +const SITEMAP_TIMEOUT_MS = 120_000; +const REQUEST_TIMEOUT_MS = 60_000; +/** Back-off for 5xx and network errors. A 429 waits for its Retry-After header instead. */ +const RETRY_DELAYS_MS = [5_000, 30_000, 120_000]; +const MAX_RETRY_AFTER_MS = 300_000; +const CHUNK_PAUSE_MS = 2_000; +const GATE_INTERVAL_MS = 60_000; +const GATE_TIMEOUT_MS = 45 * 60_000; + +export const LOCALE_CODES: ReadonlySet = new Set( + Object.keys(locales).filter((code) => code !== 'root') +); + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function compareOrdinal(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + +const defaultSleep: Sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +function escapeAnnotation(message: string): string { + return message.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); +} + +export const consoleLogger: Logger = { + info: (message) => console.log(message), + warn: (message) => + console.warn( + process.env.GITHUB_ACTIONS === 'true' + ? `::warning::${escapeAnnotation(message)}` + : `warning: ${message}` + ), + error: (message) => + console.error( + process.env.GITHUB_ACTIONS === 'true' + ? `::error::${escapeAnnotation(message)}` + : `error: ${message}` + ), +}; + +// --------------------------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------------------------- + +function isLimit(value: unknown): value is number { + return typeof value === 'number' && Number.isInteger(value) && value >= 1 && value <= MAX_LIMIT; +} + +function isHttpsUrl(value: string): boolean { + try { + const url = new URL(value); + return url.protocol === 'https:' && !url.username && !url.password; + } catch { + return false; + } +} + +export function parseConfig(value: unknown): IndexNowConfig { + if (!isRecord(value)) throw new Error('The IndexNow config must be a JSON object'); + const { host, key, endpoint, sitemapIndex, budgets } = value; + if (typeof host !== 'string' || !/^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$/.test(host)) { + throw new Error('The IndexNow config needs a lowercase "host" such as aspire.dev'); + } + if (typeof key !== 'string' || !KEY_PATTERN.test(key)) { + throw new Error('The IndexNow "key" must be 8-128 characters of letters, digits, or dashes'); + } + if (typeof endpoint !== 'string' || !isHttpsUrl(endpoint)) { + throw new Error('The IndexNow "endpoint" must be an https URL'); + } + if ( + typeof sitemapIndex !== 'string' || + !isHttpsUrl(sitemapIndex) || + new URL(sitemapIndex).hostname !== host + ) { + throw new Error('The IndexNow "sitemapIndex" must be an https URL on the configured host'); + } + if (!isRecord(budgets) || !isLimit(budgets.release) || !isLimit(budgets.rotation)) { + throw new Error(`The IndexNow "budgets" must be integers between 1 and ${MAX_LIMIT}`); + } + return { + host, + key, + endpoint, + sitemapIndex, + budgets: { release: budgets.release, rotation: budgets.rotation }, + }; +} + +export async function loadConfig(configPath = CONFIG_PATH): Promise { + const raw: unknown = JSON.parse(await readFile(configPath, 'utf8')); + return parseConfig(raw); +} + +export function keyLocation(config: Pick): string { + return `https://${config.host}/${config.key}.txt`; +} + +// --------------------------------------------------------------------------------------------- +// Sitemap +// --------------------------------------------------------------------------------------------- + +export interface ParsedSitemap { + kind: 'index' | 'urlset'; + locations: string[]; +} + +const NAMED_ENTITIES: Record = { + amp: '&', + lt: '<', + gt: '>', + quot: '"', + apos: "'", +}; + +function decodeXml(value: string): string { + return value.replace( + /&(?:#(\d+)|#x([0-9a-fA-F]+)|(amp|lt|gt|quot|apos));/g, + (match: string, decimal?: string, hex?: string, name?: string) => { + if (name) return NAMED_ENTITIES[name] ?? match; + const code = decimal ? Number(decimal) : parseInt(hex ?? '', 16); + return Number.isInteger(code) && code > 0 && code <= 0x10ffff + ? String.fromCodePoint(code) + : match; + } + ); +} + +/** + * Reads the `` values of a sitemap or sitemap index. The `xhtml:link` alternates that Astro + * adds carry their URL in an attribute and are ignored: every localized page is also listed as a + * `` of its own. + */ +export function parseSitemap(xml: string): ParsedSitemap { + const root = /<(sitemapindex|urlset)[\s>]/.exec(xml); + if (!root) throw new Error('Not a sitemap: expected a or element'); + const locations: string[] = []; + for (const match of xml.matchAll(/\s*([^<]*?)\s*<\/loc>/g)) { + const location = decodeXml(match[1]); + if (location) locations.push(location); + } + return { kind: root[1] === 'sitemapindex' ? 'index' : 'urlset', locations }; +} + +async function fetchXml(fetch: FetchLike, url: string): Promise { + const response = await fetch(url, { + headers: { 'User-Agent': USER_AGENT, Accept: 'application/xml, text/xml' }, + signal: AbortSignal.timeout(SITEMAP_TIMEOUT_MS), + }); + if (!response.ok) throw new Error(`GET ${url} failed: HTTP ${response.status}`); + const text = await response.text(); + if (text.length > MAX_SITEMAP_CHARS) throw new Error(`${url} is larger than expected`); + return text; +} + +/** Returns every `` of the sitemap, following a sitemap index one level deep. */ +export async function fetchSitemapLocations( + indexUrl: string, + fetch: FetchLike = fetchWithProxy +): Promise { + const first = parseSitemap(await fetchXml(fetch, indexUrl)); + if (first.kind === 'urlset') return first.locations; + if (first.locations.length === 0) throw new Error(`${indexUrl} lists no sitemaps`); + if (first.locations.length > MAX_CHILD_SITEMAPS) { + throw new Error(`${indexUrl} lists more than ${MAX_CHILD_SITEMAPS} sitemaps`); + } + + const host = new URL(indexUrl).hostname; + const locations: string[] = []; + for (const child of first.locations) { + if (!isHttpsUrl(child) || new URL(child).hostname !== host) { + throw new Error(`Refusing to read sitemap ${child}: it is not an https URL on ${host}`); + } + const parsed = parseSitemap(await fetchXml(fetch, child)); + if (parsed.kind !== 'urlset') throw new Error(`${child} is a nested sitemap index`); + for (const location of parsed.locations) locations.push(location); + } + return locations; +} + +/** Keeps only https URLs on `host`, drops fragments and removes duplicates. */ +export function normalizeUrls( + locations: readonly string[], + host: string +): { urls: string[]; rejected: string[] } { + const urls = new Set(); + const rejected: string[] = []; + for (const location of locations) { + let url: URL; + try { + url = new URL(location); + } catch { + rejected.push(location); + continue; + } + if ( + url.protocol !== 'https:' || + url.hostname !== host || + url.port || + url.username || + url.password + ) { + rejected.push(location); + continue; + } + url.hash = ''; + urls.add(url.href); + } + return { urls: [...urls], rejected }; +} + +// --------------------------------------------------------------------------------------------- +// Ordering +// --------------------------------------------------------------------------------------------- + +/** + * Priority of a page. Lower tiers are submitted first: English pages (docs, integrations, What's + * new), then the API reference, then localized pages. + */ +export function tierOf(pathname: string, codes: ReadonlySet = LOCALE_CODES): 0 | 1 | 2 { + const first = pathname.split('/')[1] ?? ''; + if (codes.has(first)) return 2; + if (pathname.startsWith('/reference/api/')) return 1; + return 0; +} + +interface Entry { + url: string; + tier: number; +} + +function toEntry(url: string, codes: ReadonlySet): Entry { + return { url, tier: tierOf(new URL(url).pathname, codes) }; +} + +function compareEntries(a: Entry, b: Entry): number { + return a.tier - b.tier || compareOrdinal(a.url, b.url); +} + +/** Orders URLs by (tier, URL), the order the rotation cursor walks. */ +export function compareUrls(a: string, b: string, codes: ReadonlySet = LOCALE_CODES) { + return compareEntries(toEntry(a, codes), toEntry(b, codes)); +} + +function firstIndexAfter(entries: readonly Entry[], target: Entry): number { + let low = 0; + let high = entries.length; + while (low < high) { + const middle = (low + high) >>> 1; + if (compareEntries(entries[middle], target) > 0) high = middle; + else low = middle + 1; + } + return low; +} + +/** How many of the URLs a cycle with this cursor has already covered. */ +export function countCovered( + urls: readonly string[], + cursor: string | null, + codes: ReadonlySet = LOCALE_CODES +): number { + if (cursor === null) return 0; + const target = toEntry(cursor, codes); + return urls.filter((url) => compareEntries(toEntry(url, codes), target) <= 0).length; +} + +// --------------------------------------------------------------------------------------------- +// Planning +// --------------------------------------------------------------------------------------------- + +export interface PlanInput { + /** Normalized, de-duplicated sitemap URLs. */ + current: readonly string[]; + state: IndexNowState | null; + mode: RunMode; + /** Recorded on the cycle when one is started. */ + reason: string; + budgets: IndexNowConfig['budgets']; + /** Replaces the budget for this run. */ + limit?: number; + now: Date; + allowShrink?: boolean; + localeCodes?: ReadonlySet; +} + +export interface Plan { + coldStart: boolean; + mode: RunMode; + budget: number; + /** The cycle this run works on (continued or newly started). */ + cycle: Cycle; + total: number; + /** Every URL that is in the sitemap but was not known. */ + newUrls: string[]; + /** Every known URL that left the sitemap. */ + removedUrls: string[]; + /** The removed URLs submitted this run, in submission order. */ + removed: string[]; + /** The new URLs submitted this run, in submission order. */ + fresh: string[]; + /** The rotation window submitted this run. */ + window: string[]; + /** The last URL the window scan went past, or null when it did not move. */ + windowEnd: string | null; + /** True when a fully accepted window finishes the cycle. */ + windowComplete: boolean; + /** How many sitemap URLs the cycle has covered after this run's scan. */ + covered: number; + localeCodes: ReadonlySet; +} + +/** + * Decides what to submit. Pure: it does not touch the network or the disk. + * + * - A cold start (no state) seeds the known list and never treats the whole sitemap as new. + * - A flush starts a release cycle from the top; otherwise the active cycle continues, or a new + * rotation cycle begins when the previous one has completed. + * - New and removed URLs go first. New URLs ahead of the cursor are remembered in `cycle.skip` + * (see finalizeRun) so the window does not send them a second time. + */ +export function planRun(input: PlanInput): Plan { + const localeCodes = input.localeCodes ?? LOCALE_CODES; + const { state } = input; + const entries = input.current.map((url) => toEntry(url, localeCodes)).sort(compareEntries); + const total = entries.length; + if (total === 0) throw new Error('The sitemap lists no URLs; refusing to continue'); + + const previous = state?.known ?? []; + if (state && !input.allowShrink && total < previous.length * SHRINK_THRESHOLD) { + throw new Error( + `The sitemap lists ${total} URLs, more than ${Math.round((1 - SHRINK_THRESHOLD) * 100)}% ` + + `fewer than the ${previous.length} known from the last run. Refusing to continue in case ` + + 'the site was only partially deployed; rerun with --allow-shrink if the drop is intended.' + ); + } + + const currentSet = new Set(entries.map((entry) => entry.url)); + const knownSet = new Set(previous); + const newUrls = state ? entries.filter((e) => !knownSet.has(e.url)).map((e) => e.url) : []; + const removedUrls = previous + .filter((url) => !currentSet.has(url)) + .map((url) => toEntry(url, localeCodes)) + .sort(compareEntries) + .map((entry) => entry.url); + + const startedAt = input.now.toISOString(); + let cycle: Cycle; + if (input.mode === 'flush') { + cycle = { kind: 'release', reason: input.reason, startedAt, cursor: null, skip: [] }; + } else if (state?.cycle) { + cycle = state.cycle; + } else { + cycle = { kind: 'rotation', reason: input.reason, startedAt, cursor: null, skip: [] }; + } + + const budget = + input.limit ?? (cycle.kind === 'release' ? input.budgets.release : input.budgets.rotation); + const removed = removedUrls.slice(0, MAX_PRIORITY_URLS); + const fresh = newUrls.slice(0, MAX_PRIORITY_URLS); + const priority = new Set([...removed, ...fresh]); + const skip = new Set(cycle.skip); + const room = Math.max(0, budget - removed.length - fresh.length); + + const alreadySent = (entry: Entry) => priority.has(entry.url) || skip.has(entry.url); + let index = + cycle.cursor === null ? 0 : firstIndexAfter(entries, toEntry(cycle.cursor, localeCodes)); + let windowEnd: string | null = null; + const window: string[] = []; + while (index < total && window.length < room) { + const entry = entries[index++]; + windowEnd = entry.url; + if (!alreadySent(entry)) window.push(entry.url); + } + // Trailing URLs that were already sent this cycle do not need another run to be passed over. + while (index < total && alreadySent(entries[index])) windowEnd = entries[index++].url; + + return { + coldStart: state === null, + mode: input.mode, + budget, + cycle, + total, + newUrls, + removedUrls, + removed, + fresh, + window, + windowEnd, + windowComplete: index >= total, + covered: index, + localeCodes, + }; +} + +export interface FinalizeOptions { + host: string; + /** The URLs the plan was computed from. */ + current: readonly string[]; + /** How many URLs of `[...removed, ...fresh, ...window]` were accepted, as a prefix. */ + accepted: number; + now: Date; + trigger: string; + deployment: string; + statuses: readonly number[]; +} + +/** + * Builds the state to remember after a run. Only accepted URLs count as done: a new URL that was + * not accepted stays new, a removed URL that was not accepted stays known, and the cursor stops at + * the last accepted window URL. + */ +export function finalizeRun(plan: Plan, options: FinalizeOptions): IndexNowState { + const ordered = [...plan.removed, ...plan.fresh, ...plan.window]; + const accepted = Math.min(Math.max(options.accepted, 0), ordered.length); + const acceptedUrls = new Set(ordered.slice(0, accepted)); + const acceptedWindow = Math.max(0, accepted - plan.removed.length - plan.fresh.length); + + const known = new Set(options.current); + for (const url of plan.newUrls) if (!acceptedUrls.has(url)) known.delete(url); + for (const url of plan.removedUrls) if (!acceptedUrls.has(url)) known.add(url); + + let cycle: Cycle | null = null; + const windowDone = acceptedWindow === plan.window.length; + if (!(windowDone && plan.windowComplete)) { + const cursor = windowDone + ? (plan.windowEnd ?? plan.cycle.cursor) + : (plan.window[acceptedWindow - 1] ?? plan.cycle.cursor); + const skip = new Set(plan.cycle.skip); + for (const url of plan.fresh) if (acceptedUrls.has(url)) skip.add(url); + cycle = { + ...plan.cycle, + cursor, + skip: [...skip] + .filter((url) => cursor === null || compareUrls(url, cursor, plan.localeCodes) > 0) + .sort(compareOrdinal), + }; + } + + return { + host: options.host, + updatedAt: options.now.toISOString(), + deployment: options.deployment, + cycle, + lastRun: { + at: options.now.toISOString(), + trigger: options.trigger, + mode: plan.mode, + coldStart: plan.coldStart, + newUrls: plan.newUrls.length, + removedUrls: plan.removedUrls.length, + windowUrls: acceptedWindow, + submitted: accepted, + statuses: [...options.statuses], + }, + known: [...known].sort(compareOrdinal), + }; +} + +// --------------------------------------------------------------------------------------------- +// State on disk +// --------------------------------------------------------------------------------------------- + +function parseCycle(value: unknown): Cycle | null { + if (value === null) return null; + if (!isRecord(value)) throw new Error('"cycle" must be an object or null'); + const { kind, reason, startedAt, cursor, skip } = value; + if ( + (kind !== 'release' && kind !== 'rotation') || + typeof reason !== 'string' || + typeof startedAt !== 'string' || + !(cursor === null || typeof cursor === 'string') || + !Array.isArray(skip) || + !skip.every((url): url is string => typeof url === 'string') + ) { + throw new Error('"cycle" has an unexpected shape'); + } + return { kind, reason, startedAt, cursor, skip }; +} + +function parseRunRecord(value: unknown): RunRecord | null { + if (!isRecord(value)) return null; + const count = (entry: unknown) => (typeof entry === 'number' ? entry : 0); + return { + at: typeof value.at === 'string' ? value.at : '', + trigger: typeof value.trigger === 'string' ? value.trigger : '', + mode: value.mode === 'flush' ? 'flush' : 'daily', + coldStart: value.coldStart === true, + newUrls: count(value.newUrls), + removedUrls: count(value.removedUrls), + windowUrls: count(value.windowUrls), + submitted: count(value.submitted), + statuses: Array.isArray(value.statuses) + ? value.statuses.filter((status): status is number => typeof status === 'number') + : [], + }; +} + +export async function loadState( + dir: string, + host: string, + log: Logger = consoleLogger +): Promise { + let raw: string; + try { + raw = await readFile(path.join(dir, STATE_FILE), 'utf8'); + } catch (error) { + if (isRecord(error) && error.code === 'ENOENT') return null; + throw error; + } + + try { + const file: unknown = JSON.parse(raw); + if (!isRecord(file) || file.schemaVersion !== SCHEMA_VERSION) { + throw new Error('the state file has a different schema version'); + } + if (file.host !== host) throw new Error(`the state belongs to ${String(file.host)}`); + if (typeof file.updatedAt !== 'string' || typeof file.deployment !== 'string') { + throw new Error('the state file is missing fields'); + } + const known = gunzipSync(await readFile(path.join(dir, KNOWN_FILE))) + .toString('utf8') + .split('\n') + .filter(Boolean); + if (known.length !== file.knownCount) { + throw new Error(`expected ${String(file.knownCount)} known URLs but found ${known.length}`); + } + return { + host, + updatedAt: file.updatedAt, + deployment: file.deployment, + cycle: parseCycle(file.cycle), + lastRun: parseRunRecord(file.lastRun), + known, + }; + } catch (error) { + log.warn(`Ignoring unusable IndexNow state (${errorMessage(error)}); starting cold.`); + return null; + } +} + +export async function saveState(dir: string, state: IndexNowState): Promise { + await mkdir(dir, { recursive: true }); + const stateFile = path.join(dir, STATE_FILE); + const knownFile = path.join(dir, KNOWN_FILE); + const stagedState = `${stateFile}.${process.pid}.next`; + const stagedKnown = `${knownFile}.${process.pid}.next`; + const { known, ...rest } = state; + try { + await writeFile(stagedKnown, gzipSync(`${known.join('\n')}\n`)); + await writeFile( + stagedState, + `${JSON.stringify({ schemaVersion: SCHEMA_VERSION, ...rest, knownCount: known.length }, null, 2)}\n`, + 'utf8' + ); + await rename(stagedKnown, knownFile); + await rename(stagedState, stateFile); + } finally { + await rm(stagedKnown, { force: true }); + await rm(stagedState, { force: true }); + } +} + +// --------------------------------------------------------------------------------------------- +// Submission +// --------------------------------------------------------------------------------------------- + +export interface ChunkResult { + size: number; + status: number; + attempts: number; +} + +export interface SubmitStop { + reason: 'rate-limited' | 'failed'; + status: number; + detail: string; +} + +export interface SubmitResult { + /** URLs accepted by IndexNow: always a prefix of the list that was submitted. */ + accepted: number; + chunks: ChunkResult[]; + stop: SubmitStop | null; +} + +export interface SubmitOptions { + config: IndexNowConfig; + urls: readonly string[]; + fetch?: FetchLike; + sleep?: Sleep; + now?: () => number; + log?: Logger; +} + +/** Reads a Retry-After header, either as seconds or as an HTTP date, into milliseconds. */ +export function parseRetryAfter(value: string | null, nowMs: number): number | undefined { + if (value === null) return undefined; + const trimmed = value.trim(); + if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1000; + const date = Date.parse(trimmed); + return Number.isNaN(date) ? undefined : Math.max(0, date - nowMs); +} + +interface PostAttempt { + /** HTTP status, or 0 when the request itself failed. */ + status: number; + detail: string; + retryAfter?: number; +} + +async function postOnce( + config: IndexNowConfig, + body: string, + fetch: FetchLike, + now: () => number +): Promise { + try { + const response = await fetch(config.endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': USER_AGENT }, + body, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + return { + status: response.status, + retryAfter: parseRetryAfter(response.headers.get('retry-after'), now()), + detail: (await response.text()).trim().slice(0, 300), + }; + } catch (error) { + return { status: 0, detail: errorMessage(error) }; + } +} + +async function postChunk( + config: IndexNowConfig, + urls: readonly string[], + fetch: FetchLike, + sleep: Sleep, + now: () => number, + log: Logger +): Promise { + const body = JSON.stringify({ + host: config.host, + key: config.key, + keyLocation: keyLocation(config), + urlList: urls, + }); + + for (let attempt = 1; ; attempt++) { + const { status, detail, retryAfter } = await postOnce(config, body, fetch, now); + const result = { size: urls.length, status, attempts: attempt, detail }; + if (status === 200 || status === 202) return result; + const retryable = status === 0 || status === 429 || status >= 500; + const delay = status === 429 ? retryAfter : RETRY_DELAYS_MS[attempt - 1]; + if ( + !retryable || + attempt > RETRY_DELAYS_MS.length || + delay === undefined || + delay > MAX_RETRY_AFTER_MS + ) { + return result; + } + log.warn( + `IndexNow answered ${status || 'with a network error'}; retrying in ${Math.ceil(delay / 1000)}s.` + ); + await sleep(delay); + } +} + +/** + * POSTs the URLs in chunks of at most 10,000. Chunks are all-or-nothing, so `accepted` is a + * prefix of `urls`. A 429 (after honouring a short Retry-After) or any other failure stops the + * run; the caller keeps the progress made so far. + */ +export async function submitUrls(options: SubmitOptions): Promise { + const { + config, + urls, + fetch = fetchWithProxy, + sleep = defaultSleep, + now = Date.now, + log = consoleLogger, + } = options; + const chunks: ChunkResult[] = []; + let accepted = 0; + + for (let offset = 0; offset < urls.length; offset += MAX_URLS_PER_REQUEST) { + if (offset > 0) await sleep(CHUNK_PAUSE_MS); + const chunk = urls.slice(offset, offset + MAX_URLS_PER_REQUEST); + const { detail, ...result } = await postChunk(config, chunk, fetch, sleep, now, log); + chunks.push(result); + if (result.status === 200 || result.status === 202) { + accepted += chunk.length; + log.info(`IndexNow accepted ${chunk.length} URLs (HTTP ${result.status}).`); + continue; + } + return { + accepted, + chunks, + stop: { + reason: result.status === 429 ? 'rate-limited' : 'failed', + status: result.status, + detail, + }, + }; + } + return { accepted, chunks, stop: null }; +} + +// --------------------------------------------------------------------------------------------- +// Live-site checks +// --------------------------------------------------------------------------------------------- + +/** IndexNow verifies ownership by fetching this file, so it has to be live before submitting. */ +export async function verifyKeyFile( + config: IndexNowConfig, + fetch: FetchLike = fetchWithProxy +): Promise<{ ok: true } | { ok: false; detail: string }> { + const url = keyLocation(config); + try { + const response = await fetch(url, { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) return { ok: false, detail: `GET ${url} returned HTTP ${response.status}` }; + const body = (await response.text()).trim(); + return body === config.key + ? { ok: true } + : { ok: false, detail: `${url} does not contain the configured key` }; + } catch (error) { + return { ok: false, detail: `GET ${url} failed: ${errorMessage(error)}` }; + } +} + +/** The What's new page of a release: aspire-13-6 for 13.6, aspire-14 for 14.0. */ +export function whatsNewPath(version: string): string { + const match = /^(\d+)\.(\d+)$/.exec(version); + if (!match) throw new Error(`"${version}" is not a release version such as 13.6`); + const major = Number(match[1]); + const minor = Number(match[2]); + return minor === 0 ? `/whats-new/aspire-${major}/` : `/whats-new/aspire-${major}-${minor}/`; +} + +export async function isReleaseLive( + host: string, + version: string, + fetch: FetchLike = fetchWithProxy +): Promise { + try { + const response = await fetch(`https://${host}${whatsNewPath(version)}`, { + method: 'HEAD', + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(30_000), + }); + return response.ok; + } catch { + return false; + } +} + +export interface WaitForReleaseOptions { + host: string; + version: string; + fetch?: FetchLike; + sleep?: Sleep; + now?: () => number; + timeoutMs?: number; + intervalMs?: number; + log?: Logger; +} + +/** + * Waits for the release's What's new page to be served. The release merge normally lands right + * after the slot swap; this protects against submitting URLs while the old build is still live, + * which would make engines crawl stale content. + */ +export async function waitForRelease(options: WaitForReleaseOptions): Promise { + const { + host, + version, + fetch = fetchWithProxy, + sleep = defaultSleep, + now = Date.now, + timeoutMs = GATE_TIMEOUT_MS, + intervalMs = GATE_INTERVAL_MS, + log = consoleLogger, + } = options; + const deadline = now() + timeoutMs; + for (;;) { + if (await isReleaseLive(host, version, fetch)) return true; + if (now() + intervalMs > deadline) return false; + log.info(`Release ${version} is not live yet; checking again in ${intervalMs / 1000}s.`); + await sleep(intervalMs); + } +} + +/** Reads the deployment identity the site embeds in every page. Diagnostic only. */ +export async function readDeployment( + host: string, + fetch: FetchLike = fetchWithProxy +): Promise { + try { + const response = await fetch(`https://${host}/`, { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(30_000), + }); + if (!response.ok) return ''; + const match = / + isRecord(side) && isRecord(side.repo) && typeof side.repo.full_name === 'string' + ? side.repo.full_name + : ''; + const headRepo = repo(pr.head); + if (!headRepo || headRepo !== repo(pr.base)) return null; + + const headRef = isRecord(pr.head) && typeof pr.head.ref === 'string' ? pr.head.ref : ''; + const title = typeof pr.title === 'string' ? pr.title : ''; + const match = + /^release\/(\d+\.\d+)$/.exec(headRef) ?? /^Merge release\/(\d+\.\d+) into main\b/i.exec(title); + return match ? { version: match[1], reason: `release/${match[1]}` } : null; +} + +export interface CliOptions { + stateDir: string; + mode: RunMode; + limit?: number; + release?: string; + trigger: string; + dryRun: boolean; + allowShrink: boolean; +} + +export function parseCli(argv: readonly string[]): CliOptions { + const { values } = parseArgs({ + args: [...argv], + options: { + 'state-dir': { type: 'string' }, + mode: { type: 'string' }, + limit: { type: 'string' }, + release: { type: 'string' }, + trigger: { type: 'string' }, + 'dry-run': { type: 'boolean' }, + 'allow-shrink': { type: 'boolean' }, + }, + strict: true, + allowPositionals: false, + }); + + const stateDir = values['state-dir']; + if (!stateDir) throw new Error('--state-dir is required'); + const mode = values.mode ?? 'daily'; + if (mode !== 'daily' && mode !== 'flush') throw new Error('--mode must be "daily" or "flush"'); + + let limit: number | undefined; + if (values.limit !== undefined) { + limit = Number(values.limit); + if (!isLimit(limit)) throw new Error(`--limit must be an integer between 1 and ${MAX_LIMIT}`); + } + if (values.release !== undefined && !/^\d+\.\d+$/.test(values.release)) { + throw new Error('--release must look like 13.6'); + } + + return { + stateDir: path.resolve(stateDir), + mode, + limit, + release: values.release, + trigger: values.trigger || 'manual', + dryRun: values['dry-run'] ?? false, + allowShrink: values['allow-shrink'] ?? false, + }; +} + +export interface ResolvedRun { + /** Set when the trigger does not call for a run; nothing is fetched or submitted. */ + skip?: string; + mode: RunMode; + release?: string; + reason: string; +} + +/** Turns the CLI options and the (pull_request) event payload into what this run should do. */ +export function resolveRun(options: CliOptions, event: unknown): ResolvedRun { + if (options.trigger === 'pull_request') { + const release = detectReleaseMerge(event); + if (!release) { + return { + skip: 'This pull request is not a merged release/X.Y pull request; nothing to submit.', + mode: 'flush', + reason: '', + }; + } + return { mode: 'flush', release: release.version, reason: release.reason }; + } + + let reason = 'manual'; + if (options.release) reason = `release/${options.release}`; + else if (options.mode === 'flush') reason = 'manual flush'; + else if (options.trigger === 'schedule') reason = 'schedule'; + return { mode: options.mode, release: options.release, reason }; +} + +// --------------------------------------------------------------------------------------------- +// Run +// --------------------------------------------------------------------------------------------- + +export interface RunReport { + outcome: 'completed' | 'dry-run' | 'skipped' | 'rate-limited' | 'failed'; + trigger: string; + mode: RunMode; + reason: string; + host: string; + note: string; + deployment: string; + coldStart: boolean; + budget: number; + totalUrls: number; + newUrls: number; + removedUrls: number; + /** URLs planned (dry run) or accepted (real run) from the rotation window. */ + windowUrls: number; + submitted: number; + /** New and removed URLs that did not fit in this run and stay pending. */ + deferred: number; + cycleKind: CycleKind | null; + cycleCovered: number; + cycleCompleted: boolean; + chunks: ChunkResult[]; + firstWindowUrl: string; + lastWindowUrl: string; +} + +export interface RunResult { + report: RunReport; + stateChanged: boolean; + /** Set when the run failed after it had started submitting. */ + failure?: string; +} + +export interface RunOptions { + config: IndexNowConfig; + stateDir: string; + mode: RunMode; + trigger: string; + reason: string; + dryRun?: boolean; + allowShrink?: boolean; + limit?: number; + /** Wait for this release's What's new page before submitting. */ + release?: string; + fetch?: FetchLike; + sleep?: Sleep; + now?: () => Date; + log?: Logger; + gate?: { timeoutMs?: number; intervalMs?: number }; +} + +export function emptyReport( + overrides: Partial & Pick +): RunReport { + return { + mode: 'daily', + reason: '', + note: '', + deployment: '', + coldStart: false, + budget: 0, + totalUrls: 0, + newUrls: 0, + removedUrls: 0, + windowUrls: 0, + submitted: 0, + deferred: 0, + cycleKind: null, + cycleCovered: 0, + cycleCompleted: false, + chunks: [], + firstWindowUrl: '', + lastWindowUrl: '', + ...overrides, + }; +} + +export async function run(options: RunOptions): Promise { + const { config, stateDir, mode, trigger, reason, release, limit } = options; + const dryRun = options.dryRun ?? false; + const fetch = options.fetch ?? fetchWithProxy; + const sleep = options.sleep ?? defaultSleep; + const now = options.now ?? (() => new Date()); + const log = options.log ?? consoleLogger; + const base = { trigger, mode, reason, host: config.host }; + + const state = await loadState(stateDir, config.host, log); + log.info( + state + ? `Loaded state from ${state.updatedAt}: ${state.known.length} known URLs.` + : 'No previous state; this is a cold start.' + ); + + const key = await verifyKeyFile(config, fetch); + if (!key.ok) { + if (dryRun) { + log.warn(`${key.detail}. IndexNow would reject submissions until the key file is live.`); + } else if (state === null) { + // Before the first deployment that ships the key file there is nothing to submit yet. + const note = `The IndexNow key file is not live yet (${key.detail}); skipping this run.`; + log.warn(note); + return { report: emptyReport({ ...base, outcome: 'skipped', note }), stateChanged: false }; + } else { + throw new Error(`The IndexNow key file is not reachable: ${key.detail}`); + } + } + + if (release) { + if (dryRun) { + if (!(await isReleaseLive(config.host, release, fetch))) { + log.warn(`The What's new page for release ${release} is not live yet.`); + } + } else { + const ready = await waitForRelease({ + host: config.host, + version: release, + fetch, + sleep, + now: () => now().getTime(), + log, + ...options.gate, + }); + if (!ready) { + throw new Error( + `The What's new page for release ${release} (${whatsNewPath(release)}) did not go live in ` + + 'time. Run the workflow again with the release input once the new docs are live.' + ); + } + } + } + + const { urls, rejected } = normalizeUrls( + await fetchSitemapLocations(config.sitemapIndex, fetch), + config.host + ); + if (rejected.length > 0) { + log.warn( + `Ignored ${rejected.length} sitemap URLs that are not https URLs on ${config.host} (for example ${rejected[0]}).` + ); + } + const deployment = await readDeployment(config.host, fetch); + + const plan = planRun({ + current: urls, + state, + mode, + reason, + budgets: config.budgets, + limit, + now: now(), + allowShrink: options.allowShrink, + }); + const ordered = [...plan.removed, ...plan.fresh, ...plan.window]; + const planned = { + ...base, + deployment, + coldStart: plan.coldStart, + budget: plan.budget, + totalUrls: plan.total, + newUrls: plan.newUrls.length, + removedUrls: plan.removedUrls.length, + deferred: + plan.newUrls.length + plan.removedUrls.length - plan.removed.length - plan.fresh.length, + cycleKind: plan.cycle.kind, + firstWindowUrl: plan.window[0] ?? '', + lastWindowUrl: plan.window[plan.window.length - 1] ?? '', + }; + log.info( + `Sitemap: ${plan.total} URLs; ${plan.newUrls.length} new, ${plan.removedUrls.length} removed. ` + + `Submitting ${ordered.length} (${plan.removed.length} removed, ${plan.fresh.length} new, ` + + `${plan.window.length} from the ${plan.cycle.kind} cycle) with a budget of ${plan.budget}.` + ); + + if (dryRun) { + return { + report: emptyReport({ + ...planned, + outcome: 'dry-run', + note: 'Dry run: nothing was submitted and the state was not saved.', + windowUrls: plan.window.length, + submitted: ordered.length, + cycleCovered: plan.covered, + cycleCompleted: plan.windowComplete, + }), + stateChanged: false, + }; + } + + const result: SubmitResult = + ordered.length === 0 + ? { accepted: 0, chunks: [], stop: null } + : await submitUrls({ config, urls: ordered, fetch, sleep, now: () => now().getTime(), log }); + + // Progress is kept even when a later chunk fails, so accepted URLs are never sent twice. + const stateChanged = result.stop === null || result.accepted > 0 || plan.coldStart; + let next: IndexNowState | null = null; + if (stateChanged) { + next = finalizeRun(plan, { + host: config.host, + current: urls, + accepted: result.accepted, + now: now(), + trigger, + deployment, + statuses: result.chunks.map((chunk) => chunk.status), + }); + await saveState(stateDir, next); + } + + const cycleCompleted = next !== null && next.cycle === null; + const cursor = next ? (next.cycle?.cursor ?? null) : plan.cycle.cursor; + const report = emptyReport({ + ...planned, + outcome: 'completed', + windowUrls: next?.lastRun?.windowUrls ?? 0, + submitted: result.accepted, + chunks: result.chunks, + cycleCovered: cycleCompleted ? plan.total : countCovered(urls, cursor, plan.localeCodes), + cycleCompleted, + }); + + if (result.stop?.reason === 'rate-limited') { + const note = + `IndexNow answered 429 (too many requests) after ${result.accepted} URLs. The progress was ` + + 'saved and the next run continues from there.'; + log.warn(note); + return { report: { ...report, outcome: 'rate-limited', note }, stateChanged }; + } + if (result.stop) { + const failure = + `IndexNow rejected the submission with HTTP ${result.stop.status || 'a network error'}` + + `${result.stop.detail ? `: ${result.stop.detail}` : ''}. ` + + 'For 403 or 422, check that the key file is live and matches config/indexnow.json.'; + return { report: { ...report, outcome: 'failed', note: failure }, stateChanged, failure }; + } + return { report, stateChanged }; +} + +export function renderSummary(report: RunReport): string { + const count = (value: number) => value.toLocaleString('en-US'); + const code = (value: string) => `\`${value}\``; + const rows: [string, string][] = [ + ['Outcome', report.outcome], + ['Trigger', code(report.trigger)], + ]; + if (report.outcome !== 'skipped') { + rows.push(['Mode', `${code(report.mode)}${report.reason ? ` (${report.reason})` : ''}`]); + } + if (report.totalUrls > 0) { + rows.push( + ['Sitemap URLs', count(report.totalUrls)], + ['New / removed', `${count(report.newUrls)} / ${count(report.removedUrls)}`] + ); + if (report.deferred > 0) rows.push(['Deferred to the next run', count(report.deferred)]); + rows.push([ + report.outcome === 'dry-run' ? 'Would submit' : 'Submitted', + `${count(report.submitted)} of a ${count(report.budget)} URL budget`, + ]); + if (report.cycleKind) { + const percent = Math.floor((report.cycleCovered / report.totalUrls) * 100); + rows.push([ + 'Cycle', + `${report.cycleKind}: ${count(report.cycleCovered)} of ${count(report.totalUrls)} URLs ` + + `(${percent}%)${report.cycleCompleted ? ', complete' : ''}`, + ]); + } + if (report.firstWindowUrl) { + rows.push([ + 'Window', + `${count(report.windowUrls)} URLs, ${code(report.firstWindowUrl)} to ${code(report.lastWindowUrl)}`, + ]); + } + } + if (report.chunks.length > 0) { + rows.push([ + 'Requests', + report.chunks + .map((chunk) => `HTTP ${chunk.status || 'error'} for ${count(chunk.size)} URLs`) + .join('
'), + ]); + } + if (report.deployment) rows.push(['Live deployment', code(report.deployment.slice(0, 7))]); + if (report.coldStart) { + rows.push([ + 'Cold start', + `yes: the known URL list ${report.outcome === 'dry-run' ? 'would be' : 'was'} seeded`, + ]); + } + + const lines = [ + `## IndexNow submission${report.host ? ` for ${report.host}` : ''}`, + '', + '| | |', + '|---|---|', + ...rows.map(([name, value]) => `| ${name} | ${value} |`), + ]; + if (report.note) lines.push('', report.note); + return `${lines.join('\n')}\n`; +} + +async function readEventPayload(env: NodeJS.ProcessEnv): Promise { + if (!env.GITHUB_EVENT_PATH) throw new Error('GITHUB_EVENT_PATH is not set'); + const payload: unknown = JSON.parse(await readFile(env.GITHUB_EVENT_PATH, 'utf8')); + return payload; +} + +async function publish( + env: NodeJS.ProcessEnv, + report: RunReport, + stateChanged: boolean +): Promise { + if (env.GITHUB_STEP_SUMMARY) await appendFile(env.GITHUB_STEP_SUMMARY, renderSummary(report)); + if (env.GITHUB_OUTPUT) { + await appendFile( + env.GITHUB_OUTPUT, + `state_changed=${stateChanged}\nsubmitted=${report.submitted}\noutcome=${report.outcome}\n` + ); + } +} + +/** Runs the CLI and returns the process exit code. */ +export async function main( + argv: readonly string[] = process.argv.slice(2), + env: NodeJS.ProcessEnv = process.env, + log: Logger = consoleLogger, + overrides: Pick = {} +): Promise { + let trigger = 'manual'; + let host = ''; + try { + const options = parseCli(argv); + trigger = options.trigger; + const config = await loadConfig(); + host = config.host; + const event = trigger === 'pull_request' ? await readEventPayload(env) : undefined; + const resolved = resolveRun(options, event); + + if (resolved.skip) { + log.info(resolved.skip); + await publish( + env, + emptyReport({ outcome: 'skipped', trigger, host, note: resolved.skip }), + false + ); + return 0; + } + + const result = await run({ + config, + stateDir: options.stateDir, + mode: resolved.mode, + trigger, + reason: resolved.reason, + release: resolved.release, + limit: options.limit, + dryRun: options.dryRun, + allowShrink: options.allowShrink, + log, + ...overrides, + }); + await publish(env, result.report, result.stateChanged); + if (result.failure) { + log.error(result.failure); + return 1; + } + return 0; + } catch (error) { + const message = errorMessage(error); + log.error(message); + await publish( + env, + emptyReport({ outcome: 'failed', trigger, host, note: message }), + false + ).catch(() => undefined); + return 1; + } +} + +const isMainModule = process.argv[1] + ? path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) + : false; + +if (isMainModule) { + void main().then((code) => { + process.exitCode = code; + }); +} diff --git a/src/frontend/tests/unit/indexnow.vitest.test.ts b/src/frontend/tests/unit/indexnow.vitest.test.ts new file mode 100644 index 0000000000..7f81fd4a65 --- /dev/null +++ b/src/frontend/tests/unit/indexnow.vitest.test.ts @@ -0,0 +1,1515 @@ +import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { gzipSync } from 'node:zlib'; +import { Response } from 'node-fetch'; +import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; + +import type { fetchWithProxy } from '../../scripts/fetch-with-proxy'; +import { + MAX_URLS_PER_REQUEST, + compareUrls, + countCovered, + detectReleaseMerge, + emptyReport, + fetchSitemapLocations, + finalizeRun, + isReleaseLive, + loadConfig, + loadState, + main, + normalizeUrls, + parseCli, + parseConfig, + parseRetryAfter, + parseSitemap, + planRun, + readDeployment, + renderSummary, + resolveRun, + run, + saveState, + submitUrls, + tierOf, + verifyKeyFile, + waitForRelease, + whatsNewPath, + type IndexNowConfig, + type IndexNowState, + type RunOptions, +} from '../../scripts/indexnow'; + +const HOST = 'aspire.dev'; +const KEY = 'a1b2c3d4e5f60718293a4b5c6d7e8f90'; +const NOW = new Date('2026-10-13T06:43:00.000Z'); + +const config: IndexNowConfig = { + host: HOST, + key: KEY, + endpoint: 'https://api.indexnow.org/indexnow', + sitemapIndex: `https://${HOST}/sitemap-index.xml`, + budgets: { release: 5, rotation: 3 }, +}; + +const url = (pathname: string) => `https://${HOST}${pathname}`; + +// Ten pages in (tier, path) order: four English pages, three API pages, three localized pages. +const DOCS = ['/', '/app-host/', '/get-started/', '/whats-new/aspire-13-6/']; +const API = [ + '/reference/api/csharp/a/', + '/reference/api/csharp/b/', + '/reference/api/typescript/c/', +]; +const LOCALIZED = ['/de/', '/de/app-host/', '/ja/']; +const SMALL_SITE = [...DOCS, ...API, ...LOCALIZED]; + +function createLogger() { + return { + info: vi.fn<(message: string) => void>(), + warn: vi.fn<(message: string) => void>(), + error: vi.fn<(message: string) => void>(), + }; +} + +function createClock() { + let current = NOW.getTime(); + return { + now: () => new Date(current), + sleep: vi.fn<(ms: number) => Promise>((ms) => { + current += ms; + return Promise.resolve(); + }), + }; +} + +function makeState(overrides: Partial = {}): IndexNowState { + return { + host: HOST, + updatedAt: '2026-10-12T06:43:00.000Z', + deployment: '', + cycle: null, + lastRun: null, + known: [], + ...overrides, + }; +} + +function syntheticUrls(count: number, prefix = '/reference/api/csharp/page-'): string[] { + return Array.from({ length: count }, (_, index) => + url(`${prefix}${String(index).padStart(6, '0')}/`) + ); +} + +interface PostCall { + host: string; + key: string; + keyLocation: string; + urlList: string[]; +} + +type QueuedResponse = number | { status: number; headers?: Record }; + +/** The JSON body of a request the code under test sent. */ +function postBody(init?: { body?: unknown }): PostCall { + expect(typeof init?.body).toBe('string'); + return JSON.parse(init?.body as string) as PostCall; +} + +/** A stand-in for aspire.dev and the IndexNow endpoint. */ +function createSite(paths: string[], key = KEY) { + const site = { + paths: [...paths], + keyLive: true, + liveReleases: new Set(), + deployment: 'deadbeefcafe0123456789', + /** Statuses for upcoming IndexNow POSTs; 200 when the queue is empty. */ + responses: [] as QueuedResponse[], + posts: [] as PostCall[], + fetch: vi.fn(), + }; + + const urlset = () => + '' + + site.paths + .map( + (pathname) => + `${url(pathname)}` + ) + .join('') + + ''; + + site.fetch.mockImplementation((input, init) => { + const target = new URL(String(input)); + if (target.origin === 'https://api.indexnow.org') { + const body = postBody(init); + site.posts.push(body); + const next = site.responses.shift() ?? 200; + const { status, headers } = typeof next === 'number' ? { status: next, headers: {} } : next; + return Promise.resolve(new Response('', { status, headers })); + } + if (target.pathname === `/${key}.txt`) { + return Promise.resolve( + site.keyLive ? new Response(key) : new Response('Not found', { status: 404 }) + ); + } + if (target.pathname === '/sitemap-index.xml') { + return Promise.resolve( + new Response( + '' + + `${url('/sitemap-0.xml')}` + ) + ); + } + if (target.pathname === '/sitemap-0.xml') return Promise.resolve(new Response(urlset())); + if (target.pathname === '/') { + return Promise.resolve( + new Response( + `` + ) + ); + } + const release = /^\/whats-new\/aspire-(\d+)(?:-(\d+))?\/$/.exec(target.pathname); + if (release) { + const version = `${release[1]}.${release[2] ?? '0'}`; + return Promise.resolve( + new Response('', { status: site.liveReleases.has(version) ? 200 : 404 }) + ); + } + return Promise.resolve(new Response('Not found', { status: 404 })); + }); + + return site; +} + +type Site = ReturnType; + +let stateDir: string; + +beforeEach(async () => { + stateDir = await mkdtemp(path.join(os.tmpdir(), 'indexnow-test-')); +}); + +afterEach(async () => { + await rm(stateDir, { recursive: true, force: true }); +}); + +describe('IndexNow config', () => { + test('the committed config is valid and its key file is served from the site root', async () => { + const real = await loadConfig(); + const keyFile = fileURLToPath(new URL(`../../public/${real.key}.txt`, import.meta.url)); + const contents = await readFile(keyFile, 'utf8'); + expect(contents.charCodeAt(0)).not.toBe(0xfeff); + expect(contents.trim()).toBe(real.key); + expect(real.host).toBe('aspire.dev'); + expect(new URL(real.sitemapIndex).hostname).toBe(real.host); + }); + + test.each([ + ['a key that is too short', { ...config, key: 'short' }], + ['a key with illegal characters', { ...config, key: `${KEY.slice(0, 31)}_` }], + ['an endpoint that is not https', { ...config, endpoint: 'http://api.indexnow.org/indexnow' }], + ['a sitemap on another host', { ...config, sitemapIndex: 'https://example.com/sitemap.xml' }], + ['a zero budget', { ...config, budgets: { release: 0, rotation: 3 } }], + ['a budget above the limit', { ...config, budgets: { release: 5, rotation: 50_001 } }], + ['an uppercase host', { ...config, host: 'Aspire.dev' }], + ])('rejects %s', (_name, value) => { + expect(() => parseConfig(value)).toThrow(); + }); + + test('accepts the shape used by the workflow', () => { + expect(parseConfig(config)).toEqual(config); + }); +}); + +describe('sitemap parsing', () => { + test('reads values, decodes entities and ignores xhtml alternates', () => { + const parsed = parseSitemap( + '' + + 'https://aspire.dev/' + + ' https://aspire.dev/a?b=1&c=2 ' + + 'https://aspire.dev/éé/' + + '' + ); + expect(parsed).toEqual({ + kind: 'urlset', + locations: ['https://aspire.dev/', 'https://aspire.dev/a?b=1&c=2', 'https://aspire.dev/éé/'], + }); + }); + + test('recognizes a sitemap index', () => { + const parsed = parseSitemap( + 'https://aspire.dev/sitemap-0.xml' + ); + expect(parsed).toEqual({ kind: 'index', locations: ['https://aspire.dev/sitemap-0.xml'] }); + }); + + test('rejects documents that are not sitemaps', () => { + expect(() => parseSitemap('Not found')).toThrow(/not a sitemap/i); + }); + + test('follows a sitemap index to its child sitemaps', async () => { + const site = createSite(['/', '/a/']); + expect(await fetchSitemapLocations(config.sitemapIndex, site.fetch)).toEqual([ + url('/'), + url('/a/'), + ]); + expect(site.fetch).toHaveBeenCalledTimes(2); + }); + + test('refuses child sitemaps on another host or nested indexes', async () => { + const fetch = vi.fn(); + fetch.mockResolvedValueOnce( + new Response( + 'https://evil.example/s.xml' + ) + ); + await expect(fetchSitemapLocations(config.sitemapIndex, fetch)).rejects.toThrow( + /not an https URL on aspire\.dev/ + ); + + fetch.mockResolvedValueOnce( + new Response( + 'https://aspire.dev/a.xml' + ) + ); + fetch.mockResolvedValueOnce( + new Response( + 'https://aspire.dev/b.xml' + ) + ); + await expect(fetchSitemapLocations(config.sitemapIndex, fetch)).rejects.toThrow(/nested/); + }); + + test('fails when the sitemap cannot be fetched', async () => { + const fetch = vi.fn(); + fetch.mockResolvedValueOnce(new Response('oops', { status: 503 })); + await expect(fetchSitemapLocations(config.sitemapIndex, fetch)).rejects.toThrow(/HTTP 503/); + }); + + test('keeps https URLs on the host, drops fragments and de-duplicates', () => { + const { urls, rejected } = normalizeUrls( + [ + 'https://aspire.dev/a/', + 'https://aspire.dev/a/#section', + 'https://ASPIRE.dev/b/', + 'http://aspire.dev/c/', + 'https://aspire.dev:8443/d/', + 'https://user:pass@aspire.dev/e/', + 'https://example.com/f/', + 'not a url', + ], + HOST + ); + expect(urls).toEqual(['https://aspire.dev/a/', 'https://aspire.dev/b/']); + expect(rejected).toHaveLength(5); + }); +}); + +describe('ordering', () => { + test('puts English pages first, then the API reference, then localized pages', () => { + expect(tierOf('/')).toBe(0); + expect(tierOf('/app-host/certificate-configuration/')).toBe(0); + expect(tierOf('/reference/cli/')).toBe(0); + expect(tierOf('/reference/api')).toBe(0); + expect(tierOf('/reference/api/csharp/aspire.hosting/')).toBe(1); + expect(tierOf('/zh-cn/whats-new/upgrade-aspire/')).toBe(2); + expect(tierOf('/pt-br/')).toBe(2); + expect(tierOf('/de/reference/api/csharp/x/')).toBe(2); + expect(tierOf('/deployment/')).toBe(0); + }); + + test('sorts by tier and then by URL', () => { + const shuffled = [...SMALL_SITE].reverse().map(url); + expect(shuffled.sort((a, b) => compareUrls(a, b))).toEqual(SMALL_SITE.map(url)); + }); + + test('counts how many URLs a cursor has covered', () => { + const urls = SMALL_SITE.map(url); + expect(countCovered(urls, null)).toBe(0); + expect(countCovered(urls, url('/get-started/'))).toBe(3); + expect(countCovered(urls, url('/ja/'))).toBe(10); + }); +}); + +describe('planning', () => { + const base = { + current: SMALL_SITE.map(url), + state: null as IndexNowState | null, + mode: 'daily' as const, + reason: 'schedule', + budgets: config.budgets, + now: NOW, + }; + + test('a cold start seeds the known list instead of treating every URL as new', () => { + const plan = planRun(base); + expect(plan.coldStart).toBe(true); + expect(plan.newUrls).toEqual([]); + expect(plan.removedUrls).toEqual([]); + expect(plan.cycle).toMatchObject({ kind: 'rotation', cursor: null, reason: 'schedule' }); + expect(plan.window).toEqual(['/', '/app-host/', '/get-started/'].map(url)); + expect(plan.windowComplete).toBe(false); + }); + + test('a cycle continues after its cursor', () => { + const plan = planRun({ + ...base, + state: makeState({ + known: base.current, + cycle: { + kind: 'rotation', + reason: 'schedule', + startedAt: '2026-10-12T00:00:00.000Z', + cursor: url('/get-started/'), + skip: [], + }, + }), + }); + expect(plan.coldStart).toBe(false); + expect(plan.window).toEqual( + ['/whats-new/aspire-13-6/', '/reference/api/csharp/a/', '/reference/api/csharp/b/'].map(url) + ); + }); + + test('a cursor that is no longer in the sitemap still finds the next URL', () => { + const plan = planRun({ + ...base, + state: makeState({ + known: base.current, + cycle: { + kind: 'rotation', + reason: 'schedule', + startedAt: '2026-10-12T00:00:00.000Z', + cursor: url('/get-started/gone/'), + skip: [], + }, + }), + }); + expect(plan.window[0]).toBe(url('/whats-new/aspire-13-6/')); + }); + + test('a flush restarts the cycle from the top with the release budget', () => { + const plan = planRun({ + ...base, + mode: 'flush', + reason: 'release/13.6', + state: makeState({ + known: base.current, + cycle: { + kind: 'rotation', + reason: 'schedule', + startedAt: '2026-10-01T00:00:00.000Z', + cursor: url('/ja/'), + skip: [url('/de/')], + }, + }), + }); + expect(plan.cycle).toMatchObject({ + kind: 'release', + reason: 'release/13.6', + cursor: null, + skip: [], + }); + expect(plan.budget).toBe(5); + expect(plan.window).toEqual( + [ + '/', + '/app-host/', + '/get-started/', + '/whats-new/aspire-13-6/', + '/reference/api/csharp/a/', + ].map(url) + ); + }); + + test('a release cycle keeps its larger budget until it completes', () => { + const plan = planRun({ + ...base, + state: makeState({ + known: base.current, + cycle: { + kind: 'release', + reason: 'release/13.6', + startedAt: '', + cursor: url('/ja/'), + skip: [], + }, + }), + }); + expect(plan.budget).toBe(5); + }); + + test('--limit replaces the budget', () => { + expect(planRun({ ...base, limit: 2 }).window).toHaveLength(2); + }); + + test('new and removed URLs go first and count against the budget', () => { + const known = SMALL_SITE.map(url); + const current = [...known.filter((entry) => entry !== url('/ja/')), url('/aaa/')]; + const plan = planRun({ + ...base, + current, + state: makeState({ + known, + cycle: { + kind: 'rotation', + reason: 'schedule', + startedAt: '', + cursor: url('/get-started/'), + skip: [], + }, + }), + }); + expect(plan.removed).toEqual([url('/ja/')]); + expect(plan.fresh).toEqual([url('/aaa/')]); + expect(plan.window).toEqual([url('/whats-new/aspire-13-6/')]); + }); + + test('a new URL ahead of the cursor is not sent again by the window', () => { + const known = SMALL_SITE.map(url); + const created = url('/reference/api/csharp/ab/'); + const plan = planRun({ + ...base, + current: [...known, created], + state: makeState({ + known, + cycle: { + kind: 'rotation', + reason: 'schedule', + startedAt: '', + cursor: url('/get-started/'), + skip: [], + }, + }), + limit: 10, + }); + expect(plan.fresh).toEqual([created]); + expect(plan.window).not.toContain(created); + expect(plan.window).toHaveLength(7); + }); + + test('new and removed URLs beyond the per-run cap stay pending', () => { + const known = syntheticUrls(3); + const created = syntheticUrls(10_005, '/reference/api/csharp/new-'); + const plan = planRun({ + ...base, + current: [...known, ...created], + state: makeState({ known }), + limit: 20_000, + }); + expect(plan.newUrls).toHaveLength(10_005); + expect(plan.fresh).toHaveLength(10_000); + const next = finalizeRun(plan, { + host: HOST, + current: [...known, ...created], + accepted: plan.removed.length + plan.fresh.length + plan.window.length, + now: NOW, + trigger: 'schedule', + deployment: '', + statuses: [200], + }); + const stillNew = created.filter((entry) => !next.known.includes(entry)); + expect(stillNew).toHaveLength( + 5 - plan.window.filter((entry) => created.includes(entry)).length + ); + }); + + test('refuses an empty sitemap', () => { + expect(() => planRun({ ...base, current: [] })).toThrow(/no URLs/); + }); + + test('refuses a sitemap that shrank by more than 20% unless allowed', () => { + const known = syntheticUrls(100); + const shrunk = known.slice(0, 79); + expect(() => planRun({ ...base, current: shrunk, state: makeState({ known }) })).toThrow( + /--allow-shrink/ + ); + expect(() => + planRun({ ...base, current: known.slice(0, 80), state: makeState({ known }) }) + ).not.toThrow(); + expect(() => + planRun({ ...base, current: shrunk, state: makeState({ known }), allowShrink: true }) + ).not.toThrow(); + }); +}); + +describe('finalizing a run', () => { + const finalizeOptions = { + host: HOST, + now: NOW, + trigger: 'schedule', + deployment: 'abc1234', + statuses: [200], + }; + + test('remembers the whole sitemap and advances the cursor when everything was accepted', () => { + const current = SMALL_SITE.map(url); + const plan = planRun({ + current, + state: null, + mode: 'daily', + reason: 'schedule', + budgets: config.budgets, + now: NOW, + }); + const next = finalizeRun(plan, { ...finalizeOptions, current, accepted: 3 }); + expect(next.known).toEqual([...current].sort()); + expect(next.cycle?.cursor).toBe(url('/get-started/')); + expect(next.lastRun).toMatchObject({ coldStart: true, submitted: 3, windowUrls: 3 }); + expect(next.deployment).toBe('abc1234'); + }); + + test('completes the cycle once the window reaches the end of the sitemap', () => { + const current = SMALL_SITE.map(url); + const plan = planRun({ + current, + state: makeState({ + known: current, + cycle: { + kind: 'release', + reason: 'release/13.6', + startedAt: '', + cursor: url('/reference/api/typescript/c/'), + skip: [], + }, + }), + mode: 'daily', + reason: 'schedule', + budgets: config.budgets, + now: NOW, + }); + expect(plan.window).toEqual(LOCALIZED.map(url)); + expect(plan.windowComplete).toBe(true); + expect(finalizeRun(plan, { ...finalizeOptions, current, accepted: 3 }).cycle).toBeNull(); + // A partly accepted window leaves the cycle open at the last accepted URL. + const partial = finalizeRun(plan, { ...finalizeOptions, current, accepted: 2 }); + expect(partial.cycle?.cursor).toBe(url('/de/app-host/')); + }); + + test('keeps URLs that were not accepted pending', () => { + const known = SMALL_SITE.map(url); + const created = [url('/aaa/'), url('/aab/')]; + const current = [...known.filter((entry) => entry !== url('/ja/')), ...created]; + const plan = planRun({ + current, + state: makeState({ known }), + mode: 'daily', + reason: 'schedule', + budgets: config.budgets, + now: NOW, + }); + expect(plan.removed).toEqual([url('/ja/')]); + expect(plan.fresh).toEqual(created); + + const partial = finalizeRun(plan, { ...finalizeOptions, current, accepted: 2 }); + expect(partial.known).not.toContain(url('/ja/')); + expect(partial.known).toContain(created[0]); + expect(partial.known).not.toContain(created[1]); + + const none = finalizeRun(plan, { ...finalizeOptions, current, accepted: 0 }); + expect(none.known).toContain(url('/ja/')); + expect(none.known).not.toContain(created[0]); + expect(none.known).not.toContain(created[1]); + }); + + test('forgets skipped URLs once the cursor has passed them', () => { + const known = SMALL_SITE.map(url); + const created = url('/reference/api/csharp/ab/'); + const current = [...known, created]; + const cycle = { + kind: 'rotation' as const, + reason: 'schedule', + startedAt: '', + cursor: url('/get-started/'), + skip: [], + }; + const plan = planRun({ + current, + state: makeState({ known, cycle }), + mode: 'daily', + reason: 'schedule', + budgets: config.budgets, + limit: 2, + now: NOW, + }); + expect(plan.window).toEqual([url('/whats-new/aspire-13-6/')]); + const day = finalizeRun(plan, { + ...finalizeOptions, + current, + accepted: 1 + plan.window.length, + }); + expect(day.cycle?.skip).toEqual([created]); + + const nextPlan = planRun({ + current, + state: day, + mode: 'daily', + reason: 'schedule', + budgets: config.budgets, + limit: 4, + now: NOW, + }); + expect(nextPlan.window).toEqual( + [ + '/reference/api/csharp/a/', + '/reference/api/csharp/b/', + '/reference/api/typescript/c/', + '/de/', + ].map(url) + ); + const nextDay = finalizeRun(nextPlan, { + ...finalizeOptions, + current, + accepted: nextPlan.window.length, + }); + expect(nextDay.cycle?.skip).toEqual([]); + }); + + test('a release burst over a sitemap shaped like aspire.dev takes four runs and sends each URL once', () => { + const docs = syntheticUrls(611, '/docs/page-'); + const api = syntheticUrls(24_207); + const localized = syntheticUrls(7_518, '/de/page-'); + const current = [...localized, ...api, ...docs]; + const budgets = { release: 10_000, rotation: 1_000 }; + + let state: IndexNowState = makeState({ known: current }); + const sizes: number[] = []; + const seen = new Set(); + for (let day = 0; day < 10; day++) { + const plan = planRun({ + current, + state, + mode: day === 0 ? 'flush' : 'daily', + reason: 'release/13.6', + budgets, + now: NOW, + }); + if (day === 0) expect(plan.window.slice(0, 611)).toEqual(docs); + for (const entry of plan.window) { + expect(seen.has(entry)).toBe(false); + seen.add(entry); + } + sizes.push(plan.window.length); + state = finalizeRun(plan, { + ...finalizeOptions, + current, + accepted: plan.window.length, + }); + if (state.cycle === null) break; + } + expect(sizes).toEqual([10_000, 10_000, 10_000, 2_336]); + expect(seen.size).toBe(current.length); + + // The next run starts a slow rotation cycle from the top. + const after = planRun({ current, state, mode: 'daily', reason: 'schedule', budgets, now: NOW }); + expect(after.cycle.kind).toBe('rotation'); + expect(after.window).toHaveLength(1_000); + expect(after.window[0]).toBe(docs[0]); + }); +}); + +describe('state on disk', () => { + const state = makeState({ + deployment: 'abc1234', + known: SMALL_SITE.map(url), + cycle: { + kind: 'release', + reason: 'release/13.6', + startedAt: '2026-10-13T06:43:00.000Z', + cursor: url('/get-started/'), + skip: [url('/reference/api/csharp/ab/')], + }, + lastRun: { + at: '2026-10-13T06:43:00.000Z', + trigger: 'pull_request', + mode: 'flush', + coldStart: false, + newUrls: 1, + removedUrls: 2, + windowUrls: 3, + submitted: 6, + statuses: [200], + }, + }); + + test('round-trips and leaves no staging files behind', async () => { + await saveState(stateDir, state); + expect((await readdir(stateDir)).sort()).toEqual(['known-urls.txt.gz', 'state.json']); + expect(await loadState(stateDir, HOST, createLogger())).toEqual(state); + }); + + test('a missing state is a cold start without a warning', async () => { + const log = createLogger(); + expect(await loadState(path.join(stateDir, 'missing'), HOST, log)).toBeNull(); + expect(log.warn).not.toHaveBeenCalled(); + }); + + test.each([ + ['unreadable JSON', async () => writeFile(path.join(stateDir, 'state.json'), '{nope')], + [ + 'a different schema version', + async () => { + await saveState(stateDir, state); + const file = path.join(stateDir, 'state.json'); + const parsed = JSON.parse(await readFile(file, 'utf8')) as Record; + await writeFile(file, JSON.stringify({ ...parsed, schemaVersion: 99 })); + }, + ], + [ + 'another host', + async () => { + await saveState(stateDir, { ...state, host: 'example.com' }); + }, + ], + [ + 'a known list that does not match its count', + async () => { + await saveState(stateDir, state); + await writeFile( + path.join(stateDir, 'known-urls.txt.gz'), + gzipSync('https://aspire.dev/\n') + ); + }, + ], + [ + 'a truncated known list', + async () => { + await saveState(stateDir, state); + await writeFile(path.join(stateDir, 'known-urls.txt.gz'), 'not gzip'); + }, + ], + ])('falls back to a cold start for %s', async (_name, arrange) => { + await arrange(); + const log = createLogger(); + expect(await loadState(stateDir, HOST, log)).toBeNull(); + expect(log.warn).toHaveBeenCalledWith(expect.stringContaining('starting cold')); + }); +}); + +describe('submitting URLs', () => { + function fetchWith(...responses: (Response | Error)[]) { + const fetch = vi.fn(); + for (const response of responses) { + if (response instanceof Error) fetch.mockRejectedValueOnce(response); + else fetch.mockResolvedValueOnce(response); + } + return fetch; + } + + const urls = syntheticUrls(3); + + test('POSTs the documented payload', async () => { + const fetch = fetchWith(new Response('', { status: 200 })); + const result = await submitUrls({ + config, + urls, + fetch, + sleep: createClock().sleep, + log: createLogger(), + }); + expect(result).toEqual({ + accepted: 3, + chunks: [{ size: 3, status: 200, attempts: 1 }], + stop: null, + }); + + const [target, init] = fetch.mock.calls[0]; + expect(String(target)).toBe(config.endpoint); + expect(init?.method).toBe('POST'); + expect(init?.headers).toMatchObject({ 'Content-Type': 'application/json; charset=utf-8' }); + expect(postBody(init)).toEqual({ + host: HOST, + key: KEY, + keyLocation: `https://${HOST}/${KEY}.txt`, + urlList: urls, + }); + }); + + test('accepts 202 for a key that is still being verified', async () => { + const fetch = fetchWith(new Response('', { status: 202 })); + const result = await submitUrls({ + config, + urls, + fetch, + sleep: createClock().sleep, + log: createLogger(), + }); + expect(result.accepted).toBe(3); + expect(result.stop).toBeNull(); + }); + + test('splits large lists into chunks of at most 10,000 and pauses between them', async () => { + const many = syntheticUrls(25_000); + const fetch = fetchWith( + new Response('', { status: 200 }), + new Response('', { status: 200 }), + new Response('', { status: 200 }) + ); + const clock = createClock(); + const result = await submitUrls({ + config, + urls: many, + fetch, + sleep: clock.sleep, + log: createLogger(), + }); + expect(result.chunks.map((chunk) => chunk.size)).toEqual([10_000, 10_000, 5_000]); + expect(result.accepted).toBe(25_000); + expect(MAX_URLS_PER_REQUEST).toBe(10_000); + expect(clock.sleep).toHaveBeenCalledTimes(2); + const sent = fetch.mock.calls.flatMap(([, init]) => postBody(init).urlList); + expect(sent).toEqual(many); + }); + + test('waits for a short Retry-After on 429 and then succeeds', async () => { + const fetch = fetchWith( + new Response('', { status: 429, headers: { 'retry-after': '2' } }), + new Response('', { status: 200 }) + ); + const clock = createClock(); + const result = await submitUrls({ + config, + urls, + fetch, + sleep: clock.sleep, + log: createLogger(), + }); + expect(clock.sleep).toHaveBeenCalledWith(2_000); + expect(result.chunks).toEqual([{ size: 3, status: 200, attempts: 2 }]); + }); + + test('stops at once on a 429 without a usable Retry-After and keeps earlier progress', async () => { + const many = syntheticUrls(15_000); + const fetch = fetchWith( + new Response('', { status: 200 }), + new Response('slow down', { status: 429 }) + ); + const clock = createClock(); + const result = await submitUrls({ + config, + urls: many, + fetch, + sleep: clock.sleep, + log: createLogger(), + }); + expect(result.accepted).toBe(10_000); + expect(result.stop).toEqual({ reason: 'rate-limited', status: 429, detail: 'slow down' }); + expect(fetch).toHaveBeenCalledTimes(2); + + const tooLong = fetchWith( + new Response('', { status: 429, headers: { 'retry-after': '3600' } }) + ); + const stopped = await submitUrls({ + config, + urls, + fetch: tooLong, + sleep: clock.sleep, + log: createLogger(), + }); + expect(stopped.stop?.reason).toBe('rate-limited'); + expect(tooLong).toHaveBeenCalledTimes(1); + }); + + test('retries server errors with back-off and then gives up', async () => { + const fetch = fetchWith( + new Response('', { status: 503 }), + new Response('', { status: 503 }), + new Response('', { status: 503 }), + new Response('unavailable', { status: 503 }) + ); + const clock = createClock(); + const result = await submitUrls({ + config, + urls, + fetch, + sleep: clock.sleep, + log: createLogger(), + }); + expect(clock.sleep.mock.calls.map(([ms]) => ms)).toEqual([5_000, 30_000, 120_000]); + expect(result.accepted).toBe(0); + expect(result.chunks).toEqual([{ size: 3, status: 503, attempts: 4 }]); + expect(result.stop).toEqual({ reason: 'failed', status: 503, detail: 'unavailable' }); + }); + + test('treats network errors like server errors', async () => { + const boom = new Error('socket hang up'); + const fetch = fetchWith(boom, boom, boom, boom); + const result = await submitUrls({ + config, + urls, + fetch, + sleep: createClock().sleep, + log: createLogger(), + }); + expect(result.stop).toEqual({ reason: 'failed', status: 0, detail: 'socket hang up' }); + }); + + test.each([400, 403, 422])('does not retry HTTP %s', async (status) => { + const fetch = fetchWith(new Response('bad key', { status })); + const clock = createClock(); + const result = await submitUrls({ + config, + urls, + fetch, + sleep: clock.sleep, + log: createLogger(), + }); + expect(result.stop).toEqual({ reason: 'failed', status, detail: 'bad key' }); + expect(fetch).toHaveBeenCalledTimes(1); + expect(clock.sleep).not.toHaveBeenCalled(); + }); + + test('reads Retry-After as seconds or as an HTTP date', () => { + expect(parseRetryAfter('120', 0)).toBe(120_000); + expect( + parseRetryAfter('Wed, 21 Oct 2026 07:28:00 GMT', Date.parse('Wed, 21 Oct 2026 07:27:00 GMT')) + ).toBe(60_000); + expect( + parseRetryAfter('Wed, 21 Oct 2026 07:28:00 GMT', Date.parse('Wed, 21 Oct 2026 07:29:00 GMT')) + ).toBe(0); + expect(parseRetryAfter('soon', 0)).toBeUndefined(); + expect(parseRetryAfter(null, 0)).toBeUndefined(); + }); +}); + +describe('live-site checks', () => { + test('verifies that the key file contains the key', async () => { + const site = createSite([]); + expect(await verifyKeyFile(config, site.fetch)).toEqual({ ok: true }); + + site.keyLive = false; + expect(await verifyKeyFile(config, site.fetch)).toEqual({ + ok: false, + detail: `GET https://aspire.dev/${KEY}.txt returned HTTP 404`, + }); + + const wrong = vi.fn().mockResolvedValue(new Response('another-key')); + expect(await verifyKeyFile(config, wrong)).toMatchObject({ + ok: false, + detail: expect.stringContaining('does not contain'), + }); + + const down = vi.fn().mockRejectedValue(new Error('offline')); + expect(await verifyKeyFile(config, down)).toMatchObject({ + ok: false, + detail: expect.stringContaining('offline'), + }); + }); + + test("derives the What's new page of a release", () => { + expect(whatsNewPath('13.6')).toBe('/whats-new/aspire-13-6/'); + expect(whatsNewPath('9.5')).toBe('/whats-new/aspire-9-5/'); + expect(whatsNewPath('14.0')).toBe('/whats-new/aspire-14/'); + expect(() => whatsNewPath('13')).toThrow(); + expect(() => whatsNewPath('13.6.1')).toThrow(); + }); + + test('waits until the release page is served', async () => { + const site = createSite([]); + const clock = createClock(); + let polls = 0; + clock.sleep.mockImplementation(() => { + if (++polls === 2) site.liveReleases.add('13.6'); + return Promise.resolve(); + }); + const ready = await waitForRelease({ + host: HOST, + version: '13.6', + fetch: site.fetch, + sleep: clock.sleep, + timeoutMs: 600_000, + intervalMs: 60_000, + log: createLogger(), + }); + expect(ready).toBe(true); + expect(clock.sleep).toHaveBeenCalledTimes(2); + expect(site.fetch.mock.calls.every(([, init]) => init?.method === 'HEAD')).toBe(true); + }); + + test('gives up when the release page never appears', async () => { + const site = createSite([]); + const clock = createClock(); + const start = clock.now().getTime(); + const ready = await waitForRelease({ + host: HOST, + version: '13.6', + fetch: site.fetch, + sleep: clock.sleep, + now: () => clock.now().getTime(), + timeoutMs: 180_000, + intervalMs: 60_000, + log: createLogger(), + }); + expect(ready).toBe(false); + expect(clock.now().getTime() - start).toBeLessThanOrEqual(180_000); + expect(await isReleaseLive(HOST, '13.6', site.fetch)).toBe(false); + }); + + test('reads the deployment identity and tolerates failures', async () => { + const site = createSite([]); + expect(await readDeployment(HOST, site.fetch)).toBe('deadbeefcafe0123456789'); + const down = vi.fn().mockRejectedValue(new Error('offline')); + expect(await readDeployment(HOST, down)).toBe(''); + const missing = vi.fn().mockResolvedValue(new Response('')); + expect(await readDeployment(HOST, missing)).toBe(''); + }); +}); + +describe('triggers', () => { + const pullRequest = ( + overrides: Record = {}, + head: Record = {} + ) => ({ + pull_request: { + merged: true, + title: 'Content for release: 13.6', + head: { ref: 'release/13.6', repo: { full_name: 'microsoft/aspire.dev' }, ...head }, + base: { ref: 'main', repo: { full_name: 'microsoft/aspire.dev' } }, + ...overrides, + }, + }); + + test('recognizes a merged release/X.Y pull request', () => { + expect(detectReleaseMerge(pullRequest())).toEqual({ version: '13.6', reason: 'release/13.6' }); + expect(detectReleaseMerge(pullRequest({}, { ref: 'release/14.0' }))).toEqual({ + version: '14.0', + reason: 'release/14.0', + }); + }); + + test('recognizes the "Merge release/X.Y into main" repair pull request', () => { + const event = pullRequest( + { title: 'Merge release/13.4 into main' }, + { ref: 'joperezr/merge-release-13-4-into-main' } + ); + expect(detectReleaseMerge(event)).toEqual({ version: '13.4', reason: 'release/13.4' }); + }); + + test.each([ + ['an unmerged pull request', pullRequest({ merged: false })], + [ + 'a fork whose branch is named release/13.6', + pullRequest({}, { repo: { full_name: 'someone/aspire.dev' } }), + ], + ['a head without a repository', pullRequest({}, { repo: null })], + [ + 'a branch that is not a release branch', + pullRequest({ title: 'Fix typo' }, { ref: 'release-notes' }), + ], + ['a patch release branch', pullRequest({ title: 'Fix typo' }, { ref: 'release/13.6.1' })], + ['an event without a pull request', { action: 'closed' }], + ['a non-object payload', 'closed'], + ['a missing payload', undefined], + ])('ignores %s', (_name, event) => { + expect(detectReleaseMerge(event)).toBeNull(); + }); + + const cli = (overrides: Partial> = {}) => ({ + ...parseCli(['--state-dir', stateDir]), + ...overrides, + }); + + test('a release merge flushes and gates on the release', () => { + expect(resolveRun(cli({ trigger: 'pull_request' }), pullRequest())).toEqual({ + mode: 'flush', + release: '13.6', + reason: 'release/13.6', + }); + }); + + test('a pull request that is not a release merge is skipped', () => { + const resolved = resolveRun(cli({ trigger: 'pull_request' }), pullRequest({ merged: false })); + expect(resolved.skip).toMatch(/not a merged release/); + }); + + test('describes scheduled and manual runs', () => { + expect(resolveRun(cli({ trigger: 'schedule' }), undefined)).toEqual({ + mode: 'daily', + release: undefined, + reason: 'schedule', + }); + expect(resolveRun(cli({ trigger: 'workflow_dispatch', mode: 'flush' }), undefined).reason).toBe( + 'manual flush' + ); + expect( + resolveRun(cli({ trigger: 'workflow_dispatch', mode: 'flush', release: '13.6' }), undefined) + ).toEqual({ + mode: 'flush', + release: '13.6', + reason: 'release/13.6', + }); + expect(resolveRun(cli({ trigger: 'workflow_dispatch' }), undefined).reason).toBe('manual'); + }); + + test('parses command-line options', () => { + expect(parseCli(['--state-dir', stateDir])).toEqual({ + stateDir, + mode: 'daily', + limit: undefined, + release: undefined, + trigger: 'manual', + dryRun: false, + allowShrink: false, + }); + expect( + parseCli([ + '--state-dir', + stateDir, + '--mode', + 'flush', + '--limit', + '250', + '--release', + '13.6', + '--trigger', + 'workflow_dispatch', + '--dry-run', + '--allow-shrink', + ]) + ).toMatchObject({ + mode: 'flush', + limit: 250, + release: '13.6', + dryRun: true, + allowShrink: true, + }); + }); + + test.each([ + ['no state directory', []], + ['an unknown mode', ['--state-dir', 'x', '--mode', 'weekly']], + ['a zero limit', ['--state-dir', 'x', '--limit', '0']], + ['a limit above the maximum', ['--state-dir', 'x', '--limit', '50001']], + ['a fractional limit', ['--state-dir', 'x', '--limit', '1.5']], + ['a non-numeric limit', ['--state-dir', 'x', '--limit', 'many']], + ['a malformed release', ['--state-dir', 'x', '--release', '13']], + ['an unknown flag', ['--state-dir', 'x', '--force']], + ])('rejects %s', (_name, argv) => { + expect(() => parseCli(argv)).toThrow(); + }); +}); + +describe('running end to end', () => { + let site: Site; + let log: ReturnType; + let clock: ReturnType; + + beforeEach(() => { + site = createSite(SMALL_SITE); + log = createLogger(); + clock = createClock(); + }); + + const runDay = (overrides: Partial = {}) => { + site.posts.length = 0; + return run({ + config, + stateDir, + mode: 'daily', + trigger: 'schedule', + reason: 'schedule', + fetch: site.fetch, + sleep: clock.sleep, + now: clock.now, + log, + ...overrides, + }); + }; + const submitted = () => site.posts.flatMap((post) => post.urlList); + const sent = (paths: string[]) => paths.map(url); + + test('a cold start submits the first window and seeds the state', async () => { + const result = await runDay(); + expect(result.report).toMatchObject({ + outcome: 'completed', + coldStart: true, + submitted: 3, + totalUrls: 10, + deployment: 'deadbeefcafe0123456789', + }); + expect(result.stateChanged).toBe(true); + expect(submitted()).toEqual(sent(['/', '/app-host/', '/get-started/'])); + + const saved = await loadState(stateDir, HOST, log); + expect(saved?.known).toHaveLength(10); + expect(saved?.cycle?.cursor).toBe(url('/get-started/')); + }); + + test('daily runs walk the sitemap without repeating a URL and then start over', async () => { + const days: string[][] = []; + for (let day = 0; day < 5; day++) { + await runDay(); + days.push(submitted()); + } + expect(days).toEqual([ + sent(['/', '/app-host/', '/get-started/']), + sent(['/whats-new/aspire-13-6/', '/reference/api/csharp/a/', '/reference/api/csharp/b/']), + sent(['/reference/api/typescript/c/', '/de/', '/de/app-host/']), + sent(['/ja/']), + sent(['/', '/app-host/', '/get-started/']), + ]); + }); + + test('new and removed URLs are sent first and nothing is sent twice within a cycle', async () => { + await runDay(); + const day1 = submitted(); + + site.paths = [ + ...site.paths.filter((entry) => entry !== '/ja/'), + '/aaa/', + '/reference/api/csharp/ab/', + ]; + const day2Result = await runDay(); + const day2 = submitted(); + expect(day2.slice(0, 3)).toEqual(sent(['/ja/', '/aaa/', '/reference/api/csharp/ab/'])); + expect(day2Result.report).toMatchObject({ newUrls: 2, removedUrls: 1, submitted: 3 }); + + await runDay(); + const day3 = submitted(); + await runDay(); + const day4 = submitted(); + + const cycle = [...day1, ...day2, ...day3, ...day4]; + expect(new Set(cycle).size).toBe(cycle.length); + expect(new Set(cycle)).toEqual(new Set([...site.paths.map(url), url('/ja/')])); + }); + + test('a flush restarts from the top at the release budget and then returns to the slow rotation', async () => { + await runDay(); + const flush = await runDay({ mode: 'flush', reason: 'release/13.6' }); + expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); + expect(flush.report.cycleKind).toBe('release'); + + await runDay(); + expect(submitted()).toEqual( + sent(['/reference/api/csharp/b/', '/reference/api/typescript/c/', ...LOCALIZED]) + ); + const saved = await loadState(stateDir, HOST, log); + expect(saved?.cycle).toBeNull(); + + await runDay(); + expect(submitted()).toEqual(sent(['/', '/app-host/', '/get-started/'])); + }); + + test('skips a cold start while the key file is not live yet', async () => { + site.keyLive = false; + const result = await runDay(); + expect(result.report.outcome).toBe('skipped'); + expect(result.stateChanged).toBe(false); + expect(site.posts).toHaveLength(0); + expect(log.warn).toHaveBeenCalled(); + }); + + test('fails when the key file disappears after the first successful run', async () => { + await runDay(); + site.keyLive = false; + await expect(runDay()).rejects.toThrow(/key file is not reachable/); + }); + + test('a dry run plans without submitting or saving', async () => { + const result = await runDay({ dryRun: true }); + expect(result.report).toMatchObject({ outcome: 'dry-run', submitted: 3, totalUrls: 10 }); + expect(result.stateChanged).toBe(false); + expect(site.posts).toHaveLength(0); + expect(await readdir(stateDir)).toEqual([]); + }); + + test('a dry run still plans while the key file is missing', async () => { + site.keyLive = false; + const result = await runDay({ dryRun: true }); + expect(result.report.outcome).toBe('dry-run'); + expect(log.warn).toHaveBeenCalledWith(expect.stringContaining('key file is live')); + }); + + test('waits for the release page before submitting', async () => { + let polls = 0; + clock.sleep.mockImplementation(() => { + if (++polls === 2) site.liveReleases.add('13.6'); + return Promise.resolve(); + }); + const result = await runDay({ mode: 'flush', release: '13.6', reason: 'release/13.6' }); + expect(result.report.outcome).toBe('completed'); + expect(polls).toBe(2); + expect(site.posts).toHaveLength(1); + }); + + test('does not submit when the release page never goes live', async () => { + await expect( + runDay({ + mode: 'flush', + release: '13.6', + reason: 'release/13.6', + gate: { timeoutMs: 120_000, intervalMs: 60_000 }, + }) + ).rejects.toThrow(/did not go live/); + expect(site.posts).toHaveLength(0); + expect(await readdir(stateDir)).toEqual([]); + }); + + test('keeps the progress made before a 429 and resumes from it', async () => { + site.paths = ['/', ...syntheticUrls(12_000).map((entry) => new URL(entry).pathname)]; + site.responses = [200, 429]; + const limited = await runDay({ limit: 12_001 }); + expect(limited.report.outcome).toBe('rate-limited'); + expect(limited.report.submitted).toBe(10_000); + expect(limited.failure).toBeUndefined(); + expect(limited.stateChanged).toBe(true); + const firstBatch = site.posts[0].urlList; + + const resumed = await runDay({ limit: 3 }); + expect(resumed.report.outcome).toBe('completed'); + expect(submitted()).toEqual( + [...site.paths.map(url)].sort((a, b) => compareUrls(a, b)).slice(10_000, 10_003) + ); + expect(firstBatch).toHaveLength(10_000); + }); + + test('keeps the progress made before a rejected chunk and reports the failure', async () => { + site.paths = ['/', ...syntheticUrls(12_000).map((entry) => new URL(entry).pathname)]; + site.responses = [200, 422]; + const failed = await runDay({ limit: 12_001 }); + expect(failed.report.outcome).toBe('failed'); + expect(failed.failure).toMatch(/HTTP 422/); + expect(failed.stateChanged).toBe(true); + expect((await loadState(stateDir, HOST, log))?.cycle?.cursor).not.toBeNull(); + }); + + test('a rejection of the first chunk leaves an existing state untouched', async () => { + await runDay(); + const before = await readFile(path.join(stateDir, 'state.json'), 'utf8'); + site.responses = [403]; + const failed = await runDay(); + expect(failed.failure).toMatch(/HTTP 403/); + expect(failed.stateChanged).toBe(false); + expect(await readFile(path.join(stateDir, 'state.json'), 'utf8')).toBe(before); + }); + + test('refuses a sitemap that suddenly lost most of its pages and keeps the state', async () => { + await runDay(); + const before = await readFile(path.join(stateDir, 'state.json'), 'utf8'); + site.paths = site.paths.slice(0, 5); + await expect(runDay()).rejects.toThrow(/--allow-shrink/); + expect(await readFile(path.join(stateDir, 'state.json'), 'utf8')).toBe(before); + }); +}); + +describe('summary and command line', () => { + test('renders a readable summary', () => { + const summary = renderSummary( + emptyReport({ + outcome: 'completed', + trigger: 'schedule', + host: HOST, + mode: 'daily', + reason: 'schedule', + deployment: '32f2d1c895952771c0d58924ef8c9115a37b3a0a', + budget: 1_000, + totalUrls: 32_336, + newUrls: 2, + removedUrls: 1, + windowUrls: 997, + submitted: 1_000, + cycleKind: 'rotation', + cycleCovered: 1_000, + chunks: [{ size: 1_000, status: 200, attempts: 1 }], + firstWindowUrl: url('/'), + lastWindowUrl: url('/app-host/'), + }) + ); + expect(summary).toContain('## IndexNow submission for aspire.dev'); + expect(summary).toContain('| Sitemap URLs | 32,336 |'); + expect(summary).toContain('| New / removed | 2 / 1 |'); + expect(summary).toContain('| Submitted | 1,000 of a 1,000 URL budget |'); + expect(summary).toContain('| Cycle | rotation: 1,000 of 32,336 URLs (3%) |'); + expect(summary).toContain('HTTP 200 for 1,000 URLs'); + expect(summary).toContain('`32f2d1c`'); + }); + + async function runMain(argv: string[], event?: unknown, overrides = {}) { + const files = { + GITHUB_EVENT_PATH: path.join(stateDir, 'event.json'), + GITHUB_STEP_SUMMARY: path.join(stateDir, 'summary.md'), + GITHUB_OUTPUT: path.join(stateDir, 'output.txt'), + }; + if (event !== undefined) await writeFile(files.GITHUB_EVENT_PATH, JSON.stringify(event)); + const log = createLogger(); + const code = await main( + ['--state-dir', path.join(stateDir, 'state'), ...argv], + files, + log, + overrides + ); + const read = (file: string) => readFile(file, 'utf8').catch(() => ''); + return { + code, + log, + summary: await read(files.GITHUB_STEP_SUMMARY), + output: await read(files.GITHUB_OUTPUT), + }; + } + + const pullRequest = (title: string, ref: string) => ({ + pull_request: { + merged: true, + title, + head: { ref, repo: { full_name: 'microsoft/aspire.dev' } }, + base: { ref: 'main', repo: { full_name: 'microsoft/aspire.dev' } }, + }, + }); + + test('skips a merged pull request that is not a release', async () => { + const { code, output, summary } = await runMain( + ['--trigger', 'pull_request'], + pullRequest('Fix a typo', 'someone/typo') + ); + expect(code).toBe(0); + expect(output).toContain('state_changed=false'); + expect(output).toContain('outcome=skipped'); + expect(summary).toContain('not a merged release'); + }); + + test('a merged release pull request waits for the release page and flushes', async () => { + const real = await loadConfig(); + const site = createSite(SMALL_SITE, real.key); + site.liveReleases.add('13.6'); + const clock = createClock(); + const { code, output, summary } = await runMain( + ['--trigger', 'pull_request'], + pullRequest('Content for release: 13.6', 'release/13.6'), + { fetch: site.fetch, sleep: clock.sleep, now: clock.now } + ); + expect(code).toBe(0); + expect(output).toContain('state_changed=true'); + expect(output).toContain(`submitted=${SMALL_SITE.length}`); + expect(summary).toContain('release/13.6'); + expect(site.posts).toHaveLength(1); + expect(site.posts[0]).toMatchObject({ host: real.host, key: real.key }); + expect(site.posts[0].urlList).toEqual(SMALL_SITE.map(url)); + }); + + test('reports a failed run and returns a failing exit code', async () => { + const real = await loadConfig(); + const site = createSite(SMALL_SITE, real.key); + const clock = createClock(); + // A first run seeds the state; the key then disappears, which is a hard failure. + await runMain([], undefined, { fetch: site.fetch, sleep: clock.sleep, now: clock.now }); + site.keyLive = false; + const { code, log, summary, output } = await runMain([], undefined, { + fetch: site.fetch, + sleep: clock.sleep, + now: clock.now, + }); + expect(code).toBe(1); + expect(log.error).toHaveBeenCalledWith(expect.stringContaining('key file is not reachable')); + expect(summary).toContain('failed'); + expect(output).toContain('state_changed=false'); + }); + + test('rejects invalid arguments', async () => { + const { code, log } = await runMain(['--mode', 'weekly']); + expect(code).toBe(1); + expect(log.error).toHaveBeenCalledWith(expect.stringContaining('--mode')); + }); +}); From 00b2e1f14b3602e17f47e14ede7cae81141b7238 Mon Sep 17 00:00:00 2001 From: David Pine <7679720+IEvangelist@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:50:00 -0500 Subject: [PATCH 2/6] Prioritize the pages a release added or updated in the IndexNow burst A release adds thousands of generated API pages and rewrites the data of most of the rest, so a plain walk over the sitemap sends the pages that changed no sooner than the ones that did not. - Map the files a release pull request changed to sitemap pages (docs routes, and every page of an API package or module whose data changed beyond the version, source commit and source line numbers) and rank those pages first in the cycle. - Cover the pages that are new when a flush starts with the same ranking, so docs go before the generated API reference however many API pages a release adds. - Read the changed files in the workflow from the release pull request (or the last merged release/X.Y pull request for a manual flush). - Persist state after a flush that was rate limited before anything was accepted, fail a run that a 429 stopped before any URL was accepted, validate the state read from an artifact, ignore artifacts uploaded by runs from forks, and queue runs per job so closed pull requests do not replace a pending run. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/indexnow.yml | 67 +- src/frontend/scripts/indexnow.ts | 335 +++++++-- .../tests/unit/indexnow.vitest.test.ts | 703 +++++++++++++++++- 3 files changed, 1041 insertions(+), 64 deletions(-) diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml index 6ebbad0c7b..f5f0eb1c78 100644 --- a/.github/workflows/indexnow.yml +++ b/.github/workflows/indexnow.yml @@ -6,8 +6,11 @@ name: IndexNow # window of a slow rotation through the whole sitemap. # - Release: when a release/X.Y pull request is merged into main, waits for that release's "What's # new" page to be live (the docs slot swap) and then starts a full-sitemap burst that the daily -# runs finish. -# - Manual: run it from the Actions tab (mode, limit, release, dry run). +# runs finish. The pages the release added or updated go first (docs before the generated API +# reference; the updated ones are found from the files the release pull request changed), then +# everything else. +# - Manual: run it from the Actions tab (mode, limit, release, dry run). A flush with a release +# prioritizes that release's new and updated pages the same way. # # All logic lives in src/frontend/scripts/indexnow.ts so the same flow can be reproduced locally: # cd src/frontend && pnpm indexnow --state-dir /tmp/indexnow --dry-run @@ -53,10 +56,6 @@ on: permissions: {} -concurrency: - group: indexnow - cancel-in-progress: false - jobs: submit: name: Submit URLs to IndexNow @@ -71,9 +70,18 @@ jobs: startsWith(github.event.pull_request.title, 'Merge release/')))) }} runs-on: ubuntu-latest timeout-minutes: 75 + # The group is on the job, not the workflow, so the many pull requests that are closed without + # being a release merge (their job is skipped above) never join it. A workflow-level group would + # let each of them replace a run that is waiting its turn. "queue: max" lets the daily run, a + # release run and a manual run wait in line instead of replacing each other. + concurrency: + group: indexnow + cancel-in-progress: false + queue: max permissions: contents: read actions: read + pull-requests: read defaults: run: working-directory: src/frontend @@ -110,11 +118,14 @@ jobs: mkdir -p "$state_dir" # `gh run download` lists every artifact in the repository and fails with HTTP 500 here, so - # ask for this artifact by name and take the newest one that has not expired. + # ask for this artifact by name and take the newest one that has not expired. Runs from + # forks are listed in this repository too and could upload an artifact with this name, so + # only artifacts from runs of this repository are trusted. The script validates the state + # again when it reads it. artifact_id="" for attempt in 1 2 3; do if artifact_id=$(gh api "repos/$GH_REPO/actions/artifacts?name=indexnow-state&per_page=100" \ - --jq '[.artifacts[] | select(.expired == false)] | sort_by(.created_at) | last | .id // empty'); then + --jq '[.artifacts[] | select(.expired == false and .workflow_run.head_repository_id == .workflow_run.repository_id)] | sort_by(.created_at) | last | .id // empty'); then break fi if [ "$attempt" -eq 3 ]; then @@ -133,6 +144,44 @@ jobs: gh api "repos/$GH_REPO/actions/artifacts/$artifact_id/zip" > "$RUNNER_TEMP/indexnow-state.zip" unzip -q -o "$RUNNER_TEMP/indexnow-state.zip" -d "$state_dir" + # A release rewrites most of the API reference data, so the pages worth submitting first are + # the ones whose own files changed. The sitemap has no lastmod to tell, so list the files of + # the release pull request. The script maps docs and API data to pages and leaves out API + # files that only moved to the new version. + - name: Find the files changed by the release + id: changes + if: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.mode == 'flush' && inputs.release != '') }} + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + EVENT_NAME: ${{ github.event_name }} + PR_NUMBER: ${{ github.event.pull_request.number }} + RELEASE: ${{ inputs.release }} + shell: bash + run: | + set -euo pipefail + pr="$PR_NUMBER" + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then + if ! [[ "$RELEASE" =~ ^[0-9]+\.[0-9]+$ ]]; then + echo "::error::The release input must look like 13.6." + exit 1 + fi + pr=$(gh pr list --state merged --base main --head "release/$RELEASE" \ + --json number,mergedAt --jq 'sort_by(.mergedAt) | last | .number // empty') + fi + + if [ -z "$pr" ]; then + echo "::notice::No merged release pull request was found, so only the pages that are new to the sitemap are prioritized." + exit 0 + fi + + changes="$RUNNER_TEMP/changed-files.jsonl" + gh api "repos/$GH_REPO/pulls/$pr/files" --paginate \ + --jq '.[] | {status, filename, changes, patch: (if (.filename | test("/data/(pkgs|ts-modules)/")) then .patch else null end)}' \ + > "$changes" + echo "Release pull request #$pr changed $(wc -l < "$changes") files." + echo "path=$changes" >> "$GITHUB_OUTPUT" + - name: Submit URLs id: submit env: @@ -142,10 +191,12 @@ jobs: INPUT_RELEASE: ${{ inputs.release }} INPUT_DRY_RUN: ${{ inputs.dry_run }} INPUT_ALLOW_SHRINK: ${{ inputs.allow_shrink }} + CHANGED_FILES: ${{ steps.changes.outputs.path }} shell: bash run: | set -euo pipefail args=(--state-dir "$RUNNER_TEMP/indexnow-state" --trigger "$EVENT_NAME") + [ -z "$CHANGED_FILES" ] || args+=(--changed-files "$CHANGED_FILES") if [ "$EVENT_NAME" = "workflow_dispatch" ]; then args+=(--mode "$INPUT_MODE") [ -z "$INPUT_LIMIT" ] || args+=(--limit "$INPUT_LIMIT") diff --git a/src/frontend/scripts/indexnow.ts b/src/frontend/scripts/indexnow.ts index 207a578012..92f5b7c1e0 100644 --- a/src/frontend/scripts/indexnow.ts +++ b/src/frontend/scripts/indexnow.ts @@ -5,7 +5,12 @@ * Every run reads the live sitemap, compares it with the URL list remembered from the previous run * and submits, in priority order: * 1. removed URLs (so engines drop them) and new URLs (so engines discover them), - * 2. the next window of the rotation cycle, which walks the whole sitemap in (tier, path) order. + * 2. the next window of the rotation cycle, which walks the whole sitemap in (rank, tier, path) + * order: the pages a release added or updated first (docs before the API reference), then + * everything else. + * + * A release flush covers its new URLs in step 2 instead of step 1: a release adds thousands of + * generated API pages, and sending those first would use up the budget before any updated docs page. * * The rotation cycle is a cursor over that order, so it survives URLs being added or removed. A * `flush` (a release merge or a manual dispatch) restarts the cycle from the top at the larger @@ -13,13 +18,18 @@ * which the slow rotation budget applies. IndexNow does not publish its rate limits, so both * budgets live in config/indexnow.json and a 429 simply pauses the run until the next one. * + * The sitemap has no `lastmod`, so "updated" comes from the release itself: the workflow lists the + * files the release pull request changed (--changed-files) and they are mapped to pages (docs and + * API reference data). API data files only count when something besides the version, source + * commit and source line numbers changed, because those move in every file on every release. + * * State (`state.json` and `known-urls.txt.gz`) lives in --state-dir. The workflow keeps it between * runs as an Actions artifact; losing it only causes a cold start, which seeds the known URL list * without submitting every URL as "new". * * Usage (from src/frontend): * pnpm indexnow --state-dir [--mode daily|flush] [--limit ] [--release ] - * [--trigger ] [--dry-run] [--allow-shrink] + * [--changed-files ] [--trigger ] [--dry-run] [--allow-shrink] */ import { appendFile, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; import path from 'node:path'; @@ -49,7 +59,7 @@ export interface IndexNowConfig { budgets: { release: number; rotation: number }; } -/** A pass over the sitemap. `cursor` is the last URL covered, in (tier, path) order. */ +/** A pass over the sitemap. `cursor` is the last URL covered, in (rank, tier, path) order. */ export interface Cycle { kind: CycleKind; reason: string; @@ -57,6 +67,8 @@ export interface Cycle { cursor: string | null; /** URLs ahead of the cursor that were already submitted as new, so the window skips them. */ skip: string[]; + /** Pages the release added or changed. They rank ahead of every other page in this cycle. */ + updated: string[]; } export interface RunRecord { @@ -333,20 +345,29 @@ export function tierOf(pathname: string, codes: ReadonlySet = LOCALE_COD interface Entry { url: string; + /** 0 for pages the release updated, 1 for the rest. */ + rank: number; tier: number; } -function toEntry(url: string, codes: ReadonlySet): Entry { - return { url, tier: tierOf(new URL(url).pathname, codes) }; +const NO_URLS: ReadonlySet = new Set(); + +function toEntry(url: string, codes: ReadonlySet, updated: ReadonlySet): Entry { + return { url, rank: updated.has(url) ? 0 : 1, tier: tierOf(new URL(url).pathname, codes) }; } function compareEntries(a: Entry, b: Entry): number { - return a.tier - b.tier || compareOrdinal(a.url, b.url); + return a.rank - b.rank || a.tier - b.tier || compareOrdinal(a.url, b.url); } -/** Orders URLs by (tier, URL), the order the rotation cursor walks. */ -export function compareUrls(a: string, b: string, codes: ReadonlySet = LOCALE_CODES) { - return compareEntries(toEntry(a, codes), toEntry(b, codes)); +/** Orders URLs by (rank, tier, URL), the order the rotation cursor walks. */ +export function compareUrls( + a: string, + b: string, + codes: ReadonlySet = LOCALE_CODES, + updated: ReadonlySet = NO_URLS +) { + return compareEntries(toEntry(a, codes, updated), toEntry(b, codes, updated)); } function firstIndexAfter(entries: readonly Entry[], target: Entry): number { @@ -364,11 +385,122 @@ function firstIndexAfter(entries: readonly Entry[], target: Entry): number { export function countCovered( urls: readonly string[], cursor: string | null, - codes: ReadonlySet = LOCALE_CODES + codes: ReadonlySet = LOCALE_CODES, + updated: ReadonlySet = NO_URLS ): number { if (cursor === null) return 0; - const target = toEntry(cursor, codes); - return urls.filter((url) => compareEntries(toEntry(url, codes), target) <= 0).length; + const target = toEntry(cursor, codes, updated); + return urls.filter((url) => compareEntries(toEntry(url, codes, updated), target) <= 0).length; +} + +// --------------------------------------------------------------------------------------------- +// Pages updated by a release +// --------------------------------------------------------------------------------------------- + +/** A file a release changed, as listed by GitHub's "list pull request files" API. */ +export interface ChangedFile { + status: string; + filename: string; + /** Lines added plus lines deleted, when known. */ + changes: number | null; + /** The unified diff of the file. GitHub leaves it out when the diff is very large. */ + patch: string | null; +} + +/** Reads one JSON object per line: `{"status": "...", "filename": "...", "changes": 3, "patch": "..."}`. */ +export function parseChangedFiles(text: string): ChangedFile[] { + const files: ChangedFile[] = []; + text.split('\n').forEach((line, index) => { + if (!line.trim()) return; + let value: unknown; + try { + value = JSON.parse(line); + } catch { + throw new Error(`Line ${index + 1} of the changed files is not JSON`); + } + if ( + !isRecord(value) || + typeof value.status !== 'string' || + typeof value.filename !== 'string' + ) { + throw new Error(`Line ${index + 1} of the changed files needs a "status" and a "filename"`); + } + files.push({ + status: value.status, + filename: value.filename, + changes: typeof value.changes === 'number' ? value.changes : null, + patch: typeof value.patch === 'string' ? value.patch : null, + }); + }); + return files; +} + +const DOCS_FILE = /^src\/frontend\/src\/content\/docs\/(.+)\.mdx?$/; +const API_DATA_FILE = + /^src\/frontend\/src\/data\/(pkgs|ts-modules)\/(.+?)\.\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?\.json$/; +const API_ROUTES = { pkgs: '/reference/api/csharp/', 'ts-modules': '/reference/api/typescript/' }; +/** Keys that change in every API data file on every release without changing the API. */ +const VOLATILE_API_KEY = /^[+-]\s*"(?:version|sourceCommit|sourceLines)"\s*:/; +const CONTENT_STATUSES = new Set(['added', 'modified', 'renamed', 'copied', 'changed']); + +/** + * Whether an API data file changed in a way that shows on its pages. Every file is renamed to the + * new package version and gets a new `version`, `sourceCommit` and shifted `sourceLines` on each + * release, so only a diff with other changed lines counts. A diff GitHub left out is too large to + * be one of those, except for a pure rename, which has no changes at all. + */ +export function hasApiChanges(file: ChangedFile): boolean { + if (file.status === 'added') return true; + if (file.patch === null) return file.changes !== 0; + return file.patch.split('\n').some((line) => /^[+-]/.test(line) && !VOLATILE_API_KEY.test(line)); +} + +/** + * Maps the files a release changed to the sitemap pages they produce, so those pages can be + * submitted first. Docs map to their route (`content/docs/a/b.mdx` is `/a/b/`, an `index` file is + * its folder, and localized docs live in a locale folder); API data files map to every page of + * their package or module. Files that no page comes from, such as components, are ignored. + */ +export function urlsForChangedFiles( + files: readonly ChangedFile[], + urls: readonly string[], + host: string +): string[] { + const origin = `https://${host}`; + const byPath = new Map(); + const byApiRoute = new Map(); + for (const url of urls) { + const pathname = url.slice(origin.length); + byPath.set(pathname.toLowerCase(), url); + for (const prefix of Object.values(API_ROUTES)) { + if (!pathname.startsWith(prefix)) continue; + const route = `${prefix}${pathname.slice(prefix.length).split('/')[0]}/`; + const group = byApiRoute.get(route); + if (group) group.push(url); + else byApiRoute.set(route, [url]); + } + } + + const updated = new Set(); + for (const file of files) { + if (!CONTENT_STATUSES.has(file.status)) continue; + + const docs = DOCS_FILE.exec(file.filename); + if (docs) { + const segments = docs[1].split('/'); + if (segments[segments.length - 1] === 'index') segments.pop(); + const url = byPath.get(segments.length === 0 ? '/' : `/${segments.join('/')}/`); + if (url) updated.add(url); + continue; + } + + const api = API_DATA_FILE.exec(file.filename); + if (api && hasApiChanges(file)) { + const prefix = api[1] === 'pkgs' ? API_ROUTES.pkgs : API_ROUTES['ts-modules']; + for (const url of byApiRoute.get(`${prefix}${api[2].toLowerCase()}/`) ?? []) updated.add(url); + } + } + return [...updated].sort(compareOrdinal); } // --------------------------------------------------------------------------------------------- @@ -383,8 +515,10 @@ export interface PlanInput { /** Recorded on the cycle when one is started. */ reason: string; budgets: IndexNowConfig['budgets']; - /** Replaces the budget for this run. */ + /** Replaces the budget for this run. It also caps the new and removed URLs sent first. */ limit?: number; + /** Pages the release updated. They rank first when this run starts a new cycle. */ + updatedUrls?: readonly string[]; now: Date; allowShrink?: boolean; localeCodes?: ReadonlySet; @@ -403,7 +537,7 @@ export interface Plan { removedUrls: string[]; /** The removed URLs submitted this run, in submission order. */ removed: string[]; - /** The new URLs submitted this run, in submission order. */ + /** The new URLs submitted ahead of the window this run. A flush leaves its new URLs to the cycle. */ fresh: string[]; /** The rotation window submitted this run. */ window: string[]; @@ -422,14 +556,16 @@ export interface Plan { * - A cold start (no state) seeds the known list and never treats the whole sitemap as new. * - A flush starts a release cycle from the top; otherwise the active cycle continues, or a new * rotation cycle begins when the previous one has completed. - * - New and removed URLs go first. New URLs ahead of the cursor are remembered in `cycle.skip` - * (see finalizeRun) so the window does not send them a second time. + * - Removed and new URLs go first. A flush is the exception for new URLs: it covers them in the + * release cycle it starts, ordered with the pages the release updated. New URLs ahead of the + * cursor are remembered in `cycle.skip` (see finalizeRun) so the window does not send them a + * second time. */ export function planRun(input: PlanInput): Plan { const localeCodes = input.localeCodes ?? LOCALE_CODES; const { state } = input; - const entries = input.current.map((url) => toEntry(url, localeCodes)).sort(compareEntries); - const total = entries.length; + const currentSet = new Set(input.current); + const total = currentSet.size; if (total === 0) throw new Error('The sitemap lists no URLs; refusing to continue'); const previous = state?.known ?? []; @@ -441,36 +577,54 @@ export function planRun(input: PlanInput): Plan { ); } - const currentSet = new Set(entries.map((entry) => entry.url)); - const knownSet = new Set(previous); - const newUrls = state ? entries.filter((e) => !knownSet.has(e.url)).map((e) => e.url) : []; - const removedUrls = previous - .filter((url) => !currentSet.has(url)) - .map((url) => toEntry(url, localeCodes)) - .sort(compareEntries) - .map((entry) => entry.url); - + // A flush covers the pages that are new when it starts together with the ones its files updated, + // so a release's new and updated pages are ordered by tier: docs before the generated API + // reference, however many API pages a new module adds. Without that, thousands of new API pages + // sent first would use up the budget before any updated docs page. The set is fixed when the + // cycle starts and ranks first until the cycle completes. + const startsRelease = input.mode === 'flush'; const startedAt = input.now.toISOString(); + const knownSet = new Set(previous); + const unknown = state ? [...currentSet].filter((url) => !knownSet.has(url)) : []; + const updated = [...new Set([...(input.updatedUrls ?? []), ...(startsRelease ? unknown : [])])] + .filter((url) => currentSet.has(url)) + .sort(compareOrdinal); let cycle: Cycle; - if (input.mode === 'flush') { - cycle = { kind: 'release', reason: input.reason, startedAt, cursor: null, skip: [] }; + if (startsRelease) { + cycle = { kind: 'release', reason: input.reason, startedAt, cursor: null, skip: [], updated }; } else if (state?.cycle) { cycle = state.cycle; } else { - cycle = { kind: 'rotation', reason: input.reason, startedAt, cursor: null, skip: [] }; + cycle = { kind: 'rotation', reason: input.reason, startedAt, cursor: null, skip: [], updated }; } + const updatedSet = new Set(cycle.updated); + const entries = [...currentSet] + .map((url) => toEntry(url, localeCodes, updatedSet)) + .sort(compareEntries); + const newUrls = state ? entries.filter((e) => !knownSet.has(e.url)).map((e) => e.url) : []; + const removedUrls = previous + .filter((url) => !currentSet.has(url)) + .map((url) => toEntry(url, localeCodes, updatedSet)) + .sort(compareEntries) + .map((entry) => entry.url); + const budget = input.limit ?? (cycle.kind === 'release' ? input.budgets.release : input.budgets.rotation); - const removed = removedUrls.slice(0, MAX_PRIORITY_URLS); - const fresh = newUrls.slice(0, MAX_PRIORITY_URLS); + // --limit is a hard cap, so it bounds the new and removed URLs that go first as well. A flush + // leaves its new URLs to the cycle it starts instead of sending them ahead of it. + const cap = Math.min(MAX_PRIORITY_URLS, input.limit ?? Infinity); + const removed = removedUrls.slice(0, cap); + const fresh = (startsRelease ? [] : newUrls).slice(0, Math.max(0, cap - removed.length)); const priority = new Set([...removed, ...fresh]); const skip = new Set(cycle.skip); const room = Math.max(0, budget - removed.length - fresh.length); const alreadySent = (entry: Entry) => priority.has(entry.url) || skip.has(entry.url); let index = - cycle.cursor === null ? 0 : firstIndexAfter(entries, toEntry(cycle.cursor, localeCodes)); + cycle.cursor === null + ? 0 + : firstIndexAfter(entries, toEntry(cycle.cursor, localeCodes, updatedSet)); let windowEnd: string | null = null; const window: string[] = []; while (index < total && window.length < room) { @@ -514,7 +668,8 @@ export interface FinalizeOptions { /** * Builds the state to remember after a run. Only accepted URLs count as done: a new URL that was * not accepted stays new, a removed URL that was not accepted stays known, and the cursor stops at - * the last accepted window URL. + * the last accepted window URL. The new URLs of a flush count as known right away, because the + * release cycle it starts submits them when its cursor reaches them. */ export function finalizeRun(plan: Plan, options: FinalizeOptions): IndexNowState { const ordered = [...plan.removed, ...plan.fresh, ...plan.window]; @@ -523,7 +678,9 @@ export function finalizeRun(plan: Plan, options: FinalizeOptions): IndexNowState const acceptedWindow = Math.max(0, accepted - plan.removed.length - plan.fresh.length); const known = new Set(options.current); - for (const url of plan.newUrls) if (!acceptedUrls.has(url)) known.delete(url); + if (plan.mode !== 'flush') { + for (const url of plan.newUrls) if (!acceptedUrls.has(url)) known.delete(url); + } for (const url of plan.removedUrls) if (!acceptedUrls.has(url)) known.add(url); let cycle: Cycle | null = null; @@ -534,11 +691,14 @@ export function finalizeRun(plan: Plan, options: FinalizeOptions): IndexNowState : (plan.window[acceptedWindow - 1] ?? plan.cycle.cursor); const skip = new Set(plan.cycle.skip); for (const url of plan.fresh) if (acceptedUrls.has(url)) skip.add(url); + const updatedSet = new Set(plan.cycle.updated); cycle = { ...plan.cycle, cursor, skip: [...skip] - .filter((url) => cursor === null || compareUrls(url, cursor, plan.localeCodes) > 0) + .filter( + (url) => cursor === null || compareUrls(url, cursor, plan.localeCodes, updatedSet) > 0 + ) .sort(compareOrdinal), }; } @@ -570,18 +730,20 @@ export function finalizeRun(plan: Plan, options: FinalizeOptions): IndexNowState function parseCycle(value: unknown): Cycle | null { if (value === null) return null; if (!isRecord(value)) throw new Error('"cycle" must be an object or null'); - const { kind, reason, startedAt, cursor, skip } = value; + const { kind, reason, startedAt, cursor, skip, updated = [] } = value; + const isUrlList = (list: unknown): list is string[] => + Array.isArray(list) && list.every((url) => typeof url === 'string'); if ( (kind !== 'release' && kind !== 'rotation') || typeof reason !== 'string' || typeof startedAt !== 'string' || !(cursor === null || typeof cursor === 'string') || - !Array.isArray(skip) || - !skip.every((url): url is string => typeof url === 'string') + !isUrlList(skip) || + !isUrlList(updated) ) { throw new Error('"cycle" has an unexpected shape'); } - return { kind, reason, startedAt, cursor, skip }; + return { kind, reason, startedAt, cursor, skip, updated }; } function parseRunRecord(value: unknown): RunRecord | null { @@ -602,6 +764,18 @@ function parseRunRecord(value: unknown): RunRecord | null { }; } +/** Rejects entries that are not already the canonical https URLs on `host` that normalizeUrls makes. */ +function assertCanonical(urls: readonly string[], host: string, what: string): void { + const { urls: canonical, rejected } = normalizeUrls(urls, host); + if ( + rejected.length > 0 || + canonical.length !== urls.length || + canonical.some((url, index) => url !== urls[index]) + ) { + throw new Error(`${what} has entries that are not https://${host}/ pages`); + } +} + export async function loadState( dir: string, host: string, @@ -631,11 +805,21 @@ export async function loadState( if (known.length !== file.knownCount) { throw new Error(`expected ${String(file.knownCount)} known URLs but found ${known.length}`); } + // The state comes from an artifact, so it is checked like any other input. A list that is + // empty, or holds other hosts, would otherwise make a run submit URLs it should not. + if (known.length === 0) throw new Error('the known URL list is empty'); + assertCanonical(known, host, 'the known URL list'); + const cycle = parseCycle(file.cycle); + if (cycle) { + assertCanonical(cycle.skip, host, 'the cycle skip list'); + assertCanonical(cycle.updated, host, 'the cycle updated list'); + if (cycle.cursor !== null) assertCanonical([cycle.cursor], host, 'the cycle cursor'); + } return { host, updatedAt: file.updatedAt, deployment: file.deployment, - cycle: parseCycle(file.cycle), + cycle, lastRun: parseRunRecord(file.lastRun), known, }; @@ -952,6 +1136,8 @@ export interface CliOptions { mode: RunMode; limit?: number; release?: string; + /** A file with one JSON object per line for each file the release changed. */ + changedFiles?: string; trigger: string; dryRun: boolean; allowShrink: boolean; @@ -965,6 +1151,7 @@ export function parseCli(argv: readonly string[]): CliOptions { mode: { type: 'string' }, limit: { type: 'string' }, release: { type: 'string' }, + 'changed-files': { type: 'string' }, trigger: { type: 'string' }, 'dry-run': { type: 'boolean' }, 'allow-shrink': { type: 'boolean' }, @@ -992,6 +1179,7 @@ export function parseCli(argv: readonly string[]): CliOptions { mode, limit, release: values.release, + changedFiles: values['changed-files'] ? path.resolve(values['changed-files']) : undefined, trigger: values.trigger || 'manual', dryRun: values['dry-run'] ?? false, allowShrink: values['allow-shrink'] ?? false, @@ -1044,6 +1232,8 @@ export interface RunReport { totalUrls: number; newUrls: number; removedUrls: number; + /** Pages the release added or updated, which this cycle covers before everything else. */ + updatedUrls: number; /** URLs planned (dry run) or accepted (real run) from the rotation window. */ windowUrls: number; submitted: number; @@ -1075,6 +1265,8 @@ export interface RunOptions { limit?: number; /** Wait for this release's What's new page before submitting. */ release?: string; + /** The files the release changed; their pages are submitted first when a cycle starts. */ + changedFiles?: readonly ChangedFile[]; fetch?: FetchLike; sleep?: Sleep; now?: () => Date; @@ -1095,6 +1287,7 @@ export function emptyReport( totalUrls: 0, newUrls: 0, removedUrls: 0, + updatedUrls: 0, windowUrls: 0, submitted: 0, deferred: 0, @@ -1156,7 +1349,7 @@ export async function run(options: RunOptions): Promise { if (!ready) { throw new Error( `The What's new page for release ${release} (${whatsNewPath(release)}) did not go live in ` + - 'time. Run the workflow again with the release input once the new docs are live.' + `time. Once the new docs are live, run the workflow manually with mode "flush" and release "${release}".` ); } } @@ -1172,6 +1365,14 @@ export async function run(options: RunOptions): Promise { ); } const deployment = await readDeployment(config.host, fetch); + const updatedUrls = options.changedFiles + ? urlsForChangedFiles(options.changedFiles, urls, config.host) + : []; + if (options.changedFiles) { + log.info( + `${options.changedFiles.length} files changed by the release map to ${updatedUrls.length} updated pages.` + ); + } const plan = planRun({ current: urls, @@ -1180,10 +1381,14 @@ export async function run(options: RunOptions): Promise { reason, budgets: config.budgets, limit, + updatedUrls, now: now(), allowShrink: options.allowShrink, }); const ordered = [...plan.removed, ...plan.fresh, ...plan.window]; + const updatedSet = new Set(plan.cycle.updated); + // A flush leaves its new URLs to the release cycle it starts, so none of them waits for a later run. + const pendingNew = plan.mode === 'flush' ? 0 : plan.newUrls.length - plan.fresh.length; const planned = { ...base, deployment, @@ -1192,14 +1397,15 @@ export async function run(options: RunOptions): Promise { totalUrls: plan.total, newUrls: plan.newUrls.length, removedUrls: plan.removedUrls.length, - deferred: - plan.newUrls.length + plan.removedUrls.length - plan.removed.length - plan.fresh.length, + updatedUrls: plan.cycle.updated.length, + deferred: pendingNew + plan.removedUrls.length - plan.removed.length, cycleKind: plan.cycle.kind, firstWindowUrl: plan.window[0] ?? '', lastWindowUrl: plan.window[plan.window.length - 1] ?? '', }; log.info( - `Sitemap: ${plan.total} URLs; ${plan.newUrls.length} new, ${plan.removedUrls.length} removed. ` + + `Sitemap: ${plan.total} URLs; ${plan.newUrls.length} new, ${plan.removedUrls.length} removed, ` + + `${plan.cycle.updated.length} prioritized as new or updated by the release. ` + `Submitting ${ordered.length} (${plan.removed.length} removed, ${plan.fresh.length} new, ` + `${plan.window.length} from the ${plan.cycle.kind} cycle) with a budget of ${plan.budget}.` ); @@ -1224,8 +1430,11 @@ export async function run(options: RunOptions): Promise { ? { accepted: 0, chunks: [], stop: null } : await submitUrls({ config, urls: ordered, fetch, sleep, now: () => now().getTime(), log }); - // Progress is kept even when a later chunk fails, so accepted URLs are never sent twice. - const stateChanged = result.stop === null || result.accepted > 0 || plan.coldStart; + // Progress is kept even when a later chunk fails, so accepted URLs are never sent twice. A flush + // is kept even when nothing was accepted: starting the release cycle is what it changes, and + // dropping it would lose the release burst, which the next run then picks up. + const stateChanged = + result.stop === null || result.accepted > 0 || plan.coldStart || plan.mode === 'flush'; let next: IndexNowState | null = null; if (stateChanged) { next = finalizeRun(plan, { @@ -1248,16 +1457,26 @@ export async function run(options: RunOptions): Promise { windowUrls: next?.lastRun?.windowUrls ?? 0, submitted: result.accepted, chunks: result.chunks, - cycleCovered: cycleCompleted ? plan.total : countCovered(urls, cursor, plan.localeCodes), + cycleCovered: cycleCompleted + ? plan.total + : countCovered(urls, cursor, plan.localeCodes, updatedSet), cycleCompleted, }); if (result.stop?.reason === 'rate-limited') { - const note = - `IndexNow answered 429 (too many requests) after ${result.accepted} URLs. The progress was ` + - 'saved and the next run continues from there.'; - log.warn(note); - return { report: { ...report, outcome: 'rate-limited', note }, stateChanged }; + if (result.accepted > 0) { + const note = + `IndexNow answered 429 (too many requests) after ${result.accepted} URLs. The progress was ` + + 'saved and the next run continues from there.'; + log.warn(note); + return { report: { ...report, outcome: 'rate-limited', note }, stateChanged }; + } + // Nothing went out, so this run did not do its job even though it will be retried. + const failure = + 'IndexNow answered 429 (too many requests) before accepting any URL, so nothing was ' + + `submitted. ${plan.mode === 'flush' ? 'The release cycle was started, so the next run submits it from the top. ' : 'The next run tries again. '}` + + 'If this keeps happening, lower the budgets in config/indexnow.json.'; + return { report: { ...report, outcome: 'rate-limited', note: failure }, stateChanged, failure }; } if (result.stop) { const failure = @@ -1285,6 +1504,12 @@ export function renderSummary(report: RunReport): string { ['New / removed', `${count(report.newUrls)} / ${count(report.removedUrls)}`] ); if (report.deferred > 0) rows.push(['Deferred to the next run', count(report.deferred)]); + if (report.updatedUrls > 0) { + rows.push([ + 'Prioritized pages', + `${count(report.updatedUrls)} new or updated by the release, ahead of the rest`, + ]); + } rows.push([ report.outcome === 'dry-run' ? 'Would submit' : 'Submitted', `${count(report.submitted)} of a ${count(report.budget)} URL budget`, @@ -1378,6 +1603,9 @@ export async function main( return 0; } + const changedFiles = options.changedFiles + ? parseChangedFiles(await readFile(options.changedFiles, 'utf8')) + : undefined; const result = await run({ config, stateDir: options.stateDir, @@ -1386,6 +1614,7 @@ export async function main( reason: resolved.reason, release: resolved.release, limit: options.limit, + changedFiles, dryRun: options.dryRun, allowShrink: options.allowShrink, log, diff --git a/src/frontend/tests/unit/indexnow.vitest.test.ts b/src/frontend/tests/unit/indexnow.vitest.test.ts index 7f81fd4a65..935c2ce160 100644 --- a/src/frontend/tests/unit/indexnow.vitest.test.ts +++ b/src/frontend/tests/unit/indexnow.vitest.test.ts @@ -15,11 +15,13 @@ import { emptyReport, fetchSitemapLocations, finalizeRun, + hasApiChanges, isReleaseLive, loadConfig, loadState, main, normalizeUrls, + parseChangedFiles, parseCli, parseConfig, parseRetryAfter, @@ -32,9 +34,12 @@ import { saveState, submitUrls, tierOf, + urlsForChangedFiles, verifyKeyFile, waitForRelease, whatsNewPath, + type ChangedFile, + type Cycle, type IndexNowConfig, type IndexNowState, type RunOptions, @@ -83,15 +88,20 @@ function createClock() { }; } -function makeState(overrides: Partial = {}): IndexNowState { +type CycleInput = Omit & { updated?: string[] }; + +function makeState( + overrides: Partial> & { cycle?: CycleInput | null } = {} +): IndexNowState { + const { cycle, ...rest } = overrides; return { host: HOST, updatedAt: '2026-10-12T06:43:00.000Z', deployment: '', - cycle: null, + cycle: cycle ? { updated: [], ...cycle } : null, lastRun: null, known: [], - ...overrides, + ...rest, }; } @@ -333,6 +343,171 @@ describe('ordering', () => { }); }); +describe('pages updated by a release', () => { + const file = (filename: string, overrides: Partial = {}): ChangedFile => ({ + status: 'modified', + filename, + changes: 4, + patch: null, + ...overrides, + }); + const docs = (relative: string, overrides: Partial = {}) => + file(`src/frontend/src/content/docs/${relative}`, overrides); + const apiData = ( + kind: 'pkgs' | 'ts-modules', + name: string, + overrides: Partial = {} + ) => file(`src/frontend/src/data/${kind}/${name}`, { status: 'renamed', ...overrides }); + const patchOf = (...lines: string[]) => ['@@ -4 +4 @@', ...lines].join('\n'); + const VERSION_ONLY = patchOf( + '- "version": "13.5.4",', + '+ "version": "13.6.0",', + '- "sourceCommit": "9c1b401dd677"', + '+ "sourceCommit": "f4c27f2d43dd"', + '- "sourceLines": "37-87"', + '+ "sourceLines": "39-89"' + ); + const REAL_CHANGE = patchOf( + '- "version": "13.5.4",', + '+ "version": "13.6.0",', + '+ "name": "AsExpress",' + ); + + const site = [ + '/', + '/app-host/', + '/app-host/eventing/', + '/get-started/first-app/', + '/whats-new/aspire-13-6/', + '/de/', + '/de/app-host/', + '/reference/api/csharp/aspire.hosting/', + '/reference/api/csharp/aspire.hosting/distributedapplication/', + '/reference/api/csharp/aspire.hosting.redis/', + '/reference/api/csharp/aspire.hosting.redis/addredis/', + '/reference/api/typescript/aspire.hosting.redis/', + '/reference/api/typescript/aspire.hosting.redis/addredis/', + ].map(url); + const map = (...files: ChangedFile[]) => urlsForChangedFiles(files, site, HOST); + + test('reads one JSON object per line', () => { + const text = + '{"status":"modified","filename":"a.mdx","changes":3,"patch":"@@ -1 +1 @@"}\n' + + '\n' + + '{"status":"renamed","filename":"b.json","patch":null}\n'; + expect(parseChangedFiles(text)).toEqual([ + { status: 'modified', filename: 'a.mdx', changes: 3, patch: '@@ -1 +1 @@' }, + { status: 'renamed', filename: 'b.json', changes: null, patch: null }, + ]); + expect(parseChangedFiles('')).toEqual([]); + }); + + test.each([ + ['text that is not JSON', 'not json', /Line 1 .* not JSON/], + ['an object without a filename', '{"status":"added"}', /Line 1 .* "filename"/], + ['a value that is not an object', '{"status":"added","filename":"a"}\n42', /Line 2/], + ])('rejects %s', (_name, text, message) => { + expect(() => parseChangedFiles(text)).toThrow(message); + }); + + test('API data only counts when more than the volatile keys changed', () => { + expect(hasApiChanges(file('x.json', { status: 'added' }))).toBe(true); + expect(hasApiChanges(file('x.json', { patch: VERSION_ONLY }))).toBe(false); + expect(hasApiChanges(file('x.json', { patch: REAL_CHANGE }))).toBe(true); + // GitHub leaves out the diff of a large change, but a pure rename has none to show. + expect(hasApiChanges(file('x.json', { changes: 900 }))).toBe(true); + expect(hasApiChanges(file('x.json', { changes: null }))).toBe(true); + expect(hasApiChanges(file('x.json', { status: 'renamed', changes: 0 }))).toBe(false); + }); + + test('maps docs to their route', () => { + expect( + map( + docs('app-host/eventing.mdx'), + docs('app-host/index.mdx'), + docs('de/index.mdx'), + docs('get-started/first-app.md') + ) + ).toEqual( + ['/app-host/', '/app-host/eventing/', '/de/', '/get-started/first-app/'].map(url).sort() + ); + expect(map(docs('index.mdx'))).toEqual([url('/')]); + expect(map(docs('de/app-host/index.mdx'))).toEqual([url('/de/app-host/')]); + }); + + test('ignores docs without a page, removed docs and files no page comes from', () => { + expect( + map( + docs('404.mdx'), + docs('app-host/removed.mdx'), + docs('app-host/eventing.mdx', { status: 'removed' }), + file('src/frontend/src/components/Header.astro'), + file('src/frontend/src/assets/logo.svg'), + file('src/frontend/package.json'), + file('src/frontend/src/content/docs/app-host/diagram.svg') + ) + ).toEqual([]); + }); + + test('maps an API data file to every page of its package, and only that package', () => { + expect(map(apiData('pkgs', 'Aspire.Hosting.13.6.0.json', { patch: REAL_CHANGE }))).toEqual( + [ + '/reference/api/csharp/aspire.hosting/', + '/reference/api/csharp/aspire.hosting/distributedapplication/', + ].map(url) + ); + expect( + map( + apiData('pkgs', 'Aspire.Hosting.Redis.13.6.0-preview.1.26478.8.json', { + patch: REAL_CHANGE, + }) + ) + ).toEqual( + [ + '/reference/api/csharp/aspire.hosting.redis/', + '/reference/api/csharp/aspire.hosting.redis/addredis/', + ].map(url) + ); + }); + + test('maps a TypeScript module file to its module pages', () => { + expect( + map(apiData('ts-modules', 'Aspire.Hosting.Redis.13.6.0.json', { patch: REAL_CHANGE })) + ).toEqual( + [ + '/reference/api/typescript/aspire.hosting.redis/', + '/reference/api/typescript/aspire.hosting.redis/addredis/', + ].map(url) + ); + }); + + test('skips API data files that only moved to the new version', () => { + expect( + map( + apiData('pkgs', 'Aspire.Hosting.13.6.0.json', { patch: VERSION_ONLY }), + apiData('ts-modules', 'Aspire.Hosting.Redis.13.6.0.json', { status: 'renamed', changes: 0 }) + ) + ).toEqual([]); + }); + + test('counts new API data files and ignores packages without pages', () => { + expect( + map(apiData('pkgs', 'Aspire.Hosting.Redis.13.6.0.json', { status: 'added' })) + ).toHaveLength(2); + expect(map(apiData('pkgs', 'Aspire.Unknown.13.6.0.json', { status: 'added' }))).toEqual([]); + }); + + test('returns each page once, in a stable order', () => { + const result = map( + docs('app-host/eventing.mdx'), + apiData('pkgs', 'Aspire.Hosting.Redis.13.6.0.json', { patch: REAL_CHANGE }), + docs('app-host/eventing.mdx') + ); + expect(new Set(result).size).toBe(result.length); + expect(result).toEqual([...result].sort()); + }); +}); + describe('planning', () => { const base = { current: SMALL_SITE.map(url), @@ -516,6 +691,156 @@ describe('planning', () => { ); }); + test('pages updated by the release rank ahead of every other page', () => { + const updated = [url('/reference/api/csharp/b/'), url('/de/app-host/')]; + const plan = planRun({ + ...base, + mode: 'flush', + reason: 'release/13.6', + updatedUrls: updated, + limit: 4, + }); + expect(plan.cycle.updated).toEqual([...updated].sort()); + // Updated pages first (English, then API, then localized), then the rest in the usual order. + expect(plan.window).toEqual( + ['/reference/api/csharp/b/', '/de/app-host/', '/', '/app-host/'].map(url) + ); + }); + + test('updated pages that are not in the sitemap are dropped', () => { + const plan = planRun({ + ...base, + mode: 'flush', + reason: 'release/13.6', + updatedUrls: [url('/gone/'), url('/ja/'), url('/ja/')], + }); + expect(plan.cycle.updated).toEqual([url('/ja/')]); + }); + + test('a cycle that is already running keeps its own updated pages', () => { + const plan = planRun({ + ...base, + updatedUrls: [url('/de/')], + state: makeState({ + known: base.current, + cycle: { + kind: 'release', + reason: 'release/13.6', + startedAt: '', + cursor: null, + skip: [], + updated: [url('/ja/')], + }, + }), + }); + expect(plan.cycle.updated).toEqual([url('/ja/')]); + expect(plan.window[0]).toBe(url('/ja/')); + }); + + test('the cursor walks the updated pages first and then the rest', () => { + const updated = [url('/reference/api/csharp/b/'), url('/de/app-host/')]; + const plan = planRun({ + ...base, + state: makeState({ + known: base.current, + cycle: { + kind: 'release', + reason: 'release/13.6', + startedAt: '', + cursor: url('/reference/api/csharp/b/'), + skip: [], + updated, + }, + }), + }); + expect(plan.window).toEqual( + ['/de/app-host/', '/', '/app-host/', '/get-started/', '/whats-new/aspire-13-6/'].map(url) + ); + }); + + test('a release covers its new pages with the updated ones, docs before the API reference', () => { + const known = SMALL_SITE.map(url); + const created = [url('/reference/api/csharp/ab/'), url('/reference/api/csharp/ac/')]; + const createdDocs = url('/aaa/'); + const updated = [url('/app-host/')]; + const plan = planRun({ + ...base, + current: [...known, ...created, createdDocs], + state: makeState({ known }), + mode: 'flush', + reason: 'release/13.6', + updatedUrls: updated, + }); + expect(plan.newUrls).toHaveLength(3); + expect(plan.fresh).toEqual([]); + expect(plan.cycle.updated).toEqual([...updated, ...created, createdDocs].sort()); + expect(plan.window).toEqual( + ['/aaa/', '/app-host/', '/reference/api/csharp/ab/', '/reference/api/csharp/ac/', '/'].map( + url + ) + ); + }); + + test('a new page that appears after a release cycle started still goes first', () => { + const known = SMALL_SITE.map(url); + const late = url('/reference/api/csharp/zz/'); + const plan = planRun({ + ...base, + current: [...known, late], + state: makeState({ + known, + cycle: { + kind: 'release', + reason: 'release/13.6', + startedAt: '', + cursor: url('/get-started/'), + skip: [], + updated: [url('/app-host/')], + }, + }), + }); + expect(plan.fresh).toEqual([late]); + expect(plan.cycle.updated).toEqual([url('/app-host/')]); + }); + + test('a new page the running cycle already names is still sent first on a daily run', () => { + const known = SMALL_SITE.map(url); + const returned = url('/reference/api/csharp/ab/'); + const plan = planRun({ + ...base, + current: [...known, returned], + state: makeState({ + known, + cycle: { + kind: 'release', + reason: 'release/13.6', + startedAt: '', + cursor: url('/get-started/'), + skip: [], + updated: [returned], + }, + }), + }); + // The cursor has already passed its place in the cycle, so only sending it first submits it. + expect(plan.fresh).toEqual([returned]); + expect(plan.window).not.toContain(returned); + }); + + test('--limit also caps the new and removed URLs that go first', () => { + const known = SMALL_SITE.map(url); + const current = [ + ...known.filter((entry) => entry !== url('/ja/')), + url('/aaa/'), + url('/aab/'), + url('/aac/'), + ]; + const plan = planRun({ ...base, current, state: makeState({ known }), limit: 2 }); + expect(plan.removed).toEqual([url('/ja/')]); + expect(plan.fresh).toEqual([url('/aaa/')]); + expect(plan.window).toEqual([]); + expect(plan.newUrls).toHaveLength(3); + }); + test('refuses an empty sitemap', () => { expect(() => planRun({ ...base, current: [] })).toThrow(/no URLs/); }); @@ -614,6 +939,24 @@ describe('finalizing a run', () => { expect(none.known).not.toContain(created[1]); }); + test('a new page the release cycle covers stays known, whether or not anything was accepted', () => { + const known = SMALL_SITE.map(url); + const covered = url('/reference/api/csharp/ab/'); + const current = [...known, covered]; + const plan = planRun({ + current, + state: makeState({ known }), + mode: 'flush', + reason: 'release/13.6', + budgets: config.budgets, + now: NOW, + }); + expect(plan.fresh).toEqual([]); + const none = finalizeRun(plan, { ...finalizeOptions, current, accepted: 0 }); + expect(none.known).toContain(covered); + expect(none.cycle).toMatchObject({ cursor: null, skip: [], updated: [covered] }); + }); + test('forgets skipped URLs once the cursor has passed them', () => { const known = SMALL_SITE.map(url); const created = url('/reference/api/csharp/ab/'); @@ -667,6 +1010,119 @@ describe('finalizing a run', () => { expect(nextDay.cycle?.skip).toEqual([]); }); + test('ranks the skip list the same way as the cycle, so updated pages do not hide skipped ones', () => { + const known = SMALL_SITE.map(url); + const updated = [url('/reference/api/csharp/b/')]; + const cycle = { + kind: 'release' as const, + reason: 'release/13.6', + startedAt: '', + cursor: url('/reference/api/csharp/b/'), + skip: [url('/get-started/')], + updated, + }; + const plan = planRun({ + current: known, + state: makeState({ known, cycle }), + mode: 'daily', + reason: 'schedule', + budgets: config.budgets, + limit: 1, + now: NOW, + }); + // /get-started/ is an English page, so it sorts before the API page by tier, but it ranks after + // it because it was not updated. It is still ahead of the cursor and must stay skipped. + const next = finalizeRun(plan, { ...finalizeOptions, current: known, accepted: 0 }); + expect(next.cycle?.cursor).toBe(cycle.cursor); + expect(next.cycle?.skip).toEqual([url('/get-started/')]); + expect(next.cycle?.updated).toEqual(updated); + }); + + test('a release burst sends the updated pages of a sitemap shaped like aspire.dev first, and each URL once', () => { + const docs = syntheticUrls(611, '/docs/page-'); + const api = syntheticUrls(24_207); + const localized = syntheticUrls(7_518, '/de/page-'); + const current = [...localized, ...api, ...docs]; + const budgets = { release: 10_000, rotation: 1_000 }; + // 150 updated docs, 3,000 updated API pages and 20 updated localized pages. + const updated = [ + ...docs.slice(100, 250), + ...api.slice(5_000, 8_000), + ...localized.slice(0, 20), + ]; + + let state: IndexNowState = makeState({ known: current }); + const sizes: number[] = []; + const seen = new Set(); + for (let day = 0; day < 10; day++) { + const plan = planRun({ + current, + state, + mode: day === 0 ? 'flush' : 'daily', + reason: 'release/13.6', + budgets, + updatedUrls: day === 0 ? updated : undefined, + now: NOW, + }); + if (day === 0) { + // Updated pages first, in tier order, and only then the other pages. + expect(plan.window.slice(0, 150)).toEqual(docs.slice(100, 250)); + expect(plan.window.slice(150, 3_150)).toEqual(api.slice(5_000, 8_000)); + expect(plan.window.slice(3_150, 3_170)).toEqual(localized.slice(0, 20)); + expect(plan.window[3_170]).toBe(docs[0]); + } + for (const entry of plan.window) { + expect(seen.has(entry)).toBe(false); + seen.add(entry); + } + sizes.push(plan.window.length); + state = finalizeRun(plan, { ...finalizeOptions, current, accepted: plan.window.length }); + if (state.cycle === null) break; + } + expect(sizes).toEqual([10_000, 10_000, 10_000, 2_336]); + expect(seen.size).toBe(current.length); + }); + + test('a release that adds more API pages than the budget covers them across runs, each once', () => { + const docs = syntheticUrls(50, '/docs/page-'); + const known = [...docs, ...syntheticUrls(200)]; + const created = syntheticUrls(120, '/reference/api/csharp/new-'); + const current = [...known, ...created]; + const budgets = { release: 100, rotation: 10 }; + const updated = docs.slice(10, 20); + + let state: IndexNowState = makeState({ known }); + const sizes: number[] = []; + const seen = new Set(); + for (let day = 0; day < 10; day++) { + const plan = planRun({ + current, + state, + mode: day === 0 ? 'flush' : 'daily', + reason: 'release/13.6', + budgets, + updatedUrls: day === 0 ? updated : undefined, + now: NOW, + }); + // The new pages are covered by the cycle, so none of them jumps ahead of the updated docs. + expect(plan.fresh).toEqual([]); + if (day === 0) { + expect(plan.window.slice(0, 10)).toEqual(updated); + expect(plan.window.slice(10)).toEqual(created.slice(0, 90)); + } + for (const entry of plan.window) { + expect(seen.has(entry)).toBe(false); + seen.add(entry); + } + sizes.push(plan.window.length); + state = finalizeRun(plan, { ...finalizeOptions, current, accepted: plan.window.length }); + expect(state.known).toEqual([...current].sort()); + if (state.cycle === null) break; + } + expect(sizes).toEqual([100, 100, 100, 70]); + expect(seen.size).toBe(current.length); + }); + test('a release burst over a sitemap shaped like aspire.dev takes four runs and sends each URL once', () => { const docs = syntheticUrls(611, '/docs/page-'); const api = syntheticUrls(24_207); @@ -720,6 +1176,7 @@ describe('state on disk', () => { startedAt: '2026-10-13T06:43:00.000Z', cursor: url('/get-started/'), skip: [url('/reference/api/csharp/ab/')], + updated: [url('/de/'), url('/get-started/')], }, lastRun: { at: '2026-10-13T06:43:00.000Z', @@ -788,6 +1245,49 @@ describe('state on disk', () => { }); }); +describe('state read from an artifact', () => { + const cycle = { + kind: 'release' as const, + reason: 'release/13.6', + startedAt: '2026-10-13T06:43:00.000Z', + cursor: url('/get-started/'), + skip: [], + updated: [], + }; + + test('loads states saved before cycles tracked updated pages', async () => { + await saveState(stateDir, makeState({ known: SMALL_SITE.map(url), cycle })); + const file = path.join(stateDir, 'state.json'); + const parsed = JSON.parse(await readFile(file, 'utf8')) as { cycle: Record }; + delete parsed.cycle.updated; + await writeFile(file, JSON.stringify(parsed)); + expect((await loadState(stateDir, HOST, createLogger()))?.cycle?.updated).toEqual([]); + }); + + test.each([ + ['an http URL', { known: [url('/'), 'http://aspire.dev/insecure/'] }], + ['another host', { known: [url('/'), 'https://evil.example/spam/'] }], + ['something that is not a URL', { known: [url('/'), 'javascript:alert(1)'] }], + ['a duplicate', { known: [url('/'), url('/')] }], + ['a URL with a fragment', { known: [url('/'), url('/a/#top')] }], + ['an empty known list', { known: [] }], + ['a cursor that is not a URL', { known: [url('/')], cycle: { ...cycle, cursor: 'not a url' } }], + [ + 'a skip list with another host', + { known: [url('/')], cycle: { ...cycle, skip: ['https://evil.example/'] } }, + ], + [ + 'an updated list with another host', + { known: [url('/')], cycle: { ...cycle, updated: ['https://evil.example/'] } }, + ], + ])('starts cold instead of trusting %s', async (_name, overrides) => { + await saveState(stateDir, makeState(overrides)); + const log = createLogger(); + expect(await loadState(stateDir, HOST, log)).toBeNull(); + expect(log.warn).toHaveBeenCalledWith(expect.stringContaining('starting cold')); + }); +}); + describe('submitting URLs', () => { function fetchWith(...responses: (Response | Error)[]) { const fetch = vi.fn(); @@ -1177,6 +1677,13 @@ describe('triggers', () => { }); }); + test('reads the changed files option as an absolute path', () => { + expect( + parseCli(['--state-dir', stateDir, '--changed-files', 'changes.jsonl']).changedFiles + ).toBe(path.resolve('changes.jsonl')); + expect(parseCli(['--state-dir', stateDir]).changedFiles).toBeUndefined(); + }); + test.each([ ['no state directory', []], ['an unknown mode', ['--state-dir', 'x', '--mode', 'weekly']], @@ -1393,6 +1900,146 @@ describe('running end to end', () => { await expect(runDay()).rejects.toThrow(/--allow-shrink/); expect(await readFile(path.join(stateDir, 'state.json'), 'utf8')).toBe(before); }); + + const releaseFiles: ChangedFile[] = [ + { + status: 'added', + filename: 'src/frontend/src/content/docs/get-started/new-page.mdx', + changes: 40, + patch: null, + }, + { + status: 'modified', + filename: 'src/frontend/src/content/docs/whats-new/aspire-13-6.mdx', + changes: 6, + patch: null, + }, + { + status: 'renamed', + filename: 'src/frontend/src/data/pkgs/A.13.6.0.json', + changes: 9, + patch: '@@ -9 +9 @@\n+ "name": "AsExpress",', + }, + { + // Only moved to the new version, so its pages did not change. + status: 'renamed', + filename: 'src/frontend/src/data/pkgs/B.13.6.0.json', + changes: 4, + patch: '@@ -4 +4 @@\n- "version": "13.5.4",\n+ "version": "13.6.0",', + }, + ]; + + test('a release flush submits new pages, then updated pages, then the rest, and sends nothing twice', async () => { + await runDay(); + site.paths = [...site.paths, '/get-started/new-page/']; + + const flush = await runDay({ + mode: 'flush', + reason: 'release/13.6', + changedFiles: releaseFiles, + }); + expect(flush.report).toMatchObject({ newUrls: 1, updatedUrls: 3, cycleKind: 'release' }); + expect(submitted()).toEqual( + sent([ + '/get-started/new-page/', // new + '/whats-new/aspire-13-6/', // updated docs + '/reference/api/csharp/a/', // updated API package (B only moved to the new version) + '/', // then everything else, in the usual order + '/app-host/', + ]) + ); + + // The updated pages are remembered, so the next runs finish the rest without repeating them. + const days = [submitted()]; + for (let day = 0; day < 2; day++) { + await runDay(); + days.push(submitted()); + } + expect(days[1]).toEqual( + sent([ + '/get-started/', + '/reference/api/csharp/b/', + '/reference/api/typescript/c/', + '/de/', + '/de/app-host/', + ]) + ); + expect(days[2]).toEqual(sent(['/ja/'])); + const everything = days.flat(); + expect(new Set(everything).size).toBe(everything.length); + expect(new Set(everything)).toEqual(new Set(site.paths.map(url))); + expect((await loadState(stateDir, HOST, log))?.cycle).toBeNull(); + }); + + test('a release flush puts new generated API pages after the updated docs, not before them', async () => { + await runDay(); + site.paths = [...site.paths, '/reference/api/csharp/ab/', '/reference/api/csharp/ac/']; + + const flush = await runDay({ + mode: 'flush', + reason: 'release/13.6', + changedFiles: [releaseFiles[1]], + }); + expect(flush.report).toMatchObject({ newUrls: 2, updatedUrls: 3, cycleKind: 'release' }); + expect(submitted()).toEqual( + sent([ + '/whats-new/aspire-13-6/', // updated docs + '/reference/api/csharp/ab/', // new API pages, ordered with the updated pages + '/reference/api/csharp/ac/', + '/', // then everything else, in the usual order + '/app-host/', + ]) + ); + + const days = [submitted()]; + for (let day = 0; day < 2; day++) { + await runDay(); + days.push(submitted()); + } + const everything = days.flat(); + expect(new Set(everything).size).toBe(everything.length); + expect(new Set(everything)).toEqual(new Set(site.paths.map(url))); + expect((await loadState(stateDir, HOST, log))?.cycle).toBeNull(); + }); + + test('a flush that is rate limited before anything is accepted still starts the release cycle', async () => { + await runDay(); + site.responses = [429]; + const limited = await runDay({ mode: 'flush', reason: 'release/13.6' }); + expect(limited.report.outcome).toBe('rate-limited'); + expect(limited.failure).toMatch(/before accepting any URL.*release cycle/); + expect(limited.stateChanged).toBe(true); + expect((await loadState(stateDir, HOST, log))?.cycle).toMatchObject({ + kind: 'release', + reason: 'release/13.6', + cursor: null, + }); + + // The next scheduled run carries on with the release burst from the top. + await runDay(); + expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); + }); + + test('a rate-limited daily run that accepts nothing fails and leaves the state untouched', async () => { + await runDay(); + const before = await readFile(path.join(stateDir, 'state.json'), 'utf8'); + site.responses = [429]; + const limited = await runDay(); + expect(limited.failure).toMatch(/429/); + expect(limited.stateChanged).toBe(false); + expect(await readFile(path.join(stateDir, 'state.json'), 'utf8')).toBe(before); + }); + + test('replaces a poisoned state instead of submitting what it lists', async () => { + await saveState( + stateDir, + makeState({ known: [url('/'), 'https://evil.example/spam/', 'http://aspire.dev/insecure/'] }) + ); + const result = await runDay(); + expect(result.report.coldStart).toBe(true); + expect(submitted().every((entry) => entry.startsWith('https://aspire.dev/'))).toBe(true); + expect((await loadState(stateDir, HOST, log))?.known).toEqual(SMALL_SITE.map(url).sort()); + }); }); describe('summary and command line', () => { @@ -1409,6 +2056,7 @@ describe('summary and command line', () => { totalUrls: 32_336, newUrls: 2, removedUrls: 1, + updatedUrls: 3_150, windowUrls: 997, submitted: 1_000, cycleKind: 'rotation', @@ -1421,6 +2069,9 @@ describe('summary and command line', () => { expect(summary).toContain('## IndexNow submission for aspire.dev'); expect(summary).toContain('| Sitemap URLs | 32,336 |'); expect(summary).toContain('| New / removed | 2 / 1 |'); + expect(summary).toContain( + '| Prioritized pages | 3,150 new or updated by the release, ahead of the rest |' + ); expect(summary).toContain('| Submitted | 1,000 of a 1,000 URL budget |'); expect(summary).toContain('| Cycle | rotation: 1,000 of 32,336 URLs (3%) |'); expect(summary).toContain('HTTP 200 for 1,000 URLs'); @@ -1489,6 +2140,52 @@ describe('summary and command line', () => { expect(site.posts[0].urlList).toEqual(SMALL_SITE.map(url)); }); + test('a release pull request sends the pages its files changed first', async () => { + const real = await loadConfig(); + const site = createSite(SMALL_SITE, real.key); + site.liveReleases.add('13.6'); + const clock = createClock(); + const changedFiles = path.join(stateDir, 'changes.jsonl'); + await writeFile( + changedFiles, + [ + { + status: 'modified', + filename: 'src/frontend/src/content/docs/get-started/index.mdx', + changes: 3, + }, + { + status: 'renamed', + filename: 'src/frontend/src/data/pkgs/A.13.6.0.json', + changes: 8, + patch: '@@ -9 +9 @@\n+ "name": "AsExpress",', + }, + { status: 'modified', filename: 'src/frontend/src/components/Header.astro', changes: 2 }, + ] + .map((entry) => JSON.stringify(entry)) + .join('\n') + ); + const { code, summary } = await runMain( + ['--trigger', 'pull_request', '--changed-files', changedFiles], + pullRequest('Content for release: 13.6', 'release/13.6'), + { fetch: site.fetch, sleep: clock.sleep, now: clock.now } + ); + expect(code).toBe(0); + expect(summary).toContain( + '| Prioritized pages | 2 new or updated by the release, ahead of the rest |' + ); + expect(site.posts[0].urlList.slice(0, 3)).toEqual( + ['/get-started/', '/reference/api/csharp/a/', '/'].map(url) + ); + expect(site.posts[0].urlList).toHaveLength(SMALL_SITE.length); + }); + + test('fails when the changed files cannot be read', async () => { + const { code, log } = await runMain(['--changed-files', path.join(stateDir, 'missing.jsonl')]); + expect(code).toBe(1); + expect(log.error).toHaveBeenCalledWith(expect.stringContaining('missing.jsonl')); + }); + test('reports a failed run and returns a failing exit code', async () => { const real = await loadConfig(); const site = createSite(SMALL_SITE, real.key); From 5568eecb19604bab5865eb23f4af4897ca128048 Mon Sep 17 00:00:00 2001 From: David Pine <7679720+IEvangelist@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:56:24 -0500 Subject: [PATCH 3/6] Continue a release cycle that is already running and tighten the tests - A pull request for a release whose cycle is still running continues that cycle instead of starting the burst over. The same release can arrive twice: its content pull request and, hours later, the one that merges the release branch back (13.4 had both, and the second changed no files). A manual flush still starts over. - Ignore pull requests from forks when a manual flush looks up the release pull request. - Fix a stale comment on the cap for removed and new URLs, which is joint. - Cover the removed URL cap, the deferred count, a cold start that fails to submit, and the continued cycle in tests. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/indexnow.yml | 6 +- src/frontend/scripts/indexnow.ts | 21 ++++- .../tests/unit/indexnow.vitest.test.ts | 77 ++++++++++++++++++- 3 files changed, 96 insertions(+), 8 deletions(-) diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml index f5f0eb1c78..344ea3bccf 100644 --- a/.github/workflows/indexnow.yml +++ b/.github/workflows/indexnow.yml @@ -8,7 +8,8 @@ name: IndexNow # new" page to be live (the docs slot swap) and then starts a full-sitemap burst that the daily # runs finish. The pages the release added or updated go first (docs before the generated API # reference; the updated ones are found from the files the release pull request changed), then -# everything else. +# everything else. A later pull request for the same release, such as the one that merges the +# release branch back, continues the burst that is running instead of starting another. # - Manual: run it from the Actions tab (mode, limit, release, dry run). A flush with a release # prioritizes that release's new and updated pages the same way. # @@ -167,7 +168,8 @@ jobs: exit 1 fi pr=$(gh pr list --state merged --base main --head "release/$RELEASE" \ - --json number,mergedAt --jq 'sort_by(.mergedAt) | last | .number // empty') + --json number,mergedAt,isCrossRepository \ + --jq 'map(select(.isCrossRepository == false)) | sort_by(.mergedAt) | last | .number // empty') fi if [ -z "$pr" ]; then diff --git a/src/frontend/scripts/indexnow.ts b/src/frontend/scripts/indexnow.ts index 92f5b7c1e0..fc0ec2e72b 100644 --- a/src/frontend/scripts/indexnow.ts +++ b/src/frontend/scripts/indexnow.ts @@ -104,7 +104,7 @@ const USER_AGENT = 'aspire.dev-indexnow (+https://github.com/microsoft/aspire.de /** IndexNow accepts at most 10,000 URLs per POST. */ export const MAX_URLS_PER_REQUEST = 10_000; -/** Cap on new URLs, and separately on removed URLs, submitted ahead of the window in one run. */ +/** Cap on the removed and new URLs, together, submitted ahead of the window in one run. */ const MAX_PRIORITY_URLS = 10_000; export const MAX_LIMIT = 50_000; /** A sitemap with fewer URLs than this share of the last known list is treated as broken. */ @@ -1302,13 +1302,12 @@ export function emptyReport( } export async function run(options: RunOptions): Promise { - const { config, stateDir, mode, trigger, reason, release, limit } = options; + const { config, stateDir, trigger, reason, release, limit } = options; const dryRun = options.dryRun ?? false; const fetch = options.fetch ?? fetchWithProxy; const sleep = options.sleep ?? defaultSleep; const now = options.now ?? (() => new Date()); const log = options.log ?? consoleLogger; - const base = { trigger, mode, reason, host: config.host }; const state = await loadState(stateDir, config.host, log); log.info( @@ -1317,6 +1316,21 @@ export async function run(options: RunOptions): Promise { : 'No previous state; this is a cold start.' ); + // A release can reach the workflow twice: its content pull request and, hours later, the pull + // request that merges the release branch back, which changes no files. Starting over would send + // the whole burst again, so a pull request for a release whose cycle is still running continues + // that cycle. A manual flush always starts over. + const resumes = + options.mode === 'flush' && + trigger === 'pull_request' && + state?.cycle?.kind === 'release' && + state.cycle.reason === reason; + const mode: RunMode = resumes ? 'daily' : options.mode; + const resumeNote = resumes + ? `The release cycle for ${reason} was already running, so this run continued it instead of starting over.` + : ''; + if (resumeNote) log.info(resumeNote); + const base = { trigger, mode, reason, host: config.host }; const key = await verifyKeyFile(config, fetch); if (!key.ok) { if (dryRun) { @@ -1454,6 +1468,7 @@ export async function run(options: RunOptions): Promise { const report = emptyReport({ ...planned, outcome: 'completed', + note: resumeNote, windowUrls: next?.lastRun?.windowUrls ?? 0, submitted: result.accepted, chunks: result.chunks, diff --git a/src/frontend/tests/unit/indexnow.vitest.test.ts b/src/frontend/tests/unit/indexnow.vitest.test.ts index 935c2ce160..c10304e32e 100644 --- a/src/frontend/tests/unit/indexnow.vitest.test.ts +++ b/src/frontend/tests/unit/indexnow.vitest.test.ts @@ -841,6 +841,20 @@ describe('planning', () => { expect(plan.newUrls).toHaveLength(3); }); + test('--limit also caps the removed URLs that go first', () => { + const known = SMALL_SITE.map(url); + const gone = [url('/reference/api/typescript/c/'), url('/ja/')]; + const plan = planRun({ + ...base, + current: known.filter((entry) => !gone.includes(entry)), + state: makeState({ known }), + limit: 1, + }); + expect(plan.removedUrls).toEqual(gone); + expect(plan.removed).toEqual([gone[0]]); + expect(plan.window).toEqual([]); + }); + test('refuses an empty sitemap', () => { expect(() => planRun({ ...base, current: [] })).toThrow(/no URLs/); }); @@ -1800,6 +1814,63 @@ describe('running end to end', () => { expect(submitted()).toEqual(sent(['/', '/app-host/', '/get-started/'])); }); + test('a second pull request for a release whose cycle is running continues it instead of starting over', async () => { + await runDay(); + const release = { mode: 'flush', trigger: 'pull_request', reason: 'release/13.6' } as const; + const first = await runDay(release); + expect(first.report).toMatchObject({ mode: 'flush', cycleKind: 'release' }); + expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); + + const second = await runDay(release); + expect(second.report).toMatchObject({ mode: 'daily', cycleKind: 'release' }); + expect(second.report.note).toMatch(/already running/); + expect(submitted()).toEqual( + sent(['/reference/api/csharp/b/', '/reference/api/typescript/c/', ...LOCALIZED]) + ); + expect((await loadState(stateDir, HOST, log))?.cycle).toBeNull(); + }); + + test('a pull request for another release, and a manual flush, still start the burst over', async () => { + await runDay(); + await runDay({ mode: 'flush', trigger: 'pull_request', reason: 'release/13.6' }); + + const other = await runDay({ mode: 'flush', trigger: 'pull_request', reason: 'release/13.7' }); + expect(other.report.mode).toBe('flush'); + expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); + + const manual = await runDay({ + mode: 'flush', + trigger: 'workflow_dispatch', + reason: 'release/13.7', + }); + expect(manual.report.mode).toBe('flush'); + expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); + }); + + test('reports the new URLs that did not fit in the run as deferred', async () => { + await runDay(); + site.paths = [...site.paths, '/aaa/', '/aab/', '/aac/']; + const limited = await runDay({ limit: 2 }); + expect(limited.report).toMatchObject({ newUrls: 3, deferred: 1 }); + }); + + test('a flush defers none of its new URLs, because the cycle it starts covers them', async () => { + await runDay(); + site.paths = [...site.paths, '/aaa/', '/aab/', '/aac/']; + const flush = await runDay({ mode: 'flush', reason: 'release/13.6', limit: 2 }); + expect(flush.report).toMatchObject({ newUrls: 3, deferred: 0 }); + }); + + test('a cold start that fails to submit still seeds the state', async () => { + site.responses = [422]; + const result = await runDay(); + expect(result.report.outcome).toBe('failed'); + expect(result.stateChanged).toBe(true); + const saved = await loadState(stateDir, HOST, log); + expect(saved?.known).toHaveLength(SMALL_SITE.length); + expect(saved?.cycle?.cursor).toBeNull(); + }); + test('skips a cold start while the key file is not live yet', async () => { site.keyLive = false; const result = await runDay(); @@ -1929,7 +2000,7 @@ describe('running end to end', () => { }, ]; - test('a release flush submits new pages, then updated pages, then the rest, and sends nothing twice', async () => { + test('a release flush submits its new and updated pages in tier order, then the rest, and sends nothing twice', async () => { await runDay(); site.paths = [...site.paths, '/get-started/new-page/']; @@ -1941,8 +2012,8 @@ describe('running end to end', () => { expect(flush.report).toMatchObject({ newUrls: 1, updatedUrls: 3, cycleKind: 'release' }); expect(submitted()).toEqual( sent([ - '/get-started/new-page/', // new - '/whats-new/aspire-13-6/', // updated docs + '/get-started/new-page/', // new docs page + '/whats-new/aspire-13-6/', // updated docs page '/reference/api/csharp/a/', // updated API package (B only moved to the new version) '/', // then everything else, in the usual order '/app-host/', From 2a73713e8e3ad7cfa1e5175f042976ed010de896 Mon Sep 17 00:00:00 2001 From: David Pine <7679720+IEvangelist@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:38:17 -0500 Subject: [PATCH 4/6] Explain the IndexNow schedule in the workflow Say what the cron expression means, how often it runs, what each run does and why the minute is not on the hour. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/indexnow.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml index 344ea3bccf..d5ed51de45 100644 --- a/.github/workflows/indexnow.yml +++ b/.github/workflows/indexnow.yml @@ -25,6 +25,13 @@ name: IndexNow on: schedule: + # Once a day at 06:43 UTC (the fields are minute, hour, day of month, month and day of week). + # Each run submits the new and removed URLs, then the next window of the rotation through the + # whole sitemap. The window size is in config/indexnow.json: at 1,000 URLs a run, a sitemap of + # about 32,000 URLs takes about a month to cover. If GitHub skips a day, the rotation only + # slips by a day, because its position is saved in the state artifact. The odd minute avoids + # the top of the hour, when GitHub delays scheduled runs the most. Scheduled runs use the + # workflow on the default branch, so this starts once the pull request is merged. - cron: "43 6 * * *" pull_request: types: [closed] From ac70340d47956a0b785948dc7fd5964bce1370c4 Mon Sep 17 00:00:00 2001 From: David Pine <7679720+IEvangelist@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:10:34 -0500 Subject: [PATCH 5/6] Simplify the IndexNow script The first version grew well past a simple scheduled job. Cut it back to what the job needs: - Drop the mapping from the files a release changed to pages. It hard-coded the content and API data paths and routes, and needed ranking, a skip list and extra state to support it. A release burst now sends the English pages first, which is where the new and updated docs are, then the API reference and the localized pages. - Send one request per run. The budgets fit in the 10,000 URLs IndexNow takes, so there is no chunking, partial acceptance or Retry-After handling: a failed request leaves the progress alone and the next run tries again. - Start a release only from a release/X.Y branch, and keep the state in a single file. - Trim the config and state validation, the logger and the run report to what the job needs. The script goes from 1,664 to 809 lines and the tests from 2,283 to 1,037. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/indexnow.yml | 61 +- src/frontend/scripts/indexnow.ts | 1699 ++++--------- .../tests/unit/indexnow.vitest.test.ts | 2160 ++++------------- 3 files changed, 887 insertions(+), 3033 deletions(-) diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml index d5ed51de45..182ee03fb8 100644 --- a/.github/workflows/indexnow.yml +++ b/.github/workflows/indexnow.yml @@ -6,12 +6,10 @@ name: IndexNow # window of a slow rotation through the whole sitemap. # - Release: when a release/X.Y pull request is merged into main, waits for that release's "What's # new" page to be live (the docs slot swap) and then starts a full-sitemap burst that the daily -# runs finish. The pages the release added or updated go first (docs before the generated API -# reference; the updated ones are found from the files the release pull request changed), then -# everything else. A later pull request for the same release, such as the one that merges the -# release branch back, continues the burst that is running instead of starting another. -# - Manual: run it from the Actions tab (mode, limit, release, dry run). A flush with a release -# prioritizes that release's new and updated pages the same way. +# runs finish. The sitemap has no lastmod, so the burst sends the English pages first (where the +# new and updated docs are), then the generated API reference, then the localized pages. +# - Manual: run it from the Actions tab (mode, limit, release, dry run). A flush restarts the +# full-sitemap burst, the same way a release does. # # All logic lives in src/frontend/scripts/indexnow.ts so the same flow can be reproduced locally: # cd src/frontend && pnpm indexnow --state-dir /tmp/indexnow --dry-run @@ -46,7 +44,7 @@ on: - flush default: daily limit: - description: "Number of URLs to submit in this run (1-50000). Leave empty to use the configured budget." + description: "Number of URLs to submit in this run (1-10000). Leave empty to use the configured budget." type: string required: false release: @@ -68,14 +66,13 @@ jobs: submit: name: Submit URLs to IndexNow # Pull requests only matter when they are merged release pull requests from this repository. The - # script checks the exact branch or title format again. + # script checks the exact branch format again. if: >- ${{ github.repository == 'microsoft/aspire.dev' && (github.event_name != 'pull_request' || (github.event.pull_request.merged == true && github.event.pull_request.head.repo.full_name == github.repository && - (startsWith(github.event.pull_request.head.ref, 'release/') || - startsWith(github.event.pull_request.title, 'Merge release/')))) }} + startsWith(github.event.pull_request.head.ref, 'release/'))) }} runs-on: ubuntu-latest timeout-minutes: 75 # The group is on the job, not the workflow, so the many pull requests that are closed without @@ -89,7 +86,6 @@ jobs: permissions: contents: read actions: read - pull-requests: read defaults: run: working-directory: src/frontend @@ -152,45 +148,6 @@ jobs: gh api "repos/$GH_REPO/actions/artifacts/$artifact_id/zip" > "$RUNNER_TEMP/indexnow-state.zip" unzip -q -o "$RUNNER_TEMP/indexnow-state.zip" -d "$state_dir" - # A release rewrites most of the API reference data, so the pages worth submitting first are - # the ones whose own files changed. The sitemap has no lastmod to tell, so list the files of - # the release pull request. The script maps docs and API data to pages and leaves out API - # files that only moved to the new version. - - name: Find the files changed by the release - id: changes - if: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.mode == 'flush' && inputs.release != '') }} - env: - GH_TOKEN: ${{ github.token }} - GH_REPO: ${{ github.repository }} - EVENT_NAME: ${{ github.event_name }} - PR_NUMBER: ${{ github.event.pull_request.number }} - RELEASE: ${{ inputs.release }} - shell: bash - run: | - set -euo pipefail - pr="$PR_NUMBER" - if [ "$EVENT_NAME" = "workflow_dispatch" ]; then - if ! [[ "$RELEASE" =~ ^[0-9]+\.[0-9]+$ ]]; then - echo "::error::The release input must look like 13.6." - exit 1 - fi - pr=$(gh pr list --state merged --base main --head "release/$RELEASE" \ - --json number,mergedAt,isCrossRepository \ - --jq 'map(select(.isCrossRepository == false)) | sort_by(.mergedAt) | last | .number // empty') - fi - - if [ -z "$pr" ]; then - echo "::notice::No merged release pull request was found, so only the pages that are new to the sitemap are prioritized." - exit 0 - fi - - changes="$RUNNER_TEMP/changed-files.jsonl" - gh api "repos/$GH_REPO/pulls/$pr/files" --paginate \ - --jq '.[] | {status, filename, changes, patch: (if (.filename | test("/data/(pkgs|ts-modules)/")) then .patch else null end)}' \ - > "$changes" - echo "Release pull request #$pr changed $(wc -l < "$changes") files." - echo "path=$changes" >> "$GITHUB_OUTPUT" - - name: Submit URLs id: submit env: @@ -200,12 +157,10 @@ jobs: INPUT_RELEASE: ${{ inputs.release }} INPUT_DRY_RUN: ${{ inputs.dry_run }} INPUT_ALLOW_SHRINK: ${{ inputs.allow_shrink }} - CHANGED_FILES: ${{ steps.changes.outputs.path }} shell: bash run: | set -euo pipefail args=(--state-dir "$RUNNER_TEMP/indexnow-state" --trigger "$EVENT_NAME") - [ -z "$CHANGED_FILES" ] || args+=(--changed-files "$CHANGED_FILES") if [ "$EVENT_NAME" = "workflow_dispatch" ]; then args+=(--mode "$INPUT_MODE") [ -z "$INPUT_LIMIT" ] || args+=(--limit "$INPUT_LIMIT") @@ -215,7 +170,7 @@ jobs: fi pnpm indexnow "${args[@]}" - # Runs after a failure too, so URLs that were accepted before an error are not sent again. + # Runs after a failure too, because a failed cold start or flush still saves what it started. - name: Save state for the next run if: ${{ !cancelled() && steps.submit.outputs.state_changed == 'true' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/src/frontend/scripts/indexnow.ts b/src/frontend/scripts/indexnow.ts index fc0ec2e72b..1d64963a88 100644 --- a/src/frontend/scripts/indexnow.ts +++ b/src/frontend/scripts/indexnow.ts @@ -2,54 +2,48 @@ * Tells IndexNow-participating search engines (Bing, Yandex, Naver, Seznam, Yep, Amazon) about * aspire.dev URLs. See https://www.indexnow.org/documentation. * - * Every run reads the live sitemap, compares it with the URL list remembered from the previous run - * and submits, in priority order: - * 1. removed URLs (so engines drop them) and new URLs (so engines discover them), - * 2. the next window of the rotation cycle, which walks the whole sitemap in (rank, tier, path) - * order: the pages a release added or updated first (docs before the API reference), then - * everything else. + * Every run reads the live sitemap, compares it with the URL list saved by the previous run and + * sends one request (IndexNow takes at most 10,000 URLs in one) with, in this order: + * 1. URLs that left the sitemap (so engines drop them) and URLs that are new in it, + * 2. the next window of a rotation that walks the whole sitemap: English pages first, then the + * API reference, then localized pages. * - * A release flush covers its new URLs in step 2 instead of step 1: a release adds thousands of - * generated API pages, and sending those first would use up the budget before any updated docs page. + * The rotation is a cursor over that order, so it survives pages being added or removed. A release + * is a "flush": it starts the rotation again from the top at the larger release budget, and the + * daily runs finish that pass before the small rotation budget applies again. IndexNow does not + * publish its rate limits, so both budgets are in config/indexnow.json. A request that fails (a + * 429 included) leaves the progress alone, so the next run simply tries again; only a cold start + * still seeds the known list and a flush still starts its pass. A page that is new while a pass + * is running can be sent twice: when it appears and when the rotation reaches it. * - * The rotation cycle is a cursor over that order, so it survives URLs being added or removed. A - * `flush` (a release merge or a manual dispatch) restarts the cycle from the top at the larger - * release budget; the daily runs then drain it until the whole sitemap has been covered, after - * which the slow rotation budget applies. IndexNow does not publish its rate limits, so both - * budgets live in config/indexnow.json and a 429 simply pauses the run until the next one. + * The sitemap has no `lastmod`, so the script cannot tell an updated page from an unchanged one. + * A release flush sends the English pages first, which is where the new and updated docs are, and + * the generated API reference after them. * - * The sitemap has no `lastmod`, so "updated" comes from the release itself: the workflow lists the - * files the release pull request changed (--changed-files) and they are mapped to pages (docs and - * API reference data). API data files only count when something besides the version, source - * commit and source line numbers changed, because those move in every file on every release. + * Besides config/indexnow.json, the script assumes that: + * - pages under /reference/api/ are the generated API reference, and the locale folders listed + * in config/locales.ts hold the translations; both are sent after the English pages, + * - release X.Y is merged from a release/X.Y branch and publishes /whats-new/aspire-X-Y/ (or + * aspire-X for X.0), which is how a run knows that the new docs are live. * - * State (`state.json` and `known-urls.txt.gz`) lives in --state-dir. The workflow keeps it between - * runs as an Actions artifact; losing it only causes a cold start, which seeds the known URL list - * without submitting every URL as "new". + * The state (state.json) lives in --state-dir. The workflow keeps it between runs as an Actions + * artifact; losing it only causes a cold start, which seeds the known URL list without submitting + * every URL as new. * * Usage (from src/frontend): * pnpm indexnow --state-dir [--mode daily|flush] [--limit ] [--release ] - * [--changed-files ] [--trigger ] [--dry-run] [--allow-shrink] + * [--trigger ] [--dry-run] [--allow-shrink] */ -import { appendFile, mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { appendFile, mkdir, readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; import { parseArgs } from 'node:util'; -import { gunzipSync, gzipSync } from 'node:zlib'; import { locales } from '../config/locales'; import { fetchWithProxy } from './fetch-with-proxy'; export type FetchLike = typeof fetchWithProxy; -export type Sleep = (ms: number) => Promise; export type RunMode = 'daily' | 'flush'; -export type CycleKind = 'release' | 'rotation'; - -export interface Logger { - info(message: string): void; - warn(message: string): void; - error(message: string): void; -} export interface IndexNowConfig { host: string; @@ -59,64 +53,41 @@ export interface IndexNowConfig { budgets: { release: number; rotation: number }; } -/** A pass over the sitemap. `cursor` is the last URL covered, in (rank, tier, path) order. */ +/** A pass over the sitemap. `cursor` is the last URL covered, in the order of `compareUrls`. */ export interface Cycle { - kind: CycleKind; - reason: string; - startedAt: string; + kind: 'release' | 'rotation'; cursor: string | null; - /** URLs ahead of the cursor that were already submitted as new, so the window skips them. */ - skip: string[]; - /** Pages the release added or changed. They rank ahead of every other page in this cycle. */ - updated: string[]; -} - -export interface RunRecord { - at: string; - trigger: string; - mode: RunMode; - coldStart: boolean; - newUrls: number; - removedUrls: number; - windowUrls: number; - submitted: number; - statuses: number[]; } export interface IndexNowState { host: string; updatedAt: string; - /** The `git-commit-id` meta value served by the site when the state was written. */ - deployment: string; - /** `null` once the previous cycle has covered the whole sitemap. */ + /** The pass in progress, or null when the last one finished and the next run starts a new one. */ cycle: Cycle | null; - lastRun: RunRecord | null; - /** Every sitemap URL as of the last run, sorted. */ + /** The sitemap URLs that were submitted or are left to the pass in progress. */ known: string[]; } +/** What a run needs from the outside world. Tests replace it. */ +export interface Deps { + fetch: FetchLike; + sleep: (ms: number) => Promise; + now: () => Date; +} + const CONFIG_PATH = fileURLToPath(new URL('../config/indexnow.json', import.meta.url)); const STATE_FILE = 'state.json'; -const KNOWN_FILE = 'known-urls.txt.gz'; const SCHEMA_VERSION = 1; -const KEY_PATTERN = /^[A-Za-z0-9-]{8,128}$/; const USER_AGENT = 'aspire.dev-indexnow (+https://github.com/microsoft/aspire.dev)'; +/** The generated API reference, which is sent after the English pages. */ +const API_REFERENCE_PATH = '/reference/api/'; -/** IndexNow accepts at most 10,000 URLs per POST. */ -export const MAX_URLS_PER_REQUEST = 10_000; -/** Cap on the removed and new URLs, together, submitted ahead of the window in one run. */ -const MAX_PRIORITY_URLS = 10_000; -export const MAX_LIMIT = 50_000; +/** IndexNow takes at most 10,000 URLs per request, and a run sends a single request. */ +export const MAX_URLS_PER_RUN = 10_000; /** A sitemap with fewer URLs than this share of the last known list is treated as broken. */ export const SHRINK_THRESHOLD = 0.8; -const MAX_SITEMAP_CHARS = 100_000_000; -const MAX_CHILD_SITEMAPS = 50; -const SITEMAP_TIMEOUT_MS = 120_000; -const REQUEST_TIMEOUT_MS = 60_000; -/** Back-off for 5xx and network errors. A 429 waits for its Retry-After header instead. */ +/** Waits before the retries of a request that failed with a 5xx or a network error. */ const RETRY_DELAYS_MS = [5_000, 30_000, 120_000]; -const MAX_RETRY_AFTER_MS = 300_000; -const CHUNK_PAUSE_MS = 2_000; const GATE_INTERVAL_MS = 60_000; const GATE_TIMEOUT_MS = 45 * 60_000; @@ -132,83 +103,20 @@ function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } -function compareOrdinal(a: string, b: string): number { - return a < b ? -1 : a > b ? 1 : 0; -} - -const defaultSleep: Sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); - -function escapeAnnotation(message: string): string { - return message.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); +function annotate(level: 'warning' | 'error', message: string): string { + return process.env.GITHUB_ACTIONS === 'true' + ? `::${level}::${message.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A')}` + : `${level}: ${message}`; } -export const consoleLogger: Logger = { - info: (message) => console.log(message), - warn: (message) => - console.warn( - process.env.GITHUB_ACTIONS === 'true' - ? `::warning::${escapeAnnotation(message)}` - : `warning: ${message}` - ), - error: (message) => - console.error( - process.env.GITHUB_ACTIONS === 'true' - ? `::error::${escapeAnnotation(message)}` - : `error: ${message}` - ), +const log = { + info: (message: string) => console.log(message), + warn: (message: string) => console.warn(annotate('warning', message)), + error: (message: string) => console.error(annotate('error', message)), }; -// --------------------------------------------------------------------------------------------- -// Configuration -// --------------------------------------------------------------------------------------------- - -function isLimit(value: unknown): value is number { - return typeof value === 'number' && Number.isInteger(value) && value >= 1 && value <= MAX_LIMIT; -} - -function isHttpsUrl(value: string): boolean { - try { - const url = new URL(value); - return url.protocol === 'https:' && !url.username && !url.password; - } catch { - return false; - } -} - -export function parseConfig(value: unknown): IndexNowConfig { - if (!isRecord(value)) throw new Error('The IndexNow config must be a JSON object'); - const { host, key, endpoint, sitemapIndex, budgets } = value; - if (typeof host !== 'string' || !/^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$/.test(host)) { - throw new Error('The IndexNow config needs a lowercase "host" such as aspire.dev'); - } - if (typeof key !== 'string' || !KEY_PATTERN.test(key)) { - throw new Error('The IndexNow "key" must be 8-128 characters of letters, digits, or dashes'); - } - if (typeof endpoint !== 'string' || !isHttpsUrl(endpoint)) { - throw new Error('The IndexNow "endpoint" must be an https URL'); - } - if ( - typeof sitemapIndex !== 'string' || - !isHttpsUrl(sitemapIndex) || - new URL(sitemapIndex).hostname !== host - ) { - throw new Error('The IndexNow "sitemapIndex" must be an https URL on the configured host'); - } - if (!isRecord(budgets) || !isLimit(budgets.release) || !isLimit(budgets.rotation)) { - throw new Error(`The IndexNow "budgets" must be integers between 1 and ${MAX_LIMIT}`); - } - return { - host, - key, - endpoint, - sitemapIndex, - budgets: { release: budgets.release, rotation: budgets.rotation }, - }; -} - export async function loadConfig(configPath = CONFIG_PATH): Promise { - const raw: unknown = JSON.parse(await readFile(configPath, 'utf8')); - return parseConfig(raw); + return JSON.parse(await readFile(configPath, 'utf8')) as IndexNowConfig; } export function keyLocation(config: Pick): string { @@ -219,351 +127,149 @@ export function keyLocation(config: Pick): strin // Sitemap // --------------------------------------------------------------------------------------------- -export interface ParsedSitemap { - kind: 'index' | 'urlset'; - locations: string[]; -} - -const NAMED_ENTITIES: Record = { - amp: '&', - lt: '<', - gt: '>', - quot: '"', - apos: "'", -}; - -function decodeXml(value: string): string { - return value.replace( - /&(?:#(\d+)|#x([0-9a-fA-F]+)|(amp|lt|gt|quot|apos));/g, - (match: string, decimal?: string, hex?: string, name?: string) => { - if (name) return NAMED_ENTITIES[name] ?? match; - const code = decimal ? Number(decimal) : parseInt(hex ?? '', 16); - return Number.isInteger(code) && code > 0 && code <= 0x10ffff - ? String.fromCodePoint(code) - : match; - } - ); -} +const XML_ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }; /** * Reads the `` values of a sitemap or sitemap index. The `xhtml:link` alternates that Astro * adds carry their URL in an attribute and are ignored: every localized page is also listed as a * `` of its own. */ -export function parseSitemap(xml: string): ParsedSitemap { +export function parseSitemap(xml: string): { kind: 'index' | 'urlset'; locations: string[] } { const root = /<(sitemapindex|urlset)[\s>]/.exec(xml); if (!root) throw new Error('Not a sitemap: expected a or element'); - const locations: string[] = []; - for (const match of xml.matchAll(/\s*([^<]*?)\s*<\/loc>/g)) { - const location = decodeXml(match[1]); - if (location) locations.push(location); - } + const locations = [...xml.matchAll(/\s*([^<]*?)\s*<\/loc>/g)].map((match) => + match[1].replace(/&(amp|lt|gt|quot|apos);/g, (_, name: string) => XML_ENTITIES[name]) + ); return { kind: root[1] === 'sitemapindex' ? 'index' : 'urlset', locations }; } -async function fetchXml(fetch: FetchLike, url: string): Promise { - const response = await fetch(url, { - headers: { 'User-Agent': USER_AGENT, Accept: 'application/xml, text/xml' }, - signal: AbortSignal.timeout(SITEMAP_TIMEOUT_MS), - }); - if (!response.ok) throw new Error(`GET ${url} failed: HTTP ${response.status}`); - const text = await response.text(); - if (text.length > MAX_SITEMAP_CHARS) throw new Error(`${url} is larger than expected`); - return text; +/** The https URL on `host` without a fragment, or null for anything else. */ +function canonicalUrl(value: string, host: string): string | null { + try { + const url = new URL(value); + if (url.protocol !== 'https:' || url.hostname !== host) return null; + if (url.port || url.username || url.password) return null; + url.hash = ''; + return url.href; + } catch { + return null; + } } -/** Returns every `` of the sitemap, following a sitemap index one level deep. */ -export async function fetchSitemapLocations( +/** Every page URL of the sitemap (following a sitemap index one level deep), de-duplicated. */ +export async function fetchSitemapUrls( indexUrl: string, - fetch: FetchLike = fetchWithProxy + host: string, + fetch: FetchLike ): Promise { - const first = parseSitemap(await fetchXml(fetch, indexUrl)); - if (first.kind === 'urlset') return first.locations; - if (first.locations.length === 0) throw new Error(`${indexUrl} lists no sitemaps`); - if (first.locations.length > MAX_CHILD_SITEMAPS) { - throw new Error(`${indexUrl} lists more than ${MAX_CHILD_SITEMAPS} sitemaps`); - } + const load = async (url: string) => { + const response = await fetch(url, { + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(120_000), + }); + if (!response.ok) throw new Error(`GET ${url} failed: HTTP ${response.status}`); + return parseSitemap(await response.text()); + }; - const host = new URL(indexUrl).hostname; - const locations: string[] = []; - for (const child of first.locations) { - if (!isHttpsUrl(child) || new URL(child).hostname !== host) { - throw new Error(`Refusing to read sitemap ${child}: it is not an https URL on ${host}`); + const root = await load(indexUrl); + let locations = root.locations; + if (root.kind === 'index') { + locations = []; + for (const child of root.locations) { + if (canonicalUrl(child, host) === null) { + throw new Error(`Refusing to read sitemap ${child}: it is not an https URL on ${host}`); + } + const sitemap = await load(child); + if (sitemap.kind !== 'urlset') throw new Error(`${child} is a nested sitemap index`); + for (const location of sitemap.locations) locations.push(location); } - const parsed = parseSitemap(await fetchXml(fetch, child)); - if (parsed.kind !== 'urlset') throw new Error(`${child} is a nested sitemap index`); - for (const location of parsed.locations) locations.push(location); } - return locations; -} -/** Keeps only https URLs on `host`, drops fragments and removes duplicates. */ -export function normalizeUrls( - locations: readonly string[], - host: string -): { urls: string[]; rejected: string[] } { const urls = new Set(); - const rejected: string[] = []; + let ignored = 0; for (const location of locations) { - let url: URL; - try { - url = new URL(location); - } catch { - rejected.push(location); - continue; - } - if ( - url.protocol !== 'https:' || - url.hostname !== host || - url.port || - url.username || - url.password - ) { - rejected.push(location); - continue; - } - url.hash = ''; - urls.add(url.href); + const url = canonicalUrl(location, host); + if (url) urls.add(url); + else ignored++; } - return { urls: [...urls], rejected }; + if (ignored > 0) log.warn(`Ignored ${ignored} sitemap URLs that are not https URLs on ${host}.`); + return [...urls]; } // --------------------------------------------------------------------------------------------- -// Ordering +// Ordering and planning // --------------------------------------------------------------------------------------------- /** - * Priority of a page. Lower tiers are submitted first: English pages (docs, integrations, What's - * new), then the API reference, then localized pages. + * Priority of a page. Lower tiers are sent first: English pages (docs, integrations, What's new), + * then the API reference, then localized pages. */ -export function tierOf(pathname: string, codes: ReadonlySet = LOCALE_CODES): 0 | 1 | 2 { - const first = pathname.split('/')[1] ?? ''; - if (codes.has(first)) return 2; - if (pathname.startsWith('/reference/api/')) return 1; - return 0; +export function tierOf(pathname: string): 0 | 1 | 2 { + if (LOCALE_CODES.has(pathname.split('/')[1] ?? '')) return 2; + return pathname.startsWith(API_REFERENCE_PATH) ? 1 : 0; } interface Entry { url: string; - /** 0 for pages the release updated, 1 for the rest. */ - rank: number; tier: number; } -const NO_URLS: ReadonlySet = new Set(); - -function toEntry(url: string, codes: ReadonlySet, updated: ReadonlySet): Entry { - return { url, rank: updated.has(url) ? 0 : 1, tier: tierOf(new URL(url).pathname, codes) }; -} +const toEntry = (url: string): Entry => ({ url, tier: tierOf(new URL(url).pathname) }); function compareEntries(a: Entry, b: Entry): number { - return a.rank - b.rank || a.tier - b.tier || compareOrdinal(a.url, b.url); -} - -/** Orders URLs by (rank, tier, URL), the order the rotation cursor walks. */ -export function compareUrls( - a: string, - b: string, - codes: ReadonlySet = LOCALE_CODES, - updated: ReadonlySet = NO_URLS -) { - return compareEntries(toEntry(a, codes, updated), toEntry(b, codes, updated)); -} - -function firstIndexAfter(entries: readonly Entry[], target: Entry): number { - let low = 0; - let high = entries.length; - while (low < high) { - const middle = (low + high) >>> 1; - if (compareEntries(entries[middle], target) > 0) high = middle; - else low = middle + 1; - } - return low; -} - -/** How many of the URLs a cycle with this cursor has already covered. */ -export function countCovered( - urls: readonly string[], - cursor: string | null, - codes: ReadonlySet = LOCALE_CODES, - updated: ReadonlySet = NO_URLS -): number { - if (cursor === null) return 0; - const target = toEntry(cursor, codes, updated); - return urls.filter((url) => compareEntries(toEntry(url, codes, updated), target) <= 0).length; -} - -// --------------------------------------------------------------------------------------------- -// Pages updated by a release -// --------------------------------------------------------------------------------------------- - -/** A file a release changed, as listed by GitHub's "list pull request files" API. */ -export interface ChangedFile { - status: string; - filename: string; - /** Lines added plus lines deleted, when known. */ - changes: number | null; - /** The unified diff of the file. GitHub leaves it out when the diff is very large. */ - patch: string | null; -} - -/** Reads one JSON object per line: `{"status": "...", "filename": "...", "changes": 3, "patch": "..."}`. */ -export function parseChangedFiles(text: string): ChangedFile[] { - const files: ChangedFile[] = []; - text.split('\n').forEach((line, index) => { - if (!line.trim()) return; - let value: unknown; - try { - value = JSON.parse(line); - } catch { - throw new Error(`Line ${index + 1} of the changed files is not JSON`); - } - if ( - !isRecord(value) || - typeof value.status !== 'string' || - typeof value.filename !== 'string' - ) { - throw new Error(`Line ${index + 1} of the changed files needs a "status" and a "filename"`); - } - files.push({ - status: value.status, - filename: value.filename, - changes: typeof value.changes === 'number' ? value.changes : null, - patch: typeof value.patch === 'string' ? value.patch : null, - }); - }); - return files; + return a.tier - b.tier || (a.url < b.url ? -1 : a.url > b.url ? 1 : 0); } -const DOCS_FILE = /^src\/frontend\/src\/content\/docs\/(.+)\.mdx?$/; -const API_DATA_FILE = - /^src\/frontend\/src\/data\/(pkgs|ts-modules)\/(.+?)\.\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?\.json$/; -const API_ROUTES = { pkgs: '/reference/api/csharp/', 'ts-modules': '/reference/api/typescript/' }; -/** Keys that change in every API data file on every release without changing the API. */ -const VOLATILE_API_KEY = /^[+-]\s*"(?:version|sourceCommit|sourceLines)"\s*:/; -const CONTENT_STATUSES = new Set(['added', 'modified', 'renamed', 'copied', 'changed']); - -/** - * Whether an API data file changed in a way that shows on its pages. Every file is renamed to the - * new package version and gets a new `version`, `sourceCommit` and shifted `sourceLines` on each - * release, so only a diff with other changed lines counts. A diff GitHub left out is too large to - * be one of those, except for a pure rename, which has no changes at all. - */ -export function hasApiChanges(file: ChangedFile): boolean { - if (file.status === 'added') return true; - if (file.patch === null) return file.changes !== 0; - return file.patch.split('\n').some((line) => /^[+-]/.test(line) && !VOLATILE_API_KEY.test(line)); +/** Orders URLs by (tier, URL), the order the rotation cursor walks. */ +export function compareUrls(a: string, b: string): number { + return compareEntries(toEntry(a), toEntry(b)); } -/** - * Maps the files a release changed to the sitemap pages they produce, so those pages can be - * submitted first. Docs map to their route (`content/docs/a/b.mdx` is `/a/b/`, an `index` file is - * its folder, and localized docs live in a locale folder); API data files map to every page of - * their package or module. Files that no page comes from, such as components, are ignored. - */ -export function urlsForChangedFiles( - files: readonly ChangedFile[], - urls: readonly string[], - host: string -): string[] { - const origin = `https://${host}`; - const byPath = new Map(); - const byApiRoute = new Map(); - for (const url of urls) { - const pathname = url.slice(origin.length); - byPath.set(pathname.toLowerCase(), url); - for (const prefix of Object.values(API_ROUTES)) { - if (!pathname.startsWith(prefix)) continue; - const route = `${prefix}${pathname.slice(prefix.length).split('/')[0]}/`; - const group = byApiRoute.get(route); - if (group) group.push(url); - else byApiRoute.set(route, [url]); - } - } - - const updated = new Set(); - for (const file of files) { - if (!CONTENT_STATUSES.has(file.status)) continue; - - const docs = DOCS_FILE.exec(file.filename); - if (docs) { - const segments = docs[1].split('/'); - if (segments[segments.length - 1] === 'index') segments.pop(); - const url = byPath.get(segments.length === 0 ? '/' : `/${segments.join('/')}/`); - if (url) updated.add(url); - continue; - } - - const api = API_DATA_FILE.exec(file.filename); - if (api && hasApiChanges(file)) { - const prefix = api[1] === 'pkgs' ? API_ROUTES.pkgs : API_ROUTES['ts-modules']; - for (const url of byApiRoute.get(`${prefix}${api[2].toLowerCase()}/`) ?? []) updated.add(url); - } - } - return [...updated].sort(compareOrdinal); -} - -// --------------------------------------------------------------------------------------------- -// Planning -// --------------------------------------------------------------------------------------------- - export interface PlanInput { /** Normalized, de-duplicated sitemap URLs. */ current: readonly string[]; state: IndexNowState | null; mode: RunMode; - /** Recorded on the cycle when one is started. */ - reason: string; budgets: IndexNowConfig['budgets']; - /** Replaces the budget for this run. It also caps the new and removed URLs sent first. */ + /** Replaces the budget for this run. It is a hard cap: it bounds the removed and new URLs too. */ limit?: number; - /** Pages the release updated. They rank first when this run starts a new cycle. */ - updatedUrls?: readonly string[]; - now: Date; allowShrink?: boolean; - localeCodes?: ReadonlySet; } export interface Plan { coldStart: boolean; mode: RunMode; budget: number; - /** The cycle this run works on (continued or newly started). */ + /** The pass this run works on (continued or newly started). */ cycle: Cycle; total: number; /** Every URL that is in the sitemap but was not known. */ newUrls: string[]; /** Every known URL that left the sitemap. */ removedUrls: string[]; - /** The removed URLs submitted this run, in submission order. */ + /** The removed URLs sent this run. */ removed: string[]; - /** The new URLs submitted ahead of the window this run. A flush leaves its new URLs to the cycle. */ + /** The new URLs sent ahead of the window. A flush leaves them to its window. */ fresh: string[]; - /** The rotation window submitted this run. */ + /** The next URLs of the rotation. */ window: string[]; - /** The last URL the window scan went past, or null when it did not move. */ - windowEnd: string | null; - /** True when a fully accepted window finishes the cycle. */ + /** True when sending the window finishes the pass. */ windowComplete: boolean; - /** How many sitemap URLs the cycle has covered after this run's scan. */ + /** How many sitemap URLs the pass has covered once the window is sent. */ covered: number; - localeCodes: ReadonlySet; } /** - * Decides what to submit. Pure: it does not touch the network or the disk. + * Decides what to send. Pure: it does not touch the network or the disk. * * - A cold start (no state) seeds the known list and never treats the whole sitemap as new. - * - A flush starts a release cycle from the top; otherwise the active cycle continues, or a new - * rotation cycle begins when the previous one has completed. - * - Removed and new URLs go first. A flush is the exception for new URLs: it covers them in the - * release cycle it starts, ordered with the pages the release updated. New URLs ahead of the - * cursor are remembered in `cycle.skip` (see finalizeRun) so the window does not send them a - * second time. + * - A flush starts a release pass from the top; otherwise the pass in progress continues, or a new + * rotation pass begins when the previous one has finished. + * - Removed and new URLs go first and count against the budget. A flush is the exception for new + * URLs: its pass covers the whole sitemap, so it leaves them to the window. */ export function planRun(input: PlanInput): Plan { - const localeCodes = input.localeCodes ?? LOCALE_CODES; - const { state } = input; + const { state, mode } = input; const currentSet = new Set(input.current); const total = currentSet.size; if (total === 0) throw new Error('The sitemap lists no URLs; refusing to continue'); @@ -577,67 +283,38 @@ export function planRun(input: PlanInput): Plan { ); } - // A flush covers the pages that are new when it starts together with the ones its files updated, - // so a release's new and updated pages are ordered by tier: docs before the generated API - // reference, however many API pages a new module adds. Without that, thousands of new API pages - // sent first would use up the budget before any updated docs page. The set is fixed when the - // cycle starts and ranks first until the cycle completes. - const startsRelease = input.mode === 'flush'; - const startedAt = input.now.toISOString(); + const entries = [...currentSet].map(toEntry).sort(compareEntries); const knownSet = new Set(previous); - const unknown = state ? [...currentSet].filter((url) => !knownSet.has(url)) : []; - const updated = [...new Set([...(input.updatedUrls ?? []), ...(startsRelease ? unknown : [])])] - .filter((url) => currentSet.has(url)) - .sort(compareOrdinal); - let cycle: Cycle; - if (startsRelease) { - cycle = { kind: 'release', reason: input.reason, startedAt, cursor: null, skip: [], updated }; - } else if (state?.cycle) { - cycle = state.cycle; - } else { - cycle = { kind: 'rotation', reason: input.reason, startedAt, cursor: null, skip: [], updated }; - } - - const updatedSet = new Set(cycle.updated); - const entries = [...currentSet] - .map((url) => toEntry(url, localeCodes, updatedSet)) - .sort(compareEntries); const newUrls = state ? entries.filter((e) => !knownSet.has(e.url)).map((e) => e.url) : []; const removedUrls = previous .filter((url) => !currentSet.has(url)) - .map((url) => toEntry(url, localeCodes, updatedSet)) + .map(toEntry) .sort(compareEntries) .map((entry) => entry.url); - const budget = - input.limit ?? (cycle.kind === 'release' ? input.budgets.release : input.budgets.rotation); - // --limit is a hard cap, so it bounds the new and removed URLs that go first as well. A flush - // leaves its new URLs to the cycle it starts instead of sending them ahead of it. - const cap = Math.min(MAX_PRIORITY_URLS, input.limit ?? Infinity); + const cycle: Cycle = + mode === 'flush' + ? { kind: 'release', cursor: null } + : (state?.cycle ?? { kind: 'rotation', cursor: null }); + const configured = cycle.kind === 'release' ? input.budgets.release : input.budgets.rotation; + const budget = Math.min(input.limit ?? configured, MAX_URLS_PER_RUN); + + const cap = Math.min(input.limit ?? MAX_URLS_PER_RUN, MAX_URLS_PER_RUN); const removed = removedUrls.slice(0, cap); - const fresh = (startsRelease ? [] : newUrls).slice(0, Math.max(0, cap - removed.length)); - const priority = new Set([...removed, ...fresh]); - const skip = new Set(cycle.skip); + const fresh = (mode === 'flush' ? [] : newUrls).slice(0, cap - removed.length); const room = Math.max(0, budget - removed.length - fresh.length); - const alreadySent = (entry: Entry) => priority.has(entry.url) || skip.has(entry.url); - let index = - cycle.cursor === null - ? 0 - : firstIndexAfter(entries, toEntry(cycle.cursor, localeCodes, updatedSet)); - let windowEnd: string | null = null; - const window: string[] = []; - while (index < total && window.length < room) { - const entry = entries[index++]; - windowEnd = entry.url; - if (!alreadySent(entry)) window.push(entry.url); + let start = 0; + if (cycle.cursor !== null) { + const target = toEntry(cycle.cursor); + start = entries.findIndex((entry) => compareEntries(entry, target) > 0); + if (start === -1) start = entries.length; } - // Trailing URLs that were already sent this cycle do not need another run to be passed over. - while (index < total && alreadySent(entries[index])) windowEnd = entries[index++].url; + const window = entries.slice(start, start + room).map((entry) => entry.url); return { coldStart: state === null, - mode: input.mode, + mode, budget, cycle, total, @@ -646,10 +323,8 @@ export function planRun(input: PlanInput): Plan { removed, fresh, window, - windowEnd, - windowComplete: index >= total, - covered: index, - localeCodes, + windowComplete: start + window.length >= total, + covered: start + window.length, }; } @@ -657,69 +332,32 @@ export interface FinalizeOptions { host: string; /** The URLs the plan was computed from. */ current: readonly string[]; - /** How many URLs of `[...removed, ...fresh, ...window]` were accepted, as a prefix. */ - accepted: number; + /** Whether IndexNow accepted the request. */ + submitted: boolean; now: Date; - trigger: string; - deployment: string; - statuses: readonly number[]; } /** - * Builds the state to remember after a run. Only accepted URLs count as done: a new URL that was - * not accepted stays new, a removed URL that was not accepted stays known, and the cursor stops at - * the last accepted window URL. The new URLs of a flush count as known right away, because the - * release cycle it starts submits them when its cursor reaches them. + * Builds the state to remember after a run. URLs count as done only when the request was + * accepted: a new URL that was not sent stays new, a removed URL that was not sent stays known, and + * the cursor moves to the last URL of the window. The new URLs of a flush count as known right + * away, because the pass it starts reaches them. */ export function finalizeRun(plan: Plan, options: FinalizeOptions): IndexNowState { - const ordered = [...plan.removed, ...plan.fresh, ...plan.window]; - const accepted = Math.min(Math.max(options.accepted, 0), ordered.length); - const acceptedUrls = new Set(ordered.slice(0, accepted)); - const acceptedWindow = Math.max(0, accepted - plan.removed.length - plan.fresh.length); - - const known = new Set(options.current); + const { host, current, submitted, now } = options; + const sent = new Set(submitted ? [...plan.removed, ...plan.fresh, ...plan.window] : []); + const known = new Set(current); if (plan.mode !== 'flush') { - for (const url of plan.newUrls) if (!acceptedUrls.has(url)) known.delete(url); - } - for (const url of plan.removedUrls) if (!acceptedUrls.has(url)) known.add(url); - - let cycle: Cycle | null = null; - const windowDone = acceptedWindow === plan.window.length; - if (!(windowDone && plan.windowComplete)) { - const cursor = windowDone - ? (plan.windowEnd ?? plan.cycle.cursor) - : (plan.window[acceptedWindow - 1] ?? plan.cycle.cursor); - const skip = new Set(plan.cycle.skip); - for (const url of plan.fresh) if (acceptedUrls.has(url)) skip.add(url); - const updatedSet = new Set(plan.cycle.updated); - cycle = { - ...plan.cycle, - cursor, - skip: [...skip] - .filter( - (url) => cursor === null || compareUrls(url, cursor, plan.localeCodes, updatedSet) > 0 - ) - .sort(compareOrdinal), - }; + for (const url of plan.newUrls) if (!sent.has(url)) known.delete(url); } + for (const url of plan.removedUrls) if (!sent.has(url)) known.add(url); + const cursor = submitted ? (plan.window.at(-1) ?? plan.cycle.cursor) : plan.cycle.cursor; return { - host: options.host, - updatedAt: options.now.toISOString(), - deployment: options.deployment, - cycle, - lastRun: { - at: options.now.toISOString(), - trigger: options.trigger, - mode: plan.mode, - coldStart: plan.coldStart, - newUrls: plan.newUrls.length, - removedUrls: plan.removedUrls.length, - windowUrls: acceptedWindow, - submitted: accepted, - statuses: [...options.statuses], - }, - known: [...known].sort(compareOrdinal), + host, + updatedAt: now.toISOString(), + cycle: submitted && plan.windowComplete ? null : { ...plan.cycle, cursor }, + known: [...known].sort(), }; } @@ -727,60 +365,17 @@ export function finalizeRun(plan: Plan, options: FinalizeOptions): IndexNowState // State on disk // --------------------------------------------------------------------------------------------- -function parseCycle(value: unknown): Cycle | null { - if (value === null) return null; - if (!isRecord(value)) throw new Error('"cycle" must be an object or null'); - const { kind, reason, startedAt, cursor, skip, updated = [] } = value; - const isUrlList = (list: unknown): list is string[] => - Array.isArray(list) && list.every((url) => typeof url === 'string'); - if ( - (kind !== 'release' && kind !== 'rotation') || - typeof reason !== 'string' || - typeof startedAt !== 'string' || - !(cursor === null || typeof cursor === 'string') || - !isUrlList(skip) || - !isUrlList(updated) - ) { - throw new Error('"cycle" has an unexpected shape'); - } - return { kind, reason, startedAt, cursor, skip, updated }; -} - -function parseRunRecord(value: unknown): RunRecord | null { - if (!isRecord(value)) return null; - const count = (entry: unknown) => (typeof entry === 'number' ? entry : 0); - return { - at: typeof value.at === 'string' ? value.at : '', - trigger: typeof value.trigger === 'string' ? value.trigger : '', - mode: value.mode === 'flush' ? 'flush' : 'daily', - coldStart: value.coldStart === true, - newUrls: count(value.newUrls), - removedUrls: count(value.removedUrls), - windowUrls: count(value.windowUrls), - submitted: count(value.submitted), - statuses: Array.isArray(value.statuses) - ? value.statuses.filter((status): status is number => typeof status === 'number') - : [], - }; -} - -/** Rejects entries that are not already the canonical https URLs on `host` that normalizeUrls makes. */ -function assertCanonical(urls: readonly string[], host: string, what: string): void { - const { urls: canonical, rejected } = normalizeUrls(urls, host); - if ( - rejected.length > 0 || - canonical.length !== urls.length || - canonical.some((url, index) => url !== urls[index]) - ) { - throw new Error(`${what} has entries that are not https://${host}/ pages`); - } +function isCycle(value: unknown): value is Cycle | null { + return ( + value === null || + (isRecord(value) && + (value.kind === 'release' || value.kind === 'rotation') && + (value.cursor === null || typeof value.cursor === 'string')) + ); } -export async function loadState( - dir: string, - host: string, - log: Logger = consoleLogger -): Promise { +/** Returns null for a missing state (a cold start) and for one that cannot be trusted. */ +export async function loadState(dir: string, host: string): Promise { let raw: string; try { raw = await readFile(path.join(dir, STATE_FILE), 'utf8'); @@ -791,38 +386,22 @@ export async function loadState( try { const file: unknown = JSON.parse(raw); - if (!isRecord(file) || file.schemaVersion !== SCHEMA_VERSION) { - throw new Error('the state file has a different schema version'); - } - if (file.host !== host) throw new Error(`the state belongs to ${String(file.host)}`); - if (typeof file.updatedAt !== 'string' || typeof file.deployment !== 'string') { - throw new Error('the state file is missing fields'); - } - const known = gunzipSync(await readFile(path.join(dir, KNOWN_FILE))) - .toString('utf8') - .split('\n') - .filter(Boolean); - if (known.length !== file.knownCount) { - throw new Error(`expected ${String(file.knownCount)} known URLs but found ${known.length}`); + if (!isRecord(file) || file.schemaVersion !== SCHEMA_VERSION || file.host !== host) { + throw new Error('it was written for another schema version or host'); } - // The state comes from an artifact, so it is checked like any other input. A list that is - // empty, or holds other hosts, would otherwise make a run submit URLs it should not. - if (known.length === 0) throw new Error('the known URL list is empty'); - assertCanonical(known, host, 'the known URL list'); - const cycle = parseCycle(file.cycle); - if (cycle) { - assertCanonical(cycle.skip, host, 'the cycle skip list'); - assertCanonical(cycle.updated, host, 'the cycle updated list'); - if (cycle.cursor !== null) assertCanonical([cycle.cursor], host, 'the cycle cursor'); + const { updatedAt, cycle, known } = file; + // The state comes from an artifact, so anything that is not a page of this site is not trusted. + if ( + typeof updatedAt !== 'string' || + !isCycle(cycle) || + !Array.isArray(known) || + known.length === 0 || + !known.every((url) => typeof url === 'string' && canonicalUrl(url, host) === url) || + (typeof cycle?.cursor === 'string' && canonicalUrl(cycle.cursor, host) !== cycle.cursor) + ) { + throw new Error('it is not in the expected format'); } - return { - host, - updatedAt: file.updatedAt, - deployment: file.deployment, - cycle, - lastRun: parseRunRecord(file.lastRun), - known, - }; + return { host, updatedAt, cycle, known: known as string[] }; } catch (error) { log.warn(`Ignoring unusable IndexNow state (${errorMessage(error)}); starting cold.`); return null; @@ -831,105 +410,25 @@ export async function loadState( export async function saveState(dir: string, state: IndexNowState): Promise { await mkdir(dir, { recursive: true }); - const stateFile = path.join(dir, STATE_FILE); - const knownFile = path.join(dir, KNOWN_FILE); - const stagedState = `${stateFile}.${process.pid}.next`; - const stagedKnown = `${knownFile}.${process.pid}.next`; - const { known, ...rest } = state; - try { - await writeFile(stagedKnown, gzipSync(`${known.join('\n')}\n`)); - await writeFile( - stagedState, - `${JSON.stringify({ schemaVersion: SCHEMA_VERSION, ...rest, knownCount: known.length }, null, 2)}\n`, - 'utf8' - ); - await rename(stagedKnown, knownFile); - await rename(stagedState, stateFile); - } finally { - await rm(stagedKnown, { force: true }); - await rm(stagedState, { force: true }); - } + const content = JSON.stringify({ schemaVersion: SCHEMA_VERSION, ...state }, null, 2); + await writeFile(path.join(dir, STATE_FILE), `${content}\n`, 'utf8'); } // --------------------------------------------------------------------------------------------- -// Submission +// Talking to IndexNow and the site // --------------------------------------------------------------------------------------------- -export interface ChunkResult { - size: number; - status: number; - attempts: number; -} - -export interface SubmitStop { - reason: 'rate-limited' | 'failed'; - status: number; - detail: string; -} - -export interface SubmitResult { - /** URLs accepted by IndexNow: always a prefix of the list that was submitted. */ - accepted: number; - chunks: ChunkResult[]; - stop: SubmitStop | null; -} - -export interface SubmitOptions { - config: IndexNowConfig; - urls: readonly string[]; - fetch?: FetchLike; - sleep?: Sleep; - now?: () => number; - log?: Logger; -} - -/** Reads a Retry-After header, either as seconds or as an HTTP date, into milliseconds. */ -export function parseRetryAfter(value: string | null, nowMs: number): number | undefined { - if (value === null) return undefined; - const trimmed = value.trim(); - if (/^\d+$/.test(trimmed)) return Number(trimmed) * 1000; - const date = Date.parse(trimmed); - return Number.isNaN(date) ? undefined : Math.max(0, date - nowMs); -} - -interface PostAttempt { - /** HTTP status, or 0 when the request itself failed. */ - status: number; - detail: string; - retryAfter?: number; -} - -async function postOnce( - config: IndexNowConfig, - body: string, - fetch: FetchLike, - now: () => number -): Promise { - try { - const response = await fetch(config.endpoint, { - method: 'POST', - headers: { 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': USER_AGENT }, - body, - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - }); - return { - status: response.status, - retryAfter: parseRetryAfter(response.headers.get('retry-after'), now()), - detail: (await response.text()).trim().slice(0, 300), - }; - } catch (error) { - return { status: 0, detail: errorMessage(error) }; - } -} +export type Submission = { ok: true } | { ok: false; status: number; detail: string }; -async function postChunk( +/** + * Sends the URLs in a single request. A 5xx or a network error is retried after a back-off; any + * other answer, a 429 included, ends the attempt, and the next run tries again. + */ +export async function submitUrls( config: IndexNowConfig, urls: readonly string[], - fetch: FetchLike, - sleep: Sleep, - now: () => number, - log: Logger -): Promise { + deps: Pick +): Promise { const body = JSON.stringify({ host: config.host, key: config.key, @@ -937,89 +436,53 @@ async function postChunk( urlList: urls, }); - for (let attempt = 1; ; attempt++) { - const { status, detail, retryAfter } = await postOnce(config, body, fetch, now); - const result = { size: urls.length, status, attempts: attempt, detail }; - if (status === 200 || status === 202) return result; - const retryable = status === 0 || status === 429 || status >= 500; - const delay = status === 429 ? retryAfter : RETRY_DELAYS_MS[attempt - 1]; - if ( - !retryable || - attempt > RETRY_DELAYS_MS.length || - delay === undefined || - delay > MAX_RETRY_AFTER_MS - ) { - return result; + let status = 0; + let detail = ''; + for (let attempt = 0; attempt <= RETRY_DELAYS_MS.length; attempt++) { + if (attempt > 0) { + const delay = RETRY_DELAYS_MS[attempt - 1]; + log.warn( + `IndexNow answered ${status || 'with a network error'}; retrying in ${delay / 1000}s.` + ); + await deps.sleep(delay); } - log.warn( - `IndexNow answered ${status || 'with a network error'}; retrying in ${Math.ceil(delay / 1000)}s.` - ); - await sleep(delay); - } -} - -/** - * POSTs the URLs in chunks of at most 10,000. Chunks are all-or-nothing, so `accepted` is a - * prefix of `urls`. A 429 (after honouring a short Retry-After) or any other failure stops the - * run; the caller keeps the progress made so far. - */ -export async function submitUrls(options: SubmitOptions): Promise { - const { - config, - urls, - fetch = fetchWithProxy, - sleep = defaultSleep, - now = Date.now, - log = consoleLogger, - } = options; - const chunks: ChunkResult[] = []; - let accepted = 0; - - for (let offset = 0; offset < urls.length; offset += MAX_URLS_PER_REQUEST) { - if (offset > 0) await sleep(CHUNK_PAUSE_MS); - const chunk = urls.slice(offset, offset + MAX_URLS_PER_REQUEST); - const { detail, ...result } = await postChunk(config, chunk, fetch, sleep, now, log); - chunks.push(result); - if (result.status === 200 || result.status === 202) { - accepted += chunk.length; - log.info(`IndexNow accepted ${chunk.length} URLs (HTTP ${result.status}).`); - continue; + try { + const response = await deps.fetch(config.endpoint, { + method: 'POST', + headers: { 'Content-Type': 'application/json; charset=utf-8', 'User-Agent': USER_AGENT }, + body, + signal: AbortSignal.timeout(60_000), + }); + status = response.status; + detail = (await response.text()).trim().slice(0, 300); + } catch (error) { + status = 0; + detail = errorMessage(error); } - return { - accepted, - chunks, - stop: { - reason: result.status === 429 ? 'rate-limited' : 'failed', - status: result.status, - detail, - }, - }; + // 202 is what IndexNow answers while it is still verifying a new key. + if (status === 200 || status === 202) return { ok: true }; + if (status !== 0 && status < 500) break; } - return { accepted, chunks, stop: null }; + return { ok: false, status, detail }; } -// --------------------------------------------------------------------------------------------- -// Live-site checks -// --------------------------------------------------------------------------------------------- - -/** IndexNow verifies ownership by fetching this file, so it has to be live before submitting. */ -export async function verifyKeyFile( +/** IndexNow verifies ownership by fetching the key file, so it has to be live before submitting. */ +export async function keyFileProblem( config: IndexNowConfig, - fetch: FetchLike = fetchWithProxy -): Promise<{ ok: true } | { ok: false; detail: string }> { + fetch: FetchLike +): Promise { const url = keyLocation(config); try { const response = await fetch(url, { headers: { 'User-Agent': USER_AGENT }, signal: AbortSignal.timeout(30_000), }); - if (!response.ok) return { ok: false, detail: `GET ${url} returned HTTP ${response.status}` }; - const body = (await response.text()).trim(); - return body === config.key - ? { ok: true } - : { ok: false, detail: `${url} does not contain the configured key` }; + if (!response.ok) return `GET ${url} returned HTTP ${response.status}`; + return (await response.text()).trim() === config.key + ? null + : `${url} does not contain the configured key`; } catch (error) { - return { ok: false, detail: `GET ${url} failed: ${errorMessage(error)}` }; + return `GET ${url} failed: ${errorMessage(error)}`; } } @@ -1027,615 +490,305 @@ export async function verifyKeyFile( export function whatsNewPath(version: string): string { const match = /^(\d+)\.(\d+)$/.exec(version); if (!match) throw new Error(`"${version}" is not a release version such as 13.6`); - const major = Number(match[1]); - const minor = Number(match[2]); + const [major, minor] = [Number(match[1]), Number(match[2])]; return minor === 0 ? `/whats-new/aspire-${major}/` : `/whats-new/aspire-${major}-${minor}/`; } -export async function isReleaseLive( - host: string, - version: string, - fetch: FetchLike = fetchWithProxy -): Promise { - try { - const response = await fetch(`https://${host}${whatsNewPath(version)}`, { - method: 'HEAD', - headers: { 'User-Agent': USER_AGENT }, - signal: AbortSignal.timeout(30_000), - }); - return response.ok; - } catch { - return false; - } -} - -export interface WaitForReleaseOptions { - host: string; - version: string; - fetch?: FetchLike; - sleep?: Sleep; - now?: () => number; - timeoutMs?: number; - intervalMs?: number; - log?: Logger; -} - /** * Waits for the release's What's new page to be served. The release merge normally lands right - * after the slot swap; this protects against submitting URLs while the old build is still live, - * which would make engines crawl stale content. + * after the slot swap; this protects against sending URLs while the old build is still live, which + * would make engines crawl stale content. */ -export async function waitForRelease(options: WaitForReleaseOptions): Promise { - const { - host, - version, - fetch = fetchWithProxy, - sleep = defaultSleep, - now = Date.now, - timeoutMs = GATE_TIMEOUT_MS, - intervalMs = GATE_INTERVAL_MS, - log = consoleLogger, - } = options; - const deadline = now() + timeoutMs; +export async function waitForRelease( + host: string, + version: string, + deps: Deps, + { timeoutMs = GATE_TIMEOUT_MS, intervalMs = GATE_INTERVAL_MS } = {} +): Promise { + const url = `https://${host}${whatsNewPath(version)}`; + const deadline = deps.now().getTime() + timeoutMs; for (;;) { - if (await isReleaseLive(host, version, fetch)) return true; - if (now() + intervalMs > deadline) return false; + try { + const response = await deps.fetch(url, { + method: 'HEAD', + headers: { 'User-Agent': USER_AGENT }, + signal: AbortSignal.timeout(30_000), + }); + if (response.ok) return true; + } catch { + // Not reachable yet counts as not live. + } + if (deps.now().getTime() + intervalMs > deadline) return false; log.info(`Release ${version} is not live yet; checking again in ${intervalMs / 1000}s.`); - await sleep(intervalMs); + await deps.sleep(intervalMs); } } -/** Reads the deployment identity the site embeds in every page. Diagnostic only. */ -export async function readDeployment( - host: string, - fetch: FetchLike = fetchWithProxy -): Promise { - try { - const response = await fetch(`https://${host}/`, { - headers: { 'User-Agent': USER_AGENT }, - signal: AbortSignal.timeout(30_000), - }); - if (!response.ok) return ''; - const match = / - isRecord(side) && isRecord(side.repo) && typeof side.repo.full_name === 'string' - ? side.repo.full_name - : ''; - const headRepo = repo(pr.head); - if (!headRepo || headRepo !== repo(pr.base)) return null; - - const headRef = isRecord(pr.head) && typeof pr.head.ref === 'string' ? pr.head.ref : ''; - const title = typeof pr.title === 'string' ? pr.title : ''; - const match = - /^release\/(\d+\.\d+)$/.exec(headRef) ?? /^Merge release\/(\d+\.\d+) into main\b/i.exec(title); - return match ? { version: match[1], reason: `release/${match[1]}` } : null; -} - -export interface CliOptions { - stateDir: string; - mode: RunMode; - limit?: number; - release?: string; - /** A file with one JSON object per line for each file the release changed. */ - changedFiles?: string; - trigger: string; - dryRun: boolean; - allowShrink: boolean; -} - -export function parseCli(argv: readonly string[]): CliOptions { - const { values } = parseArgs({ - args: [...argv], - options: { - 'state-dir': { type: 'string' }, - mode: { type: 'string' }, - limit: { type: 'string' }, - release: { type: 'string' }, - 'changed-files': { type: 'string' }, - trigger: { type: 'string' }, - 'dry-run': { type: 'boolean' }, - 'allow-shrink': { type: 'boolean' }, - }, - strict: true, - allowPositionals: false, - }); - - const stateDir = values['state-dir']; - if (!stateDir) throw new Error('--state-dir is required'); - const mode = values.mode ?? 'daily'; - if (mode !== 'daily' && mode !== 'flush') throw new Error('--mode must be "daily" or "flush"'); - - let limit: number | undefined; - if (values.limit !== undefined) { - limit = Number(values.limit); - if (!isLimit(limit)) throw new Error(`--limit must be an integer between 1 and ${MAX_LIMIT}`); - } - if (values.release !== undefined && !/^\d+\.\d+$/.test(values.release)) { - throw new Error('--release must look like 13.6'); - } - - return { - stateDir: path.resolve(stateDir), - mode, - limit, - release: values.release, - changedFiles: values['changed-files'] ? path.resolve(values['changed-files']) : undefined, - trigger: values.trigger || 'manual', - dryRun: values['dry-run'] ?? false, - allowShrink: values['allow-shrink'] ?? false, - }; -} - -export interface ResolvedRun { - /** Set when the trigger does not call for a run; nothing is fetched or submitted. */ - skip?: string; - mode: RunMode; - release?: string; - reason: string; -} - -/** Turns the CLI options and the (pull_request) event payload into what this run should do. */ -export function resolveRun(options: CliOptions, event: unknown): ResolvedRun { - if (options.trigger === 'pull_request') { - const release = detectReleaseMerge(event); - if (!release) { - return { - skip: 'This pull request is not a merged release/X.Y pull request; nothing to submit.', - mode: 'flush', - reason: '', - }; - } - return { mode: 'flush', release: release.version, reason: release.reason }; - } - - let reason = 'manual'; - if (options.release) reason = `release/${options.release}`; - else if (options.mode === 'flush') reason = 'manual flush'; - else if (options.trigger === 'schedule') reason = 'schedule'; - return { mode: options.mode, release: options.release, reason }; +export function releaseVersion(event: unknown): string | null { + const pull = isRecord(event) ? event.pull_request : undefined; + if (!isRecord(pull) || pull.merged !== true) return null; + const repository = (side: unknown) => + isRecord(side) && isRecord(side.repo) ? side.repo.full_name : undefined; + const ref = isRecord(pull.head) ? pull.head.ref : undefined; + if (typeof ref !== 'string' || typeof repository(pull.head) !== 'string') return null; + if (repository(pull.head) !== repository(pull.base)) return null; + return /^release\/(\d+\.\d+)$/.exec(ref)?.[1] ?? null; } // --------------------------------------------------------------------------------------------- // Run // --------------------------------------------------------------------------------------------- -export interface RunReport { - outcome: 'completed' | 'dry-run' | 'skipped' | 'rate-limited' | 'failed'; - trigger: string; +export interface RunOptions extends Partial { + config: IndexNowConfig; + stateDir: string; mode: RunMode; - reason: string; - host: string; - note: string; - deployment: string; - coldStart: boolean; - budget: number; - totalUrls: number; - newUrls: number; - removedUrls: number; - /** Pages the release added or updated, which this cycle covers before everything else. */ - updatedUrls: number; - /** URLs planned (dry run) or accepted (real run) from the rotation window. */ - windowUrls: number; - submitted: number; - /** New and removed URLs that did not fit in this run and stay pending. */ - deferred: number; - cycleKind: CycleKind | null; - cycleCovered: number; - cycleCompleted: boolean; - chunks: ChunkResult[]; - firstWindowUrl: string; - lastWindowUrl: string; + trigger: string; + /** Wait for this release's What's new page before submitting. */ + release?: string; + limit?: number; + dryRun?: boolean; + allowShrink?: boolean; } export interface RunResult { - report: RunReport; + outcome: 'completed' | 'dry-run' | 'skipped' | 'failed'; + /** URLs accepted by IndexNow, or planned in a dry run. */ + submitted: number; stateChanged: boolean; + /** Markdown for the job summary page. */ + summary: string; /** Set when the run failed after it had started submitting. */ failure?: string; } -export interface RunOptions { - config: IndexNowConfig; - stateDir: string; - mode: RunMode; - trigger: string; - reason: string; - dryRun?: boolean; - allowShrink?: boolean; - limit?: number; - /** Wait for this release's What's new page before submitting. */ - release?: string; - /** The files the release changed; their pages are submitted first when a cycle starts. */ - changedFiles?: readonly ChangedFile[]; - fetch?: FetchLike; - sleep?: Sleep; - now?: () => Date; - log?: Logger; - gate?: { timeoutMs?: number; intervalMs?: number }; +const count = (value: number) => value.toLocaleString('en-US'); + +function summarize(host: string, outcome: RunResult['outcome'], lines: (string | false)[]): string { + const bullets = lines.filter(Boolean).map((line) => `- ${line}`); + return [`## IndexNow submission for ${host}: ${outcome}`, '', ...bullets, ''].join('\n'); } -export function emptyReport( - overrides: Partial & Pick -): RunReport { - return { - mode: 'daily', - reason: '', - note: '', - deployment: '', - coldStart: false, - budget: 0, - totalUrls: 0, - newUrls: 0, - removedUrls: 0, - updatedUrls: 0, - windowUrls: 0, - submitted: 0, - deferred: 0, - cycleKind: null, - cycleCovered: 0, - cycleCompleted: false, - chunks: [], - firstWindowUrl: '', - lastWindowUrl: '', - ...overrides, - }; +function skipped(host: string, note: string): RunResult { + const summary = summarize(host, 'skipped', [note]); + return { outcome: 'skipped', submitted: 0, stateChanged: false, summary }; +} + +function failureMessage(submission: { status: number; detail: string }, mode: RunMode): string { + const reason = + submission.status === 429 + ? 'IndexNow answered 429 (too many requests), so nothing was submitted. If this keeps happening, lower the budgets in config/indexnow.json.' + : `IndexNow rejected the submission with HTTP ${submission.status || 'a network error'}` + + `${submission.detail ? `: ${submission.detail}` : ''}. ` + + 'For 403 or 422, check that the key file is live and matches config/indexnow.json.'; + return mode === 'flush' + ? `${reason} The release pass was started, so the next run sends it from the top.` + : `${reason} The next run tries again.`; } export async function run(options: RunOptions): Promise { - const { config, stateDir, trigger, reason, release, limit } = options; - const dryRun = options.dryRun ?? false; - const fetch = options.fetch ?? fetchWithProxy; - const sleep = options.sleep ?? defaultSleep; - const now = options.now ?? (() => new Date()); - const log = options.log ?? consoleLogger; - - const state = await loadState(stateDir, config.host, log); + const { config, stateDir, mode, trigger, release, dryRun = false } = options; + const { host } = config; + const deps: Deps = { + fetch: options.fetch ?? fetchWithProxy, + sleep: options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))), + now: options.now ?? (() => new Date()), + }; + + const state = await loadState(stateDir, host); log.info( state ? `Loaded state from ${state.updatedAt}: ${state.known.length} known URLs.` : 'No previous state; this is a cold start.' ); - // A release can reach the workflow twice: its content pull request and, hours later, the pull - // request that merges the release branch back, which changes no files. Starting over would send - // the whole burst again, so a pull request for a release whose cycle is still running continues - // that cycle. A manual flush always starts over. - const resumes = - options.mode === 'flush' && - trigger === 'pull_request' && - state?.cycle?.kind === 'release' && - state.cycle.reason === reason; - const mode: RunMode = resumes ? 'daily' : options.mode; - const resumeNote = resumes - ? `The release cycle for ${reason} was already running, so this run continued it instead of starting over.` - : ''; - if (resumeNote) log.info(resumeNote); - const base = { trigger, mode, reason, host: config.host }; - const key = await verifyKeyFile(config, fetch); - if (!key.ok) { + const keyProblem = await keyFileProblem(config, deps.fetch); + if (keyProblem) { if (dryRun) { - log.warn(`${key.detail}. IndexNow would reject submissions until the key file is live.`); + log.warn(`${keyProblem}. IndexNow would reject submissions until the key file is live.`); } else if (state === null) { // Before the first deployment that ships the key file there is nothing to submit yet. - const note = `The IndexNow key file is not live yet (${key.detail}); skipping this run.`; + const note = `The IndexNow key file is not live yet (${keyProblem}); skipping this run.`; log.warn(note); - return { report: emptyReport({ ...base, outcome: 'skipped', note }), stateChanged: false }; + return skipped(host, note); } else { - throw new Error(`The IndexNow key file is not reachable: ${key.detail}`); + throw new Error(`The IndexNow key file is not reachable: ${keyProblem}`); } } if (release) { - if (dryRun) { - if (!(await isReleaseLive(config.host, release, fetch))) { - log.warn(`The What's new page for release ${release} is not live yet.`); - } - } else { - const ready = await waitForRelease({ - host: config.host, - version: release, - fetch, - sleep, - now: () => now().getTime(), - log, - ...options.gate, - }); - if (!ready) { + // A dry run only checks once. + const gate = dryRun ? { timeoutMs: 0 } : undefined; + if (!(await waitForRelease(host, release, deps, gate))) { + if (!dryRun) { throw new Error( `The What's new page for release ${release} (${whatsNewPath(release)}) did not go live in ` + `time. Once the new docs are live, run the workflow manually with mode "flush" and release "${release}".` ); } + log.warn(`The What's new page for release ${release} is not live yet.`); } } - const { urls, rejected } = normalizeUrls( - await fetchSitemapLocations(config.sitemapIndex, fetch), - config.host - ); - if (rejected.length > 0) { - log.warn( - `Ignored ${rejected.length} sitemap URLs that are not https URLs on ${config.host} (for example ${rejected[0]}).` - ); - } - const deployment = await readDeployment(config.host, fetch); - const updatedUrls = options.changedFiles - ? urlsForChangedFiles(options.changedFiles, urls, config.host) - : []; - if (options.changedFiles) { - log.info( - `${options.changedFiles.length} files changed by the release map to ${updatedUrls.length} updated pages.` - ); - } - + const urls = await fetchSitemapUrls(config.sitemapIndex, host, deps.fetch); const plan = planRun({ current: urls, state, mode, - reason, budgets: config.budgets, - limit, - updatedUrls, - now: now(), + limit: options.limit, allowShrink: options.allowShrink, }); const ordered = [...plan.removed, ...plan.fresh, ...plan.window]; - const updatedSet = new Set(plan.cycle.updated); - // A flush leaves its new URLs to the release cycle it starts, so none of them waits for a later run. - const pendingNew = plan.mode === 'flush' ? 0 : plan.newUrls.length - plan.fresh.length; - const planned = { - ...base, - deployment, - coldStart: plan.coldStart, - budget: plan.budget, - totalUrls: plan.total, - newUrls: plan.newUrls.length, - removedUrls: plan.removedUrls.length, - updatedUrls: plan.cycle.updated.length, - deferred: pendingNew + plan.removedUrls.length - plan.removed.length, - cycleKind: plan.cycle.kind, - firstWindowUrl: plan.window[0] ?? '', - lastWindowUrl: plan.window[plan.window.length - 1] ?? '', - }; - log.info( - `Sitemap: ${plan.total} URLs; ${plan.newUrls.length} new, ${plan.removedUrls.length} removed, ` + - `${plan.cycle.updated.length} prioritized as new or updated by the release. ` + - `Submitting ${ordered.length} (${plan.removed.length} removed, ${plan.fresh.length} new, ` + - `${plan.window.length} from the ${plan.cycle.kind} cycle) with a budget of ${plan.budget}.` - ); - + const facts = [ + `Sitemap: ${count(plan.total)} URLs, ${count(plan.newUrls.length)} new and ${count(plan.removedUrls.length)} removed since the last run`, + `Plan: ${count(ordered.length)} URLs (${count(plan.removed.length)} removed, ${count(plan.fresh.length)} new, ${count(plan.window.length)} from the ${plan.cycle.kind} pass) with a budget of ${count(plan.budget)}`, + ]; + for (const fact of facts) log.info(fact); + const describe = (outcome: RunResult['outcome'], covered: number, finished: boolean, note = '') => + summarize(host, outcome, [ + `Trigger \`${trigger}\`, mode \`${mode}\`${release ? `, release ${release}` : ''}`, + ...facts, + `Pass: ${plan.cycle.kind}, ${count(covered)} of ${count(plan.total)} URLs covered (${Math.floor((covered / plan.total) * 100)}%)${finished ? ', complete' : ''}`, + plan.coldStart && `Cold start: the known URL list ${dryRun ? 'would be' : 'was'} seeded`, + note, + ]); if (dryRun) { - return { - report: emptyReport({ - ...planned, - outcome: 'dry-run', - note: 'Dry run: nothing was submitted and the state was not saved.', - windowUrls: plan.window.length, - submitted: ordered.length, - cycleCovered: plan.covered, - cycleCompleted: plan.windowComplete, - }), - stateChanged: false, - }; + const note = 'Dry run: nothing was submitted and the state was not saved.'; + const summary = describe('dry-run', plan.covered, plan.windowComplete, note); + return { outcome: 'dry-run', submitted: ordered.length, stateChanged: false, summary }; } - const result: SubmitResult = - ordered.length === 0 - ? { accepted: 0, chunks: [], stop: null } - : await submitUrls({ config, urls: ordered, fetch, sleep, now: () => now().getTime(), log }); - - // Progress is kept even when a later chunk fails, so accepted URLs are never sent twice. A flush - // is kept even when nothing was accepted: starting the release cycle is what it changes, and - // dropping it would lose the release burst, which the next run then picks up. - const stateChanged = - result.stop === null || result.accepted > 0 || plan.coldStart || plan.mode === 'flush'; - let next: IndexNowState | null = null; + const submission: Submission = + ordered.length === 0 ? { ok: true } : await submitUrls(config, ordered, deps); + + // A failed run leaves the state alone, except that a cold start still seeds the known list and a + // flush still starts the release pass: dropping that would lose the burst, which the next run + // then carries out. + const stateChanged = submission.ok || plan.coldStart || plan.mode === 'flush'; if (stateChanged) { - next = finalizeRun(plan, { - host: config.host, + const next = finalizeRun(plan, { + host, current: urls, - accepted: result.accepted, - now: now(), - trigger, - deployment, - statuses: result.chunks.map((chunk) => chunk.status), + submitted: submission.ok, + now: deps.now(), }); await saveState(stateDir, next); } - const cycleCompleted = next !== null && next.cycle === null; - const cursor = next ? (next.cycle?.cursor ?? null) : plan.cycle.cursor; - const report = emptyReport({ - ...planned, - outcome: 'completed', - note: resumeNote, - windowUrls: next?.lastRun?.windowUrls ?? 0, - submitted: result.accepted, - chunks: result.chunks, - cycleCovered: cycleCompleted - ? plan.total - : countCovered(urls, cursor, plan.localeCodes, updatedSet), - cycleCompleted, - }); - - if (result.stop?.reason === 'rate-limited') { - if (result.accepted > 0) { - const note = - `IndexNow answered 429 (too many requests) after ${result.accepted} URLs. The progress was ` + - 'saved and the next run continues from there.'; - log.warn(note); - return { report: { ...report, outcome: 'rate-limited', note }, stateChanged }; - } - // Nothing went out, so this run did not do its job even though it will be retried. - const failure = - 'IndexNow answered 429 (too many requests) before accepting any URL, so nothing was ' + - `submitted. ${plan.mode === 'flush' ? 'The release cycle was started, so the next run submits it from the top. ' : 'The next run tries again. '}` + - 'If this keeps happening, lower the budgets in config/indexnow.json.'; - return { report: { ...report, outcome: 'rate-limited', note: failure }, stateChanged, failure }; + if (submission.ok) { + const summary = describe('completed', plan.covered, plan.windowComplete); + return { outcome: 'completed', submitted: ordered.length, stateChanged, summary }; } - if (result.stop) { - const failure = - `IndexNow rejected the submission with HTTP ${result.stop.status || 'a network error'}` + - `${result.stop.detail ? `: ${result.stop.detail}` : ''}. ` + - 'For 403 or 422, check that the key file is live and matches config/indexnow.json.'; - return { report: { ...report, outcome: 'failed', note: failure }, stateChanged, failure }; - } - return { report, stateChanged }; + const failure = failureMessage(submission, plan.mode); + const summary = describe('failed', plan.covered - plan.window.length, false, failure); + return { outcome: 'failed', submitted: 0, stateChanged, summary, failure }; } -export function renderSummary(report: RunReport): string { - const count = (value: number) => value.toLocaleString('en-US'); - const code = (value: string) => `\`${value}\``; - const rows: [string, string][] = [ - ['Outcome', report.outcome], - ['Trigger', code(report.trigger)], - ]; - if (report.outcome !== 'skipped') { - rows.push(['Mode', `${code(report.mode)}${report.reason ? ` (${report.reason})` : ''}`]); - } - if (report.totalUrls > 0) { - rows.push( - ['Sitemap URLs', count(report.totalUrls)], - ['New / removed', `${count(report.newUrls)} / ${count(report.removedUrls)}`] - ); - if (report.deferred > 0) rows.push(['Deferred to the next run', count(report.deferred)]); - if (report.updatedUrls > 0) { - rows.push([ - 'Prioritized pages', - `${count(report.updatedUrls)} new or updated by the release, ahead of the rest`, - ]); - } - rows.push([ - report.outcome === 'dry-run' ? 'Would submit' : 'Submitted', - `${count(report.submitted)} of a ${count(report.budget)} URL budget`, - ]); - if (report.cycleKind) { - const percent = Math.floor((report.cycleCovered / report.totalUrls) * 100); - rows.push([ - 'Cycle', - `${report.cycleKind}: ${count(report.cycleCovered)} of ${count(report.totalUrls)} URLs ` + - `(${percent}%)${report.cycleCompleted ? ', complete' : ''}`, - ]); - } - if (report.firstWindowUrl) { - rows.push([ - 'Window', - `${count(report.windowUrls)} URLs, ${code(report.firstWindowUrl)} to ${code(report.lastWindowUrl)}`, - ]); - } - } - if (report.chunks.length > 0) { - rows.push([ - 'Requests', - report.chunks - .map((chunk) => `HTTP ${chunk.status || 'error'} for ${count(chunk.size)} URLs`) - .join('
'), - ]); - } - if (report.deployment) rows.push(['Live deployment', code(report.deployment.slice(0, 7))]); - if (report.coldStart) { - rows.push([ - 'Cold start', - `yes: the known URL list ${report.outcome === 'dry-run' ? 'would be' : 'was'} seeded`, - ]); - } +// --------------------------------------------------------------------------------------------- +// Command line +// --------------------------------------------------------------------------------------------- - const lines = [ - `## IndexNow submission${report.host ? ` for ${report.host}` : ''}`, - '', - '| | |', - '|---|---|', - ...rows.map(([name, value]) => `| ${name} | ${value} |`), - ]; - if (report.note) lines.push('', report.note); - return `${lines.join('\n')}\n`; +export interface CliOptions { + stateDir: string; + mode: RunMode; + limit?: number; + release?: string; + trigger: string; + dryRun: boolean; + allowShrink: boolean; } -async function readEventPayload(env: NodeJS.ProcessEnv): Promise { - if (!env.GITHUB_EVENT_PATH) throw new Error('GITHUB_EVENT_PATH is not set'); - const payload: unknown = JSON.parse(await readFile(env.GITHUB_EVENT_PATH, 'utf8')); - return payload; -} +export function parseCli(argv: readonly string[]): CliOptions { + const { values } = parseArgs({ + args: [...argv], + options: { + 'state-dir': { type: 'string' }, + mode: { type: 'string' }, + limit: { type: 'string' }, + release: { type: 'string' }, + trigger: { type: 'string' }, + 'dry-run': { type: 'boolean' }, + 'allow-shrink': { type: 'boolean' }, + }, + strict: true, + allowPositionals: false, + }); -async function publish( - env: NodeJS.ProcessEnv, - report: RunReport, - stateChanged: boolean -): Promise { - if (env.GITHUB_STEP_SUMMARY) await appendFile(env.GITHUB_STEP_SUMMARY, renderSummary(report)); - if (env.GITHUB_OUTPUT) { - await appendFile( - env.GITHUB_OUTPUT, - `state_changed=${stateChanged}\nsubmitted=${report.submitted}\noutcome=${report.outcome}\n` - ); + if (!values['state-dir']) throw new Error('--state-dir is required'); + const mode = values.mode ?? 'daily'; + if (mode !== 'daily' && mode !== 'flush') throw new Error('--mode must be "daily" or "flush"'); + + let limit: number | undefined; + if (values.limit !== undefined) { + limit = Number(values.limit); + if (!Number.isInteger(limit) || limit < 1 || limit > MAX_URLS_PER_RUN) { + throw new Error(`--limit must be an integer between 1 and ${MAX_URLS_PER_RUN}`); + } + } + if (values.release !== undefined && !/^\d+\.\d+$/.test(values.release)) { + throw new Error('--release must look like 13.6'); } + + return { + stateDir: path.resolve(values['state-dir']), + mode, + limit, + release: values.release, + trigger: values.trigger || 'manual', + dryRun: values['dry-run'] ?? false, + allowShrink: values['allow-shrink'] ?? false, + }; } /** Runs the CLI and returns the process exit code. */ export async function main( argv: readonly string[] = process.argv.slice(2), env: NodeJS.ProcessEnv = process.env, - log: Logger = consoleLogger, - overrides: Pick = {} + overrides: Partial = {} ): Promise { - let trigger = 'manual'; let host = ''; + const publish = async (summary: string, stateChanged: boolean) => { + if (env.GITHUB_STEP_SUMMARY) await appendFile(env.GITHUB_STEP_SUMMARY, summary); + if (env.GITHUB_OUTPUT) await appendFile(env.GITHUB_OUTPUT, `state_changed=${stateChanged}\n`); + }; + try { const options = parseCli(argv); - trigger = options.trigger; const config = await loadConfig(); host = config.host; - const event = trigger === 'pull_request' ? await readEventPayload(env) : undefined; - const resolved = resolveRun(options, event); - - if (resolved.skip) { - log.info(resolved.skip); - await publish( - env, - emptyReport({ outcome: 'skipped', trigger, host, note: resolved.skip }), - false - ); - return 0; + + let { mode, release } = options; + if (options.trigger === 'pull_request') { + if (!env.GITHUB_EVENT_PATH) throw new Error('GITHUB_EVENT_PATH is not set'); + const event: unknown = JSON.parse(await readFile(env.GITHUB_EVENT_PATH, 'utf8')); + const version = releaseVersion(event); + if (!version) { + const note = + 'This pull request is not a merged release/X.Y pull request; nothing to submit.'; + log.info(note); + await publish(skipped(host, note).summary, false); + return 0; + } + [mode, release] = ['flush', version]; } - const changedFiles = options.changedFiles - ? parseChangedFiles(await readFile(options.changedFiles, 'utf8')) - : undefined; const result = await run({ + ...overrides, config, stateDir: options.stateDir, - mode: resolved.mode, - trigger, - reason: resolved.reason, - release: resolved.release, + mode, + trigger: options.trigger, + release, limit: options.limit, - changedFiles, dryRun: options.dryRun, allowShrink: options.allowShrink, - log, - ...overrides, }); - await publish(env, result.report, result.stateChanged); + await publish(result.summary, result.stateChanged); if (result.failure) { log.error(result.failure); return 1; @@ -1644,20 +797,12 @@ export async function main( } catch (error) { const message = errorMessage(error); log.error(message); - await publish( - env, - emptyReport({ outcome: 'failed', trigger, host, note: message }), - false - ).catch(() => undefined); + await publish(summarize(host, 'failed', [message]), false).catch(() => undefined); return 1; } } -const isMainModule = process.argv[1] - ? path.resolve(process.argv[1]) === fileURLToPath(import.meta.url) - : false; - -if (isMainModule) { +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { void main().then((code) => { process.exitCode = code; }); diff --git a/src/frontend/tests/unit/indexnow.vitest.test.ts b/src/frontend/tests/unit/indexnow.vitest.test.ts index c10304e32e..b232bd838e 100644 --- a/src/frontend/tests/unit/indexnow.vitest.test.ts +++ b/src/frontend/tests/unit/indexnow.vitest.test.ts @@ -2,44 +2,29 @@ import { mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { gzipSync } from 'node:zlib'; import { Response } from 'node-fetch'; import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import type { fetchWithProxy } from '../../scripts/fetch-with-proxy'; import { - MAX_URLS_PER_REQUEST, + MAX_URLS_PER_RUN, compareUrls, - countCovered, - detectReleaseMerge, - emptyReport, - fetchSitemapLocations, + fetchSitemapUrls, finalizeRun, - hasApiChanges, - isReleaseLive, + keyFileProblem, loadConfig, loadState, main, - normalizeUrls, - parseChangedFiles, parseCli, - parseConfig, - parseRetryAfter, parseSitemap, planRun, - readDeployment, - renderSummary, - resolveRun, + releaseVersion, run, saveState, submitUrls, tierOf, - urlsForChangedFiles, - verifyKeyFile, waitForRelease, whatsNewPath, - type ChangedFile, - type Cycle, type IndexNowConfig, type IndexNowState, type RunOptions, @@ -58,8 +43,9 @@ const config: IndexNowConfig = { }; const url = (pathname: string) => `https://${HOST}${pathname}`; +const urls = (paths: string[]) => paths.map(url); -// Ten pages in (tier, path) order: four English pages, three API pages, three localized pages. +// Ten pages in the order they are sent: four English pages, three API pages, three localized pages. const DOCS = ['/', '/app-host/', '/get-started/', '/whats-new/aspire-13-6/']; const API = [ '/reference/api/csharp/a/', @@ -69,14 +55,17 @@ const API = [ const LOCALIZED = ['/de/', '/de/app-host/', '/ja/']; const SMALL_SITE = [...DOCS, ...API, ...LOCALIZED]; -function createLogger() { - return { - info: vi.fn<(message: string) => void>(), - warn: vi.fn<(message: string) => void>(), - error: vi.fn<(message: string) => void>(), - }; +function syntheticUrls(count: number, prefix = '/reference/api/csharp/page-'): string[] { + return Array.from({ length: count }, (_, index) => + url(`${prefix}${String(index).padStart(6, '0')}/`) + ); +} + +function makeState(overrides: Partial = {}): IndexNowState { + return { host: HOST, updatedAt: NOW.toISOString(), cycle: null, known: [], ...overrides }; } +/** A clock that only moves when the code under test sleeps. */ function createClock() { let current = NOW.getTime(); return { @@ -88,29 +77,6 @@ function createClock() { }; } -type CycleInput = Omit & { updated?: string[] }; - -function makeState( - overrides: Partial> & { cycle?: CycleInput | null } = {} -): IndexNowState { - const { cycle, ...rest } = overrides; - return { - host: HOST, - updatedAt: '2026-10-12T06:43:00.000Z', - deployment: '', - cycle: cycle ? { updated: [], ...cycle } : null, - lastRun: null, - known: [], - ...rest, - }; -} - -function syntheticUrls(count: number, prefix = '/reference/api/csharp/page-'): string[] { - return Array.from({ length: count }, (_, index) => - url(`${prefix}${String(index).padStart(6, '0')}/`) - ); -} - interface PostCall { host: string; key: string; @@ -118,29 +84,20 @@ interface PostCall { urlList: string[]; } -type QueuedResponse = number | { status: number; headers?: Record }; - -/** The JSON body of a request the code under test sent. */ -function postBody(init?: { body?: unknown }): PostCall { - expect(typeof init?.body).toBe('string'); - return JSON.parse(init?.body as string) as PostCall; -} - /** A stand-in for aspire.dev and the IndexNow endpoint. */ function createSite(paths: string[], key = KEY) { const site = { paths: [...paths], keyLive: true, liveReleases: new Set(), - deployment: 'deadbeefcafe0123456789', - /** Statuses for upcoming IndexNow POSTs; 200 when the queue is empty. */ - responses: [] as QueuedResponse[], + /** Statuses for the upcoming IndexNow POSTs; 200 when the queue is empty. */ + responses: [] as number[], posts: [] as PostCall[], fetch: vi.fn(), }; const urlset = () => - '' + site.paths .map( @@ -153,11 +110,8 @@ function createSite(paths: string[], key = KEY) { site.fetch.mockImplementation((input, init) => { const target = new URL(String(input)); if (target.origin === 'https://api.indexnow.org') { - const body = postBody(init); - site.posts.push(body); - const next = site.responses.shift() ?? 200; - const { status, headers } = typeof next === 'number' ? { status: next, headers: {} } : next; - return Promise.resolve(new Response('', { status, headers })); + site.posts.push(JSON.parse(init?.body as string) as PostCall); + return Promise.resolve(new Response('', { status: site.responses.shift() ?? 200 })); } if (target.pathname === `/${key}.txt`) { return Promise.resolve( @@ -167,25 +121,15 @@ function createSite(paths: string[], key = KEY) { if (target.pathname === '/sitemap-index.xml') { return Promise.resolve( new Response( - '' + - `${url('/sitemap-0.xml')}` + `${url('/sitemap-0.xml')}` ) ); } if (target.pathname === '/sitemap-0.xml') return Promise.resolve(new Response(urlset())); - if (target.pathname === '/') { - return Promise.resolve( - new Response( - `` - ) - ); - } const release = /^\/whats-new\/aspire-(\d+)(?:-(\d+))?\/$/.exec(target.pathname); if (release) { - const version = `${release[1]}.${release[2] ?? '0'}`; - return Promise.resolve( - new Response('', { status: site.liveReleases.has(version) ? 200 : 404 }) - ); + const live = site.liveReleases.has(`${release[1]}.${release[2] ?? '0'}`); + return Promise.resolve(new Response('', { status: live ? 200 : 404 })); } return Promise.resolve(new Response('Not found', { status: 404 })); }); @@ -193,127 +137,110 @@ function createSite(paths: string[], key = KEY) { return site; } -type Site = ReturnType; - let stateDir: string; beforeEach(async () => { stateDir = await mkdtemp(path.join(os.tmpdir(), 'indexnow-test-')); + for (const method of ['log', 'warn', 'error'] as const) { + vi.spyOn(console, method).mockImplementation(() => undefined); + } }); afterEach(async () => { + vi.restoreAllMocks(); await rm(stateDir, { recursive: true, force: true }); }); -describe('IndexNow config', () => { +describe('config', () => { test('the committed config is valid and its key file is served from the site root', async () => { const real = await loadConfig(); + expect(real.host).toBe('aspire.dev'); + expect(real.key).toMatch(/^[A-Za-z0-9-]{8,128}$/); + expect(new URL(real.endpoint).protocol).toBe('https:'); + expect(new URL(real.sitemapIndex)).toMatchObject({ protocol: 'https:', hostname: real.host }); + // One request carries a whole run, so a budget cannot exceed what a request may hold. + for (const budget of Object.values(real.budgets)) { + expect(Number.isInteger(budget)).toBe(true); + expect(budget).toBeGreaterThanOrEqual(1); + expect(budget).toBeLessThanOrEqual(MAX_URLS_PER_RUN); + } + const keyFile = fileURLToPath(new URL(`../../public/${real.key}.txt`, import.meta.url)); const contents = await readFile(keyFile, 'utf8'); expect(contents.charCodeAt(0)).not.toBe(0xfeff); expect(contents.trim()).toBe(real.key); - expect(real.host).toBe('aspire.dev'); - expect(new URL(real.sitemapIndex).hostname).toBe(real.host); - }); - - test.each([ - ['a key that is too short', { ...config, key: 'short' }], - ['a key with illegal characters', { ...config, key: `${KEY.slice(0, 31)}_` }], - ['an endpoint that is not https', { ...config, endpoint: 'http://api.indexnow.org/indexnow' }], - ['a sitemap on another host', { ...config, sitemapIndex: 'https://example.com/sitemap.xml' }], - ['a zero budget', { ...config, budgets: { release: 0, rotation: 3 } }], - ['a budget above the limit', { ...config, budgets: { release: 5, rotation: 50_001 } }], - ['an uppercase host', { ...config, host: 'Aspire.dev' }], - ])('rejects %s', (_name, value) => { - expect(() => parseConfig(value)).toThrow(); - }); - - test('accepts the shape used by the workflow', () => { - expect(parseConfig(config)).toEqual(config); }); }); -describe('sitemap parsing', () => { +describe('sitemap', () => { + const urlset = (...locations: string[]) => + `${locations.map((location) => `${location}`).join('')}`; + test('reads values, decodes entities and ignores xhtml alternates', () => { const parsed = parseSitemap( - '' + + '' + 'https://aspire.dev/' + ' https://aspire.dev/a?b=1&c=2 ' + - 'https://aspire.dev/éé/' + '' ); expect(parsed).toEqual({ kind: 'urlset', - locations: ['https://aspire.dev/', 'https://aspire.dev/a?b=1&c=2', 'https://aspire.dev/éé/'], + locations: ['https://aspire.dev/', 'https://aspire.dev/a?b=1&c=2'], }); }); - test('recognizes a sitemap index', () => { - const parsed = parseSitemap( - 'https://aspire.dev/sitemap-0.xml' - ); - expect(parsed).toEqual({ kind: 'index', locations: ['https://aspire.dev/sitemap-0.xml'] }); - }); - - test('rejects documents that are not sitemaps', () => { + test('recognizes a sitemap index and rejects documents that are not sitemaps', () => { + expect( + parseSitemap( + 'https://aspire.dev/s.xml' + ) + ).toEqual({ kind: 'index', locations: ['https://aspire.dev/s.xml'] }); expect(() => parseSitemap('Not found')).toThrow(/not a sitemap/i); }); test('follows a sitemap index to its child sitemaps', async () => { const site = createSite(['/', '/a/']); - expect(await fetchSitemapLocations(config.sitemapIndex, site.fetch)).toEqual([ - url('/'), - url('/a/'), - ]); + expect(await fetchSitemapUrls(config.sitemapIndex, HOST, site.fetch)).toEqual( + urls(['/', '/a/']) + ); expect(site.fetch).toHaveBeenCalledTimes(2); }); - test('refuses child sitemaps on another host or nested indexes', async () => { + test('keeps https pages on the host, drops fragments and de-duplicates', async () => { + const fetch = vi + .fn() + .mockResolvedValue( + new Response( + urlset( + 'https://aspire.dev/a/', + 'https://aspire.dev/a/#section', + 'https://ASPIRE.dev/b/', + 'http://aspire.dev/c/', + 'https://aspire.dev:8443/d/', + 'https://example.com/f/', + 'not a url' + ) + ) + ); + expect(await fetchSitemapUrls(config.sitemapIndex, HOST, fetch)).toEqual(urls(['/a/', '/b/'])); + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('Ignored 4')); + }); + + test('refuses child sitemaps on another host, nested indexes and failed requests', async () => { + const index = (child: string) => + new Response(`${child}`); const fetch = vi.fn(); - fetch.mockResolvedValueOnce( - new Response( - 'https://evil.example/s.xml' - ) - ); - await expect(fetchSitemapLocations(config.sitemapIndex, fetch)).rejects.toThrow( + + fetch.mockResolvedValueOnce(index('https://evil.example/s.xml')); + await expect(fetchSitemapUrls(config.sitemapIndex, HOST, fetch)).rejects.toThrow( /not an https URL on aspire\.dev/ ); - fetch.mockResolvedValueOnce( - new Response( - 'https://aspire.dev/a.xml' - ) - ); - fetch.mockResolvedValueOnce( - new Response( - 'https://aspire.dev/b.xml' - ) - ); - await expect(fetchSitemapLocations(config.sitemapIndex, fetch)).rejects.toThrow(/nested/); - }); + fetch.mockResolvedValueOnce(index(url('/a.xml'))).mockResolvedValueOnce(index(url('/b.xml'))); + await expect(fetchSitemapUrls(config.sitemapIndex, HOST, fetch)).rejects.toThrow(/nested/); - test('fails when the sitemap cannot be fetched', async () => { - const fetch = vi.fn(); fetch.mockResolvedValueOnce(new Response('oops', { status: 503 })); - await expect(fetchSitemapLocations(config.sitemapIndex, fetch)).rejects.toThrow(/HTTP 503/); - }); - - test('keeps https URLs on the host, drops fragments and de-duplicates', () => { - const { urls, rejected } = normalizeUrls( - [ - 'https://aspire.dev/a/', - 'https://aspire.dev/a/#section', - 'https://ASPIRE.dev/b/', - 'http://aspire.dev/c/', - 'https://aspire.dev:8443/d/', - 'https://user:pass@aspire.dev/e/', - 'https://example.com/f/', - 'not a url', - ], - HOST - ); - expect(urls).toEqual(['https://aspire.dev/a/', 'https://aspire.dev/b/']); - expect(rejected).toHaveLength(5); + await expect(fetchSitemapUrls(config.sitemapIndex, HOST, fetch)).rejects.toThrow(/HTTP 503/); }); }); @@ -323,536 +250,110 @@ describe('ordering', () => { expect(tierOf('/app-host/certificate-configuration/')).toBe(0); expect(tierOf('/reference/cli/')).toBe(0); expect(tierOf('/reference/api')).toBe(0); + expect(tierOf('/deployment/')).toBe(0); expect(tierOf('/reference/api/csharp/aspire.hosting/')).toBe(1); expect(tierOf('/zh-cn/whats-new/upgrade-aspire/')).toBe(2); expect(tierOf('/pt-br/')).toBe(2); expect(tierOf('/de/reference/api/csharp/x/')).toBe(2); - expect(tierOf('/deployment/')).toBe(0); }); test('sorts by tier and then by URL', () => { - const shuffled = [...SMALL_SITE].reverse().map(url); - expect(shuffled.sort((a, b) => compareUrls(a, b))).toEqual(SMALL_SITE.map(url)); - }); - - test('counts how many URLs a cursor has covered', () => { - const urls = SMALL_SITE.map(url); - expect(countCovered(urls, null)).toBe(0); - expect(countCovered(urls, url('/get-started/'))).toBe(3); - expect(countCovered(urls, url('/ja/'))).toBe(10); - }); -}); - -describe('pages updated by a release', () => { - const file = (filename: string, overrides: Partial = {}): ChangedFile => ({ - status: 'modified', - filename, - changes: 4, - patch: null, - ...overrides, - }); - const docs = (relative: string, overrides: Partial = {}) => - file(`src/frontend/src/content/docs/${relative}`, overrides); - const apiData = ( - kind: 'pkgs' | 'ts-modules', - name: string, - overrides: Partial = {} - ) => file(`src/frontend/src/data/${kind}/${name}`, { status: 'renamed', ...overrides }); - const patchOf = (...lines: string[]) => ['@@ -4 +4 @@', ...lines].join('\n'); - const VERSION_ONLY = patchOf( - '- "version": "13.5.4",', - '+ "version": "13.6.0",', - '- "sourceCommit": "9c1b401dd677"', - '+ "sourceCommit": "f4c27f2d43dd"', - '- "sourceLines": "37-87"', - '+ "sourceLines": "39-89"' - ); - const REAL_CHANGE = patchOf( - '- "version": "13.5.4",', - '+ "version": "13.6.0",', - '+ "name": "AsExpress",' - ); - - const site = [ - '/', - '/app-host/', - '/app-host/eventing/', - '/get-started/first-app/', - '/whats-new/aspire-13-6/', - '/de/', - '/de/app-host/', - '/reference/api/csharp/aspire.hosting/', - '/reference/api/csharp/aspire.hosting/distributedapplication/', - '/reference/api/csharp/aspire.hosting.redis/', - '/reference/api/csharp/aspire.hosting.redis/addredis/', - '/reference/api/typescript/aspire.hosting.redis/', - '/reference/api/typescript/aspire.hosting.redis/addredis/', - ].map(url); - const map = (...files: ChangedFile[]) => urlsForChangedFiles(files, site, HOST); - - test('reads one JSON object per line', () => { - const text = - '{"status":"modified","filename":"a.mdx","changes":3,"patch":"@@ -1 +1 @@"}\n' + - '\n' + - '{"status":"renamed","filename":"b.json","patch":null}\n'; - expect(parseChangedFiles(text)).toEqual([ - { status: 'modified', filename: 'a.mdx', changes: 3, patch: '@@ -1 +1 @@' }, - { status: 'renamed', filename: 'b.json', changes: null, patch: null }, - ]); - expect(parseChangedFiles('')).toEqual([]); - }); - - test.each([ - ['text that is not JSON', 'not json', /Line 1 .* not JSON/], - ['an object without a filename', '{"status":"added"}', /Line 1 .* "filename"/], - ['a value that is not an object', '{"status":"added","filename":"a"}\n42', /Line 2/], - ])('rejects %s', (_name, text, message) => { - expect(() => parseChangedFiles(text)).toThrow(message); - }); - - test('API data only counts when more than the volatile keys changed', () => { - expect(hasApiChanges(file('x.json', { status: 'added' }))).toBe(true); - expect(hasApiChanges(file('x.json', { patch: VERSION_ONLY }))).toBe(false); - expect(hasApiChanges(file('x.json', { patch: REAL_CHANGE }))).toBe(true); - // GitHub leaves out the diff of a large change, but a pure rename has none to show. - expect(hasApiChanges(file('x.json', { changes: 900 }))).toBe(true); - expect(hasApiChanges(file('x.json', { changes: null }))).toBe(true); - expect(hasApiChanges(file('x.json', { status: 'renamed', changes: 0 }))).toBe(false); - }); - - test('maps docs to their route', () => { - expect( - map( - docs('app-host/eventing.mdx'), - docs('app-host/index.mdx'), - docs('de/index.mdx'), - docs('get-started/first-app.md') - ) - ).toEqual( - ['/app-host/', '/app-host/eventing/', '/de/', '/get-started/first-app/'].map(url).sort() - ); - expect(map(docs('index.mdx'))).toEqual([url('/')]); - expect(map(docs('de/app-host/index.mdx'))).toEqual([url('/de/app-host/')]); - }); - - test('ignores docs without a page, removed docs and files no page comes from', () => { - expect( - map( - docs('404.mdx'), - docs('app-host/removed.mdx'), - docs('app-host/eventing.mdx', { status: 'removed' }), - file('src/frontend/src/components/Header.astro'), - file('src/frontend/src/assets/logo.svg'), - file('src/frontend/package.json'), - file('src/frontend/src/content/docs/app-host/diagram.svg') - ) - ).toEqual([]); - }); - - test('maps an API data file to every page of its package, and only that package', () => { - expect(map(apiData('pkgs', 'Aspire.Hosting.13.6.0.json', { patch: REAL_CHANGE }))).toEqual( - [ - '/reference/api/csharp/aspire.hosting/', - '/reference/api/csharp/aspire.hosting/distributedapplication/', - ].map(url) - ); - expect( - map( - apiData('pkgs', 'Aspire.Hosting.Redis.13.6.0-preview.1.26478.8.json', { - patch: REAL_CHANGE, - }) - ) - ).toEqual( - [ - '/reference/api/csharp/aspire.hosting.redis/', - '/reference/api/csharp/aspire.hosting.redis/addredis/', - ].map(url) - ); - }); - - test('maps a TypeScript module file to its module pages', () => { - expect( - map(apiData('ts-modules', 'Aspire.Hosting.Redis.13.6.0.json', { patch: REAL_CHANGE })) - ).toEqual( - [ - '/reference/api/typescript/aspire.hosting.redis/', - '/reference/api/typescript/aspire.hosting.redis/addredis/', - ].map(url) - ); - }); - - test('skips API data files that only moved to the new version', () => { - expect( - map( - apiData('pkgs', 'Aspire.Hosting.13.6.0.json', { patch: VERSION_ONLY }), - apiData('ts-modules', 'Aspire.Hosting.Redis.13.6.0.json', { status: 'renamed', changes: 0 }) - ) - ).toEqual([]); - }); - - test('counts new API data files and ignores packages without pages', () => { - expect( - map(apiData('pkgs', 'Aspire.Hosting.Redis.13.6.0.json', { status: 'added' })) - ).toHaveLength(2); - expect(map(apiData('pkgs', 'Aspire.Unknown.13.6.0.json', { status: 'added' }))).toEqual([]); - }); - - test('returns each page once, in a stable order', () => { - const result = map( - docs('app-host/eventing.mdx'), - apiData('pkgs', 'Aspire.Hosting.Redis.13.6.0.json', { patch: REAL_CHANGE }), - docs('app-host/eventing.mdx') - ); - expect(new Set(result).size).toBe(result.length); - expect(result).toEqual([...result].sort()); + const shuffled = urls([...SMALL_SITE].reverse()); + expect(shuffled.sort(compareUrls)).toEqual(urls(SMALL_SITE)); }); }); describe('planning', () => { const base = { - current: SMALL_SITE.map(url), + current: urls(SMALL_SITE), state: null as IndexNowState | null, mode: 'daily' as const, - reason: 'schedule', budgets: config.budgets, - now: NOW, }; + const rotation = (cursor: string | null) => ({ kind: 'rotation' as const, cursor }); + const known = urls(SMALL_SITE); test('a cold start seeds the known list instead of treating every URL as new', () => { const plan = planRun(base); - expect(plan.coldStart).toBe(true); - expect(plan.newUrls).toEqual([]); - expect(plan.removedUrls).toEqual([]); - expect(plan.cycle).toMatchObject({ kind: 'rotation', cursor: null, reason: 'schedule' }); - expect(plan.window).toEqual(['/', '/app-host/', '/get-started/'].map(url)); - expect(plan.windowComplete).toBe(false); - }); - - test('a cycle continues after its cursor', () => { - const plan = planRun({ - ...base, - state: makeState({ - known: base.current, - cycle: { - kind: 'rotation', - reason: 'schedule', - startedAt: '2026-10-12T00:00:00.000Z', - cursor: url('/get-started/'), - skip: [], - }, - }), + expect(plan).toMatchObject({ + coldStart: true, + newUrls: [], + removedUrls: [], + fresh: [], + windowComplete: false, + cycle: { kind: 'rotation', cursor: null }, }); - expect(plan.coldStart).toBe(false); - expect(plan.window).toEqual( - ['/whats-new/aspire-13-6/', '/reference/api/csharp/a/', '/reference/api/csharp/b/'].map(url) - ); + expect(plan.window).toEqual(urls(['/', '/app-host/', '/get-started/'])); }); - test('a cursor that is no longer in the sitemap still finds the next URL', () => { - const plan = planRun({ - ...base, - state: makeState({ - known: base.current, - cycle: { - kind: 'rotation', - reason: 'schedule', - startedAt: '2026-10-12T00:00:00.000Z', - cursor: url('/get-started/gone/'), - skip: [], - }, - }), - }); - expect(plan.window[0]).toBe(url('/whats-new/aspire-13-6/')); + test('a pass continues after its cursor, even when the cursor left the sitemap', () => { + for (const cursor of ['/get-started/', '/get-started/gone/']) { + const state = makeState({ known, cycle: rotation(url(cursor)) }); + const plan = planRun({ ...base, state }); + expect(plan.coldStart).toBe(false); + expect(plan.window).toEqual(urls([DOCS[3], API[0], API[1]])); + } }); - test('a flush restarts the cycle from the top with the release budget', () => { - const plan = planRun({ - ...base, - mode: 'flush', - reason: 'release/13.6', - state: makeState({ - known: base.current, - cycle: { - kind: 'rotation', - reason: 'schedule', - startedAt: '2026-10-01T00:00:00.000Z', - cursor: url('/ja/'), - skip: [url('/de/')], - }, - }), - }); - expect(plan.cycle).toMatchObject({ - kind: 'release', - reason: 'release/13.6', - cursor: null, - skip: [], - }); + test('a flush starts a release pass from the top and leaves new URLs to its window', () => { + const created = url('/reference/api/csharp/ab/'); + const state = makeState({ known, cycle: rotation(url('/ja/')) }); + const plan = planRun({ ...base, current: [...known, created], state, mode: 'flush' }); + expect(plan.cycle).toEqual({ kind: 'release', cursor: null }); expect(plan.budget).toBe(5); - expect(plan.window).toEqual( - [ - '/', - '/app-host/', - '/get-started/', - '/whats-new/aspire-13-6/', - '/reference/api/csharp/a/', - ].map(url) - ); + expect(plan.newUrls).toEqual([created]); + expect(plan.fresh).toEqual([]); + expect(plan.window).toEqual(urls([...DOCS, API[0]])); }); - test('a release cycle keeps its larger budget until it completes', () => { - const plan = planRun({ - ...base, - state: makeState({ - known: base.current, - cycle: { - kind: 'release', - reason: 'release/13.6', - startedAt: '', - cursor: url('/ja/'), - skip: [], - }, - }), - }); + test('a release pass keeps its larger budget until it finishes', () => { + const state = makeState({ known, cycle: { kind: 'release', cursor: url('/get-started/') } }); + const plan = planRun({ ...base, state }); expect(plan.budget).toBe(5); + expect(plan.window).toEqual(urls([DOCS[3], ...API, LOCALIZED[0]])); }); - test('--limit replaces the budget', () => { - expect(planRun({ ...base, limit: 2 }).window).toHaveLength(2); - }); - - test('new and removed URLs go first and count against the budget', () => { - const known = SMALL_SITE.map(url); + test('removed and new URLs go first and count against the budget', () => { const current = [...known.filter((entry) => entry !== url('/ja/')), url('/aaa/')]; - const plan = planRun({ - ...base, - current, - state: makeState({ - known, - cycle: { - kind: 'rotation', - reason: 'schedule', - startedAt: '', - cursor: url('/get-started/'), - skip: [], - }, - }), - }); + const state = makeState({ known, cycle: rotation(url('/get-started/')) }); + const plan = planRun({ ...base, current, state }); expect(plan.removed).toEqual([url('/ja/')]); expect(plan.fresh).toEqual([url('/aaa/')]); - expect(plan.window).toEqual([url('/whats-new/aspire-13-6/')]); + expect(plan.window).toEqual([url(DOCS[3])]); }); - test('a new URL ahead of the cursor is not sent again by the window', () => { - const known = SMALL_SITE.map(url); - const created = url('/reference/api/csharp/ab/'); - const plan = planRun({ - ...base, - current: [...known, created], - state: makeState({ - known, - cycle: { - kind: 'rotation', - reason: 'schedule', - startedAt: '', - cursor: url('/get-started/'), - skip: [], - }, - }), - limit: 10, - }); - expect(plan.fresh).toEqual([created]); - expect(plan.window).not.toContain(created); - expect(plan.window).toHaveLength(7); - }); + test('--limit replaces the budget and also caps the removed and new URLs', () => { + expect(planRun({ ...base, limit: 2 }).window).toHaveLength(2); - test('new and removed URLs beyond the per-run cap stay pending', () => { - const known = syntheticUrls(3); - const created = syntheticUrls(10_005, '/reference/api/csharp/new-'); - const plan = planRun({ + const gone = [url(API[2]), url('/ja/')]; + const removing = planRun({ ...base, - current: [...known, ...created], + current: known.filter((entry) => !gone.includes(entry)), state: makeState({ known }), - limit: 20_000, - }); - expect(plan.newUrls).toHaveLength(10_005); - expect(plan.fresh).toHaveLength(10_000); - const next = finalizeRun(plan, { - host: HOST, - current: [...known, ...created], - accepted: plan.removed.length + plan.fresh.length + plan.window.length, - now: NOW, - trigger: 'schedule', - deployment: '', - statuses: [200], - }); - const stillNew = created.filter((entry) => !next.known.includes(entry)); - expect(stillNew).toHaveLength( - 5 - plan.window.filter((entry) => created.includes(entry)).length - ); - }); - - test('pages updated by the release rank ahead of every other page', () => { - const updated = [url('/reference/api/csharp/b/'), url('/de/app-host/')]; - const plan = planRun({ - ...base, - mode: 'flush', - reason: 'release/13.6', - updatedUrls: updated, - limit: 4, - }); - expect(plan.cycle.updated).toEqual([...updated].sort()); - // Updated pages first (English, then API, then localized), then the rest in the usual order. - expect(plan.window).toEqual( - ['/reference/api/csharp/b/', '/de/app-host/', '/', '/app-host/'].map(url) - ); - }); - - test('updated pages that are not in the sitemap are dropped', () => { - const plan = planRun({ - ...base, - mode: 'flush', - reason: 'release/13.6', - updatedUrls: [url('/gone/'), url('/ja/'), url('/ja/')], - }); - expect(plan.cycle.updated).toEqual([url('/ja/')]); - }); - - test('a cycle that is already running keeps its own updated pages', () => { - const plan = planRun({ - ...base, - updatedUrls: [url('/de/')], - state: makeState({ - known: base.current, - cycle: { - kind: 'release', - reason: 'release/13.6', - startedAt: '', - cursor: null, - skip: [], - updated: [url('/ja/')], - }, - }), - }); - expect(plan.cycle.updated).toEqual([url('/ja/')]); - expect(plan.window[0]).toBe(url('/ja/')); - }); - - test('the cursor walks the updated pages first and then the rest', () => { - const updated = [url('/reference/api/csharp/b/'), url('/de/app-host/')]; - const plan = planRun({ - ...base, - state: makeState({ - known: base.current, - cycle: { - kind: 'release', - reason: 'release/13.6', - startedAt: '', - cursor: url('/reference/api/csharp/b/'), - skip: [], - updated, - }, - }), + limit: 1, }); - expect(plan.window).toEqual( - ['/de/app-host/', '/', '/app-host/', '/get-started/', '/whats-new/aspire-13-6/'].map(url) - ); - }); + expect(removing.removedUrls).toEqual(gone); + expect(removing.removed).toEqual([gone[0]]); + expect(removing.window).toEqual([]); - test('a release covers its new pages with the updated ones, docs before the API reference', () => { - const known = SMALL_SITE.map(url); - const created = [url('/reference/api/csharp/ab/'), url('/reference/api/csharp/ac/')]; - const createdDocs = url('/aaa/'); - const updated = [url('/app-host/')]; - const plan = planRun({ + const created = urls(['/aaa/', '/aab/', '/aac/']); + const adding = planRun({ ...base, - current: [...known, ...created, createdDocs], + current: [...known, ...created], state: makeState({ known }), - mode: 'flush', - reason: 'release/13.6', - updatedUrls: updated, - }); - expect(plan.newUrls).toHaveLength(3); - expect(plan.fresh).toEqual([]); - expect(plan.cycle.updated).toEqual([...updated, ...created, createdDocs].sort()); - expect(plan.window).toEqual( - ['/aaa/', '/app-host/', '/reference/api/csharp/ab/', '/reference/api/csharp/ac/', '/'].map( - url - ) - ); - }); - - test('a new page that appears after a release cycle started still goes first', () => { - const known = SMALL_SITE.map(url); - const late = url('/reference/api/csharp/zz/'); - const plan = planRun({ - ...base, - current: [...known, late], - state: makeState({ - known, - cycle: { - kind: 'release', - reason: 'release/13.6', - startedAt: '', - cursor: url('/get-started/'), - skip: [], - updated: [url('/app-host/')], - }, - }), + limit: 2, }); - expect(plan.fresh).toEqual([late]); - expect(plan.cycle.updated).toEqual([url('/app-host/')]); + expect(adding.newUrls).toHaveLength(3); + expect(adding.fresh).toEqual(created.slice(0, 2)); + expect(adding.window).toEqual([]); }); - test('a new page the running cycle already names is still sent first on a daily run', () => { - const known = SMALL_SITE.map(url); - const returned = url('/reference/api/csharp/ab/'); - const plan = planRun({ - ...base, - current: [...known, returned], - state: makeState({ - known, - cycle: { - kind: 'release', - reason: 'release/13.6', - startedAt: '', - cursor: url('/get-started/'), - skip: [], - updated: [returned], - }, - }), - }); - // The cursor has already passed its place in the cycle, so only sending it first submits it. - expect(plan.fresh).toEqual([returned]); - expect(plan.window).not.toContain(returned); - }); - - test('--limit also caps the new and removed URLs that go first', () => { - const known = SMALL_SITE.map(url); - const current = [ - ...known.filter((entry) => entry !== url('/ja/')), - url('/aaa/'), - url('/aab/'), - url('/aac/'), - ]; - const plan = planRun({ ...base, current, state: makeState({ known }), limit: 2 }); - expect(plan.removed).toEqual([url('/ja/')]); - expect(plan.fresh).toEqual([url('/aaa/')]); - expect(plan.window).toEqual([]); - expect(plan.newUrls).toHaveLength(3); - }); - - test('--limit also caps the removed URLs that go first', () => { - const known = SMALL_SITE.map(url); - const gone = [url('/reference/api/typescript/c/'), url('/ja/')]; - const plan = planRun({ - ...base, - current: known.filter((entry) => !gone.includes(entry)), - state: makeState({ known }), - limit: 1, - }); - expect(plan.removedUrls).toEqual(gone); - expect(plan.removed).toEqual([gone[0]]); - expect(plan.window).toEqual([]); + test('never plans more than one request, whatever the budget', () => { + const current = syntheticUrls(12_000, '/docs/page-'); + const budgets = { release: 50_000, rotation: 50_000 }; + const plan = planRun({ ...base, current, budgets, mode: 'flush' }); + expect(plan.budget).toBe(MAX_URLS_PER_RUN); + expect(plan.window).toHaveLength(MAX_URLS_PER_RUN); }); test('refuses an empty sitemap', () => { @@ -860,281 +361,91 @@ describe('planning', () => { }); test('refuses a sitemap that shrank by more than 20% unless allowed', () => { - const known = syntheticUrls(100); - const shrunk = known.slice(0, 79); - expect(() => planRun({ ...base, current: shrunk, state: makeState({ known }) })).toThrow( + const before = syntheticUrls(100); + const state = makeState({ known: before }); + expect(() => planRun({ ...base, current: before.slice(0, 79), state })).toThrow( /--allow-shrink/ ); - expect(() => - planRun({ ...base, current: known.slice(0, 80), state: makeState({ known }) }) - ).not.toThrow(); - expect(() => - planRun({ ...base, current: shrunk, state: makeState({ known }), allowShrink: true }) - ).not.toThrow(); + expect(() => planRun({ ...base, current: before.slice(0, 80), state })).not.toThrow(); + const allowed = planRun({ ...base, current: before.slice(0, 79), state, allowShrink: true }); + expect(allowed.removedUrls).toHaveLength(21); }); }); describe('finalizing a run', () => { - const finalizeOptions = { + const known = urls(SMALL_SITE); + const base = { current: known, mode: 'daily' as const, budgets: config.budgets }; + const options = (current: string[], submitted = true) => ({ host: HOST, + current, + submitted, now: NOW, - trigger: 'schedule', - deployment: 'abc1234', - statuses: [200], - }; + }); - test('remembers the whole sitemap and advances the cursor when everything was accepted', () => { - const current = SMALL_SITE.map(url); - const plan = planRun({ - current, - state: null, - mode: 'daily', - reason: 'schedule', - budgets: config.budgets, - now: NOW, - }); - const next = finalizeRun(plan, { ...finalizeOptions, current, accepted: 3 }); - expect(next.known).toEqual([...current].sort()); - expect(next.cycle?.cursor).toBe(url('/get-started/')); - expect(next.lastRun).toMatchObject({ coldStart: true, submitted: 3, windowUrls: 3 }); - expect(next.deployment).toBe('abc1234'); + test('remembers the whole sitemap and moves the cursor once the request was accepted', () => { + const plan = planRun({ ...base, state: null }); + const next = finalizeRun(plan, options(known)); + expect(next.known).toEqual([...known].sort()); + expect(next.cycle).toEqual({ kind: 'rotation', cursor: url('/get-started/') }); + expect(next.updatedAt).toBe(NOW.toISOString()); }); - test('completes the cycle once the window reaches the end of the sitemap', () => { - const current = SMALL_SITE.map(url); - const plan = planRun({ - current, - state: makeState({ - known: current, - cycle: { - kind: 'release', - reason: 'release/13.6', - startedAt: '', - cursor: url('/reference/api/typescript/c/'), - skip: [], - }, - }), - mode: 'daily', - reason: 'schedule', - budgets: config.budgets, - now: NOW, - }); - expect(plan.window).toEqual(LOCALIZED.map(url)); + test('finishes the pass when the window reaches the end of the sitemap', () => { + const state = makeState({ known, cycle: { kind: 'release', cursor: url(API[2]) } }); + const plan = planRun({ ...base, state }); + expect(plan.window).toEqual(urls(LOCALIZED)); expect(plan.windowComplete).toBe(true); - expect(finalizeRun(plan, { ...finalizeOptions, current, accepted: 3 }).cycle).toBeNull(); - // A partly accepted window leaves the cycle open at the last accepted URL. - const partial = finalizeRun(plan, { ...finalizeOptions, current, accepted: 2 }); - expect(partial.cycle?.cursor).toBe(url('/de/app-host/')); + expect(finalizeRun(plan, options(known)).cycle).toBeNull(); + expect(finalizeRun(plan, options(known, false)).cycle).toEqual(state.cycle); }); - test('keeps URLs that were not accepted pending', () => { - const known = SMALL_SITE.map(url); - const created = [url('/aaa/'), url('/aab/')]; + test('keeps URLs that were not sent pending', () => { + const created = urls(['/aaa/', '/aab/']); const current = [...known.filter((entry) => entry !== url('/ja/')), ...created]; - const plan = planRun({ - current, - state: makeState({ known }), - mode: 'daily', - reason: 'schedule', - budgets: config.budgets, - now: NOW, - }); + const plan = planRun({ ...base, current, state: makeState({ known }) }); expect(plan.removed).toEqual([url('/ja/')]); expect(plan.fresh).toEqual(created); - const partial = finalizeRun(plan, { ...finalizeOptions, current, accepted: 2 }); - expect(partial.known).not.toContain(url('/ja/')); - expect(partial.known).toContain(created[0]); - expect(partial.known).not.toContain(created[1]); + const failed = finalizeRun(plan, options(current, false)); + expect(failed.known).toContain(url('/ja/')); + expect(failed.known).not.toContain(created[0]); + expect(failed.known).not.toContain(created[1]); + expect(failed.cycle).toEqual({ kind: 'rotation', cursor: null }); - const none = finalizeRun(plan, { ...finalizeOptions, current, accepted: 0 }); - expect(none.known).toContain(url('/ja/')); - expect(none.known).not.toContain(created[0]); - expect(none.known).not.toContain(created[1]); + const accepted = finalizeRun(plan, options(current)); + expect(accepted.known).not.toContain(url('/ja/')); + expect(accepted.known).toEqual(expect.arrayContaining(created)); }); - test('a new page the release cycle covers stays known, whether or not anything was accepted', () => { - const known = SMALL_SITE.map(url); - const covered = url('/reference/api/csharp/ab/'); - const current = [...known, covered]; - const plan = planRun({ - current, - state: makeState({ known }), - mode: 'flush', - reason: 'release/13.6', - budgets: config.budgets, - now: NOW, - }); - expect(plan.fresh).toEqual([]); - const none = finalizeRun(plan, { ...finalizeOptions, current, accepted: 0 }); - expect(none.known).toContain(covered); - expect(none.cycle).toMatchObject({ cursor: null, skip: [], updated: [covered] }); + test('keeps the new URLs beyond the per-run cap pending', () => { + const before = syntheticUrls(3); + const created = syntheticUrls(10_005, '/reference/api/csharp/new-'); + const current = [...before, ...created]; + const plan = planRun({ ...base, current, state: makeState({ known: before }) }); + expect(plan.fresh).toHaveLength(MAX_URLS_PER_RUN); + + const next = finalizeRun(plan, options(current)); + expect(current.filter((entry) => !next.known.includes(entry))).toEqual(created.slice(10_000)); }); - test('forgets skipped URLs once the cursor has passed them', () => { - const known = SMALL_SITE.map(url); + test('a flush counts its new URLs as known even when the request failed', () => { const created = url('/reference/api/csharp/ab/'); const current = [...known, created]; - const cycle = { - kind: 'rotation' as const, - reason: 'schedule', - startedAt: '', - cursor: url('/get-started/'), - skip: [], - }; - const plan = planRun({ - current, - state: makeState({ known, cycle }), - mode: 'daily', - reason: 'schedule', - budgets: config.budgets, - limit: 2, - now: NOW, - }); - expect(plan.window).toEqual([url('/whats-new/aspire-13-6/')]); - const day = finalizeRun(plan, { - ...finalizeOptions, - current, - accepted: 1 + plan.window.length, - }); - expect(day.cycle?.skip).toEqual([created]); - - const nextPlan = planRun({ - current, - state: day, - mode: 'daily', - reason: 'schedule', - budgets: config.budgets, - limit: 4, - now: NOW, - }); - expect(nextPlan.window).toEqual( - [ - '/reference/api/csharp/a/', - '/reference/api/csharp/b/', - '/reference/api/typescript/c/', - '/de/', - ].map(url) - ); - const nextDay = finalizeRun(nextPlan, { - ...finalizeOptions, - current, - accepted: nextPlan.window.length, - }); - expect(nextDay.cycle?.skip).toEqual([]); + const plan = planRun({ ...base, current, state: makeState({ known }), mode: 'flush' }); + const next = finalizeRun(plan, options(current, false)); + expect(next.known).toContain(created); + expect(next.cycle).toEqual({ kind: 'release', cursor: null }); }); - test('ranks the skip list the same way as the cycle, so updated pages do not hide skipped ones', () => { - const known = SMALL_SITE.map(url); - const updated = [url('/reference/api/csharp/b/')]; - const cycle = { - kind: 'release' as const, - reason: 'release/13.6', - startedAt: '', - cursor: url('/reference/api/csharp/b/'), - skip: [url('/get-started/')], - updated, - }; - const plan = planRun({ - current: known, - state: makeState({ known, cycle }), - mode: 'daily', - reason: 'schedule', - budgets: config.budgets, - limit: 1, - now: NOW, - }); - // /get-started/ is an English page, so it sorts before the API page by tier, but it ranks after - // it because it was not updated. It is still ahead of the cursor and must stay skipped. - const next = finalizeRun(plan, { ...finalizeOptions, current: known, accepted: 0 }); - expect(next.cycle?.cursor).toBe(cycle.cursor); - expect(next.cycle?.skip).toEqual([url('/get-started/')]); - expect(next.cycle?.updated).toEqual(updated); - }); - - test('a release burst sends the updated pages of a sitemap shaped like aspire.dev first, and each URL once', () => { - const docs = syntheticUrls(611, '/docs/page-'); - const api = syntheticUrls(24_207); - const localized = syntheticUrls(7_518, '/de/page-'); - const current = [...localized, ...api, ...docs]; - const budgets = { release: 10_000, rotation: 1_000 }; - // 150 updated docs, 3,000 updated API pages and 20 updated localized pages. - const updated = [ - ...docs.slice(100, 250), - ...api.slice(5_000, 8_000), - ...localized.slice(0, 20), - ]; - - let state: IndexNowState = makeState({ known: current }); - const sizes: number[] = []; - const seen = new Set(); - for (let day = 0; day < 10; day++) { - const plan = planRun({ - current, - state, - mode: day === 0 ? 'flush' : 'daily', - reason: 'release/13.6', - budgets, - updatedUrls: day === 0 ? updated : undefined, - now: NOW, - }); - if (day === 0) { - // Updated pages first, in tier order, and only then the other pages. - expect(plan.window.slice(0, 150)).toEqual(docs.slice(100, 250)); - expect(plan.window.slice(150, 3_150)).toEqual(api.slice(5_000, 8_000)); - expect(plan.window.slice(3_150, 3_170)).toEqual(localized.slice(0, 20)); - expect(plan.window[3_170]).toBe(docs[0]); - } - for (const entry of plan.window) { - expect(seen.has(entry)).toBe(false); - seen.add(entry); - } - sizes.push(plan.window.length); - state = finalizeRun(plan, { ...finalizeOptions, current, accepted: plan.window.length }); - if (state.cycle === null) break; - } - expect(sizes).toEqual([10_000, 10_000, 10_000, 2_336]); - expect(seen.size).toBe(current.length); - }); - - test('a release that adds more API pages than the budget covers them across runs, each once', () => { - const docs = syntheticUrls(50, '/docs/page-'); - const known = [...docs, ...syntheticUrls(200)]; - const created = syntheticUrls(120, '/reference/api/csharp/new-'); - const current = [...known, ...created]; - const budgets = { release: 100, rotation: 10 }; - const updated = docs.slice(10, 20); + test('a new page ahead of the cursor is sent now and again when the pass reaches it', () => { + const created = url('/reference/api/csharp/ab/'); + const current = [...known, created]; + const state = makeState({ known, cycle: { kind: 'rotation', cursor: url('/get-started/') } }); + const plan = planRun({ ...base, current, state }); + expect(plan.fresh).toEqual([created]); - let state: IndexNowState = makeState({ known }); - const sizes: number[] = []; - const seen = new Set(); - for (let day = 0; day < 10; day++) { - const plan = planRun({ - current, - state, - mode: day === 0 ? 'flush' : 'daily', - reason: 'release/13.6', - budgets, - updatedUrls: day === 0 ? updated : undefined, - now: NOW, - }); - // The new pages are covered by the cycle, so none of them jumps ahead of the updated docs. - expect(plan.fresh).toEqual([]); - if (day === 0) { - expect(plan.window.slice(0, 10)).toEqual(updated); - expect(plan.window.slice(10)).toEqual(created.slice(0, 90)); - } - for (const entry of plan.window) { - expect(seen.has(entry)).toBe(false); - seen.add(entry); - } - sizes.push(plan.window.length); - state = finalizeRun(plan, { ...finalizeOptions, current, accepted: plan.window.length }); - expect(state.known).toEqual([...current].sort()); - if (state.cycle === null) break; - } - expect(sizes).toEqual([100, 100, 100, 70]); - expect(seen.size).toBe(current.length); + const later = planRun({ ...base, current, state: finalizeRun(plan, options(current)) }); + expect(later.window).toContain(created); }); test('a release burst over a sitemap shaped like aspire.dev takes four runs and sends each URL once', () => { @@ -1144,7 +455,7 @@ describe('finalizing a run', () => { const current = [...localized, ...api, ...docs]; const budgets = { release: 10_000, rotation: 1_000 }; - let state: IndexNowState = makeState({ known: current }); + let state = makeState({ known: current }); const sizes: number[] = []; const seen = new Set(); for (let day = 0; day < 10; day++) { @@ -1152,9 +463,7 @@ describe('finalizing a run', () => { current, state, mode: day === 0 ? 'flush' : 'daily', - reason: 'release/13.6', budgets, - now: NOW, }); if (day === 0) expect(plan.window.slice(0, 611)).toEqual(docs); for (const entry of plan.window) { @@ -1162,18 +471,14 @@ describe('finalizing a run', () => { seen.add(entry); } sizes.push(plan.window.length); - state = finalizeRun(plan, { - ...finalizeOptions, - current, - accepted: plan.window.length, - }); + state = finalizeRun(plan, options(current)); if (state.cycle === null) break; } expect(sizes).toEqual([10_000, 10_000, 10_000, 2_336]); expect(seen.size).toBe(current.length); - // The next run starts a slow rotation cycle from the top. - const after = planRun({ current, state, mode: 'daily', reason: 'schedule', budgets, now: NOW }); + // The next run starts a slow rotation pass from the top. + const after = planRun({ current, state, mode: 'daily', budgets }); expect(after.cycle.kind).toBe('rotation'); expect(after.window).toHaveLength(1_000); expect(after.window[0]).toBe(docs[0]); @@ -1181,128 +486,55 @@ describe('finalizing a run', () => { }); describe('state on disk', () => { - const state = makeState({ - deployment: 'abc1234', - known: SMALL_SITE.map(url), - cycle: { - kind: 'release', - reason: 'release/13.6', - startedAt: '2026-10-13T06:43:00.000Z', - cursor: url('/get-started/'), - skip: [url('/reference/api/csharp/ab/')], - updated: [url('/de/'), url('/get-started/')], - }, - lastRun: { - at: '2026-10-13T06:43:00.000Z', - trigger: 'pull_request', - mode: 'flush', - coldStart: false, - newUrls: 1, - removedUrls: 2, - windowUrls: 3, - submitted: 6, - statuses: [200], - }, - }); + const valid = { + schemaVersion: 1, + host: HOST, + updatedAt: NOW.toISOString(), + cycle: null, + known: [url('/')], + }; + const stateFile = () => path.join(stateDir, 'state.json'); - test('round-trips and leaves no staging files behind', async () => { + test('round-trips in a single file', async () => { + const state = makeState({ + cycle: { kind: 'release', cursor: url('/de/') }, + known: urls(SMALL_SITE), + }); await saveState(stateDir, state); - expect((await readdir(stateDir)).sort()).toEqual(['known-urls.txt.gz', 'state.json']); - expect(await loadState(stateDir, HOST, createLogger())).toEqual(state); - }); - - test('a missing state is a cold start without a warning', async () => { - const log = createLogger(); - expect(await loadState(path.join(stateDir, 'missing'), HOST, log)).toBeNull(); - expect(log.warn).not.toHaveBeenCalled(); - }); - - test.each([ - ['unreadable JSON', async () => writeFile(path.join(stateDir, 'state.json'), '{nope')], - [ - 'a different schema version', - async () => { - await saveState(stateDir, state); - const file = path.join(stateDir, 'state.json'); - const parsed = JSON.parse(await readFile(file, 'utf8')) as Record; - await writeFile(file, JSON.stringify({ ...parsed, schemaVersion: 99 })); - }, - ], - [ - 'another host', - async () => { - await saveState(stateDir, { ...state, host: 'example.com' }); - }, - ], - [ - 'a known list that does not match its count', - async () => { - await saveState(stateDir, state); - await writeFile( - path.join(stateDir, 'known-urls.txt.gz'), - gzipSync('https://aspire.dev/\n') - ); - }, - ], - [ - 'a truncated known list', - async () => { - await saveState(stateDir, state); - await writeFile(path.join(stateDir, 'known-urls.txt.gz'), 'not gzip'); - }, - ], - ])('falls back to a cold start for %s', async (_name, arrange) => { - await arrange(); - const log = createLogger(); - expect(await loadState(stateDir, HOST, log)).toBeNull(); - expect(log.warn).toHaveBeenCalledWith(expect.stringContaining('starting cold')); + expect(await readdir(stateDir)).toEqual(['state.json']); + expect(await loadState(stateDir, HOST)).toEqual(state); }); -}); - -describe('state read from an artifact', () => { - const cycle = { - kind: 'release' as const, - reason: 'release/13.6', - startedAt: '2026-10-13T06:43:00.000Z', - cursor: url('/get-started/'), - skip: [], - updated: [], - }; - test('loads states saved before cycles tracked updated pages', async () => { - await saveState(stateDir, makeState({ known: SMALL_SITE.map(url), cycle })); - const file = path.join(stateDir, 'state.json'); - const parsed = JSON.parse(await readFile(file, 'utf8')) as { cycle: Record }; - delete parsed.cycle.updated; - await writeFile(file, JSON.stringify(parsed)); - expect((await loadState(stateDir, HOST, createLogger()))?.cycle?.updated).toEqual([]); + test('a missing state is a cold start without a warning', async () => { + expect(await loadState(stateDir, HOST)).toBeNull(); + expect(console.warn).not.toHaveBeenCalled(); }); test.each([ - ['an http URL', { known: [url('/'), 'http://aspire.dev/insecure/'] }], - ['another host', { known: [url('/'), 'https://evil.example/spam/'] }], - ['something that is not a URL', { known: [url('/'), 'javascript:alert(1)'] }], - ['a duplicate', { known: [url('/'), url('/')] }], - ['a URL with a fragment', { known: [url('/'), url('/a/#top')] }], - ['an empty known list', { known: [] }], - ['a cursor that is not a URL', { known: [url('/')], cycle: { ...cycle, cursor: 'not a url' } }], + ['invalid JSON', '{not json'], + ['another schema version', JSON.stringify({ ...valid, schemaVersion: 2 })], + ['another host', JSON.stringify({ ...valid, host: 'example.com' })], + ['an empty known list', JSON.stringify({ ...valid, known: [] })], [ - 'a skip list with another host', - { known: [url('/')], cycle: { ...cycle, skip: ['https://evil.example/'] } }, + 'URLs of another host', + JSON.stringify({ ...valid, known: [url('/'), 'https://evil.example/'] }), ], + ['an insecure URL', JSON.stringify({ ...valid, known: ['http://aspire.dev/'] })], [ - 'an updated list with another host', - { known: [url('/')], cycle: { ...cycle, updated: ['https://evil.example/'] } }, + 'a cursor of another host', + JSON.stringify({ ...valid, cycle: { kind: 'release', cursor: 'https://evil.example/' } }), ], - ])('starts cold instead of trusting %s', async (_name, overrides) => { - await saveState(stateDir, makeState(overrides)); - const log = createLogger(); - expect(await loadState(stateDir, HOST, log)).toBeNull(); - expect(log.warn).toHaveBeenCalledWith(expect.stringContaining('starting cold')); + ['an empty cursor', JSON.stringify({ ...valid, cycle: { kind: 'release', cursor: '' } })], + ['a malformed pass', JSON.stringify({ ...valid, cycle: { kind: 'weekly', cursor: null } })], + ['a missing pass', JSON.stringify({ ...valid, cycle: undefined })], + ])('ignores a state with %s and starts cold', async (_name, content) => { + await writeFile(stateFile(), content); + expect(await loadState(stateDir, HOST)).toBeNull(); + expect(console.warn).toHaveBeenCalledWith(expect.stringContaining('Ignoring unusable')); }); }); -describe('submitting URLs', () => { +describe('submitting', () => { function fetchWith(...responses: (Response | Error)[]) { const fetch = vi.fn(); for (const response of responses) { @@ -1311,122 +543,34 @@ describe('submitting URLs', () => { } return fetch; } + const batch = syntheticUrls(3); - const urls = syntheticUrls(3); - - test('POSTs the documented payload', async () => { + test('POSTs the documented payload in a single request', async () => { const fetch = fetchWith(new Response('', { status: 200 })); - const result = await submitUrls({ - config, - urls, - fetch, - sleep: createClock().sleep, - log: createLogger(), - }); - expect(result).toEqual({ - accepted: 3, - chunks: [{ size: 3, status: 200, attempts: 1 }], - stop: null, - }); + const clock = createClock(); + expect(await submitUrls(config, batch, { fetch, sleep: clock.sleep })).toEqual({ ok: true }); + expect(fetch).toHaveBeenCalledTimes(1); const [target, init] = fetch.mock.calls[0]; expect(String(target)).toBe(config.endpoint); expect(init?.method).toBe('POST'); expect(init?.headers).toMatchObject({ 'Content-Type': 'application/json; charset=utf-8' }); - expect(postBody(init)).toEqual({ + expect(JSON.parse(init?.body as string)).toEqual({ host: HOST, key: KEY, keyLocation: `https://${HOST}/${KEY}.txt`, - urlList: urls, + urlList: batch, }); }); - test('accepts 202 for a key that is still being verified', async () => { + test('accepts 202, which IndexNow answers while it verifies a new key', async () => { const fetch = fetchWith(new Response('', { status: 202 })); - const result = await submitUrls({ - config, - urls, - fetch, - sleep: createClock().sleep, - log: createLogger(), - }); - expect(result.accepted).toBe(3); - expect(result.stop).toBeNull(); - }); - - test('splits large lists into chunks of at most 10,000 and pauses between them', async () => { - const many = syntheticUrls(25_000); - const fetch = fetchWith( - new Response('', { status: 200 }), - new Response('', { status: 200 }), - new Response('', { status: 200 }) - ); - const clock = createClock(); - const result = await submitUrls({ - config, - urls: many, - fetch, - sleep: clock.sleep, - log: createLogger(), - }); - expect(result.chunks.map((chunk) => chunk.size)).toEqual([10_000, 10_000, 5_000]); - expect(result.accepted).toBe(25_000); - expect(MAX_URLS_PER_REQUEST).toBe(10_000); - expect(clock.sleep).toHaveBeenCalledTimes(2); - const sent = fetch.mock.calls.flatMap(([, init]) => postBody(init).urlList); - expect(sent).toEqual(many); - }); - - test('waits for a short Retry-After on 429 and then succeeds', async () => { - const fetch = fetchWith( - new Response('', { status: 429, headers: { 'retry-after': '2' } }), - new Response('', { status: 200 }) - ); - const clock = createClock(); - const result = await submitUrls({ - config, - urls, - fetch, - sleep: clock.sleep, - log: createLogger(), - }); - expect(clock.sleep).toHaveBeenCalledWith(2_000); - expect(result.chunks).toEqual([{ size: 3, status: 200, attempts: 2 }]); - }); - - test('stops at once on a 429 without a usable Retry-After and keeps earlier progress', async () => { - const many = syntheticUrls(15_000); - const fetch = fetchWith( - new Response('', { status: 200 }), - new Response('slow down', { status: 429 }) - ); - const clock = createClock(); - const result = await submitUrls({ - config, - urls: many, - fetch, - sleep: clock.sleep, - log: createLogger(), - }); - expect(result.accepted).toBe(10_000); - expect(result.stop).toEqual({ reason: 'rate-limited', status: 429, detail: 'slow down' }); - expect(fetch).toHaveBeenCalledTimes(2); - - const tooLong = fetchWith( - new Response('', { status: 429, headers: { 'retry-after': '3600' } }) - ); - const stopped = await submitUrls({ - config, - urls, - fetch: tooLong, - sleep: clock.sleep, - log: createLogger(), + expect(await submitUrls(config, batch, { fetch, sleep: createClock().sleep })).toEqual({ + ok: true, }); - expect(stopped.stop?.reason).toBe('rate-limited'); - expect(tooLong).toHaveBeenCalledTimes(1); }); - test('retries server errors with back-off and then gives up', async () => { + test('retries server errors with back-off, then gives up', async () => { const fetch = fetchWith( new Response('', { status: 503 }), new Response('', { status: 503 }), @@ -1434,82 +578,61 @@ describe('submitting URLs', () => { new Response('unavailable', { status: 503 }) ); const clock = createClock(); - const result = await submitUrls({ - config, - urls, - fetch, - sleep: clock.sleep, - log: createLogger(), + expect(await submitUrls(config, batch, { fetch, sleep: clock.sleep })).toEqual({ + ok: false, + status: 503, + detail: 'unavailable', }); expect(clock.sleep.mock.calls.map(([ms]) => ms)).toEqual([5_000, 30_000, 120_000]); - expect(result.accepted).toBe(0); - expect(result.chunks).toEqual([{ size: 3, status: 503, attempts: 4 }]); - expect(result.stop).toEqual({ reason: 'failed', status: 503, detail: 'unavailable' }); }); - test('treats network errors like server errors', async () => { + test('succeeds when a retry gets through, and treats network errors like server errors', async () => { + const clock = createClock(); + const recovered = fetchWith( + new Response('', { status: 502 }), + new Response('', { status: 200 }) + ); + expect(await submitUrls(config, batch, { fetch: recovered, sleep: clock.sleep })).toEqual({ + ok: true, + }); + expect(clock.sleep).toHaveBeenCalledTimes(1); + const boom = new Error('socket hang up'); - const fetch = fetchWith(boom, boom, boom, boom); - const result = await submitUrls({ - config, - urls, - fetch, - sleep: createClock().sleep, - log: createLogger(), + const offline = fetchWith(boom, boom, boom, boom); + expect(await submitUrls(config, batch, { fetch: offline, sleep: clock.sleep })).toEqual({ + ok: false, + status: 0, + detail: 'socket hang up', }); - expect(result.stop).toEqual({ reason: 'failed', status: 0, detail: 'socket hang up' }); }); - test.each([400, 403, 422])('does not retry HTTP %s', async (status) => { - const fetch = fetchWith(new Response('bad key', { status })); + test.each([400, 403, 422, 429])('does not retry HTTP %s', async (status) => { + const fetch = fetchWith(new Response('nope', { status })); const clock = createClock(); - const result = await submitUrls({ - config, - urls, - fetch, - sleep: clock.sleep, - log: createLogger(), + expect(await submitUrls(config, batch, { fetch, sleep: clock.sleep })).toEqual({ + ok: false, + status, + detail: 'nope', }); - expect(result.stop).toEqual({ reason: 'failed', status, detail: 'bad key' }); expect(fetch).toHaveBeenCalledTimes(1); expect(clock.sleep).not.toHaveBeenCalled(); }); - - test('reads Retry-After as seconds or as an HTTP date', () => { - expect(parseRetryAfter('120', 0)).toBe(120_000); - expect( - parseRetryAfter('Wed, 21 Oct 2026 07:28:00 GMT', Date.parse('Wed, 21 Oct 2026 07:27:00 GMT')) - ).toBe(60_000); - expect( - parseRetryAfter('Wed, 21 Oct 2026 07:28:00 GMT', Date.parse('Wed, 21 Oct 2026 07:29:00 GMT')) - ).toBe(0); - expect(parseRetryAfter('soon', 0)).toBeUndefined(); - expect(parseRetryAfter(null, 0)).toBeUndefined(); - }); }); -describe('live-site checks', () => { - test('verifies that the key file contains the key', async () => { +describe('the live site', () => { + test('checks that the key file is served and contains the key', async () => { const site = createSite([]); - expect(await verifyKeyFile(config, site.fetch)).toEqual({ ok: true }); + expect(await keyFileProblem(config, site.fetch)).toBeNull(); site.keyLive = false; - expect(await verifyKeyFile(config, site.fetch)).toEqual({ - ok: false, - detail: `GET https://aspire.dev/${KEY}.txt returned HTTP 404`, - }); + expect(await keyFileProblem(config, site.fetch)).toBe( + `GET https://aspire.dev/${KEY}.txt returned HTTP 404` + ); const wrong = vi.fn().mockResolvedValue(new Response('another-key')); - expect(await verifyKeyFile(config, wrong)).toMatchObject({ - ok: false, - detail: expect.stringContaining('does not contain'), - }); - + expect(await keyFileProblem(config, wrong)).toMatch(/does not contain/); const down = vi.fn().mockRejectedValue(new Error('offline')); - expect(await verifyKeyFile(config, down)).toMatchObject({ - ok: false, - detail: expect.stringContaining('offline'), - }); + expect(await keyFileProblem(config, down)).toMatch(/offline/); }); test("derives the What's new page of a release", () => { @@ -1520,7 +643,7 @@ describe('live-site checks', () => { expect(() => whatsNewPath('13.6.1')).toThrow(); }); - test('waits until the release page is served', async () => { + test('waits until the release page is served, and gives up when it never is', async () => { const site = createSite([]); const clock = createClock(); let polls = 0; @@ -1528,46 +651,20 @@ describe('live-site checks', () => { if (++polls === 2) site.liveReleases.add('13.6'); return Promise.resolve(); }); - const ready = await waitForRelease({ - host: HOST, - version: '13.6', - fetch: site.fetch, - sleep: clock.sleep, - timeoutMs: 600_000, - intervalMs: 60_000, - log: createLogger(), - }); - expect(ready).toBe(true); + const deps = { fetch: site.fetch, sleep: clock.sleep, now: clock.now }; + expect( + await waitForRelease(HOST, '13.6', deps, { timeoutMs: 600_000, intervalMs: 60_000 }) + ).toBe(true); expect(clock.sleep).toHaveBeenCalledTimes(2); expect(site.fetch.mock.calls.every(([, init]) => init?.method === 'HEAD')).toBe(true); - }); - - test('gives up when the release page never appears', async () => { - const site = createSite([]); - const clock = createClock(); - const start = clock.now().getTime(); - const ready = await waitForRelease({ - host: HOST, - version: '13.6', - fetch: site.fetch, - sleep: clock.sleep, - now: () => clock.now().getTime(), - timeoutMs: 180_000, - intervalMs: 60_000, - log: createLogger(), - }); - expect(ready).toBe(false); - expect(clock.now().getTime() - start).toBeLessThanOrEqual(180_000); - expect(await isReleaseLive(HOST, '13.6', site.fetch)).toBe(false); - }); - test('reads the deployment identity and tolerates failures', async () => { - const site = createSite([]); - expect(await readDeployment(HOST, site.fetch)).toBe('deadbeefcafe0123456789'); - const down = vi.fn().mockRejectedValue(new Error('offline')); - expect(await readDeployment(HOST, down)).toBe(''); - const missing = vi.fn().mockResolvedValue(new Response('')); - expect(await readDeployment(HOST, missing)).toBe(''); + const idle = createClock(); + const start = idle.now().getTime(); + const never = { fetch: site.fetch, sleep: idle.sleep, now: idle.now }; + expect( + await waitForRelease(HOST, '14.0', never, { timeoutMs: 180_000, intervalMs: 60_000 }) + ).toBe(false); + expect(idle.now().getTime() - start).toBeLessThanOrEqual(180_000); }); }); @@ -1585,79 +682,29 @@ describe('triggers', () => { }, }); - test('recognizes a merged release/X.Y pull request', () => { - expect(detectReleaseMerge(pullRequest())).toEqual({ version: '13.6', reason: 'release/13.6' }); - expect(detectReleaseMerge(pullRequest({}, { ref: 'release/14.0' }))).toEqual({ - version: '14.0', - reason: 'release/14.0', - }); - }); - - test('recognizes the "Merge release/X.Y into main" repair pull request', () => { - const event = pullRequest( - { title: 'Merge release/13.4 into main' }, - { ref: 'joperezr/merge-release-13-4-into-main' } - ); - expect(detectReleaseMerge(event)).toEqual({ version: '13.4', reason: 'release/13.4' }); + test('recognizes a merged release/X.Y pull request from this repository', () => { + expect(releaseVersion(pullRequest())).toBe('13.6'); + expect(releaseVersion(pullRequest({}, { ref: 'release/14.0' }))).toBe('14.0'); }); test.each([ - ['an unmerged pull request', pullRequest({ merged: false })], - [ - 'a fork whose branch is named release/13.6', - pullRequest({}, { repo: { full_name: 'someone/aspire.dev' } }), - ], - ['a head without a repository', pullRequest({}, { repo: null })], + ['one that was closed without merging', pullRequest({ merged: false })], + ['one from a fork', pullRequest({}, { repo: { full_name: 'someone/aspire.dev' } })], + ['one from another branch', pullRequest({}, { ref: 'someone/fix-typo' })], + ['one from release/X.Y.Z', pullRequest({}, { ref: 'release/13.6.1' })], [ - 'a branch that is not a release branch', - pullRequest({ title: 'Fix typo' }, { ref: 'release-notes' }), + 'one that only has a release title', + pullRequest({ title: 'Merge release/13.4 into main' }, { ref: 'joperezr/merge-13-4' }), ], - ['a patch release branch', pullRequest({ title: 'Fix typo' }, { ref: 'release/13.6.1' })], - ['an event without a pull request', { action: 'closed' }], - ['a non-object payload', 'closed'], - ['a missing payload', undefined], + ['a payload without a pull request', {}], + ['no payload', undefined], ])('ignores %s', (_name, event) => { - expect(detectReleaseMerge(event)).toBeNull(); - }); - - const cli = (overrides: Partial> = {}) => ({ - ...parseCli(['--state-dir', stateDir]), - ...overrides, - }); - - test('a release merge flushes and gates on the release', () => { - expect(resolveRun(cli({ trigger: 'pull_request' }), pullRequest())).toEqual({ - mode: 'flush', - release: '13.6', - reason: 'release/13.6', - }); - }); - - test('a pull request that is not a release merge is skipped', () => { - const resolved = resolveRun(cli({ trigger: 'pull_request' }), pullRequest({ merged: false })); - expect(resolved.skip).toMatch(/not a merged release/); - }); - - test('describes scheduled and manual runs', () => { - expect(resolveRun(cli({ trigger: 'schedule' }), undefined)).toEqual({ - mode: 'daily', - release: undefined, - reason: 'schedule', - }); - expect(resolveRun(cli({ trigger: 'workflow_dispatch', mode: 'flush' }), undefined).reason).toBe( - 'manual flush' - ); - expect( - resolveRun(cli({ trigger: 'workflow_dispatch', mode: 'flush', release: '13.6' }), undefined) - ).toEqual({ - mode: 'flush', - release: '13.6', - reason: 'release/13.6', - }); - expect(resolveRun(cli({ trigger: 'workflow_dispatch' }), undefined).reason).toBe('manual'); + expect(releaseVersion(event)).toBeNull(); }); +}); - test('parses command-line options', () => { +describe('command line', () => { + test('reads the options and applies the defaults', () => { expect(parseCli(['--state-dir', stateDir])).toEqual({ stateDir, mode: 'daily', @@ -1669,57 +716,39 @@ describe('triggers', () => { }); expect( parseCli([ - '--state-dir', - stateDir, - '--mode', - 'flush', - '--limit', - '250', - '--release', - '13.6', - '--trigger', - 'workflow_dispatch', - '--dry-run', - '--allow-shrink', + ...['--state-dir', stateDir, '--mode', 'flush', '--limit', '500', '--release', '13.6'], + ...['--trigger', 'workflow_dispatch', '--dry-run', '--allow-shrink'], ]) - ).toMatchObject({ + ).toEqual({ + stateDir, mode: 'flush', - limit: 250, + limit: 500, release: '13.6', + trigger: 'workflow_dispatch', dryRun: true, allowShrink: true, }); }); - test('reads the changed files option as an absolute path', () => { - expect( - parseCli(['--state-dir', stateDir, '--changed-files', 'changes.jsonl']).changedFiles - ).toBe(path.resolve('changes.jsonl')); - expect(parseCli(['--state-dir', stateDir]).changedFiles).toBeUndefined(); - }); - test.each([ ['no state directory', []], ['an unknown mode', ['--state-dir', 'x', '--mode', 'weekly']], - ['a zero limit', ['--state-dir', 'x', '--limit', '0']], - ['a limit above the maximum', ['--state-dir', 'x', '--limit', '50001']], - ['a fractional limit', ['--state-dir', 'x', '--limit', '1.5']], - ['a non-numeric limit', ['--state-dir', 'x', '--limit', 'many']], - ['a malformed release', ['--state-dir', 'x', '--release', '13']], - ['an unknown flag', ['--state-dir', 'x', '--force']], + ['a limit below 1', ['--state-dir', 'x', '--limit', '0']], + ['a limit above one request', ['--state-dir', 'x', '--limit', '10001']], + ['a limit that is not a number', ['--state-dir', 'x', '--limit', 'many']], + ['a release that is not X.Y', ['--state-dir', 'x', '--release', '13']], + ['an unknown flag', ['--state-dir', 'x', '--changed-files', 'f.jsonl']], ])('rejects %s', (_name, argv) => { expect(() => parseCli(argv)).toThrow(); }); }); describe('running end to end', () => { - let site: Site; - let log: ReturnType; + let site: ReturnType; let clock: ReturnType; beforeEach(() => { site = createSite(SMALL_SITE); - log = createLogger(); clock = createClock(); }); @@ -1730,32 +759,28 @@ describe('running end to end', () => { stateDir, mode: 'daily', trigger: 'schedule', - reason: 'schedule', fetch: site.fetch, sleep: clock.sleep, now: clock.now, - log, ...overrides, }); }; const submitted = () => site.posts.flatMap((post) => post.urlList); - const sent = (paths: string[]) => paths.map(url); + const savedState = () => loadState(stateDir, HOST); + const readState = () => readFile(path.join(stateDir, 'state.json'), 'utf8'); test('a cold start submits the first window and seeds the state', async () => { const result = await runDay(); - expect(result.report).toMatchObject({ - outcome: 'completed', - coldStart: true, - submitted: 3, - totalUrls: 10, - deployment: 'deadbeefcafe0123456789', - }); - expect(result.stateChanged).toBe(true); - expect(submitted()).toEqual(sent(['/', '/app-host/', '/get-started/'])); + expect(result).toMatchObject({ outcome: 'completed', submitted: 3, stateChanged: true }); + expect(result.summary).toContain('## IndexNow submission for aspire.dev: completed'); + expect(result.summary).toContain('Sitemap: 10 URLs'); + expect(result.summary).toContain('Pass: rotation, 3 of 10 URLs covered (30%)'); + expect(result.summary).toContain('Cold start'); + expect(submitted()).toEqual(urls(['/', '/app-host/', '/get-started/'])); - const saved = await loadState(stateDir, HOST, log); - expect(saved?.known).toHaveLength(10); - expect(saved?.cycle?.cursor).toBe(url('/get-started/')); + const state = await savedState(); + expect(state?.known).toHaveLength(10); + expect(state?.cycle?.cursor).toBe(url('/get-started/')); }); test('daily runs walk the sitemap without repeating a URL and then start over', async () => { @@ -1765,119 +790,61 @@ describe('running end to end', () => { days.push(submitted()); } expect(days).toEqual([ - sent(['/', '/app-host/', '/get-started/']), - sent(['/whats-new/aspire-13-6/', '/reference/api/csharp/a/', '/reference/api/csharp/b/']), - sent(['/reference/api/typescript/c/', '/de/', '/de/app-host/']), - sent(['/ja/']), - sent(['/', '/app-host/', '/get-started/']), + urls(['/', '/app-host/', '/get-started/']), + urls([DOCS[3], API[0], API[1]]), + urls([API[2], '/de/', '/de/app-host/']), + urls(['/ja/']), + urls(['/', '/app-host/', '/get-started/']), ]); }); - test('new and removed URLs are sent first and nothing is sent twice within a cycle', async () => { + test('removed and new URLs are sent first, and only a new page ahead of the cursor repeats', async () => { await runDay(); - const day1 = submitted(); + const days = [submitted()]; site.paths = [ ...site.paths.filter((entry) => entry !== '/ja/'), '/aaa/', '/reference/api/csharp/ab/', ]; - const day2Result = await runDay(); - const day2 = submitted(); - expect(day2.slice(0, 3)).toEqual(sent(['/ja/', '/aaa/', '/reference/api/csharp/ab/'])); - expect(day2Result.report).toMatchObject({ newUrls: 2, removedUrls: 1, submitted: 3 }); - - await runDay(); - const day3 = submitted(); - await runDay(); - const day4 = submitted(); - - const cycle = [...day1, ...day2, ...day3, ...day4]; - expect(new Set(cycle).size).toBe(cycle.length); - expect(new Set(cycle)).toEqual(new Set([...site.paths.map(url), url('/ja/')])); - }); - - test('a flush restarts from the top at the release budget and then returns to the slow rotation', async () => { - await runDay(); - const flush = await runDay({ mode: 'flush', reason: 'release/13.6' }); - expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); - expect(flush.report.cycleKind).toBe('release'); + const second = await runDay(); + expect(second.summary).toContain('2 new and 1 removed'); + expect(submitted()).toEqual(urls(['/ja/', '/aaa/', '/reference/api/csharp/ab/'])); + days.push(submitted()); - await runDay(); - expect(submitted()).toEqual( - sent(['/reference/api/csharp/b/', '/reference/api/typescript/c/', ...LOCALIZED]) - ); - const saved = await loadState(stateDir, HOST, log); - expect(saved?.cycle).toBeNull(); - - await runDay(); - expect(submitted()).toEqual(sent(['/', '/app-host/', '/get-started/'])); - }); - - test('a second pull request for a release whose cycle is running continues it instead of starting over', async () => { - await runDay(); - const release = { mode: 'flush', trigger: 'pull_request', reason: 'release/13.6' } as const; - const first = await runDay(release); - expect(first.report).toMatchObject({ mode: 'flush', cycleKind: 'release' }); - expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); - - const second = await runDay(release); - expect(second.report).toMatchObject({ mode: 'daily', cycleKind: 'release' }); - expect(second.report.note).toMatch(/already running/); - expect(submitted()).toEqual( - sent(['/reference/api/csharp/b/', '/reference/api/typescript/c/', ...LOCALIZED]) + for (let day = 0; day < 3; day++) { + await runDay(); + days.push(submitted()); + } + const everything = days.flat(); + const seen = new Set(everything); + expect(seen).toEqual(new Set([...urls(site.paths), url('/ja/')])); + expect(everything).toHaveLength(seen.size + 1); + expect(everything.filter((entry) => entry === url('/reference/api/csharp/ab/'))).toHaveLength( + 2 ); - expect((await loadState(stateDir, HOST, log))?.cycle).toBeNull(); }); - test('a pull request for another release, and a manual flush, still start the burst over', async () => { + test('a flush restarts from the top at the release budget and then returns to the rotation', async () => { await runDay(); - await runDay({ mode: 'flush', trigger: 'pull_request', reason: 'release/13.6' }); - - const other = await runDay({ mode: 'flush', trigger: 'pull_request', reason: 'release/13.7' }); - expect(other.report.mode).toBe('flush'); - expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); - - const manual = await runDay({ - mode: 'flush', - trigger: 'workflow_dispatch', - reason: 'release/13.7', - }); - expect(manual.report.mode).toBe('flush'); - expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); - }); + const flush = await runDay({ mode: 'flush', release: undefined }); + expect(flush.summary).toContain('Pass: release'); + expect(submitted()).toEqual(urls([...DOCS, API[0]])); - test('reports the new URLs that did not fit in the run as deferred', async () => { await runDay(); - site.paths = [...site.paths, '/aaa/', '/aab/', '/aac/']; - const limited = await runDay({ limit: 2 }); - expect(limited.report).toMatchObject({ newUrls: 3, deferred: 1 }); - }); + expect(submitted()).toEqual(urls([API[1], API[2], ...LOCALIZED])); + expect((await savedState())?.cycle).toBeNull(); - test('a flush defers none of its new URLs, because the cycle it starts covers them', async () => { await runDay(); - site.paths = [...site.paths, '/aaa/', '/aab/', '/aac/']; - const flush = await runDay({ mode: 'flush', reason: 'release/13.6', limit: 2 }); - expect(flush.report).toMatchObject({ newUrls: 3, deferred: 0 }); - }); - - test('a cold start that fails to submit still seeds the state', async () => { - site.responses = [422]; - const result = await runDay(); - expect(result.report.outcome).toBe('failed'); - expect(result.stateChanged).toBe(true); - const saved = await loadState(stateDir, HOST, log); - expect(saved?.known).toHaveLength(SMALL_SITE.length); - expect(saved?.cycle?.cursor).toBeNull(); + expect(submitted()).toEqual(urls(['/', '/app-host/', '/get-started/'])); }); test('skips a cold start while the key file is not live yet', async () => { site.keyLive = false; const result = await runDay(); - expect(result.report.outcome).toBe('skipped'); - expect(result.stateChanged).toBe(false); + expect(result).toMatchObject({ outcome: 'skipped', submitted: 0, stateChanged: false }); expect(site.posts).toHaveLength(0); - expect(log.warn).toHaveBeenCalled(); + expect(await savedState()).toBeNull(); }); test('fails when the key file disappears after the first successful run', async () => { @@ -1886,287 +853,122 @@ describe('running end to end', () => { await expect(runDay()).rejects.toThrow(/key file is not reachable/); }); - test('a dry run plans without submitting or saving', async () => { - const result = await runDay({ dryRun: true }); - expect(result.report).toMatchObject({ outcome: 'dry-run', submitted: 3, totalUrls: 10 }); - expect(result.stateChanged).toBe(false); - expect(site.posts).toHaveLength(0); - expect(await readdir(stateDir)).toEqual([]); - }); - - test('a dry run still plans while the key file is missing', async () => { + test('a dry run plans without submitting or saving, even while the key file is missing', async () => { site.keyLive = false; const result = await runDay({ dryRun: true }); - expect(result.report.outcome).toBe('dry-run'); - expect(log.warn).toHaveBeenCalledWith(expect.stringContaining('key file is live')); + expect(result).toMatchObject({ outcome: 'dry-run', submitted: 3, stateChanged: false }); + expect(result.summary).toContain('Dry run'); + expect(site.posts).toHaveLength(0); + expect(await savedState()).toBeNull(); }); test('waits for the release page before submitting', async () => { - let polls = 0; - clock.sleep.mockImplementation(() => { - if (++polls === 2) site.liveReleases.add('13.6'); + clock.sleep.mockImplementationOnce(() => { + site.liveReleases.add('13.6'); return Promise.resolve(); }); - const result = await runDay({ mode: 'flush', release: '13.6', reason: 'release/13.6' }); - expect(result.report.outcome).toBe('completed'); - expect(polls).toBe(2); + const result = await runDay({ mode: 'flush', release: '13.6' }); + expect(result.outcome).toBe('completed'); + expect(result.summary).toContain('release 13.6'); + expect(clock.sleep).toHaveBeenCalledTimes(1); expect(site.posts).toHaveLength(1); }); test('does not submit when the release page never goes live', async () => { - await expect( - runDay({ - mode: 'flush', - release: '13.6', - reason: 'release/13.6', - gate: { timeoutMs: 120_000, intervalMs: 60_000 }, - }) - ).rejects.toThrow(/did not go live/); + await expect(runDay({ mode: 'flush', release: '13.6' })).rejects.toThrow(/did not go live/); expect(site.posts).toHaveLength(0); - expect(await readdir(stateDir)).toEqual([]); - }); - - test('keeps the progress made before a 429 and resumes from it', async () => { - site.paths = ['/', ...syntheticUrls(12_000).map((entry) => new URL(entry).pathname)]; - site.responses = [200, 429]; - const limited = await runDay({ limit: 12_001 }); - expect(limited.report.outcome).toBe('rate-limited'); - expect(limited.report.submitted).toBe(10_000); - expect(limited.failure).toBeUndefined(); - expect(limited.stateChanged).toBe(true); - const firstBatch = site.posts[0].urlList; - - const resumed = await runDay({ limit: 3 }); - expect(resumed.report.outcome).toBe('completed'); - expect(submitted()).toEqual( - [...site.paths.map(url)].sort((a, b) => compareUrls(a, b)).slice(10_000, 10_003) - ); - expect(firstBatch).toHaveLength(10_000); - }); - - test('keeps the progress made before a rejected chunk and reports the failure', async () => { - site.paths = ['/', ...syntheticUrls(12_000).map((entry) => new URL(entry).pathname)]; - site.responses = [200, 422]; - const failed = await runDay({ limit: 12_001 }); - expect(failed.report.outcome).toBe('failed'); - expect(failed.failure).toMatch(/HTTP 422/); - expect(failed.stateChanged).toBe(true); - expect((await loadState(stateDir, HOST, log))?.cycle?.cursor).not.toBeNull(); - }); - - test('a rejection of the first chunk leaves an existing state untouched', async () => { - await runDay(); - const before = await readFile(path.join(stateDir, 'state.json'), 'utf8'); - site.responses = [403]; - const failed = await runDay(); - expect(failed.failure).toMatch(/HTTP 403/); - expect(failed.stateChanged).toBe(false); - expect(await readFile(path.join(stateDir, 'state.json'), 'utf8')).toBe(before); - }); - - test('refuses a sitemap that suddenly lost most of its pages and keeps the state', async () => { - await runDay(); - const before = await readFile(path.join(stateDir, 'state.json'), 'utf8'); - site.paths = site.paths.slice(0, 5); - await expect(runDay()).rejects.toThrow(/--allow-shrink/); - expect(await readFile(path.join(stateDir, 'state.json'), 'utf8')).toBe(before); + expect(await savedState()).toBeNull(); }); - const releaseFiles: ChangedFile[] = [ - { - status: 'added', - filename: 'src/frontend/src/content/docs/get-started/new-page.mdx', - changes: 40, - patch: null, - }, - { - status: 'modified', - filename: 'src/frontend/src/content/docs/whats-new/aspire-13-6.mdx', - changes: 6, - patch: null, - }, - { - status: 'renamed', - filename: 'src/frontend/src/data/pkgs/A.13.6.0.json', - changes: 9, - patch: '@@ -9 +9 @@\n+ "name": "AsExpress",', - }, - { - // Only moved to the new version, so its pages did not change. - status: 'renamed', - filename: 'src/frontend/src/data/pkgs/B.13.6.0.json', - changes: 4, - patch: '@@ -4 +4 @@\n- "version": "13.5.4",\n+ "version": "13.6.0",', - }, - ]; - - test('a release flush submits its new and updated pages in tier order, then the rest, and sends nothing twice', async () => { - await runDay(); - site.paths = [...site.paths, '/get-started/new-page/']; - - const flush = await runDay({ - mode: 'flush', - reason: 'release/13.6', - changedFiles: releaseFiles, - }); - expect(flush.report).toMatchObject({ newUrls: 1, updatedUrls: 3, cycleKind: 'release' }); - expect(submitted()).toEqual( - sent([ - '/get-started/new-page/', // new docs page - '/whats-new/aspire-13-6/', // updated docs page - '/reference/api/csharp/a/', // updated API package (B only moved to the new version) - '/', // then everything else, in the usual order - '/app-host/', - ]) - ); - - // The updated pages are remembered, so the next runs finish the rest without repeating them. - const days = [submitted()]; - for (let day = 0; day < 2; day++) { + test.each([429, 422])( + 'a request answered with HTTP %s fails and leaves the state alone', + async (status) => { await runDay(); - days.push(submitted()); - } - expect(days[1]).toEqual( - sent([ - '/get-started/', - '/reference/api/csharp/b/', - '/reference/api/typescript/c/', - '/de/', - '/de/app-host/', - ]) - ); - expect(days[2]).toEqual(sent(['/ja/'])); - const everything = days.flat(); - expect(new Set(everything).size).toBe(everything.length); - expect(new Set(everything)).toEqual(new Set(site.paths.map(url))); - expect((await loadState(stateDir, HOST, log))?.cycle).toBeNull(); - }); - - test('a release flush puts new generated API pages after the updated docs, not before them', async () => { - await runDay(); - site.paths = [...site.paths, '/reference/api/csharp/ab/', '/reference/api/csharp/ac/']; - - const flush = await runDay({ - mode: 'flush', - reason: 'release/13.6', - changedFiles: [releaseFiles[1]], - }); - expect(flush.report).toMatchObject({ newUrls: 2, updatedUrls: 3, cycleKind: 'release' }); - expect(submitted()).toEqual( - sent([ - '/whats-new/aspire-13-6/', // updated docs - '/reference/api/csharp/ab/', // new API pages, ordered with the updated pages - '/reference/api/csharp/ac/', - '/', // then everything else, in the usual order - '/app-host/', - ]) - ); + const before = await readState(); + site.responses = [status]; + const result = await runDay(); + expect(result).toMatchObject({ outcome: 'failed', submitted: 0, stateChanged: false }); + expect(result.failure).toContain(String(status)); + expect(result.failure).toContain('The next run tries again.'); + expect(await readState()).toBe(before); - const days = [submitted()]; - for (let day = 0; day < 2; day++) { await runDay(); - days.push(submitted()); + expect(submitted()).toEqual(urls([DOCS[3], API[0], API[1]])); } - const everything = days.flat(); - expect(new Set(everything).size).toBe(everything.length); - expect(new Set(everything)).toEqual(new Set(site.paths.map(url))); - expect((await loadState(stateDir, HOST, log))?.cycle).toBeNull(); - }); + ); - test('a flush that is rate limited before anything is accepted still starts the release cycle', async () => { + test('a flush whose request failed still starts the release pass', async () => { await runDay(); site.responses = [429]; - const limited = await runDay({ mode: 'flush', reason: 'release/13.6' }); - expect(limited.report.outcome).toBe('rate-limited'); - expect(limited.failure).toMatch(/before accepting any URL.*release cycle/); - expect(limited.stateChanged).toBe(true); - expect((await loadState(stateDir, HOST, log))?.cycle).toMatchObject({ - kind: 'release', - reason: 'release/13.6', - cursor: null, - }); + const result = await runDay({ mode: 'flush' }); + expect(result).toMatchObject({ outcome: 'failed', stateChanged: true }); + expect(result.failure).toContain('release pass was started'); + expect((await savedState())?.cycle).toEqual({ kind: 'release', cursor: null }); - // The next scheduled run carries on with the release burst from the top. await runDay(); - expect(submitted()).toEqual(sent([...DOCS, '/reference/api/csharp/a/'])); + expect(submitted()).toEqual(urls([...DOCS, API[0]])); + }); + + test('a cold start whose request failed still seeds the state', async () => { + site.responses = [422]; + const result = await runDay(); + expect(result).toMatchObject({ outcome: 'failed', stateChanged: true }); + const state = await savedState(); + expect(state?.known).toHaveLength(10); + expect(state?.cycle?.cursor).toBeNull(); }); - test('a rate-limited daily run that accepts nothing fails and leaves the state untouched', async () => { + test('refuses a sitemap that suddenly lost most of its pages and keeps the state', async () => { await runDay(); - const before = await readFile(path.join(stateDir, 'state.json'), 'utf8'); - site.responses = [429]; - const limited = await runDay(); - expect(limited.failure).toMatch(/429/); - expect(limited.stateChanged).toBe(false); - expect(await readFile(path.join(stateDir, 'state.json'), 'utf8')).toBe(before); + const before = await readState(); + site.paths = site.paths.slice(0, 5); + await expect(runDay()).rejects.toThrow(/--allow-shrink/); + expect(await readState()).toBe(before); }); - test('replaces a poisoned state instead of submitting what it lists', async () => { + test('replaces a state that lists other sites instead of submitting what it lists', async () => { await saveState( stateDir, makeState({ known: [url('/'), 'https://evil.example/spam/', 'http://aspire.dev/insecure/'] }) ); const result = await runDay(); - expect(result.report.coldStart).toBe(true); + expect(result.summary).toContain('Cold start'); expect(submitted().every((entry) => entry.startsWith('https://aspire.dev/'))).toBe(true); - expect((await loadState(stateDir, HOST, log))?.known).toEqual(SMALL_SITE.map(url).sort()); + expect((await savedState())?.known).toEqual([...urls(SMALL_SITE)].sort()); }); -}); -describe('summary and command line', () => { - test('renders a readable summary', () => { - const summary = renderSummary( - emptyReport({ - outcome: 'completed', - trigger: 'schedule', - host: HOST, - mode: 'daily', - reason: 'schedule', - deployment: '32f2d1c895952771c0d58924ef8c9115a37b3a0a', - budget: 1_000, - totalUrls: 32_336, - newUrls: 2, - removedUrls: 1, - updatedUrls: 3_150, - windowUrls: 997, - submitted: 1_000, - cycleKind: 'rotation', - cycleCovered: 1_000, - chunks: [{ size: 1_000, status: 200, attempts: 1 }], - firstWindowUrl: url('/'), - lastWindowUrl: url('/app-host/'), - }) - ); - expect(summary).toContain('## IndexNow submission for aspire.dev'); - expect(summary).toContain('| Sitemap URLs | 32,336 |'); - expect(summary).toContain('| New / removed | 2 / 1 |'); - expect(summary).toContain( - '| Prioritized pages | 3,150 new or updated by the release, ahead of the rest |' + test('sends a single request even when the budget is larger than a request can hold', async () => { + site = createSite( + Array.from({ length: 12_000 }, (_, index) => `/docs/page-${String(index).padStart(5, '0')}/`) ); - expect(summary).toContain('| Submitted | 1,000 of a 1,000 URL budget |'); - expect(summary).toContain('| Cycle | rotation: 1,000 of 32,336 URLs (3%) |'); - expect(summary).toContain('HTTP 200 for 1,000 URLs'); - expect(summary).toContain('`32f2d1c`'); + const budgets = { release: 50_000, rotation: 50_000 }; + await runDay({ config: { ...config, budgets }, mode: 'flush' }); + expect(site.posts).toHaveLength(1); + expect(site.posts[0].urlList).toHaveLength(MAX_URLS_PER_RUN); }); +}); - async function runMain(argv: string[], event?: unknown, overrides = {}) { +describe('the command line entry point', () => { + async function runMain(argv: string[], event?: unknown) { const files = { GITHUB_EVENT_PATH: path.join(stateDir, 'event.json'), GITHUB_STEP_SUMMARY: path.join(stateDir, 'summary.md'), GITHUB_OUTPUT: path.join(stateDir, 'output.txt'), }; if (event !== undefined) await writeFile(files.GITHUB_EVENT_PATH, JSON.stringify(event)); - const log = createLogger(); - const code = await main( - ['--state-dir', path.join(stateDir, 'state'), ...argv], - files, - log, - overrides - ); + const real = await loadConfig(); + const site = createSite(SMALL_SITE, real.key); + const clock = createClock(); + const code = await main(['--state-dir', path.join(stateDir, 'state'), ...argv], files, { + fetch: site.fetch, + sleep: clock.sleep, + now: clock.now, + }); const read = (file: string) => readFile(file, 'utf8').catch(() => ''); return { code, - log, + site, + real, summary: await read(files.GITHUB_STEP_SUMMARY), output: await read(files.GITHUB_OUTPUT), }; @@ -2182,102 +984,54 @@ describe('summary and command line', () => { }); test('skips a merged pull request that is not a release', async () => { - const { code, output, summary } = await runMain( + const { code, output, summary, site } = await runMain( ['--trigger', 'pull_request'], pullRequest('Fix a typo', 'someone/typo') ); expect(code).toBe(0); - expect(output).toContain('state_changed=false'); - expect(output).toContain('outcome=skipped'); + expect(output).toBe('state_changed=false\n'); + expect(summary).toContain('skipped'); expect(summary).toContain('not a merged release'); + expect(site.fetch).not.toHaveBeenCalled(); }); - test('a merged release pull request waits for the release page and flushes', async () => { + test('a merged release pull request starts a burst once the release page is live', async () => { + const files = { GITHUB_EVENT_PATH: path.join(stateDir, 'event.json') }; + await writeFile( + files.GITHUB_EVENT_PATH, + JSON.stringify(pullRequest('Content for release: 13.6', 'release/13.6')) + ); const real = await loadConfig(); const site = createSite(SMALL_SITE, real.key); site.liveReleases.add('13.6'); const clock = createClock(); - const { code, output, summary } = await runMain( - ['--trigger', 'pull_request'], - pullRequest('Content for release: 13.6', 'release/13.6'), - { fetch: site.fetch, sleep: clock.sleep, now: clock.now } - ); + const env = { ...files, GITHUB_OUTPUT: path.join(stateDir, 'output.txt') }; + const code = await main(['--state-dir', stateDir, '--trigger', 'pull_request'], env, { + fetch: site.fetch, + sleep: clock.sleep, + now: clock.now, + }); + expect(code).toBe(0); - expect(output).toContain('state_changed=true'); - expect(output).toContain(`submitted=${SMALL_SITE.length}`); - expect(summary).toContain('release/13.6'); + expect(await readFile(env.GITHUB_OUTPUT, 'utf8')).toBe('state_changed=true\n'); expect(site.posts).toHaveLength(1); expect(site.posts[0]).toMatchObject({ host: real.host, key: real.key }); - expect(site.posts[0].urlList).toEqual(SMALL_SITE.map(url)); - }); - - test('a release pull request sends the pages its files changed first', async () => { - const real = await loadConfig(); - const site = createSite(SMALL_SITE, real.key); - site.liveReleases.add('13.6'); - const clock = createClock(); - const changedFiles = path.join(stateDir, 'changes.jsonl'); - await writeFile( - changedFiles, - [ - { - status: 'modified', - filename: 'src/frontend/src/content/docs/get-started/index.mdx', - changes: 3, - }, - { - status: 'renamed', - filename: 'src/frontend/src/data/pkgs/A.13.6.0.json', - changes: 8, - patch: '@@ -9 +9 @@\n+ "name": "AsExpress",', - }, - { status: 'modified', filename: 'src/frontend/src/components/Header.astro', changes: 2 }, - ] - .map((entry) => JSON.stringify(entry)) - .join('\n') - ); - const { code, summary } = await runMain( - ['--trigger', 'pull_request', '--changed-files', changedFiles], - pullRequest('Content for release: 13.6', 'release/13.6'), - { fetch: site.fetch, sleep: clock.sleep, now: clock.now } - ); - expect(code).toBe(0); - expect(summary).toContain( - '| Prioritized pages | 2 new or updated by the release, ahead of the rest |' - ); - expect(site.posts[0].urlList.slice(0, 3)).toEqual( - ['/get-started/', '/reference/api/csharp/a/', '/'].map(url) - ); - expect(site.posts[0].urlList).toHaveLength(SMALL_SITE.length); - }); - - test('fails when the changed files cannot be read', async () => { - const { code, log } = await runMain(['--changed-files', path.join(stateDir, 'missing.jsonl')]); - expect(code).toBe(1); - expect(log.error).toHaveBeenCalledWith(expect.stringContaining('missing.jsonl')); + expect(site.posts[0].urlList).toEqual(urls(SMALL_SITE)); + expect((await loadState(stateDir, real.host))?.cycle).toBeNull(); }); test('reports a failed run and returns a failing exit code', async () => { - const real = await loadConfig(); - const site = createSite(SMALL_SITE, real.key); - const clock = createClock(); - // A first run seeds the state; the key then disappears, which is a hard failure. - await runMain([], undefined, { fetch: site.fetch, sleep: clock.sleep, now: clock.now }); - site.keyLive = false; - const { code, log, summary, output } = await runMain([], undefined, { - fetch: site.fetch, - sleep: clock.sleep, - now: clock.now, - }); + const { code, output, summary } = await runMain(['--mode', 'flush', '--release', '13.6']); expect(code).toBe(1); - expect(log.error).toHaveBeenCalledWith(expect.stringContaining('key file is not reachable')); + expect(output).toBe('state_changed=false\n'); expect(summary).toContain('failed'); - expect(output).toContain('state_changed=false'); + expect(summary).toContain('did not go live'); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining('did not go live')); }); test('rejects invalid arguments', async () => { - const { code, log } = await runMain(['--mode', 'weekly']); + const { code } = await runMain(['--mode', 'weekly']); expect(code).toBe(1); - expect(log.error).toHaveBeenCalledWith(expect.stringContaining('--mode')); + expect(console.error).toHaveBeenCalledWith(expect.stringContaining('--mode')); }); }); From 0b40c7c595be2efab4018aa20a774bc1026e7687 Mon Sep 17 00:00:00 2001 From: David Pine <7679720+IEvangelist@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:16:54 -0500 Subject: [PATCH 6/6] Explain the permissions of the IndexNow job Say why the job declares permissions although the workflow sets none: a job's own block replaces the workflow default, and these two are what the checkout and the state artifact restore need. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/indexnow.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml index 182ee03fb8..1234ce25e1 100644 --- a/.github/workflows/indexnow.yml +++ b/.github/workflows/indexnow.yml @@ -83,6 +83,10 @@ jobs: group: indexnow cancel-in-progress: false queue: max + # The workflow-level "permissions: {}" gives every job no access by default. A job's own block + # replaces that default instead of adding to it, so this job gets exactly these two: contents to + # check out the repository, and actions to list and download the state artifact. Uploading the + # artifact needs no token permission. permissions: contents: read actions: read