From ef1e23f78aafda1d92a80342dcd37f33e99a82dc Mon Sep 17 00:00:00 2001 From: Leif Hagen Date: Mon, 21 Sep 2026 10:05:14 -0700 Subject: [PATCH] fix: emit absolute URLs in llms.txt, llms-full.txt, sitemap Bots read llms.txt and llms-full.txt as standalone documents, so the root-relative targets in them resolved against the host root. The site is served under /docs, so every one of those links 404'd. Prefix page targets with the docs base path in llms.txt, and rewrite root-relative markdown targets and HTML hrefs in llms-full.txt as the text leaves the app. Content files keep writing root-relative links, which Next resolves against basePath for in-app navigation. Collapse the base URL into one constant that sitemap.ts and both routes share. Drop changefreq and priority from sitemap entries, which Google ignores, and the hardcoded root entry that duplicated the index page. The index is emitted without a trailing slash so it resolves in one redirect. --- src/app/llms-full.txt/route.ts | 3 +- src/app/llms.txt/route.test.ts | 35 ++++++++++++++++ src/app/llms.txt/route.ts | 5 ++- src/app/sitemap.ts | 18 ++------ src/lib/site-url.test.ts | 75 ++++++++++++++++++++++++++++++++++ src/lib/site-url.ts | 34 +++++++++++++++ 6 files changed, 153 insertions(+), 17 deletions(-) create mode 100644 src/app/llms.txt/route.test.ts create mode 100644 src/lib/site-url.test.ts create mode 100644 src/lib/site-url.ts diff --git a/src/app/llms-full.txt/route.ts b/src/app/llms-full.txt/route.ts index fcccaee..e784ed3 100644 --- a/src/app/llms-full.txt/route.ts +++ b/src/app/llms-full.txt/route.ts @@ -1,4 +1,5 @@ import { getLLMText, source } from "@/lib/source"; +import { absolutizeRootRelativeLinks } from "@/lib/site-url"; export const revalidate = false; @@ -6,5 +7,5 @@ export async function GET() { const scan = source.getPages().map(getLLMText); const scanned = await Promise.all(scan); - return new Response(scanned.join("\n\n")); + return new Response(absolutizeRootRelativeLinks(scanned.join("\n\n"))); } diff --git a/src/app/llms.txt/route.test.ts b/src/app/llms.txt/route.test.ts new file mode 100644 index 0000000..312f34e --- /dev/null +++ b/src/app/llms.txt/route.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from "bun:test"; +import { SITE_BASE_URL } from "@/lib/site-url"; +import { GET } from "./route"; + +async function llmsTxt(): Promise { + return await (await GET()).text(); +} + +function linkTargets(body: string): string[] { + return [...body.matchAll(/\]\(([^)]+)\)/g)].map((match) => match[1]); +} + +describe("llms.txt", () => { + test("lists every documentation page", async () => { + expect(linkTargets(await llmsTxt()).length).toBeGreaterThan(50); + }); + + test("gives every link an absolute target under the docs base path", async () => { + const targets = linkTargets(await llmsTxt()); + + expect(targets).not.toBeEmpty(); + for (const target of targets) { + const underBasePath = + target === SITE_BASE_URL || target.startsWith(`${SITE_BASE_URL}/`); + + expect(underBasePath).toBe(true); + } + }); + + test("points at llms-full.txt absolutely", async () => { + expect(await llmsTxt()).toContain( + `(${SITE_BASE_URL}/llms-full.txt): Complete documentation content`, + ); + }); +}); diff --git a/src/app/llms.txt/route.ts b/src/app/llms.txt/route.ts index 36dfeed..7e703c1 100644 --- a/src/app/llms.txt/route.ts +++ b/src/app/llms.txt/route.ts @@ -1,4 +1,5 @@ import { source } from "@/lib/source"; +import { absoluteDocsUrl } from "@/lib/site-url"; export const revalidate = false; @@ -6,7 +7,7 @@ export async function GET() { const pages = source.getPages(); const lines = pages.map((page) => { - const url = page.url; + const url = absoluteDocsUrl(page.url); const title = page.data.title; const description = page.data.description || ""; @@ -21,7 +22,7 @@ ${lines.join("\n")} ## Full Documentation -- [/llms-full.txt](/llms-full.txt): Complete documentation content +- [llms-full.txt](${absoluteDocsUrl("/llms-full.txt")}): Complete documentation content `; return new Response(content); diff --git a/src/app/sitemap.ts b/src/app/sitemap.ts index bed0bed..a179b9e 100644 --- a/src/app/sitemap.ts +++ b/src/app/sitemap.ts @@ -1,19 +1,9 @@ import type { MetadataRoute } from "next"; import { source } from "@/lib/source"; - -const baseUrl = "https://api.whitepages.com/docs"; +import { absoluteDocsUrl } from "@/lib/site-url"; export default function sitemap(): MetadataRoute.Sitemap { - return [ - { - url: baseUrl, - changeFrequency: "monthly", - priority: 1, - }, - ...source.getPages().map((page) => ({ - url: `${baseUrl}${page.url}`, - changeFrequency: "weekly" as const, - priority: 0.5, - })), - ]; + return source.getPages().map((page) => ({ + url: absoluteDocsUrl(page.url), + })); } diff --git a/src/lib/site-url.test.ts b/src/lib/site-url.test.ts new file mode 100644 index 0000000..03ecfde --- /dev/null +++ b/src/lib/site-url.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from "bun:test"; +import { + SITE_BASE_URL, + absoluteDocsUrl, + absolutizeRootRelativeLinks, +} from "./site-url"; + +describe("absoluteDocsUrl", () => { + test("prefixes a page path with the docs base path", () => { + expect(absoluteDocsUrl("/documentation/mcp")).toBe( + "https://api.whitepages.com/docs/documentation/mcp", + ); + }); + + test("emits the index page without a trailing slash", () => { + expect(absoluteDocsUrl("/")).toBe("https://api.whitepages.com/docs"); + }); +}); + +describe("absolutizeRootRelativeLinks", () => { + test("rewrites a root-relative markdown target", () => { + expect( + absolutizeRootRelativeLinks( + "See [MCP Server](/documentation/mcp) for details.", + ), + ).toBe(`See [MCP Server](${SITE_BASE_URL}/documentation/mcp) for details.`); + }); + + test("rewrites a root-relative html href", () => { + expect( + absolutizeRootRelativeLinks('Billing'), + ).toBe(`Billing`); + }); + + test("keeps the fragment when a target carries one", () => { + expect( + absolutizeRootRelativeLinks("[Limits](/references/rate-limits#monthly)"), + ).toBe(`[Limits](${SITE_BASE_URL}/references/rate-limits#monthly)`); + }); + + test("rewrites every occurrence, not just the first", () => { + const result = absolutizeRootRelativeLinks( + '[A](/a) and [B](/b) and C', + ); + + expect(result).toBe( + `[A](${SITE_BASE_URL}/a) and [B](${SITE_BASE_URL}/b) and C`, + ); + }); + + test("leaves an already-absolute target untouched", () => { + const text = "[Whitepages](https://www.whitepages.com/about)"; + + expect(absolutizeRootRelativeLinks(text)).toBe(text); + }); + + test("leaves a protocol-relative target untouched", () => { + const text = + '[CDN](//cdn.example.com/a) b'; + + expect(absolutizeRootRelativeLinks(text)).toBe(text); + }); + + test("leaves a bare fragment and a mailto target untouched", () => { + const text = "[Top](#overview) and [Support](mailto:api@whitepages.com)"; + + expect(absolutizeRootRelativeLinks(text)).toBe(text); + }); + + test("leaves prose containing a slash alone", () => { + const text = "Rates are measured in requests/second (see the table)."; + + expect(absolutizeRootRelativeLinks(text)).toBe(text); + }); +}); diff --git a/src/lib/site-url.ts b/src/lib/site-url.ts new file mode 100644 index 0000000..388b534 --- /dev/null +++ b/src/lib/site-url.ts @@ -0,0 +1,34 @@ +/** + * Absolute URLs for machine-readable outputs. + * + * Crawlers fetch sitemap.xml, llms.txt, and llms-full.txt as standalone + * documents, so a root-relative target in one of them resolves against the + * host root rather than the `/docs` base path the site is served under. + * Content files keep writing root-relative links because Next prepends + * `basePath` itself for in-app navigation; the rewrite belongs here, at the + * point the text leaves the app, not in the MDX. + * + * The index page is emitted as the bare base path. Serving normalizes + * `/docs/` to `/docs` with a 308 before the 307 on to the getting-started + * page, so keeping the trailing slash would publish a two-hop redirect chain. + * + * `absolutizeRootRelativeLinks` is intentionally narrow. It rewrites a target + * only where a single leading `/` follows a markdown `](` or an HTML `href="`, + * which leaves absolute URLs, protocol-relative `//host` targets, `mailto:`, + * and bare `#anchor` fragments untouched. + */ + +export const SITE_BASE_URL = "https://api.whitepages.com/docs"; + +export function absoluteDocsUrl(path: string): string { + return path === "/" ? SITE_BASE_URL : `${SITE_BASE_URL}${path}`; +} + +export function absolutizeRootRelativeLinks(text: string): string { + const markdownTarget = /\]\(\/(?!\/)/g; + const htmlHref = /href="\/(?!\/)/g; + + return text + .replace(markdownTarget, () => `](${SITE_BASE_URL}/`) + .replace(htmlHref, () => `href="${SITE_BASE_URL}/`); +}