diff --git a/src/app/llms-full.txt/route.ts b/src/app/llms-full.txt/route.ts index fcccaee..e784ed3 100644 --- a/src/app/llms-full.txt/route.ts +++ b/src/app/llms-full.txt/route.ts @@ -1,4 +1,5 @@ import { getLLMText, source } from "@/lib/source"; +import { absolutizeRootRelativeLinks } from "@/lib/site-url"; export const revalidate = false; @@ -6,5 +7,5 @@ export async function GET() { const scan = source.getPages().map(getLLMText); const scanned = await Promise.all(scan); - return new Response(scanned.join("\n\n")); + return new Response(absolutizeRootRelativeLinks(scanned.join("\n\n"))); } diff --git a/src/app/llms.txt/route.test.ts b/src/app/llms.txt/route.test.ts new file mode 100644 index 0000000..312f34e --- /dev/null +++ b/src/app/llms.txt/route.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, test } from "bun:test"; +import { SITE_BASE_URL } from "@/lib/site-url"; +import { GET } from "./route"; + +async function llmsTxt(): Promise { + return await (await GET()).text(); +} + +function linkTargets(body: string): string[] { + return [...body.matchAll(/\]\(([^)]+)\)/g)].map((match) => match[1]); +} + +describe("llms.txt", () => { + test("lists every documentation page", async () => { + expect(linkTargets(await llmsTxt()).length).toBeGreaterThan(50); + }); + + test("gives every link an absolute target under the docs base path", async () => { + const targets = linkTargets(await llmsTxt()); + + expect(targets).not.toBeEmpty(); + for (const target of targets) { + const underBasePath = + target === SITE_BASE_URL || target.startsWith(`${SITE_BASE_URL}/`); + + expect(underBasePath).toBe(true); + } + }); + + test("points at llms-full.txt absolutely", async () => { + expect(await llmsTxt()).toContain( + `(${SITE_BASE_URL}/llms-full.txt): Complete documentation content`, + ); + }); +}); diff --git a/src/app/llms.txt/route.ts b/src/app/llms.txt/route.ts index 36dfeed..7e703c1 100644 --- a/src/app/llms.txt/route.ts +++ b/src/app/llms.txt/route.ts @@ -1,4 +1,5 @@ import { source } from "@/lib/source"; +import { absoluteDocsUrl } from "@/lib/site-url"; export const revalidate = false; @@ -6,7 +7,7 @@ export async function GET() { const pages = source.getPages(); const lines = pages.map((page) => { - const url = page.url; + const url = absoluteDocsUrl(page.url); const title = page.data.title; const description = page.data.description || ""; @@ -21,7 +22,7 @@ ${lines.join("\n")} ## Full Documentation -- [/llms-full.txt](/llms-full.txt): Complete documentation content +- [llms-full.txt](${absoluteDocsUrl("/llms-full.txt")}): Complete documentation content `; return new Response(content); diff --git a/src/app/sitemap.ts b/src/app/sitemap.ts index bed0bed..a179b9e 100644 --- a/src/app/sitemap.ts +++ b/src/app/sitemap.ts @@ -1,19 +1,9 @@ import type { MetadataRoute } from "next"; import { source } from "@/lib/source"; - -const baseUrl = "https://api.whitepages.com/docs"; +import { absoluteDocsUrl } from "@/lib/site-url"; export default function sitemap(): MetadataRoute.Sitemap { - return [ - { - url: baseUrl, - changeFrequency: "monthly", - priority: 1, - }, - ...source.getPages().map((page) => ({ - url: `${baseUrl}${page.url}`, - changeFrequency: "weekly" as const, - priority: 0.5, - })), - ]; + return source.getPages().map((page) => ({ + url: absoluteDocsUrl(page.url), + })); } diff --git a/src/lib/site-url.test.ts b/src/lib/site-url.test.ts new file mode 100644 index 0000000..03ecfde --- /dev/null +++ b/src/lib/site-url.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from "bun:test"; +import { + SITE_BASE_URL, + absoluteDocsUrl, + absolutizeRootRelativeLinks, +} from "./site-url"; + +describe("absoluteDocsUrl", () => { + test("prefixes a page path with the docs base path", () => { + expect(absoluteDocsUrl("/documentation/mcp")).toBe( + "https://api.whitepages.com/docs/documentation/mcp", + ); + }); + + test("emits the index page without a trailing slash", () => { + expect(absoluteDocsUrl("/")).toBe("https://api.whitepages.com/docs"); + }); +}); + +describe("absolutizeRootRelativeLinks", () => { + test("rewrites a root-relative markdown target", () => { + expect( + absolutizeRootRelativeLinks( + "See [MCP Server](/documentation/mcp) for details.", + ), + ).toBe(`See [MCP Server](${SITE_BASE_URL}/documentation/mcp) for details.`); + }); + + test("rewrites a root-relative html href", () => { + expect( + absolutizeRootRelativeLinks('Billing'), + ).toBe(`Billing`); + }); + + test("keeps the fragment when a target carries one", () => { + expect( + absolutizeRootRelativeLinks("[Limits](/references/rate-limits#monthly)"), + ).toBe(`[Limits](${SITE_BASE_URL}/references/rate-limits#monthly)`); + }); + + test("rewrites every occurrence, not just the first", () => { + const result = absolutizeRootRelativeLinks( + '[A](/a) and [B](/b) and C', + ); + + expect(result).toBe( + `[A](${SITE_BASE_URL}/a) and [B](${SITE_BASE_URL}/b) and C`, + ); + }); + + test("leaves an already-absolute target untouched", () => { + const text = "[Whitepages](https://www.whitepages.com/about)"; + + expect(absolutizeRootRelativeLinks(text)).toBe(text); + }); + + test("leaves a protocol-relative target untouched", () => { + const text = + '[CDN](//cdn.example.com/a) b'; + + expect(absolutizeRootRelativeLinks(text)).toBe(text); + }); + + test("leaves a bare fragment and a mailto target untouched", () => { + const text = "[Top](#overview) and [Support](mailto:api@whitepages.com)"; + + expect(absolutizeRootRelativeLinks(text)).toBe(text); + }); + + test("leaves prose containing a slash alone", () => { + const text = "Rates are measured in requests/second (see the table)."; + + expect(absolutizeRootRelativeLinks(text)).toBe(text); + }); +}); diff --git a/src/lib/site-url.ts b/src/lib/site-url.ts new file mode 100644 index 0000000..388b534 --- /dev/null +++ b/src/lib/site-url.ts @@ -0,0 +1,34 @@ +/** + * Absolute URLs for machine-readable outputs. + * + * Crawlers fetch sitemap.xml, llms.txt, and llms-full.txt as standalone + * documents, so a root-relative target in one of them resolves against the + * host root rather than the `/docs` base path the site is served under. + * Content files keep writing root-relative links because Next prepends + * `basePath` itself for in-app navigation; the rewrite belongs here, at the + * point the text leaves the app, not in the MDX. + * + * The index page is emitted as the bare base path. Serving normalizes + * `/docs/` to `/docs` with a 308 before the 307 on to the getting-started + * page, so keeping the trailing slash would publish a two-hop redirect chain. + * + * `absolutizeRootRelativeLinks` is intentionally narrow. It rewrites a target + * only where a single leading `/` follows a markdown `](` or an HTML `href="`, + * which leaves absolute URLs, protocol-relative `//host` targets, `mailto:`, + * and bare `#anchor` fragments untouched. + */ + +export const SITE_BASE_URL = "https://api.whitepages.com/docs"; + +export function absoluteDocsUrl(path: string): string { + return path === "/" ? SITE_BASE_URL : `${SITE_BASE_URL}${path}`; +} + +export function absolutizeRootRelativeLinks(text: string): string { + const markdownTarget = /\]\(\/(?!\/)/g; + const htmlHref = /href="\/(?!\/)/g; + + return text + .replace(markdownTarget, () => `](${SITE_BASE_URL}/`) + .replace(htmlHref, () => `href="${SITE_BASE_URL}/`); +}