fix: emit absolute URLs in llms.txt, llms-full.txt, and sitemap

src/app/llms-full.txt/route.ts

@@ -1,10 +1,11 @@
 import { getLLMText, source } from "@/lib/source";
+import { absolutizeRootRelativeLinks } from "@/lib/site-url";
 
 export const revalidate = false;
 
 export async function GET() {
   const scan = source.getPages().map(getLLMText);
   const scanned = await Promise.all(scan);
 
-  return new Response(scanned.join("\n\n"));
+  return new Response(absolutizeRootRelativeLinks(scanned.join("\n\n")));
 }

src/app/llms.txt/route.test.ts

@@ -0,0 +1,35 @@
+import { describe, expect, test } from "bun:test";
+import { SITE_BASE_URL } from "@/lib/site-url";
+import { GET } from "./route";
+
+async function llmsTxt(): Promise<string> {
+  return await (await GET()).text();
+}
+
+function linkTargets(body: string): string[] {
+  return [...body.matchAll(/\]\(([^)]+)\)/g)].map((match) => match[1]);
+}
+
+describe("llms.txt", () => {
+  test("lists every documentation page", async () => {
+    expect(linkTargets(await llmsTxt()).length).toBeGreaterThan(50);
+  });
+
+  test("gives every link an absolute target under the docs base path", async () => {
+    const targets = linkTargets(await llmsTxt());
+
+    expect(targets).not.toBeEmpty();
+    for (const target of targets) {
+      const underBasePath =
+        target === SITE_BASE_URL || target.startsWith(`${SITE_BASE_URL}/`);
+
+      expect(underBasePath).toBe(true);
+    }
+  });
+
+  test("points at llms-full.txt absolutely", async () => {
+    expect(await llmsTxt()).toContain(
+      `(${SITE_BASE_URL}/llms-full.txt): Complete documentation content`,
+    );
+  });
+});

src/app/llms.txt/route.ts

@@ -1,12 +1,13 @@
 import { source } from "@/lib/source";
+import { absoluteDocsUrl } from "@/lib/site-url";
 
 export const revalidate = false;
 
 export async function GET() {
   const pages = source.getPages();
 
   const lines = pages.map((page) => {
-    const url = page.url;
+    const url = absoluteDocsUrl(page.url);
     const title = page.data.title;
     const description = page.data.description || "";
 
@@ -21,7 +22,7 @@ ${lines.join("\n")}
 
 ## Full Documentation
 
-- [/llms-full.txt](/llms-full.txt): Complete documentation content
+- [llms-full.txt](${absoluteDocsUrl("/llms-full.txt")}): Complete documentation content
 `;
 
   return new Response(content);

src/app/sitemap.ts

@@ -1,19 +1,9 @@
 import type { MetadataRoute } from "next";
 import { source } from "@/lib/source";
-
-const baseUrl = "https://api.whitepages.com/docs";
+import { absoluteDocsUrl } from "@/lib/site-url";
 
 export default function sitemap(): MetadataRoute.Sitemap {
-  return [
-    {
-      url: baseUrl,
-      changeFrequency: "monthly",
-      priority: 1,
-    },
-    ...source.getPages().map((page) => ({
-      url: `${baseUrl}${page.url}`,
-      changeFrequency: "weekly" as const,
-      priority: 0.5,
-    })),
-  ];
+  return source.getPages().map((page) => ({
+    url: absoluteDocsUrl(page.url),
+  }));
 }

src/lib/site-url.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, test } from "bun:test";
+import {
+  SITE_BASE_URL,
+  absoluteDocsUrl,
+  absolutizeRootRelativeLinks,
+} from "./site-url";
+
+describe("absoluteDocsUrl", () => {
+  test("prefixes a page path with the docs base path", () => {
+    expect(absoluteDocsUrl("/documentation/mcp")).toBe(
+      "https://api.whitepages.com/docs/documentation/mcp",
+    );
+  });
+
+  test("emits the index page without a trailing slash", () => {
+    expect(absoluteDocsUrl("/")).toBe("https://api.whitepages.com/docs");
+  });
+});
+
+describe("absolutizeRootRelativeLinks", () => {
+  test("rewrites a root-relative markdown target", () => {
+    expect(
+      absolutizeRootRelativeLinks(
+        "See [MCP Server](/documentation/mcp) for details.",
+      ),
+    ).toBe(`See [MCP Server](${SITE_BASE_URL}/documentation/mcp) for details.`);
+  });
+
+  test("rewrites a root-relative html href", () => {
+    expect(
+      absolutizeRootRelativeLinks('<a href="/references/billing">Billing</a>'),
+    ).toBe(`<a href="${SITE_BASE_URL}/references/billing">Billing</a>`);
+  });
+
+  test("keeps the fragment when a target carries one", () => {
+    expect(
+      absolutizeRootRelativeLinks("[Limits](/references/rate-limits#monthly)"),
+    ).toBe(`[Limits](${SITE_BASE_URL}/references/rate-limits#monthly)`);
+  });
+
+  test("rewrites every occurrence, not just the first", () => {
+    const result = absolutizeRootRelativeLinks(
+      '[A](/a) and [B](/b) and <a href="/c">C</a>',
+    );
+
+    expect(result).toBe(
+      `[A](${SITE_BASE_URL}/a) and [B](${SITE_BASE_URL}/b) and <a href="${SITE_BASE_URL}/c">C</a>`,
+    );
+  });
+
+  test("leaves an already-absolute target untouched", () => {
+    const text = "[Whitepages](https://www.whitepages.com/about)";
+
+    expect(absolutizeRootRelativeLinks(text)).toBe(text);
+  });
+
+  test("leaves a protocol-relative target untouched", () => {
+    const text =
+      '[CDN](//cdn.example.com/a) <a href="//cdn.example.com/b">b</a>';
+
+    expect(absolutizeRootRelativeLinks(text)).toBe(text);
+  });
+
+  test("leaves a bare fragment and a mailto target untouched", () => {
+    const text = "[Top](#overview) and [Support](mailto:api@whitepages.com)";
+
+    expect(absolutizeRootRelativeLinks(text)).toBe(text);
+  });
+
+  test("leaves prose containing a slash alone", () => {
+    const text = "Rates are measured in requests/second (see the table).";
+
+    expect(absolutizeRootRelativeLinks(text)).toBe(text);
+  });
+});

src/lib/site-url.ts

@@ -0,0 +1,34 @@
+/**
+ * Absolute URLs for machine-readable outputs.
+ *
+ * Crawlers fetch sitemap.xml, llms.txt, and llms-full.txt as standalone
+ * documents, so a root-relative target in one of them resolves against the
+ * host root rather than the `/docs` base path the site is served under.
+ * Content files keep writing root-relative links because Next prepends
+ * `basePath` itself for in-app navigation; the rewrite belongs here, at the
+ * point the text leaves the app, not in the MDX.
+ *
+ * The index page is emitted as the bare base path. Serving normalizes
+ * `/docs/` to `/docs` with a 308 before the 307 on to the getting-started
+ * page, so keeping the trailing slash would publish a two-hop redirect chain.
+ *
+ * `absolutizeRootRelativeLinks` is intentionally narrow. It rewrites a target
+ * only where a single leading `/` follows a markdown `](` or an HTML `href="`,
+ * which leaves absolute URLs, protocol-relative `//host` targets, `mailto:`,
+ * and bare `#anchor` fragments untouched.
+ */
+
+export const SITE_BASE_URL = "https://api.whitepages.com/docs";
+
+export function absoluteDocsUrl(path: string): string {
+  return path === "/" ? SITE_BASE_URL : `${SITE_BASE_URL}${path}`;
+}
+
+export function absolutizeRootRelativeLinks(text: string): string {
+  const markdownTarget = /\]\(\/(?!\/)/g;
+  const htmlHref = /href="\/(?!\/)/g;
+
+  return text
+    .replace(markdownTarget, () => `](${SITE_BASE_URL}/`)
+    .replace(htmlHref, () => `href="${SITE_BASE_URL}/`);
+}