diff --git a/docs/ONLINE.md b/docs/ONLINE.md index e14ac1b..15d5984 100644 --- a/docs/ONLINE.md +++ b/docs/ONLINE.md @@ -100,6 +100,24 @@ the community site. Poster links are limited to those hosts plus the exact image-mirror host currently used by the upstream talks archive; unrelated, social, document-sharing, and deceptive lookalike domains are omitted. +### Mirrors and restricted networks + +Some networks cannot reach `raw.githubusercontent.com`. Set +`SUSTECH_ONLINE_RAW_BASE` to a plain HTTPS base that serves the same repository; +the default is +`https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master`. + +```bash +export SUSTECH_ONLINE_RAW_BASE="https://ghfast.top/https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master" +sustech online talks list --limit 5 +``` + +The raw allowlist is derived from this base. Redirects are still rejected and +the final URL must still match the constructed target exactly, so mirrors that +answer with a 3xx redirect cannot be used. Values that are not HTTPS URLs +without a query string or fragment are rejected with +`ONLINE_RAW_BASE_INVALID`. + Rendered-page metadata is optional; raw Markdown is required. Tests use frozen synthetic fixtures and do not copy the upstream contact or talks content into the package. diff --git a/src/online/manual.ts b/src/online/manual.ts index 7b21374..75409d2 100644 --- a/src/online/manual.ts +++ b/src/online/manual.ts @@ -7,11 +7,10 @@ import { ONLINE_DEFAULT_STALE_AFTER_DAYS, ONLINE_DEFAULT_TIMEOUT_MS, ONLINE_MAX_DOCUMENT_BYTES, + ONLINE_RAW_BASE_PATH, ONLINE_RAW_ORIGIN, - ONLINE_REPO_BRANCH, - ONLINE_REPO_NAME, - ONLINE_REPO_OWNER, ONLINE_SITE_ORIGIN, + onlineRawBase, scoreSearchMatch, type OnlineFetchOptions, } from "./shared.js"; @@ -242,7 +241,7 @@ export function createOnlineManualAdapter(fetchImpl: typeof fetch = globalThis.f export function onlineManualRawUrl(repoPath: string): string { assertAllowedManualRepoPath(repoPath); const segments = repoPath.split("/").map((segment) => encodeURIComponent(segment)); - return `${ONLINE_RAW_ORIGIN}/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/${segments.join("/")}`; + return `${onlineRawBase()}/${segments.join("/")}`; } export function onlineManualSiteUrl(sitePath: string): string { @@ -774,13 +773,12 @@ function validateFetchedManualUrl(value: string, expected: string, kind: "raw" | ); } if (kind === "raw") { - const expectedPrefix = `/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/docs/`; - if (url.origin !== ONLINE_RAW_ORIGIN || !url.pathname.startsWith(expectedPrefix)) { + if (url.origin !== ONLINE_RAW_ORIGIN || !url.pathname.startsWith(`${ONLINE_RAW_BASE_PATH}docs/`)) { throw new CliError("The fetched SUSTech Online manual source escaped the raw allowlist.", "ONLINE_SOURCE_NOT_ALLOWED", 2, { sourceUrl: value, }); } - assertAllowedManualRepoPath(decodeOnlinePath(url.pathname.slice(expectedPrefix.length - "docs/".length))); + assertAllowedManualRepoPath(decodeOnlinePath(url.pathname.slice(ONLINE_RAW_BASE_PATH.length))); return; } if (url.origin !== ONLINE_SITE_ORIGIN) { diff --git a/src/online/shared.ts b/src/online/shared.ts index 52695d9..5811ab6 100644 --- a/src/online/shared.ts +++ b/src/online/shared.ts @@ -13,7 +13,39 @@ export const ONLINE_SITE_ORIGIN = "https://sustech.online"; export const ONLINE_REPO_OWNER = "SUSTech-CRA"; export const ONLINE_REPO_NAME = "sustech-online-ng"; export const ONLINE_REPO_BRANCH = "master"; -export const ONLINE_RAW_ORIGIN = "https://raw.githubusercontent.com"; +const DEFAULT_ONLINE_RAW_BASE = `https://raw.githubusercontent.com/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}`; + +function normaliseOnlineRawBase(value: string): string | undefined { + const candidate = value.trim().replace(/\/+$/, ""); + if (!candidate) return undefined; + try { + const url = new URL(candidate); + if (url.protocol !== "https:" || url.search || url.hash) return undefined; + } catch { + return undefined; + } + return candidate; +} + +export function onlineRawBase(env: NodeJS.ProcessEnv = process.env): string { + const configured = env.SUSTECH_ONLINE_RAW_BASE?.trim(); + if (!configured) return DEFAULT_ONLINE_RAW_BASE; + const normalised = normaliseOnlineRawBase(configured); + if (!normalised) { + throw new CliError( + "SUSTECH_ONLINE_RAW_BASE must be a plain HTTPS URL without a query string or fragment.", + "ONLINE_RAW_BASE_INVALID", + 2, + { variable: "SUSTECH_ONLINE_RAW_BASE" }, + ); + } + return normalised; +} + +export const ONLINE_RAW_BASE = normaliseOnlineRawBase(process.env.SUSTECH_ONLINE_RAW_BASE ?? "") ?? DEFAULT_ONLINE_RAW_BASE; +const ONLINE_RAW_BASE_URL = new URL(ONLINE_RAW_BASE); +export const ONLINE_RAW_ORIGIN = ONLINE_RAW_BASE_URL.origin; +export const ONLINE_RAW_BASE_PATH = `${ONLINE_RAW_BASE_URL.pathname.replace(/\/+$/, "")}/`; export const ONLINE_TALKS_INDEX_REPO_PATH = "docs/study/talks/README.md"; export const ONLINE_TALKS_DIRECTORY_REPO_PATH = "docs/study/talks"; export const ONLINE_CONTACT_REPO_PATH = "docs/contact/README.md"; @@ -95,7 +127,7 @@ export function buildOnlineProvenance( export function onlineRawUrl(repoPath: string): string { assertAllowedOnlineRepoPath(repoPath); const segments = repoPath.split("/").map((segment) => encodeURIComponent(segment)); - return `${ONLINE_RAW_ORIGIN}/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/${segments.join("/")}`; + return `${onlineRawBase()}/${segments.join("/")}`; } export function onlineSiteUrl(sitePath: string): string { @@ -387,8 +419,7 @@ function validateFetchedUrl(value: string, expected: string, kind: "raw" | "site }); } if (kind === "raw") { - const expectedPrefix = `/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/docs/`; - if (url.origin !== ONLINE_RAW_ORIGIN || !url.pathname.startsWith(expectedPrefix)) { + if (url.origin !== ONLINE_RAW_ORIGIN || !url.pathname.startsWith(`${ONLINE_RAW_BASE_PATH}docs/`)) { throw new CliError("The fetched SUSTech Online source escaped the raw allowlist.", "ONLINE_SOURCE_NOT_ALLOWED", 2, { sourceUrl: value, }); diff --git a/src/services/sustech-online.ts b/src/services/sustech-online.ts index 94fe2ea..33ea23b 100644 --- a/src/services/sustech-online.ts +++ b/src/services/sustech-online.ts @@ -1,10 +1,7 @@ import { ONLINE_CONTACT_REPO_PATH, ONLINE_CONTACT_SITE_PATH, - ONLINE_RAW_ORIGIN, - ONLINE_REPO_BRANCH, - ONLINE_REPO_NAME, - ONLINE_REPO_OWNER, + ONLINE_RAW_BASE, ONLINE_SITE_ORIGIN, ONLINE_TALKS_INDEX_REPO_PATH, ONLINE_TALKS_INDEX_SITE_PATH, @@ -24,8 +21,8 @@ export const SUSTECH_ONLINE_STATUS: ServiceStatus = { "Handbook ingestion is constrained to a fixed allowlist; high-stakes, financial, personal, dining/chat, and professor-list content is excluded.", ], endpoints: [ - `${ONLINE_RAW_ORIGIN}/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/${ONLINE_TALKS_INDEX_REPO_PATH}`, - `${ONLINE_RAW_ORIGIN}/${ONLINE_REPO_OWNER}/${ONLINE_REPO_NAME}/${ONLINE_REPO_BRANCH}/${ONLINE_CONTACT_REPO_PATH}`, + `${ONLINE_RAW_BASE}/${ONLINE_TALKS_INDEX_REPO_PATH}`, + `${ONLINE_RAW_BASE}/${ONLINE_CONTACT_REPO_PATH}`, `${ONLINE_SITE_ORIGIN}${ONLINE_TALKS_INDEX_SITE_PATH}`, `${ONLINE_SITE_ORIGIN}${ONLINE_CONTACT_SITE_PATH}`, ...ONLINE_MANUAL_ENDPOINTS, diff --git a/src/test/online-raw-base.test.ts b/src/test/online-raw-base.test.ts new file mode 100644 index 0000000..20bc970 --- /dev/null +++ b/src/test/online-raw-base.test.ts @@ -0,0 +1,70 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { resolve } from "node:path"; +import test from "node:test"; +import { pathToFileURL } from "node:url"; +import { CliError } from "../core/errors.js"; +import { + ONLINE_RAW_BASE, + ONLINE_RAW_BASE_PATH, + ONLINE_RAW_ORIGIN, + onlineRawBase, + onlineRawUrl, +} from "../online/index.js"; + +const DEFAULT_BASE = "https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master"; +const MIRROR_BASE = "https://ghfast.top/https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master"; +const CONTACT_PATH = "docs/contact/README.md"; + +test("defaults to the public GitHub raw base", () => { + assert.equal(onlineRawBase({}), DEFAULT_BASE); + assert.equal(onlineRawUrl(CONTACT_PATH), `${DEFAULT_BASE}/${CONTACT_PATH}`); +}); + +test("accepts an HTTPS mirror override and trims surrounding slashes and spaces", () => { + assert.equal(onlineRawBase({ SUSTECH_ONLINE_RAW_BASE: MIRROR_BASE }), MIRROR_BASE); + assert.equal(onlineRawBase({ SUSTECH_ONLINE_RAW_BASE: `${MIRROR_BASE}/` }), MIRROR_BASE); + assert.equal(onlineRawBase({ SUSTECH_ONLINE_RAW_BASE: ` ${MIRROR_BASE} ` }), MIRROR_BASE); +}); + +test("rejects unsafe or malformed overrides", () => { + const values = [ + "http://mirror.example/repo", + "https://mirror.example/repo?token=1", + "https://mirror.example/repo#fragment", + "not a url", + ]; + for (const value of values) { + assert.throws( + () => onlineRawBase({ SUSTECH_ONLINE_RAW_BASE: value }), + (error: unknown) => error instanceof CliError && error.code === "ONLINE_RAW_BASE_INVALID", + ); + } +}); + +test( + "import-time allowlist derives from the default base", + { skip: process.env.SUSTECH_ONLINE_RAW_BASE ? "SUSTECH_ONLINE_RAW_BASE is set in this environment" : false }, + () => { + assert.equal(ONLINE_RAW_BASE, DEFAULT_BASE); + assert.equal(ONLINE_RAW_ORIGIN, "https://raw.githubusercontent.com"); + assert.equal(ONLINE_RAW_BASE_PATH, "/SUSTech-CRA/sustech-online-ng/master/"); + }, +); + +test("mirror override is applied when the module loads", () => { + const sharedModule = pathToFileURL(resolve("dist", "online", "shared.js")).href; + const script = [ + `const m = await import(${JSON.stringify(sharedModule)});`, + `console.log(m.onlineRawUrl(${JSON.stringify(CONTACT_PATH)}));`, + "console.log(m.ONLINE_RAW_BASE_PATH);", + ].join(""); + const [builtUrl, basePath] = execFileSync(process.execPath, ["--input-type=module", "-e", script], { + encoding: "utf8", + env: { ...process.env, SUSTECH_ONLINE_RAW_BASE: MIRROR_BASE }, + }) + .trim() + .split(/\r?\n/); + assert.equal(builtUrl, `${MIRROR_BASE}/${CONTACT_PATH}`); + assert.equal(basePath, "/https://raw.githubusercontent.com/SUSTech-CRA/sustech-online-ng/master/"); +});