diff --git a/.github/release-notes.md.template b/.github/release-notes.md.template new file mode 100644 index 0000000..e25afef --- /dev/null +++ b/.github/release-notes.md.template @@ -0,0 +1,3 @@ +## Release resources + +- [Release website](https://mboworks.github.io/bashtest/site/tag/@TAG@/) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b318dd7..3a8405c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,26 @@ name: Test on: [push] jobs: + release-site-tests: + name: Release site tests + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.13' + - run: python3 -m unittest discover -s tools -p release_site_test.py + - run: python3 -m unittest discover -s tools -p release_notes_test.py + - name: Verify configured documentation and generated links + env: + GH_TOKEN: ${{ github.token }} + run: | + # This is a disposable build on the runner, never a Pages publication. + python3 tools/release_site.py . "${RUNNER_TEMP}/release-site-check" \ + --repository "${GITHUB_REPOSITORY}" --tag 0.0.0-verification --latest "" + pre-commit: runs-on: ubuntu-latest steps: @@ -37,7 +57,7 @@ jobs: bazel test //... done: - needs: [pre-commit, test] + needs: [release-site-tests, pre-commit, test] if: always() runs-on: ubuntu-latest steps: diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..3bd9185 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +name: Publish release site + +on: + # Releases created with GITHUB_TOKEN do not trigger release events. + workflow_run: + workflows: [Release] + types: [completed] + workflow_dispatch: + inputs: + tag: + description: Published release tag (empty selects latest) + type: string + required: false + + config_path: + description: Optional config file on main for backfill (empty uses the tag config) + type: string + required: false + +permissions: {} + +# Share the lock and retained branch with coverage: every deployment includes both. +concurrency: + group: coverage-pages + queue: max + cancel-in-progress: false + +jobs: + publish: + if: >- + github.event_name != 'workflow_run' || + (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push') + runs-on: ubuntu-latest + permissions: + contents: write + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v7 + with: + ref: main + path: source + - name: Resolve published release + id: release + env: + GH_TOKEN: ${{ github.token }} + REQUESTED_TAG: ${{ inputs.tag }} + RUN_TAG: ${{ github.event.workflow_run.head_branch }} + run: | + set -euo pipefail + if gh api "repos/${GITHUB_REPOSITORY}/releases/latest" > latest.json; then + latest="$(jq -r .tag_name latest.json)" + elif jq --exit-status '.status == "404"' latest.json >/dev/null; then + latest="" # A repository may have only prereleases so far. + else + exit 1 + fi + tag="${REQUESTED_TAG:-${RUN_TAG:-${latest}}}" + if [[ ! "${tag}" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "Invalid release tag: ${tag}" >&2 + exit 1 + fi + gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${tag}" > release.json + jq --exit-status '.draft == false' release.json + echo "tag=${tag}" >> "${GITHUB_OUTPUT}" + echo "latest=${latest}" >> "${GITHUB_OUTPUT}" + - uses: actions/checkout@v7 + with: + ref: refs/tags/${{ steps.release.outputs.tag }} + path: release + persist-credentials: false + - name: Restore retained Pages tree + working-directory: source + run: | + set -euo pipefail + git fetch origin + if git show-ref --verify --quiet refs/remotes/origin/coverage-pages; then + git worktree add ../site -B coverage-pages origin/coverage-pages + else + git worktree add --detach ../site + git -C ../site checkout --orphan coverage-pages + git -C ../site rm -rf . + fi + - uses: actions/setup-python@v7 + with: + python-version: "3.13" + - name: Test site builder + working-directory: source + run: python3 -m unittest discover -s tools -p release_site_test.py + - name: Convert release documentation and update latest redirect + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ steps.release.outputs.tag }} + LATEST_TAG: ${{ steps.release.outputs.latest }} + CONFIG_PATH: ${{ inputs.config_path }} + run: | + set -euo pipefail + config_args=() + if [[ -n "${CONFIG_PATH}" ]]; then + config_file="$(realpath --canonicalize-existing "source/${CONFIG_PATH}")" + if [[ "${config_file}" != "${GITHUB_WORKSPACE}/source/"* ]]; then + echo "Configuration must be a tracked file inside the main checkout" >&2 + exit 1 + fi + git -C source ls-files --error-unmatch -- "${CONFIG_PATH}" >/dev/null + config_args=(--config "${config_file}") + fi + python3 source/tools/release_site.py release site \ + --repository "${GITHUB_REPOSITORY}" --tag "${RELEASE_TAG}" --latest "${LATEST_TAG}" \ + "${config_args[@]}" + - name: Retain and stage complete Pages tree + env: + RELEASE_TAG: ${{ steps.release.outputs.tag }} + run: | + set -euo pipefail + git -C site config user.name 'github-actions[bot]' + git -C site config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git -C site add --all + if ! git -C site diff --cached --quiet; then + git -C site commit -m "site: publish ${RELEASE_TAG}" + git -C site push origin HEAD:coverage-pages + fi + mkdir -p public + rsync --archive --exclude='.git' site/ public/ + - uses: actions/configure-pages@v6 + - uses: actions/upload-pages-artifact@v5 + with: + path: public + - name: Deploy complete site + id: deployment + uses: actions/deploy-pages@v5 diff --git a/.github/workflows/release_prep.sh b/.github/workflows/release_prep.sh index bb59e99..2a73dbf 100755 --- a/.github/workflows/release_prep.sh +++ b/.github/workflows/release_prep.sh @@ -111,3 +111,6 @@ cat </`, preserving the exact Git tag name. +Each release keeps its converted HTML, images, and configured files. Retrying +publication leaves an existing snapshot unchanged; a different commit cannot +replace it. Older versions remain directly accessible. + +[`release-site.json`](release-site.json) defines the layout. Source names are +relative to the repository root; destinations are relative to that release's +site directory. For example: + +```json +{ + "pages": { + "README.md": "index.html", + "docs/guide.md": "guide/index.html" + }, + "files": { + "schema/example.json": "schema/v1.json" + }, + "links": [ + { + "label": "Release", + "href": "https://github.com/{owner}/{repo}/releases/tag/{tag}" + } + ] +} +``` + +Use existing source files in the actual configuration. `pages` converts Markdown; +optional `files` copies other files unchanged. `README.md` must map to `index.html`. +The generated `documents.html`, `release.json`, `release-site.json`, and `assets/` +paths are reserved. Destination paths cannot have hidden components (names starting +with a dot), because the Pages artifact uploader excludes them. Hidden source +paths remain valid; for example, `.github/workflows/README.md` maps to +`workflows/index.html`. +Navigation links support `{owner}`, `{repo}`, `{tag}`, `{version}`, and `{commit}`. +`{version}` omits a leading `v` for compatibility with coverage report paths. +By default, the configuration and content come from the release tag. Every linked +local Markdown page (including directory README links) must have a `pages` mapping. +Publication fails for an omitted mapping, a missing generated file, or a broken +anchor within the snapshot. Links to configured pages follow their destination +mappings; other local source links use the exact release commit. Embedded images are copied, including remote badges. Markdown +conversion uses the [GitHub Markdown API](https://docs.github.com/en/rest/markdown/markdown) +at publication time; browsing the result requires no Markdown renderer or CDN. + +After the Release workflow succeeds, `Publish release site` retains the snapshot +on `coverage-pages` and deploys the complete Pages tree. Coverage and site +publication share a concurrency group to preserve both trees. GitHub's latest +stable release selects the root redirect; backfilling an older release does not +make it latest. The workflow can also be dispatched with a published tag to retry +publication. Enable GitHub Pages with +**GitHub Actions** as its source, and set the repository's About website to +`https://mboworks.github.io/bashtest/`. + +### Backfill a historical release + +No new release or tag change is needed. Manually dispatch `Publish release site` +with `tag` set to the historical release and `config_path` set to a tracked JSON +file on `main`. Leave `config_path` empty to use a configuration already in the tag. +For example, after selecting a compatible configuration and an existing tag: + +```sh +gh workflow run pages.yml --repo mboworks/bashtest --ref main \ + -f tag="$RELEASE_TAG" -f config_path=release-site.json +``` + +The override controls only publication layout; all Markdown and copied files come +from the selected tag. Each new snapshot retains the exact configuration as +`release-site.json`, with its SHA-256, origin, and source commit in `release.json`. +A configuration can serve several historical tags when its sources exist in each. +For another layout, commit another configuration and select its path. Missing +sources or links fail publication instead of using newer content. Retrying a +published tag preserves its original HTML and configuration. + +Local regression tests: `python3 -m unittest discover -s tools -p release_site_test.py`. +CI also converts the configured documentation and checks the generated links in +a disposable runner directory. It never commits, retains, or deploys that preview. diff --git a/release-site.json b/release-site.json new file mode 100644 index 0000000..4b1b47d --- /dev/null +++ b/release-site.json @@ -0,0 +1,11 @@ +{ + "pages": { + "README.md": "index.html", + "CHANGELOG.md": "CHANGELOG.html", + "CONTRIBUTING.md": "CONTRIBUTING.html", + "bashtest/README.md": "bashtest/README.html", + "CODE_OF_CONDUCT.md": "CODE_OF_CONDUCT.html", + "RULES.md": "RULES.html" + }, + "links": [] +} diff --git a/tools/release_notes.sh b/tools/release_notes.sh new file mode 100755 index 0000000..b0b6ee3 --- /dev/null +++ b/tools/release_notes.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render release-note links without building, publishing, or changing repository state. +set -euo pipefail + +TAG="${1:?Usage: release_notes.sh TAG}" +if [[ ! "${TAG}" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]; then + echo "Invalid release tag: ${TAG}" >&2 + exit 1 +fi +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +sed -e "s|@TAG@|${TAG}|g" -e "s|@VERSION@|${TAG#v}|g" \ + "${ROOT}/.github/release-notes.md.template" diff --git a/tools/release_notes_test.py b/tools/release_notes_test.py new file mode 100644 index 0000000..322a8fe --- /dev/null +++ b/tools/release_notes_test.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Exercise the release-note renderer with the actual repository template.""" + +from pathlib import Path +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[1] +REPO = "bashtest" + + +class ReleaseNotesTest(unittest.TestCase): + def render(self, tag): + with tempfile.TemporaryDirectory() as cwd: + return subprocess.run( + ["bash", str(ROOT / "tools/release_notes.sh"), tag], + cwd=cwd, text=True, capture_output=True, check=False) + + def test_release_resources(self): + for tag in ("1.2.3", "v1.2.3", "v1.2.3-rc.1"): + with self.subTest(tag=tag): + result = self.render(tag) + self.assertEqual(result.returncode, 0, result.stderr) + notes = result.stdout + base = f"https://mboworks.github.io/{REPO}" + self.assertIn(f"{base}/site/tag/{tag}/", notes) + self.assertNotIn("@TAG@", notes) + self.assertNotIn("@VERSION@", notes) + version = tag.removeprefix("v") + if REPO in ("mbo", "xff", "carve"): + self.assertIn(f"{base}/coverage/tag/{version}/", notes) + else: + self.assertNotIn("/coverage/", notes) + if REPO == "xff": + self.assertIn(f"{base}/releases/{version}/)", notes) + self.assertIn(f"{base}/releases/{version}/XFF.md", notes) + if REPO == "coderef": + self.assertIn(f"/blob/{tag}/CHANGELOG.md", notes) + self.assertIn(f"{base}/site/tag/{tag}/schema/v1.json", notes) + + def test_reject_invalid_tags_without_partial_notes(self): + for tag in ("", "main", "../1.2.3", "v1.2.3|bad", "1.2.3\nmain"): + with self.subTest(tag=tag): + result = self.render(tag) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release_site.py b/tools/release_site.py new file mode 100644 index 0000000..f5634cd --- /dev/null +++ b/tools/release_site.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Convert a release checkout to a retained, self-contained documentation site.""" + +import argparse +import hashlib +import html +from html.parser import HTMLParser +import json +from pathlib import Path +import posixpath +import re +import subprocess +import tempfile +from urllib.parse import quote, unquote, urlsplit, urlunsplit +from urllib.request import urlopen + + +STYLE = """ +:root { color-scheme: light dark; font: 17px/1.6 system-ui, sans-serif; } +body { max-width: 76rem; margin: auto; padding: 2rem; } +a { color: light-dark(#075da8, #8cc8ff); } +nav { display: flex; flex-wrap: wrap; gap: 1rem; border-bottom: 1px solid #888; } +pre { padding: 1rem; overflow: auto; background: light-dark(#f3f5f7, #20252b); } +code { font-size: .9em; } img { max-width: 100%; } +table { display: block; overflow: auto; border-collapse: collapse; } +th, td { border: 1px solid #888; padding: .4rem .7rem; } +blockquote { border-left: 4px solid #888; margin-left: 0; padding-left: 1rem; } +""" + + +def git(source, *args): + return subprocess.check_output(["git", "-C", str(source), *args], text=True).strip() + + +def render(markdown, repository): + return subprocess.check_output( + ["gh", "api", "markdown", "--input", "-"], + input=json.dumps({"text": markdown, "mode": "gfm", "context": repository}), + text=True, + ) + + +def version(tag): + if not re.fullmatch(r"v?[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?", tag): + raise ValueError(f"Invalid release tag: {tag!r}") + return tag.removeprefix("v") + + +def configuration(source, override=None): + data = (override if override is not None else source / "release-site.json").read_bytes() + config = json.loads(data) + pages = config["pages"] + if not pages or pages.get("README.md") != "index.html": + raise ValueError("README.md must map to index.html") + destinations = set() + for src, dst in pages.items(): + for path in (src, dst): + if (not isinstance(path, str) or path.startswith("/") + or ".." in Path(path).parts or str(Path(path)) != path + or any(char in path for char in "\\?#")): + raise ValueError(f"Unsafe site path: {path!r}") + if not src.endswith(".md") or not dst.endswith(".html"): + raise ValueError("Page mappings must convert .md sources to .html destinations") + if dst in destinations or dst == "documents.html" or dst.startswith("assets/"): + raise ValueError(f"Duplicate or reserved destination: {dst}") + if any(part.startswith(".") for part in Path(dst).parts): + raise ValueError(f"Pages excludes hidden destinations: {dst}") + destinations.add(dst) + for src, dst in config.get("files", {}).items(): + if src in pages: + raise ValueError(f"Source is mapped as both a page and a file: {src}") + if src.lower().endswith(".md"): + raise ValueError(f"Markdown must be converted through pages: {src}") + for path in (src, dst): + if (not isinstance(path, str) or path.startswith("/") + or ".." in Path(path).parts or str(Path(path)) != path + or any(char in path for char in "\\?#")): + raise ValueError(f"Unsafe asset path: {path!r}") + if dst in destinations or dst in ("documents.html", "release.json", "release-site.json") or dst.startswith("assets/"): + raise ValueError(f"Duplicate or reserved destination: {dst}") + if any(part.startswith(".") for part in Path(dst).parts): + raise ValueError(f"Pages excludes hidden destinations: {dst}") + destinations.add(dst) + return config, data + + +def headings(body): + """The Markdown API omits GitHub's heading anchors; restore their slugs.""" + used = set() + + def heading(match): + level, attrs, text = match.groups() + plain = html.unescape(re.sub(r"<[^>]*>", "", text)).lower() + slug = re.sub(r"[^\w\- ]", "", plain).replace(" ", "-") + anchor = slug + count = 0 + while anchor in used: + count += 1 + anchor = f"{slug}-{count}" + used.add(anchor) + return f'{text}' + + return re.sub(r"]*)>(.*?)", heading, body, flags=re.DOTALL) + + +class Links(HTMLParser): + def __init__(self, source, output, document, documents, repository, sha, tag): + super().__init__(convert_charrefs=False) + self.source = source + self.output = output + self.document = document + self.documents = documents + self.repository = repository + self.sha = sha + self.tag = tag + self.parts = [] + + def local(self, value): + parsed = urlsplit(value) + prefix = f"https://github.com/{self.repository}/" + if value.startswith(prefix): + rest = value[len(prefix):] + for kind in ("blob/", "tree/"): + for ref in ("main", "master", self.tag, self.sha): + start = f"{kind}{ref}/" + if rest.startswith(start): + return posixpath.normpath(unquote(urlsplit(rest[len(start):]).path)) + if parsed.scheme or parsed.netloc or not parsed.path: + return None + path = unquote(parsed.path) + if path.startswith("/"): + return posixpath.normpath(path.lstrip("/")) + return posixpath.normpath(posixpath.join(posixpath.dirname(self.document), path)) + + def rewrite(self, value, image=False): + parsed = urlsplit(value) + owner, repo = self.repository.split("/") + coverage = f"/{repo}/coverage/" + if not image and (value.startswith(coverage) or value.startswith(f"https://{owner}.github.io{coverage}")): + return f"/{repo}/coverage/tag/{version(self.tag)}/" + local = self.local(value) + if local is not None: + if local == ".." or local.startswith("../"): + raise ValueError(f"Link escapes repository: {value}") + readme = f"{local.rstrip('/')}/README.md" + if local not in self.documents and (readme in self.documents or (self.source / readme).is_file()): + local = readme + if local == ".": + local = "README.md" + if local in self.documents and not image: + target = posixpath.relpath(self.documents[local], posixpath.dirname(self.documents[self.document]) or ".") + return urlunsplit(("", "", quote(target), parsed.query, parsed.fragment)) + if not image and local.lower().endswith(".md"): + raise ValueError(f"{self.document}: linked Markdown has no page mapping: {local}") + path = self.source / local + if image: + if not path.is_file() or path.resolve() != path.absolute(): + raise ValueError(f"Missing or symlinked image: {local}") + data = path.read_bytes() + suffix = path.suffix + else: + kind = "tree" if path.is_dir() else "blob" + return f"https://github.com/{self.repository}/{kind}/{self.sha}/{quote(local)}" + ( + f"#{parsed.fragment}" if parsed.fragment else "" + ) + elif image: + if parsed.scheme not in ("https", "http"): + raise ValueError(f"Unsupported image URL: {value}") + with urlopen(value, timeout=60) as response: + data = response.read() + media = response.headers.get_content_type() + suffix = {"image/svg+xml": ".svg", "image/png": ".png", "image/jpeg": ".jpg", + "image/gif": ".gif", "image/webp": ".webp"}.get(media) + if suffix is None: + raise ValueError(f"Unsupported image content type: {media}") + else: + return value + asset = f"assets/{hashlib.sha256(data).hexdigest()}{suffix}" + (self.output / "assets").mkdir(exist_ok=True) + (self.output / asset).write_bytes(data) + return posixpath.relpath(asset, posixpath.dirname(self.documents[self.document]) or ".") + + def handle_starttag(self, tag, attrs): + rewritten = [] + for key, value in attrs: + if key in ("data-canonical-src", "srcset"): + continue + if value is not None and key in ("href", "src"): + value = self.rewrite(value, image=tag == "img" and key == "src") + rewritten.append(key if value is None else f'{key}="{html.escape(value, quote=True)}"') + self.parts.append(f"<{tag}{' ' if rewritten else ''}{' '.join(rewritten)}>") + + def handle_startendtag(self, tag, attrs): + self.handle_starttag(tag, attrs) + + def handle_endtag(self, tag): + self.parts.append(f"") + + def handle_data(self, data): + self.parts.append(data) + + def handle_entityref(self, name): + self.parts.append(f"&{name};") + + def handle_charref(self, name): + self.parts.append(f"&#{name};") + + +class PageReferences(HTMLParser): + """Collect browser-visible anchors and links from final HTML.""" + + def __init__(self, text): + super().__init__() + self.anchors = set() + self.links = [] + self.feed(text) + + def handle_starttag(self, tag, attrs): + for key, value in attrs: + if value is None: + continue + if key == "id" or (tag == "a" and key == "name"): + self.anchors.add(value) + if key in ("href", "src"): + self.links.append(value) + + handle_startendtag = handle_starttag + + +def validate_site(output, repository, tag): + """Reject broken links inside the snapshot before retaining or deploying it.""" + owner, repo = repository.split("/") + base = f"/{repo}/site/tag/{tag}/" + host = f"{owner}.github.io" + pages = {path.relative_to(output).as_posix(): PageReferences(path.read_text()) + for path in output.rglob("*.html")} + for document, page in pages.items(): + for link in page.links: + parsed = urlsplit(link) + if parsed.scheme or parsed.netloc: + if parsed.scheme not in ("http", "https") or parsed.netloc != host: + continue + if not parsed.path.startswith(base): + continue + path = unquote(parsed.path) + if path.startswith("/"): + if not path.startswith(base): + continue # Coverage and other explicitly separate Pages trees. + target = posixpath.normpath(path[len(base):]) + elif path: + target = posixpath.normpath(posixpath.join(posixpath.dirname(document), path)) + else: + target = document + if target == ".." or target.startswith("../"): + raise ValueError(f"{document}: link escapes release snapshot: {link}") + destination = output / target + if destination.is_dir(): + target = posixpath.join(target, "index.html") + destination = output / target + target = posixpath.normpath(target) + if not destination.is_file(): + raise ValueError(f"{document}: missing generated link target: {link}") + fragment = unquote(parsed.fragment) + if fragment and target in pages and fragment not in pages[target].anchors: + raise ValueError(f"{document}: missing generated anchor: {link}") + + +def build(source, retained, repository, tag, renderer=render, config_path=None): + source = source.resolve() + release_version = version(tag) + destination = retained / "site" / "tag" / tag + sha = git(source, "rev-parse", "HEAD") + if destination.exists(): + metadata = json.loads((destination / "release.json").read_text()) + if metadata["commit"] != sha or metadata["tag"] != tag: + raise ValueError("A retained release cannot be replaced by a different commit or tag") + return + config, config_data = configuration(source, config_path) + documents = config["pages"] + files = config.get("files", {}) + tracked = set(git(source, "ls-files", "-z").split("\0")) + for document in [*documents, *files]: + path = source / document + if document not in tracked or not path.is_file() or path.resolve() != path.absolute(): + raise ValueError(f"Missing, untracked, or symlinked document: {document}") + destination.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=destination.parent) as temporary: + output = Path(temporary) + for src, dst in files.items(): + target = output / dst + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes((source / src).read_bytes()) + owner, repo = repository.split("/") + base = f"/{repo}/site/tag/{tag}/" + links = [("Home", base), ("Documentation", base + "documents.html"), + ("Release & downloads", f"https://github.com/{repository}/releases/tag/{tag}"), + ("Source", f"https://github.com/{repository}/tree/{sha}")] + for link in config.get("links", []): + links.append((link["label"], link["href"].format( + repo=repo, owner=owner, tag=tag, version=release_version, commit=sha))) + navigation = "".join(f'{html.escape(label)}' for label, url in links) + + def page(title, body): + return (f'' + f'' + f'{html.escape(title)} - {repo} {tag}' + f'

{repo} {tag}

{body}
' + f'
Release snapshot ยท {sha}
\n') + + for document in sorted(documents): + parser = Links(source, output, document, {**documents, **files}, repository, sha, tag) + parser.feed(headings(renderer((source / document).read_text(), repository))) + target = output / documents[document] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(page(document, "".join(parser.parts))) + index = "

Documentation

" + (output / "documents.html").write_text(page("Documentation", index)) + (output / "release-site.json").write_bytes(config_data) + (output / "release.json").write_text(json.dumps({ + "tag": tag, "commit": sha, + "configuration": {"origin": "override" if config_path is not None else "tag", + "sha256": hashlib.sha256(config_data).hexdigest()}, + }) + "\n") + validate_site(output, repository, tag) + # Rename only after every document and image has been converted successfully. + output.rename(destination) + + +def redirect(retained, tag): + version(tag) # Validate the exact Git tag before using it as a path. + target = f"site/tag/{tag}/" + if not (retained / target / "index.html").is_file(): + return # An older backfill must not redirect to an unpublished latest site. + (retained / "index.html").write_text( + '' + f'' + f'Latest releaseLatest release\n' + ) + (retained / ".nojekyll").touch() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("source", type=Path) + parser.add_argument("retained", type=Path) + parser.add_argument("--repository", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--latest", required=True) + parser.add_argument("--config", type=Path, help="Explicit configuration override for historical tags") + args = parser.parse_args() + build(args.source, args.retained, args.repository, args.tag, config_path=args.config) + if args.latest: + redirect(args.retained, args.latest) + + +if __name__ == "__main__": + main() diff --git a/tools/release_site_test.py b/tools/release_site_test.py new file mode 100644 index 0000000..07553d7 --- /dev/null +++ b/tools/release_site_test.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors +# SPDX-License-Identifier: Apache-2.0 +"""Regression tests for release snapshots, configured links, and retained history.""" + +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +import release_site as site + + +class ReleaseSiteTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.source = self.root / "source" + self.retained = self.root / "retained" + self.source.mkdir() + subprocess.run(["git", "init", "-q", str(self.source)], check=True) + self.write("README.md", "release readme") + self.write("docs/guide.md", "release guide") + self.write("image.svg", '') + self.config = {"pages": {"README.md": "index.html", "docs/guide.md": "guide/start.html"}, + "links": [{"label": "Coverage", "href": "/{repo}/coverage/tag/{version}/"}]} + self.write("release-site.json", json.dumps(self.config)) + subprocess.run(["git", "-C", str(self.source), "add", "."], check=True) + subprocess.run(["git", "-C", str(self.source), "-c", "user.name=Test", + "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", + "commit", "-qm", "fixture"], check=True) + + def write(self, name, text): + target = self.source / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text) + + def render(self, markdown, repository): + self.assertEqual(repository, "mboworks/mbo") + if markdown == "release readme": + return ('

A title!

A title!

' + 'Guide' + 'Guide' + 'Coverage' + 'Source') + self.assertEqual(markdown, "release guide") + return '

Usage

Home' + + def build(self, tag="v1.2.3", renderer=None): + site.build(self.source, self.retained, "mboworks/mbo", tag, renderer or self.render) + return self.retained / "site/tag" / tag + + def test_configured_destinations_resolve_local_and_github_links(self): + output = self.build() + home = (output / "index.html").read_text() + guide = (output / "guide/start.html").read_text() + self.assertIn('href="guide/start.html#usage"', home) + self.assertIn('href="guide/start.html"', home) + self.assertIn('href="../index.html#a-title"', guide) + self.assertIn('id="a-title"', home) + self.assertIn('id="a-title-1"', home) + self.assertIn('/mbo/coverage/tag/1.2.3/', home) + self.assertNotIn('/coverage/main/', home) + sha = site.git(self.source, "rev-parse", "HEAD") + self.assertIn(f'/blob/{sha}/src/code.cc', home) + self.assertEqual(len(list((output / "assets").iterdir())), 1) + self.assertIn('guide/start.html', (output / "documents.html").read_text()) + + def test_rerun_retains_original_bytes_without_rendering(self): + output = self.build() + before = {p.relative_to(output): p.read_bytes() for p in output.rglob("*") if p.is_file()} + self.build(renderer=mock.Mock(side_effect=AssertionError("must not render again"))) + self.assertEqual(before, {p.relative_to(output): p.read_bytes() + for p in output.rglob("*") if p.is_file()}) + + def test_changed_commit_cannot_replace_retained_release(self): + self.build() + with mock.patch.object(site, "git", return_value="different commit"): + with self.assertRaisesRegex(ValueError, "cannot be replaced"): + self.build() + + def test_new_release_and_backfill_preserve_coverage_schema_and_history(self): + for name in ("coverage/tag/1.2.3/index.html", "schema/v1.json", "releases/1.2.3/index.html"): + target = self.retained / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text("retained") + self.build("1.2.3") + self.build("2.0.0") + site.redirect(self.retained, "2.0.0") + self.build("1.0.0") + site.redirect(self.retained, "2.0.0") + self.assertIn('url=site/tag/2.0.0/', (self.retained / "index.html").read_text()) + for name in ("coverage/tag/1.2.3/index.html", "schema/v1.json", "releases/1.2.3/index.html"): + self.assertEqual((self.retained / name).read_text(), "retained") + for tag in ("1.0.0", "1.2.3", "2.0.0"): + self.assertTrue((self.retained / f"site/tag/{tag}/index.html").is_file()) + + def test_unpublished_latest_does_not_break_existing_redirect(self): + self.build() + site.redirect(self.retained, "v1.2.3") + before = (self.retained / "index.html").read_bytes() + site.redirect(self.retained, "9.0.0") + self.assertEqual((self.retained / "index.html").read_bytes(), before) + + def test_conversion_failure_leaves_no_partial_release(self): + with self.assertRaisesRegex(RuntimeError, "renderer unavailable"): + self.build(renderer=mock.Mock(side_effect=RuntimeError("renderer unavailable"))) + self.assertFalse((self.retained / "site/tag/v1.2.3").exists()) + + def test_external_images_are_snapshotted(self): + response = mock.MagicMock() + response.__enter__.return_value = response + response.read.return_value = b"image bytes" + response.headers.get_content_type.return_value = "image/png" + with mock.patch.object(site, "urlopen", return_value=response): + output = self.build(renderer=lambda *_: '') + self.assertNotIn('src="https:', (output / "index.html").read_text()) + self.assertEqual(next((output / "assets").iterdir()).read_bytes(), b"image bytes") + + def test_invalid_paths_and_duplicate_destinations_fail(self): + for pages in ({"README.md": "index.html", "../bad.md": "bad.html"}, + {"README.md": "index.html", "docs/guide.md": "../bad.html"}, + {"README.md": "index.html", "docs/guide.md": "index.html"}): + with self.subTest(pages=pages): + self.write("release-site.json", json.dumps({"pages": pages})) + with self.assertRaises(ValueError): + self.build() + + def test_configured_files_are_copied_and_links_use_their_destinations(self): + self.config["files"] = {"image.svg": "static/logo.svg"} + self.write("release-site.json", json.dumps(self.config)) + output = self.build(renderer=lambda *_: 'Download') + self.assertEqual((output / "static/logo.svg").read_bytes(), + (self.source / "image.svg").read_bytes()) + self.assertIn('href="static/logo.svg"', (output / "index.html").read_text()) + self.assertIn('href="../static/logo.svg"', (output / "guide/start.html").read_text()) + + def test_missing_configured_source_fails_before_rendering(self): + self.config["pages"]["missing.md"] = "missing.html" + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "Missing"): + self.build(renderer=mock.Mock(side_effect=AssertionError("must not render"))) + + def test_root_relative_coverage_links_select_release(self): + output = self.build(renderer=lambda *_: 'Report') + home = (output / "index.html").read_text() + self.assertIn('href="/mbo/coverage/tag/1.2.3/"', home) + self.assertNotIn('/coverage/main/', home) + + def test_heading_suffix_collisions_get_unique_anchors(self): + rendered = site.headings("

Title

Title

Title-1

") + self.assertIn('id="title"', rendered) + self.assertIn('id="title-1"', rendered) + self.assertIn('id="title-1-1"', rendered) + + def test_one_source_cannot_have_page_and_file_destinations(self): + self.config["files"] = {"README.md": "readme.txt"} + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "both a page and a file"): + self.build() + + def test_backfill_uses_old_source_and_retains_exact_override(self): + override = self.root / "backfill.json" + data = json.dumps(self.config, indent=4).encode() + b"\n" + override.write_bytes(data) + subprocess.run(["git", "-C", str(self.source), "rm", "-q", "release-site.json"], check=True) + subprocess.run(["git", "-C", str(self.source), "-c", "user.name=Test", + "-c", "user.email=test@example.com", "-c", "commit.gpgsign=false", + "commit", "-qm", "Historical source without a site config"], check=True) + site.build(self.source, self.retained, "mboworks/mbo", "v1.2.3", + self.render, config_path=override) + output = self.retained / "site/tag/v1.2.3" + self.assertEqual((output / "release-site.json").read_bytes(), data) + metadata = json.loads((output / "release.json").read_text()) + self.assertEqual(metadata["commit"], site.git(self.source, "rev-parse", "HEAD")) + self.assertEqual(metadata["configuration"]["origin"], "override") + self.assertEqual(metadata["configuration"]["sha256"], site.hashlib.sha256(data).hexdigest()) + self.assertFalse((self.source / "release-site.json").exists()) + before = (output / "index.html").read_bytes() + override.write_text("invalid replacement config") + site.build(self.source, self.retained, "mboworks/mbo", "v1.2.3", + mock.Mock(side_effect=AssertionError("must not render")), config_path=override) + self.assertEqual((output / "index.html").read_bytes(), before) + self.assertEqual((output / "release-site.json").read_bytes(), data) + + def test_override_never_reads_missing_content_from_publisher_checkout(self): + override = self.root / "backfill.json" + self.config["pages"]["newer.md"] = "newer.html" + override.write_text(json.dumps(self.config)) + (self.root / "newer.md").write_text("content only present alongside publisher config") + with self.assertRaisesRegex(ValueError, "Missing.*newer.md"): + site.build(self.source, self.retained, "mboworks/mbo", "v1.2.3", + mock.Mock(side_effect=AssertionError("must not render")), config_path=override) + + def test_tag_config_remains_default_and_is_retained(self): + output = self.build() + metadata = json.loads((output / "release.json").read_text()) + self.assertEqual(metadata["configuration"]["origin"], "tag") + self.assertEqual((output / "release-site.json").read_bytes(), + (self.source / "release-site.json").read_bytes()) + + def test_unmapped_markdown_links_fail_publication(self): + del self.config["pages"]["docs/guide.md"] + self.write("release-site.json", json.dumps(self.config)) + for link in ("docs/guide.md", "/docs/guide.md", + "https://github.com/mboworks/mbo/blob/main/docs/guide.md"): + with self.subTest(link=link), self.assertRaisesRegex(ValueError, "no page mapping"): + self.build(renderer=lambda *_: f'Guide') + self.assertFalse((self.retained / "site/tag/v1.2.3").exists()) + + def test_directory_readme_requires_a_mapping(self): + self.write("docs/README.md", "directory guide") + with self.assertRaisesRegex(ValueError, "no page mapping"): + self.build(renderer=lambda *_: 'Guide') + + def test_missing_generated_anchor_aborts_without_retaining_snapshot(self): + with self.assertRaisesRegex(ValueError, "missing generated anchor"): + self.build(renderer=lambda *_: 'Guide') + self.assertFalse((self.retained / "site/tag/v1.2.3").exists()) + + def test_generated_audit_checks_files_anchors_and_snapshot_boundaries(self): + output = self.root / "html" + output.mkdir() + (output / "guide.html").write_text('

Guide

') + for link, error in (("missing.html", "missing generated link target"), + ("guide.html#absent", "missing generated anchor"), + ("../outside.html", "escapes release snapshot"), + ("/mbo/site/tag/1.2.3/missing.html", "missing generated link target"), + ("https://mboworks.github.io/mbo/site/tag/1.2.3/guide.html#absent", + "missing generated anchor")): + with self.subTest(link=link): + (output / "index.html").write_text(f'Target') + with self.assertRaisesRegex(ValueError, error): + site.validate_site(output, "mboworks/mbo", "1.2.3") + (output / "index.html").write_text( + 'GuideSelf' + 'Coverage' + 'External') + site.validate_site(output, "mboworks/mbo", "1.2.3") + + def test_reserved_config_destination_cannot_be_overwritten(self): + self.config["files"] = {"image.svg": "release-site.json"} + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "reserved destination"): + self.build() + + def test_pages_packaging_exclusions_are_rejected_before_rendering(self): + for destination in (".github/guide.html", "guide/.hidden.html", ".hidden/guide.html"): + with self.subTest(destination=destination): + self.config["pages"]["docs/guide.md"] = destination + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "excludes hidden"): + self.build(renderer=mock.Mock(side_effect=AssertionError("must not render"))) + self.config["pages"]["docs/guide.md"] = "guide.html" + self.config["files"] = {"image.svg": "assets-custom/.hidden.svg"} + self.write("release-site.json", json.dumps(self.config)) + with self.assertRaisesRegex(ValueError, "excludes hidden"): + self.build() + + def test_invalid_tag_cannot_escape_site_directory(self): + for tag in ("../bad", "v1.2.3/evil", "1.2.3\nextra", "main"): + with self.subTest(tag=tag), self.assertRaises(ValueError): + self.build(tag) + + def test_symlinked_document_is_rejected(self): + target = self.source / "docs/guide.md" + target.unlink() + target.symlink_to(self.source / "README.md") + with self.assertRaisesRegex(ValueError, "symlinked"): + self.build() + + +if __name__ == "__main__": + unittest.main()