diff --git a/doc/developer-handbook/development-and-validation.md b/doc/developer-handbook/development-and-validation.md index 977d74e..75881e3 100644 --- a/doc/developer-handbook/development-and-validation.md +++ b/doc/developer-handbook/development-and-validation.md @@ -10,7 +10,7 @@ Use repository-relative commands. Do not put user-specific absolute paths in PR | Compile check | Python sources in `src` and `test` compile without syntax errors. | `python3 -m compileall -q src test` | | Focused package governance | Explicit package lists, repository-only namespaces, and source bootstraps remain reviewable. | `python3 -m pytest -q test/test_repository_governance.py test/test_namespace_layout.py` | | Editable install | The configured package set remains importable from an editable installation without changing package discovery. | `python3 -m pip install -e . --no-deps` followed by `python3 -m pytest -q test/test_namespace_layout.py` | -| Wheel build | The project builds a wheel from the checked-out source without build isolation when `build` is installed. | `python3 -m build --no-isolation` | +| Tracked-source release chain | Export only files tracked at the revision, build a direct wheel and an sdist, then build the release wheel from the unpacked sdist without `.git` metadata. | `python3 -m pytest -q -s test/test_wheel_install.py -m "not network"` | | Non-network clean wheel | Runtime packages install from the built wheel without dependency resolution or source checkout leakage. | `python3 -m pytest -q -s test/test_wheel_install.py -m "not network"` | | Full isolated wheel | Declared dependencies install in a clean venv when a package index or local wheelhouse is available. | `CACHEROUTE_RUN_NETWORK_TESTS=1 python3 -m pytest -q -s test/test_wheel_install.py` | | Offline full isolated wheel | The network-marked wheel test can use a local wheelhouse instead of indexes. | `CACHEROUTE_RUN_NETWORK_TESTS=1 CACHEROUTE_TEST_WHEELHOUSE="$PWD/wheelhouse" python3 -m pytest -q -s test/test_wheel_install.py` | @@ -18,6 +18,25 @@ Use repository-relative commands. Do not put user-specific absolute paths in PR | Contract foundations | KDN/Gateway v1 contracts and migration shims preserve object identity and validation. | `python3 -m pytest -q test/test_contract_foundation.py test/test_contract_service_migration.py` | | Diff hygiene | The final diff has no whitespace errors. | `git diff --check "$(git merge-base origin/main HEAD)" HEAD` | +## Release artifact boundaries + +Release validation distinguishes five layers: the active source checkout, a +Git-tracked source export, the sdist made from that export, the wheel made from +the sdist, and an isolated installation of that final wheel. A raw copy of the +active worktree is not release evidence because ignored or untracked files can +mask a stale explicit package declaration. The non-network wheel test also +builds a direct tracked-source wheel as a separate check, but uses the +sdist-derived wheel for content and isolated-install assertions. +When the same test module runs from an unpacked sdist, it skips only the +checkout-specific tracked-export assertion and builds the validation wheel +directly from that Git-free sdist tree; it must not attempt to resolve `HEAD`. + +For a manual release check, create a clean tracked-file export without `.git`, +run `python3 -m build --sdist --no-isolation` there, unpack the generated sdist +into a separate clean location, and build the wheel from that unpacked tree. Do +not substitute a wheel built from the original checkout for the sdist-derived +wheel. + ## Required final-head checklist ```bash diff --git a/doc/developer-handbook/package-and-module-map.md b/doc/developer-handbook/package-and-module-map.md index 3c6c1c6..5677ead 100644 --- a/doc/developer-handbook/package-and-module-map.md +++ b/doc/developer-handbook/package-and-module-map.md @@ -1,6 +1,6 @@ # Package and module map -Installed package set is derived from the explicit `[tool.setuptools].packages` list in [pyproject.toml](../../pyproject.toml); package discovery must not change in this handbook PR. +Installed package set is derived from the explicit `[tool.setuptools].packages` list in [pyproject.toml](../../pyproject.toml); package discovery remains explicit; each configured entry must map to reviewed, Git-tracked Python source. | Concept / responsibility | Canonical package/module | Current transitional implementation | Dependency direction | Public/internal | Design link | Relevant tests | Migration status | Status | |---|---|---|---|---|---|---|---|---| @@ -33,4 +33,13 @@ Installed package set is derived from the explicit `[tool.setuptools].packages` ## Explicitly configured package coverage -`cacheroute`, `cacheroute.compat`, `cacheroute.observability`, `cacheroute.observability.v1`, `cacheroute.runtime`, `cacheroute.topology`, `cacheroute.cache`, `cacheroute.routing`, `cacheroute.contracts`, `cacheroute.contracts.v1`, `cacheroute_compat`, `UI`, `UI.client_ui`, `UI.client_ui.static`, `UI.client_ui.templates`, `UI.kdn_ui`, `UI.proxy_ui`, `UI.proxy_ui.static`, `client`, `client.taskset`, `core`, `data`, `data.CacheRoute_dataset`, `data.CacheRoute_dataset.knowledge_document`, `instance`, `instance.TPOT_predictor`, `instance.TTFT_predictor`, `instance.TTFT_predictor.data`, `instance.pclient`, `instance.resource_agent`, `instance.resource_agent.src`, `instance.resource_dashboard`, `instance.resource_dashboard.static`, `kdn_server`, `kdn_server.KV_database`, `kdn_server.contracts`, `kdn_server.domain`, `kdn_server.gateway`, `kdn_server.sclient`, `kdn_server.text_database`, `kdn_server.text_database.blocks`, `kdn_server.util`, `model`, `proxy`, `proxy.metrics`, `proxy.metrics.data`, `proxy.queue`, `proxy.resource`, `proxy.sclient`, `proxy.strategy`, `scheduler`, `scheduler.knowledge`, `scheduler.resource`, `scheduler.strategy`, `store`, `util`. +`cacheroute`, `cacheroute.compat`, `cacheroute.observability`, `cacheroute.observability.v1`, `cacheroute.runtime`, `cacheroute.topology`, `cacheroute.cache`, `cacheroute.routing`, `cacheroute.contracts`, `cacheroute.contracts.v1`, `cacheroute_compat`, `UI`, `UI.client_ui`, `UI.kdn_ui`, `UI.proxy_ui`, `client`, `client.taskset`, `core`, `data`, `instance`, `instance.TPOT_predictor`, `instance.TTFT_predictor`, `instance.pclient`, `instance.resource_agent`, `instance.resource_dashboard`, `kdn_server`, `kdn_server.contracts`, `kdn_server.domain`, `kdn_server.gateway`, `kdn_server.sclient`, `kdn_server.util`, `model`, `proxy`, `proxy.metrics`, `proxy.queue`, `proxy.resource`, `proxy.sclient`, `proxy.strategy`, `scheduler`, `scheduler.knowledge`, `scheduler.resource`, `scheduler.strategy`, `store`, `util`. + + +Tracked datasets, runtime state, browser assets, calibration tables, native-source +directories, and other package data are not Python packages merely because they +are nested below an installed package. Package-data patterns retain the reviewed +runtime assets; the explicit package list contains only directories with tracked +Python source. In particular, repository datasets under `data/CacheRoute_dataset` +and KDN state under `kdn_server/{KV_database,text_database}` remain outside the +wheel and are not package declarations. diff --git a/pyproject.toml b/pyproject.toml index aa549a0..21a155c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,18 +45,13 @@ packages = [ "cacheroute.runtime", "cacheroute.topology", "cacheroute.cache", "cacheroute.routing", "cacheroute.contracts", "cacheroute.contracts.v1", "cacheroute_compat", - "UI", "UI.client_ui", "UI.client_ui.static", "UI.client_ui.templates", - "UI.kdn_ui", "UI.proxy_ui", "UI.proxy_ui.static", - "client", "client.taskset", "core", "data", "data.CacheRoute_dataset", - "data.CacheRoute_dataset.knowledge_document", "instance", "instance.TPOT_predictor", - "instance.TTFT_predictor", "instance.TTFT_predictor.data", "instance.pclient", - "instance.resource_agent", "instance.resource_agent.src", - "instance.resource_dashboard", "instance.resource_dashboard.static", - "kdn_server", "kdn_server.KV_database", "kdn_server.contracts", + "UI", "UI.client_ui", "UI.kdn_ui", "UI.proxy_ui", + "client", "client.taskset", "core", "data", "instance", "instance.TPOT_predictor", + "instance.TTFT_predictor", "instance.pclient", "instance.resource_agent", + "instance.resource_dashboard", "kdn_server", "kdn_server.contracts", "kdn_server.domain", "kdn_server.gateway", "kdn_server.sclient", - "kdn_server.text_database", "kdn_server.text_database.blocks", "kdn_server.util", "model", "proxy", - "proxy.metrics", "proxy.metrics.data", "proxy.queue", "proxy.resource", + "proxy.metrics", "proxy.queue", "proxy.resource", "proxy.sclient", "proxy.strategy", "scheduler", "scheduler.knowledge", "scheduler.resource", "scheduler.strategy", "store", "util", ] diff --git a/test/test_repository_governance.py b/test/test_repository_governance.py index 31f8015..cbd2ba9 100644 --- a/test/test_repository_governance.py +++ b/test/test_repository_governance.py @@ -2,7 +2,6 @@ import ast from fnmatch import fnmatchcase -import os from pathlib import Path import re import subprocess @@ -134,20 +133,48 @@ def test_transitional_explicit_packages_match_root_discovery(): def _discover_root_namespace_packages(): packages = set() - for directory, child_directories, _files in os.walk(ROOT): - relative = Path(directory).relative_to(ROOT) - if relative == Path("."): - child_directories[:] = [name for name in child_directories if not name.startswith(".")] - continue - package = ".".join(relative.parts) - if any(fnmatchcase(package, pattern) for pattern in GENERATED_PACKAGE_EXCLUDES): - child_directories[:] = [] + for path in _tracked_files(): + relative = path.relative_to(ROOT) + if path.suffix != ".py" or relative.name == "__init__.py" and len(relative.parts) == 1: continue - if not any(part.startswith(".") for part in relative.parts): + package = ".".join(relative.parent.parts) + if package and not any( + fnmatchcase(package, pattern) for pattern in GENERATED_PACKAGE_EXCLUDES + ): packages.add(package) return packages +def _configured_package_path(package, package_directories): + mapped = [ + prefix for prefix in package_directories + if package == prefix or package.startswith(f"{prefix}.") + ] + if not mapped: + return Path(*package.split(".")) + prefix = max(mapped, key=len) + suffix = package.removeprefix(prefix).lstrip(".") + return Path(package_directories[prefix], *suffix.split(".")) if suffix else Path(package_directories[prefix]) + + +def test_explicit_packages_have_reviewed_tracked_python_sources(): + configuration = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) + setuptools = configuration["tool"]["setuptools"] + tracked = {path.relative_to(ROOT) for path in _tracked_files()} + invalid = [] + for package in setuptools["packages"]: + directory = _configured_package_path(package, setuptools.get("package-dir", {})) + if not any( + path.parent == directory and path.suffix == ".py" + for path in tracked + ): + invalid.append(package) + assert not invalid, ( + "explicit setuptools packages without tracked Python source: " + + ", ".join(sorted(invalid)) + ) + + def test_legacy_compatibility_references_are_narrowly_allowlisted(): references = _files_containing("cacheroute_compat") assert references <= LEGACY_REFERENCE_ALLOWLIST diff --git a/test/test_wheel_install.py b/test/test_wheel_install.py index 728ac2c..2963526 100644 --- a/test/test_wheel_install.py +++ b/test/test_wheel_install.py @@ -1,10 +1,11 @@ """Verify wheel contents and imports without using the source checkout.""" from pathlib import Path +import io import os -import shutil import subprocess import sys +import tarfile import venv import zipfile @@ -64,24 +65,142 @@ } +def _tracked_paths(repo: Path) -> set[Path]: + result = subprocess.run( + ["git", "ls-tree", "-rz", "--name-only", "HEAD"], + cwd=repo, check=True, capture_output=True, + ) + return { + Path(os.fsdecode(name)) + for name in result.stdout.split(b"\0") if name + } + + +def _extract_our_tar(archive: tarfile.TarFile, destination: Path) -> None: + """Safely extract an archive generated by Git or the build backend.""" + destination = destination.resolve() + if sys.version_info >= (3, 12): + archive.extractall(destination, filter="data") + return + + # Python 3.10 and 3.11 do not expose tarfile's extraction filters. These + # tests create both accepted archives themselves, so rejecting links and + # paths outside the destination provides the compatible narrow fallback. + for member in archive.getmembers(): + target = (destination / member.name).resolve() + if not target.is_relative_to(destination): + raise ValueError(f"tar member escapes extraction root: {member.name}") + if member.issym() or member.islnk(): + raise ValueError(f"tar links are not supported by this test: {member.name}") + archive.extractall(destination) + + +def _export_tracked_source(repo: Path, destination: Path) -> None: + archive = subprocess.run( + ["git", "archive", "--format=tar", "HEAD"], + cwd=repo, check=True, capture_output=True, + ).stdout + destination.mkdir() + with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as source: + _extract_our_tar(source, destination) + + +def _pip_wheel(source: Path, wheelhouse: Path) -> Path: + subprocess.run( + [sys.executable, "-m", "pip", "wheel", str(source), "--no-deps", + "--no-build-isolation", "-w", str(wheelhouse)], + check=True, + ) + return next(wheelhouse.glob("cacheroute-*.whl")) + + @pytest.fixture(scope="module") -def built_wheel(tmp_path_factory): +def release_artifacts(tmp_path_factory): repo = Path(__file__).resolve().parents[1] - wheelhouse = tmp_path_factory.mktemp("wheelhouse") - source = tmp_path_factory.mktemp("wheel-source") / "CacheRoute" - shutil.copytree( - repo, - source, - ignore=shutil.ignore_patterns( - ".git", "build", "dist", "*.egg-info", "__pycache__", - ".pytest_cache", ".mypy_cache", ".ruff_cache", ".venv", "wheelhouse", - ), + if not (repo / ".git").exists(): + wheelhouse = tmp_path_factory.mktemp("sdist-wheelhouse") + return { + "source": repo, + "direct_wheel": None, + "sdist": None, + "sdist_source": repo, + "wheel": _pip_wheel(repo, wheelhouse), + } + + source = tmp_path_factory.mktemp("tracked-source") / "CacheRoute" + _export_tracked_source(repo, source) + exported = { + path.relative_to(source) for path in source.rglob("*") + if path.is_file() or path.is_symlink() + } + assert exported == _tracked_paths(repo) + assert not (source / ".git").exists() + + direct_wheelhouse = tmp_path_factory.mktemp("direct-wheelhouse") + direct_wheel = _pip_wheel(source, direct_wheelhouse) + + sdist_dir = tmp_path_factory.mktemp("sdist") + backend_script = ( + "import setuptools.build_meta as backend, sys; " + "print(backend.build_sdist(sys.argv[1]))" ) subprocess.run( - [sys.executable, "-m", "pip", "wheel", str(source), "--no-deps", "--no-build-isolation", "-w", str(wheelhouse)], + [sys.executable, "-c", backend_script, str(sdist_dir)], + cwd=source, check=True, + ) + sdist = next(sdist_dir.glob("cacheroute-*.tar.gz")) + + sdist_source_root = tmp_path_factory.mktemp("sdist-source") + with tarfile.open(sdist, mode="r:gz") as archive: + _extract_our_tar(archive, sdist_source_root) + sdist_source = next(path for path in sdist_source_root.iterdir() if path.is_dir()) + assert not (sdist_source / ".git").exists() + final_wheelhouse = tmp_path_factory.mktemp("sdist-wheelhouse") + final_wheel = _pip_wheel(sdist_source, final_wheelhouse) + return { + "source": source, + "direct_wheel": direct_wheel, + "sdist": sdist, + "sdist_source": sdist_source, + "wheel": final_wheel, + } + + +@pytest.fixture(scope="module") +def built_wheel(release_artifacts): + return release_artifacts["wheel"] + + +def test_release_artifacts_use_tracked_source_and_both_build_paths(release_artifacts): + if release_artifacts["direct_wheel"] is None: + pytest.skip("tracked-source generation requires a Git checkout") + assert release_artifacts["direct_wheel"].is_file() + assert release_artifacts["sdist"].is_file() + assert release_artifacts["wheel"].is_file() + assert not (release_artifacts["source"] / ".git").exists() + + +def test_generated_sdist_runs_git_free_wheel_validation(release_artifacts): + if release_artifacts["sdist"] is None: + pytest.skip("generated-sdist regression is run by the Git checkout") + result = subprocess.run( + [ + sys.executable, "-m", "pytest", "-q", + "test/test_wheel_install.py::test_git_free_source_builds_wheel_without_head", + ], + cwd=release_artifacts["sdist_source"], check=True, + text=True, + capture_output=True, ) - return next(wheelhouse.glob("cacheroute-*.whl")) + assert "1 passed" in result.stdout + + +def test_git_free_source_builds_wheel_without_head(release_artifacts): + if release_artifacts["sdist"] is not None: + pytest.skip("Git-free wheel validation runs from the generated sdist") + assert not (release_artifacts["source"] / ".git").exists() + assert release_artifacts["wheel"].is_file() def _create_isolated_environment(path: Path) -> Path: