From e55abdd8c5f2ae28bd2bdda535d3a58f226d0067 Mon Sep 17 00:00:00 2001 From: Austin Gregg-Smith Date: Sun, 30 Aug 2026 00:59:59 +0100 Subject: [PATCH 1/2] Reclaim the tagged derivative subtrees Most of a full cache is inside worktrees that have to stand: 18 of the 72 measured on the reference host carried a whole `.pixi/envs/default`, about 94.5 GB of the 104.5, and a site that is dirty, unpushed or unprovable keeps every byte of it. What makes reaching inside one legitimate is what the standing verdict is a statement about. Every reason a site stands, except a claimant's, is an answer about git's account of that site's content, and nothing under `.pixi/envs/` has ever been in that account: pixi's own `.pixi/.gitignore` of `*` and `!config.toml` puts it outside every index, every status and every commit. The two sets of bytes are disjoint, and a file the installer wrote is what separates them. The gate is the Cache Directory Tagging Specification's published 43 byte signature, and it never reads a directory's name: rattler, cargo, uv and pytest write one, `python -m venv` and npm write none, so the same name lands on both sides. The walk stops at the outermost tag and at every site the forest holds. A tag says regenerable and not by what, so one reader answers with the thing that re-derives it: `conda-meta/pixi` for the environment name, then a `pixi.lock` found by walking up inside the site. `manifest_path` is never stored, because it is a container path on every host environment. An absent lock, a lock that no longer names the environment, and a tag no reader recognises all stand and are named with their bytes. No new flag. It rides `--prune`'s own plan and its own y/N, and the acting pass re-reads both records under the lock before anything goes. What goes is the tagged directory alone, never `.pixi`. Closes #472. --- CHANGELOG.md | 48 ++ docs/cleanup.md | 91 ++- docs/workspace-tools.md | 5 + rust/devlaunch-core/public-api.rest.txt | 107 +++ .../src/flows/agent_worktrees.rs | 252 ++++++- .../src/flows/agent_worktrees/derivatives.rs | 638 ++++++++++++++++++ .../agent_worktrees/derivatives/tests.rs | 559 +++++++++++++++ .../src/flows/agent_worktrees/tests.rs | 439 ++++++++++++ .../tests/public_api_snapshots.rs | 39 ++ rust/dl/src/render.rs | 77 ++- 10 files changed, 2227 insertions(+), 28 deletions(-) create mode 100644 rust/devlaunch-core/src/flows/agent_worktrees/derivatives.rs create mode 100644 rust/devlaunch-core/src/flows/agent_worktrees/derivatives/tests.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 591f359a..20e2965b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -146,6 +146,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `dl --ls --size` and `--ls --json`'s `disk` object also name the worktree share of a clone's figure, as a part of it and never an addition. +- **`dl --prune` now reclaims the regenerable subtrees inside the agent worktrees + it is leaving standing.** Most of a full cache is in worktrees that have to + stay: 18 of the 72 measured on the reference host carried a whole + `.pixi/envs/default`, about 94.5 GB of the 104.5, and a site that is dirty, + unpushed or unprovable keeps every byte of it. + + What makes reaching inside one legitimate is what the standing verdict is a + statement *about*. Every reason a site stands, except a claimant's, is an answer + about git's account of that site's content, and nothing under `.pixi/envs/` has + ever been in that account: pixi writes a `.pixi/.gitignore` of `*` and + `!config.toml`, which puts it outside every index, every status and every commit + in every clone. The two sets of bytes are disjoint, and a file the installer + wrote is what separates them. Refusing would have meant a report that says "this + holds work that exists nowhere else, 0 bytes" and declines to give back 5 GB in + the same breath. + + The gate is a declaration and not a directory name. It is the Cache Directory + Tagging Specification's published 43 byte signature at the front of a + `CACHEDIR.TAG`, written by the program that made the directory. Measured: + rattler, cargo, uv and pytest write one; `python -m venv` writes none and npm + writes none anywhere beneath `node_modules`. So a `.venv` is taken or left + depending on which program made it, and `.pixi` appears nowhere in the + predicate. + + A tag says regenerable and not by what, so one reader has to answer with the + thing that re-derives it: pixi's own `conda-meta/pixi` for the environment name, + then a `pixi.lock` found by walking up inside the worktree whose `environments:` + map names it. A lock that names the environment restores 5507 of 5507 files in + 0.52 s with every proxy pointed at a dead port, and a stale one restores what + was there. A lock that is absent, one that no longer names the environment, and + a tag no reader recognises all stand, are named with their bytes, and the middle + one gets `pixi clean -e ` as its pointer. + + A claim reaches the subtree and an account of content does not. A + `git worktree lock`, or a repository lock `dl` could not take, pins it; dirty, + unpushed and unprovable do not, because they were never about these bytes. + + **No new flag.** It rides `--prune`'s own plan and its own `y/N`, which names + each directory and its size before the question, and the acting pass reads the + tag, the record and the lockfile again under the lock. Hanging it on + `--force-worktrees` would have made one flag carry two consents, since that flag + also carries past a lock and past another repository's worktree. What goes is + the tagged directory alone, never `.pixi`, which holds `config.toml`. + + `dl --ls` does not cost these, deliberately: weighing one is a full walk of a + worktree plus a walk of a 12000 file environment, and a derivative is never a + reason a site stands. `docs/cleanup.md` carries the whole of it. + ### Changed - **A workspace id is derived once, and the three signatures that had a triple in diff --git a/docs/cleanup.md b/docs/cleanup.md index 4e9fb112..66f9bc70 100644 --- a/docs/cleanup.md +++ b/docs/cleanup.md @@ -521,14 +521,17 @@ Clone directories under /home/you/.cache/devlaunch/repos: Leaving 1: - /home/you/.cache/devlaunch/repos/blooop/devlaunch/devlaunch-main-zovo: workspace devlaunch-main-zovo still opens it -Agent git worktrees inside the clones above -- 6.0 GiB: +Agent git worktrees inside the clones above -- 6.0 GiB in worktrees that go, and 2.3 GiB in regenerable subtrees inside the ones that stay: /home/you/.cache/devlaunch/repos/blooop/devlaunch/devlaunch-main-zovo: - removing .../.claude/worktrees/agent-a49a (5.8 GiB), and dropping its 1 registration(s) - removing .../.claude/worktrees/agent-a8da (204.0 MiB), and dropping its 2 registration(s) - leaving .../.claude/worktrees/agent-b120: git is holding it locked (claude session) -- add --force-worktrees to remove it anyway + - leaving .../.claude/worktrees/agent-c771: holds 3 uncommitted change(s) -- add --force-worktrees to remove it anyway + - reclaiming .../.claude/worktrees/agent-c771/.pixi/envs/default (2.3 GiB): a pixi environment, re-derived by `pixi install -e default` from .claude/worktrees/agent-c771/pixi.lock Whether a worktree's commits are anywhere else is as of the last fetch into the repository cache; --prune does not fetch. +A regenerable subtree is one whose creator wrote a CACHEDIR.TAG into it and whose lockfile is still beside it; putting one back is one command and no network beyond the shared package cache. Are you sure? [y/N] ``` @@ -550,9 +553,89 @@ The 18 duplicated `.pixi/envs/default` copies are the reason the figure is 104 G rather than about 10, and they cannot be pointed at the shared package cache: only the pixi *download* cache is shared, because installed environments bake absolute paths (see "The shared pixi package cache" in -[workspace-tools.md](workspace-tools.md)). Removing the worktree is the way those -bytes come back, which is what this does: an env is gitignored content, and by -the limit above it does not by itself keep a finished worktree standing. +[workspace-tools.md](workspace-tools.md)). Removing the worktree is one way those +bytes come back: an env is gitignored content, and by the limit above it does not +by itself keep a finished worktree standing. The next section is the other way, +and it is the one that reaches the worktrees that have to stay. + +#### The regenerable subtrees inside a worktree it keeps + +| artifact | who reclaims it | what makes that safe | +| --- | --- | --- | +| a directory carrying a `CACHEDIR.TAG` inside an agent worktree that is staying | `--prune`, on the same plan and the same `y/N` as everything above it | the tag plus the lockfile: the program that made the directory declared it regenerable, and a lockfile still inside the worktree names the environment it re-derives | + +Most of the 104.5 GB is in worktrees that have to stand. 18 of the 72 carried a +whole `.pixi/envs/default`, and a site that is dirty, unpushed or unprovable +keeps every byte of it. About 94.5 GB of the measured total is in there, which is +roughly 90%. + +**Reaching into a standing worktree to delete part of it sounds like exactly the +thing the rules above exist to stop, and the answer is what the standing verdict +is a statement about.** Every reason a site stands, except a claimant's, is an +answer about git's account of that site's content: what `git status` says through +its admin directory, what a ref reaches, or the fact that neither could be +obtained. Nothing under `.pixi/envs/` has ever been in that account, because pixi +writes a `.pixi/.gitignore` of `*` and `!config.toml` and puts it outside every +index, every status and every commit, in every clone, forever. The bytes the +verdict is uncertain about and the bytes under the tag are disjoint sets, and a +file the installer wrote is what separates them. The alternative was a report +that says "this holds work that exists nowhere else, 0 bytes" and refuses to give +back 5 GB in the same breath. + +**What names it is a declaration, and it never reads a directory's name.** The +gate is the [Cache Directory Tagging Specification](https://bford.info/cachedir/): +a file called `CACHEDIR.TAG` whose first 43 bytes are the published signature, +written by the program that created the directory to say the contents are +regenerable. Measured: rattler, cargo, uv and pytest write one; `python -m venv` +writes none and npm writes none anywhere beneath `node_modules`. So a `.venv` is +taken or left depending on which program made it, which no rule keyed on the name +could express, and `.pixi` appears nowhere in the predicate. The walk stops at +the outermost tag rather than descending, so a `target/` inside an environment is +never a second line for the same bytes. + +**A tag says regenerable; it does not say by what.** So a tagged directory is +reclaimed only when a reader on this side answers with the thing that re-derives +it, and one reader is implemented. It reads pixi's own `conda-meta/pixi` for the +environment name, then walks up inside the worktree for a `pixi.lock` whose +`environments:` map names it. Measured: a lock that names the environment +restores 5507 of 5507 files in 0.52 s with no network, and does it with every +proxy variable pointed at a dead port; a *stale* lock still restores what was +there, because the environment on disk came from that lock; a lock that is +**absent** restores nothing; and an environment the lock **no longer names** is +reproducible from nothing on disk. The last two stand, are named with their +bytes, and the second gets `pixi clean -e ` as the pointer. A tag no reader +recognises stands the same way. + +**A claim reaches the subtree; an account of content does not.** A +`git worktree lock`, or a repository lock `dl` could not take, is somebody +asserting a claim over the directory and drawing no line between its parts, and a +lock may mean *running right now*. Those pin the tagged subtree along with +everything else. Dirty, unpushed and unprovable do not, because they were never +statements about these bytes. A worktree of **another** repository is in the +second group and not the first: whose repository an environment belongs to was +never part of the argument. + +**No new flag, and it does not ride on `--force-worktrees`.** This is a removal +with a proof rather than a force, and `--force-worktrees` also carries past a +lock and past another repository's worktree, so hanging this on it would make one +flag carry two consents. It rides `--prune`'s own question: the plan names each +directory and its size before the `y/N`, and the acting pass reads the tag, the +record and the lockfile again under the lock before anything goes. + +What goes is the tagged directory alone. Never `.pixi`, which carries no tag and +holds `config.toml`, the one file `.pixi/.gitignore` un-ignores. + +**One thing the tag does not promise, said out loud.** pixi does not defend its +own declaration: a file planted by hand inside an environment survives +`pixi install --frozen` unmentioned, so the tag is a claim about what the +directory is *for* and not a proof about what is in it now. The case rests on the +disjoint sets above, and anything somebody put inside an environment goes with +it. + +`dl --ls` does not cost these. Weighing one is a full walk of a worktree plus a +walk of a 12000-file environment, and the listing is a read-only command people +run casually; a derivative is never a reason a site stands, so the listing loses +no answer by not asking. #### The disk neither command frees diff --git a/docs/workspace-tools.md b/docs/workspace-tools.md index 09a1fa7f..93a6b351 100644 --- a/docs/workspace-tools.md +++ b/docs/workspace-tools.md @@ -690,6 +690,11 @@ trampolines are baked with absolute paths, and two containers sharing one environment tree is [pixi#5476](https://github.com/prefix-dev/pixi/issues/5476). Only the download cache is shared, which is the part that is safe to share. +That is why a project environment installed inside an agent worktree is a real +copy and costs what it costs, and it is why `dl --prune` reclaims those copies +rather than pointing them at anything: see "The regenerable subtrees inside a +worktree it keeps" in [cleanup.md](cleanup.md). + If the directory cannot be created, or is not there when the launch reaches it, because of a full disk, a read-only cache home, or a cache swept between the two, the launch goes ahead without the mount and the container downloads its own packages, diff --git a/rust/devlaunch-core/public-api.rest.txt b/rust/devlaunch-core/public-api.rest.txt index e3b2e54a..4e0c6020 100644 --- a/rust/devlaunch-core/public-api.rest.txt +++ b/rust/devlaunch-core/public-api.rest.txt @@ -647,6 +647,7 @@ pub fn devlaunch_core::domain::xdg::devlaunch_cache() -> core::result::Result) -> core::fmt::Result impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::Collectable +pub enum devlaunch_core::flows::agent_worktrees::NoRecipe +pub devlaunch_core::flows::agent_worktrees::NoRecipe::CouldNotRead(core::io::error::ErrorKind) +pub devlaunch_core::flows::agent_worktrees::NoRecipe::LockfileAbsent +pub devlaunch_core::flows::agent_worktrees::NoRecipe::LockfileDoesNotNameIt +pub devlaunch_core::flows::agent_worktrees::NoRecipe::LockfileDoesNotNameIt::environment: alloc::string::String +pub devlaunch_core::flows::agent_worktrees::NoRecipe::NoReaderRecognisedIt +impl devlaunch_core::flows::agent_worktrees::NoRecipe +pub fn devlaunch_core::flows::agent_worktrees::NoRecipe::describe(&self) -> alloc::string::String +impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::NoRecipe +pub fn devlaunch_core::flows::agent_worktrees::NoRecipe::clone(&self) -> devlaunch_core::flows::agent_worktrees::NoRecipe +impl core::cmp::Eq for devlaunch_core::flows::agent_worktrees::NoRecipe +impl core::cmp::PartialEq for devlaunch_core::flows::agent_worktrees::NoRecipe +pub fn devlaunch_core::flows::agent_worktrees::NoRecipe::eq(&self, &devlaunch_core::flows::agent_worktrees::NoRecipe) -> bool +impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::NoRecipe +pub fn devlaunch_core::flows::agent_worktrees::NoRecipe::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::NoRecipe +pub enum devlaunch_core::flows::agent_worktrees::NotDerivableNow +pub devlaunch_core::flows::agent_worktrees::NotDerivableNow::Answered(alloc::boxed::Box) +pub devlaunch_core::flows::agent_worktrees::NotDerivableNow::NoTagThere +impl devlaunch_core::flows::agent_worktrees::NotDerivableNow +pub fn devlaunch_core::flows::agent_worktrees::NotDerivableNow::describe(&self) -> alloc::string::String +impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::NotDerivableNow +pub fn devlaunch_core::flows::agent_worktrees::NotDerivableNow::clone(&self) -> devlaunch_core::flows::agent_worktrees::NotDerivableNow +impl core::cmp::Eq for devlaunch_core::flows::agent_worktrees::NotDerivableNow +impl core::cmp::PartialEq for devlaunch_core::flows::agent_worktrees::NotDerivableNow +pub fn devlaunch_core::flows::agent_worktrees::NotDerivableNow::eq(&self, &devlaunch_core::flows::agent_worktrees::NotDerivableNow) -> bool +impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::NotDerivableNow +pub fn devlaunch_core::flows::agent_worktrees::NotDerivableNow::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::NotDerivableNow pub enum devlaunch_core::flows::agent_worktrees::Place pub devlaunch_core::flows::agent_worktrees::Place::ASite(devlaunch_core::flows::agent_worktrees::Inside) pub devlaunch_core::flows::agent_worktrees::Place::TheCloneItself @@ -702,6 +732,20 @@ pub fn devlaunch_core::flows::agent_worktrees::Reason::eq(&self, &devlaunch_core impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::Reason pub fn devlaunch_core::flows::agent_worktrees::Reason::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::Reason +pub enum devlaunch_core::flows::agent_worktrees::Recipe +pub devlaunch_core::flows::agent_worktrees::Recipe::PixiEnvironment +pub devlaunch_core::flows::agent_worktrees::Recipe::PixiEnvironment::environment: alloc::string::String +pub devlaunch_core::flows::agent_worktrees::Recipe::PixiEnvironment::lock: devlaunch_core::flows::agent_worktrees::Inside +impl devlaunch_core::flows::agent_worktrees::Recipe +pub fn devlaunch_core::flows::agent_worktrees::Recipe::describe(&self) -> alloc::string::String +impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::Recipe +pub fn devlaunch_core::flows::agent_worktrees::Recipe::clone(&self) -> devlaunch_core::flows::agent_worktrees::Recipe +impl core::cmp::Eq for devlaunch_core::flows::agent_worktrees::Recipe +impl core::cmp::PartialEq for devlaunch_core::flows::agent_worktrees::Recipe +pub fn devlaunch_core::flows::agent_worktrees::Recipe::eq(&self, &devlaunch_core::flows::agent_worktrees::Recipe) -> bool +impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::Recipe +pub fn devlaunch_core::flows::agent_worktrees::Recipe::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::Recipe pub enum devlaunch_core::flows::agent_worktrees::Subject pub devlaunch_core::flows::agent_worktrees::Subject::AClaim pub devlaunch_core::flows::agent_worktrees::Subject::GitsAccountOfContent @@ -714,6 +758,29 @@ impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::Subject pub fn devlaunch_core::flows::agent_worktrees::Subject::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::Copy for devlaunch_core::flows::agent_worktrees::Subject impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::Subject +pub enum devlaunch_core::flows::agent_worktrees::Tagged +pub devlaunch_core::flows::agent_worktrees::Tagged::Claimed +pub devlaunch_core::flows::agent_worktrees::Tagged::Claimed::at: devlaunch_core::flows::agent_worktrees::Inside +pub devlaunch_core::flows::agent_worktrees::Tagged::Claimed::by: alloc::boxed::Box +pub devlaunch_core::flows::agent_worktrees::Tagged::Claimed::bytes: devlaunch_core::flows::disk_usage::DiskUsage +pub devlaunch_core::flows::agent_worktrees::Tagged::CouldNotCost +pub devlaunch_core::flows::agent_worktrees::Tagged::CouldNotCost::at: devlaunch_core::flows::agent_worktrees::Inside +pub devlaunch_core::flows::agent_worktrees::Tagged::CouldNotCost::bytes: devlaunch_core::flows::disk_usage::DiskUsage +pub devlaunch_core::flows::agent_worktrees::Tagged::CouldNotCost::why: devlaunch_core::flows::agent_worktrees::NoRecipe +pub devlaunch_core::flows::agent_worktrees::Tagged::Derivable(devlaunch_core::flows::agent_worktrees::Derivative) +impl devlaunch_core::flows::agent_worktrees::Tagged +pub fn devlaunch_core::flows::agent_worktrees::Tagged::at(&self) -> &devlaunch_core::flows::agent_worktrees::Inside +pub fn devlaunch_core::flows::agent_worktrees::Tagged::derivable(&self) -> core::option::Option<&devlaunch_core::flows::agent_worktrees::Derivative> +pub fn devlaunch_core::flows::agent_worktrees::Tagged::standing(&self) -> core::option::Option +pub fn devlaunch_core::flows::agent_worktrees::Tagged::usage(&self) -> &devlaunch_core::flows::disk_usage::DiskUsage +impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::Tagged +pub fn devlaunch_core::flows::agent_worktrees::Tagged::clone(&self) -> devlaunch_core::flows::agent_worktrees::Tagged +impl core::cmp::Eq for devlaunch_core::flows::agent_worktrees::Tagged +impl core::cmp::PartialEq for devlaunch_core::flows::agent_worktrees::Tagged +pub fn devlaunch_core::flows::agent_worktrees::Tagged::eq(&self, &devlaunch_core::flows::agent_worktrees::Tagged) -> bool +impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::Tagged +pub fn devlaunch_core::flows::agent_worktrees::Tagged::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::Tagged pub enum devlaunch_core::flows::agent_worktrees::Unaccountable pub devlaunch_core::flows::agent_worktrees::Unaccountable::GitfileUnreadable pub devlaunch_core::flows::agent_worktrees::Unaccountable::PlainDirectory @@ -774,6 +841,7 @@ impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktree pub struct devlaunch_core::flows::agent_worktrees::CloneWorktrees impl devlaunch_core::flows::agent_worktrees::CloneWorktrees pub fn devlaunch_core::flows::agent_worktrees::CloneWorktrees::clone_path(&self) -> &std::path::Path +pub fn devlaunch_core::flows::agent_worktrees::CloneWorktrees::derivatives(&self) -> &[devlaunch_core::flows::agent_worktrees::Tagged] pub fn devlaunch_core::flows::agent_worktrees::CloneWorktrees::going(&self) -> &[devlaunch_core::flows::agent_worktrees::Going] pub fn devlaunch_core::flows::agent_worktrees::CloneWorktrees::owner(&self) -> &str pub fn devlaunch_core::flows::agent_worktrees::CloneWorktrees::repo(&self) -> &str @@ -786,6 +854,19 @@ pub fn devlaunch_core::flows::agent_worktrees::CloneWorktrees::eq(&self, &devlau impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::CloneWorktrees pub fn devlaunch_core::flows::agent_worktrees::CloneWorktrees::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::CloneWorktrees +pub struct devlaunch_core::flows::agent_worktrees::Derivative +impl devlaunch_core::flows::agent_worktrees::Derivative +pub fn devlaunch_core::flows::agent_worktrees::Derivative::at(&self) -> &devlaunch_core::flows::agent_worktrees::Inside +pub fn devlaunch_core::flows::agent_worktrees::Derivative::recipe(&self) -> &devlaunch_core::flows::agent_worktrees::Recipe +pub fn devlaunch_core::flows::agent_worktrees::Derivative::usage(&self) -> &devlaunch_core::flows::disk_usage::DiskUsage +impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::Derivative +pub fn devlaunch_core::flows::agent_worktrees::Derivative::clone(&self) -> devlaunch_core::flows::agent_worktrees::Derivative +impl core::cmp::Eq for devlaunch_core::flows::agent_worktrees::Derivative +impl core::cmp::PartialEq for devlaunch_core::flows::agent_worktrees::Derivative +pub fn devlaunch_core::flows::agent_worktrees::Derivative::eq(&self, &devlaunch_core::flows::agent_worktrees::Derivative) -> bool +impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::Derivative +pub fn devlaunch_core::flows::agent_worktrees::Derivative::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::Derivative pub struct devlaunch_core::flows::agent_worktrees::ForgetRefused pub devlaunch_core::flows::agent_worktrees::ForgetRefused::reason: alloc::string::String pub devlaunch_core::flows::agent_worktrees::ForgetRefused::registered: std::path::PathBuf @@ -869,6 +950,17 @@ pub fn devlaunch_core::flows::agent_worktrees::Proof::eq(&self, &devlaunch_core: impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::Proof pub fn devlaunch_core::flows::agent_worktrees::Proof::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::Proof +pub struct devlaunch_core::flows::agent_worktrees::ReclaimedDerivative +pub devlaunch_core::flows::agent_worktrees::ReclaimedDerivative::path: std::path::PathBuf +pub devlaunch_core::flows::agent_worktrees::ReclaimedDerivative::usage: devlaunch_core::flows::disk_usage::DiskUsage +impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::ReclaimedDerivative +pub fn devlaunch_core::flows::agent_worktrees::ReclaimedDerivative::clone(&self) -> devlaunch_core::flows::agent_worktrees::ReclaimedDerivative +impl core::cmp::Eq for devlaunch_core::flows::agent_worktrees::ReclaimedDerivative +impl core::cmp::PartialEq for devlaunch_core::flows::agent_worktrees::ReclaimedDerivative +pub fn devlaunch_core::flows::agent_worktrees::ReclaimedDerivative::eq(&self, &devlaunch_core::flows::agent_worktrees::ReclaimedDerivative) -> bool +impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::ReclaimedDerivative +pub fn devlaunch_core::flows::agent_worktrees::ReclaimedDerivative::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::ReclaimedDerivative pub struct devlaunch_core::flows::agent_worktrees::Recorded impl devlaunch_core::flows::agent_worktrees::Recorded pub fn devlaunch_core::flows::agent_worktrees::Recorded::as_path(&self) -> &std::path::Path @@ -929,13 +1021,27 @@ pub fn devlaunch_core::flows::agent_worktrees::WithheldWorktree::eq(&self, &devl impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::WithheldWorktree pub fn devlaunch_core::flows::agent_worktrees::WithheldWorktree::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::WithheldWorktree +pub struct devlaunch_core::flows::agent_worktrees::WithheldDerivative +pub devlaunch_core::flows::agent_worktrees::WithheldDerivative::because: devlaunch_core::flows::agent_worktrees::NotDerivableNow +pub devlaunch_core::flows::agent_worktrees::WithheldDerivative::path: std::path::PathBuf +impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::WithheldDerivative +pub fn devlaunch_core::flows::agent_worktrees::WithheldDerivative::clone(&self) -> devlaunch_core::flows::agent_worktrees::WithheldDerivative +impl core::cmp::Eq for devlaunch_core::flows::agent_worktrees::WithheldDerivative +impl core::cmp::PartialEq for devlaunch_core::flows::agent_worktrees::WithheldDerivative +pub fn devlaunch_core::flows::agent_worktrees::WithheldDerivative::eq(&self, &devlaunch_core::flows::agent_worktrees::WithheldDerivative) -> bool +impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::WithheldDerivative +pub fn devlaunch_core::flows::agent_worktrees::WithheldDerivative::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::WithheldDerivative pub struct devlaunch_core::flows::agent_worktrees::WorktreeReport pub devlaunch_core::flows::agent_worktrees::WorktreeReport::forget_refused: alloc::vec::Vec pub devlaunch_core::flows::agent_worktrees::WorktreeReport::forgotten: usize +pub devlaunch_core::flows::agent_worktrees::WorktreeReport::reclaimed: alloc::vec::Vec pub devlaunch_core::flows::agent_worktrees::WorktreeReport::refused: alloc::vec::Vec pub devlaunch_core::flows::agent_worktrees::WorktreeReport::removed: alloc::vec::Vec pub devlaunch_core::flows::agent_worktrees::WorktreeReport::withheld: alloc::vec::Vec +pub devlaunch_core::flows::agent_worktrees::WorktreeReport::withheld_derivatives: alloc::vec::Vec impl devlaunch_core::flows::agent_worktrees::WorktreeReport +pub fn devlaunch_core::flows::agent_worktrees::WorktreeReport::derivatives_freed(&self) -> devlaunch_core::flows::disk_usage::DiskUsage pub fn devlaunch_core::flows::agent_worktrees::WorktreeReport::freed(&self) -> devlaunch_core::flows::disk_usage::DiskUsage pub fn devlaunch_core::flows::agent_worktrees::WorktreeReport::nothing_to_say(&self) -> bool impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::WorktreeReport @@ -951,6 +1057,7 @@ impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktree pub struct devlaunch_core::flows::agent_worktrees::WorktreeSweep impl devlaunch_core::flows::agent_worktrees::WorktreeSweep pub fn devlaunch_core::flows::agent_worktrees::WorktreeSweep::clones(&self) -> &[devlaunch_core::flows::agent_worktrees::CloneWorktrees] +pub fn devlaunch_core::flows::agent_worktrees::WorktreeSweep::derivatives_freed(&self) -> devlaunch_core::flows::disk_usage::DiskUsage pub fn devlaunch_core::flows::agent_worktrees::WorktreeSweep::freed(&self) -> devlaunch_core::flows::disk_usage::DiskUsage pub fn devlaunch_core::flows::agent_worktrees::WorktreeSweep::nothing_to_do(&self) -> bool pub fn devlaunch_core::flows::agent_worktrees::WorktreeSweep::nothing_to_say(&self) -> bool diff --git a/rust/devlaunch-core/src/flows/agent_worktrees.rs b/rust/devlaunch-core/src/flows/agent_worktrees.rs index 292e3014..7e80c9a1 100644 --- a/rust/devlaunch-core/src/flows/agent_worktrees.rs +++ b/rust/devlaunch-core/src/flows/agent_worktrees.rs @@ -144,6 +144,13 @@ use crate::flows::disk_usage::{self, DiskUsage}; use crate::flows::lifecycle::Insistence; use crate::flows::repo_manager::{Refusal, TreeSweep, remove_tree_as_far_as_it_goes}; +mod derivatives; + +pub use derivatives::{ + Derivative, NoRecipe, NotDerivableNow, Recipe, ReclaimedDerivative, Tagged, WithheldDerivative, +}; +use derivatives::{Derivatives, claims_over, tagged_in}; + /// The directory an agent harness puts its worktrees in, relative to a clone. const WORKTREES_DIR: [&str; 2] = [".claude", "worktrees"]; @@ -896,6 +903,9 @@ impl Reason { // reason it was not in the plan. A claimant, so #468's // derivative reclaim does not reach into it either. Blank::AppearedAfterThePlan => Subject::AClaim, + // A worktree, which is a claim on the directory holding it by + // something the tag does not speak for. + Blank::ASiteSitsInside => Subject::AClaim, Blank::NothingToAskThrough | Blank::GitWouldNotSay(_) // Decided on devlaunch#468: another repository's env, tagged, is @@ -1010,6 +1020,11 @@ impl Blank { said yes to removing it" .to_owned() } + Self::ASiteSitsInside => { + "a git worktree sits inside it, and whatever declared it regenerable was not \ + speaking for that" + .to_owned() + } Self::NotThisClonesToAccountFor(why) => why.describe().to_owned(), } } @@ -1089,6 +1104,11 @@ pub enum Blank { /// plan meant this one, so the whole unit is withheld and offered again next /// run — by which time it is in the plan they read. AppearedAfterThePlan, + /// A site sits inside a tagged directory, so whatever declared that + /// directory regenerable was not speaking for what is in it (devlaunch#468 + /// §6). Produced by the derivative fold alone and never by a site's own + /// verdict: a site is never *this* to itself. + ASiteSitsInside, } // =========================================================================== @@ -1157,6 +1177,9 @@ struct Weigher<'a, 'r> { /// it. `Result` rather than the witness alone: a refusal is an answer and /// re-asking it would not make it a different one. reachability: RefCell>>, + /// Whether this pass costs the tagged derivatives inside the sites it + /// stands. See [`Derivatives`]. + derivatives: Derivatives, } impl Weigher<'_, '_> { @@ -1605,6 +1628,11 @@ pub struct CloneWorktrees { repo: String, going: Vec, standing: Vec, + /// The tagged derivative subtrees inside the sites this run is leaving + /// standing (devlaunch#468). A derivative inside a site that is itself + /// going is not in here: the site's own removal accounts for it, which is + /// devlaunch#446 §6's two-recursions rule extended one artifact over. + derivatives: Vec, } impl CloneWorktrees { @@ -1632,12 +1660,18 @@ impl CloneWorktrees { &self.standing } + /// Every tagged derivative inside the sites this run leaves standing, the + /// ones it will reclaim and the ones it will not, each with its bytes. + pub fn derivatives(&self) -> &[Tagged] { + &self.derivatives + } + fn nothing_to_do(&self) -> bool { - self.going.is_empty() + self.going.is_empty() && !self.derivatives.iter().any(|it| it.derivable().is_some()) } fn nothing_to_say(&self) -> bool { - self.going.is_empty() && self.standing.is_empty() + self.going.is_empty() && self.standing.is_empty() && self.derivatives.is_empty() } } @@ -1655,6 +1689,23 @@ impl WorktreeSweep { &self.clones } + /// What reclaiming the tagged derivatives inside the *standing* sites would + /// free. + /// + /// Its own figure beside [`Self::freed`] rather than folded into it, for + /// the reason `PrunePlan::clones_freed` gives about the clones: these are a + /// different claim about a different set of directories — every one of them + /// is inside a site this run has just said it is leaving — and one number + /// covering both would describe neither. + pub fn derivatives_freed(&self) -> DiskUsage { + disk_usage::total_usage(self.clones.iter().flat_map(|clone| { + clone + .derivatives + .iter() + .filter_map(|it| it.derivable().map(|one| one.usage().clone())) + })) + } + /// What the whole sweep would free. pub fn freed(&self) -> DiskUsage { disk_usage::total_usage(self.clones.iter().flat_map(|clone| { @@ -1691,6 +1742,11 @@ struct Weighed { /// Standing sites in this subtree, own reasons only. Empty while /// `removable` is `Some` — an insisted subtree's reasons ride in `despite`. standing: Vec, + /// The tagged derivatives inside this subtree. Empty while `removable` is + /// `Some`, and that emptiness is the two-recursions rule rather than an + /// omission: a subtree that is going takes its derivatives with it, and + /// billing them a second time is exactly the double count R3 forbids. + derivatives: Vec, } /// The whole subtree, ready to be one [`Going`] if the parent absorbs it or to @@ -1712,18 +1768,26 @@ struct Removable { /// it: the collectable arm is reachable only when every child's recursion /// handed one back, and the caller passes no child list — so "a parent goes /// while a nested site stands" is not guarded against, it has no path. +/// +/// `claims` is every claimant reason in force from an ancestor. It flows *down* +/// while the verdict folds up, which is why this site's own verdict is taken +/// before its children are weighed rather than after: a lock is a claim over +/// everything inside the directory it names, and the derivative fold one level +/// in has to know about it. fn weigh( weigher: &Weigher<'_, '_>, site: &Site, insistence: Insistence, forest: &[PathBuf], + claims: &[Reason], ) -> Weighed { + let own = weigher.own_verdict(site, forest); + let claims_here = claims_over(claims, &own); let children: Vec = site .nested .iter() - .map(|child| weigh(weigher, child, insistence, forest)) + .map(|child| weigh(weigher, child, insistence, forest, &claims_here)) .collect(); - let own = weigher.own_verdict(site, forest); let own_removable: Option> = match &own { Verdict::Collectable(_) => Some(Vec::new()), Verdict::Stands(standing) => match insistence { @@ -1753,6 +1817,7 @@ fn weigh( removable: Some(merged), going: Vec::new(), standing: Vec::new(), + derivatives: Vec::new(), }; } // Something here stands, so nothing above this site can go: materialize the @@ -1760,12 +1825,14 @@ fn weigh( // sites that stand of their own accord. let mut going = Vec::new(); let mut standing = Vec::new(); + let mut derivatives = own_derivatives(weigher, site, &claims_here, forest); for child in children { if let Some(removable) = child.removable { going.push(materialize(removable)); } going.extend(child.going); standing.extend(child.standing); + derivatives.extend(child.derivatives); } if let Verdict::Stands(reasons) = own { standing.push(StandingSite { @@ -1777,6 +1844,40 @@ fn weigh( removable: None, going, standing, + derivatives, + } +} + +/// The tagged derivatives inside one standing site's own directory. +/// +/// Two arms take none. A registration with nothing at its place has no +/// directory to walk. And a **symlink** in the worktrees place is never walked: +/// following it is how a removal leaves the tree `--prune` is scoped to, which +/// is the same reason [`walk_sites`] never follows one either. +/// +/// A worktree of another repository *does* get walked, and that is decided +/// rather than overlooked: devlaunch#468 §6 names +/// [`Blank::NotThisClonesToAccountFor`] explicitly, because whose repository a +/// tagged environment belongs to was never part of the argument — the tag and +/// the lockfile beside it say what they say either way. +fn own_derivatives( + weigher: &Weigher<'_, '_>, + site: &Site, + claims: &[Reason], + forest: &[PathBuf], +) -> Vec { + if weigher.derivatives == Derivatives::NotAsked { + return Vec::new(); + } + match &site.kind { + SiteKind::OursGone { .. } => Vec::new(), + SiteKind::NotOurs { + why: Unaccountable::SymlinkInThePlace, + .. + } => Vec::new(), + SiteKind::OursHere { at, .. } | SiteKind::NotOurs { at, .. } => { + tagged_in(weigher.clone, at, claims, forest) + } } } @@ -1835,29 +1936,45 @@ pub(crate) fn sweep_clone( return None; } let picture = ClonePicture::of(git, clone)?; - let (going, standing) = weigh_clone(git, clone, bare, &picture, |_| insistence); + let weighed = weigh_clone(git, clone, bare, &picture, Derivatives::Weighed, |_| { + insistence + }); Some(CloneWorktrees { clone: clone.to_path_buf(), owner: owner.to_owned(), repo: repo.to_owned(), - going, - standing, + going: weighed.going, + standing: weighed.standing, + derivatives: weighed.derivatives, }) } +/// One clone's whole weighing: what goes, what stands, and the tagged +/// derivatives inside what stands. +/// +/// A struct rather than a tuple because the third member arrived and a +/// three-tuple of `Vec`s is three chances to bind the wrong one. +struct Weighing { + going: Vec, + standing: Vec, + derivatives: Vec, +} + /// Weigh every root in one clone's forest, with an insistence per going root. fn weigh_clone( git: &Git<'_>, clone: &Path, bare: Option<&Path>, picture: &ClonePicture, + want: Derivatives, insist: impl Fn(&Site) -> Insistence, -) -> (Vec, Vec) { +) -> Weighing { let weigher = Weigher { git, clone, bare, reachability: RefCell::new(HashMap::new()), + derivatives: want, }; let roots = forest_of(clone, picture); let mut forest_paths = Vec::new(); @@ -1866,13 +1983,15 @@ fn weigh_clone( } let mut going = Vec::new(); let mut standing = Vec::new(); + let mut derivatives = Vec::new(); for root in &roots { - let weighed = weigh(&weigher, root, insist(root), &forest_paths); + let weighed = weigh(&weigher, root, insist(root), &forest_paths, &[]); if let Some(removable) = weighed.removable { going.push(materialize(removable)); } going.extend(weighed.going); standing.extend(weighed.standing); + derivatives.extend(weighed.derivatives); } going.sort_by(|left, right| { let bytes = |unit: &Going| match &unit.what { @@ -1888,7 +2007,18 @@ fn weigh_clone( .then_with(|| path(left).cmp(&path(right))) }); standing.sort_by(|left, right| left.at.cmp(&right.at)); - (going, standing) + derivatives.sort_by(|left, right| { + right + .usage() + .known_bytes() + .cmp(&left.usage().known_bytes()) + .then_with(|| left.at().cmp(right.at())) + }); + Weighing { + going, + standing, + derivatives, + } } // =========================================================================== @@ -1935,20 +2065,33 @@ pub struct WorktreeReport { /// Registrations dropped, by name, each one read from a listing. pub forgotten: usize, pub forget_refused: Vec, + /// The tagged derivative subtrees this run reclaimed (devlaunch#468). + pub reclaimed: Vec, + /// The ones the plan named that the re-read would not hand back. + pub withheld_derivatives: Vec, } impl WorktreeReport { - /// What this run actually freed. + /// What this run actually freed by removing agent worktrees. pub fn freed(&self) -> DiskUsage { disk_usage::total_usage(self.removed.iter().map(|it| it.usage.clone())) } + /// What reclaiming the tagged derivatives freed, with the plan's own + /// figures. Its own number for [`WorktreeSweep::derivatives_freed`]'s + /// reason: two claims about two disjoint sets of directories. + pub fn derivatives_freed(&self) -> DiskUsage { + disk_usage::total_usage(self.reclaimed.iter().map(|it| it.usage.clone())) + } + pub fn nothing_to_say(&self) -> bool { self.removed.is_empty() && self.withheld.is_empty() && self.refused.is_empty() && self.forgotten == 0 && self.forget_refused.is_empty() + && self.reclaimed.is_empty() + && self.withheld_derivatives.is_empty() } } @@ -2009,13 +2152,25 @@ pub(crate) fn reclaim( }; // The same weighing as the plan, with the plan's own per-unit insistence: a // root the plan promoted is re-weighed as promoted, everything else as not. - let (fresh, fresh_standing) = weigh_clone(git, &plan.clone, bare, &picture, |root| { - plan.going - .iter() - .find(|going| approves(going, root, &plan.clone)) - .map(|going| going.promotion.insistence()) - .unwrap_or(Insistence::NotInsisted) - }); + let weighed = weigh_clone( + git, + &plan.clone, + bare, + &picture, + Derivatives::Weighed, + |root| { + plan.going + .iter() + .find(|going| approves(going, root, &plan.clone)) + .map(|going| going.promotion.insistence()) + .unwrap_or(Insistence::NotInsisted) + }, + ); + let Weighing { + going: fresh, + standing: fresh_standing, + derivatives: fresh_derivatives, + } = weighed; for planned in &plan.going { let Some(confirmed) = fresh .iter() @@ -2053,6 +2208,56 @@ pub(crate) fn reclaim( } act_on(git, &plan.clone, planned, confirmed, report); } + reclaim_derivatives(&plan.clone, &plan.derivatives, &fresh_derivatives, report); +} + +/// Reclaim the tagged derivatives the plan named, each one re-read first. +/// +/// **Both records are read again, and they are read by the same pass that read +/// them for the plan.** `fresh` is [`weigh_clone`]'s answer taken under the +/// lock a moment ago, so the tag, the `conda-meta/pixi` record, the lockfile and +/// the claimant fold have all been put a second time, by one implementation. A +/// plan line and the act on it therefore cannot be answering different +/// questions — the defect this map has punished three times. +/// +/// The approved set can shrink and can never grow: only a place the plan named +/// is looked at, and only where the re-read *also* says derivable is anything +/// removed. +/// +/// What is removed is the tagged directory alone. Never `.pixi`, which carries +/// no tag and holds the one file `.pixi/.gitignore` un-ignores; never anything +/// above it. +fn reclaim_derivatives( + clone: &Path, + planned: &[Tagged], + fresh: &[Tagged], + report: &mut WorktreeReport, +) { + for derivative in planned.iter().filter_map(Tagged::derivable) { + let path = clone.join(derivative.at().as_str()); + let confirmed = fresh.iter().find(|it| it.at() == derivative.at()); + let Some(_) = confirmed.and_then(Tagged::derivable) else { + report.withheld_derivatives.push(WithheldDerivative { + path, + because: match confirmed { + Some(tagged) => NotDerivableNow::Answered(Box::new(tagged.clone())), + None => NotDerivableNow::NoTagThere, + }, + }); + continue; + }; + match remove_tree_as_far_as_it_goes(&path) { + TreeSweep::Everything => report.reclaimed.push(ReclaimedDerivative { + path, + // The plan's figure, so what somebody is told they got back is + // what they said yes to — the same rule `act_on` follows. + usage: derivative.usage().clone(), + }), + TreeSweep::WhatItCould(refused) | TreeSweep::Nothing(refused) => { + report.refused.extend(refused.iter().cloned()); + } + } + } } /// The first registration `confirmed` names that `planned` did not, or nothing @@ -2279,8 +2484,15 @@ fn site_reasons(git: &Git<'_>, clone: &Path) -> Vec { }; let bare = clone.parent().map(|parent| parent.join(".bare")); let bare = bare.as_deref().filter(|path| path.is_dir()); - let (_, standing) = weigh_clone(git, clone, bare, &picture, |_| Insistence::NotInsisted); - standing + // `Derivatives::NotAsked`: this is `dl --ls`, and costing a derivative is a + // full walk of a site plus an `exclusive_usage` over a 12000-file + // environment. The field it leaves empty is discarded here rather than read + // as an answer. + let weighed = weigh_clone(git, clone, bare, &picture, Derivatives::NotAsked, |_| { + Insistence::NotInsisted + }); + weighed + .standing .into_iter() .flat_map(|site| site.reasons.iter().cloned().collect::>()) .collect() diff --git a/rust/devlaunch-core/src/flows/agent_worktrees/derivatives.rs b/rust/devlaunch-core/src/flows/agent_worktrees/derivatives.rs new file mode 100644 index 00000000..b3fa7495 --- /dev/null +++ b/rust/devlaunch-core/src/flows/agent_worktrees/derivatives.rs @@ -0,0 +1,638 @@ +//! The tagged derivative subtrees inside a site that has to stand +//! (devlaunch#468). +//! +//! # Why reaching inside a standing site is not the wedge it looks like +//! +//! A site stands because something about it could not be proved. Removing part +//! of it therefore looks like exactly the act principle 1 exists to stop — until +//! you ask what the standing verdict is a statement *about*. Every reason in +//! [`Standing`](super::Standing) except a claimant's is an answer about git's +//! account of the site's content: `status --porcelain` through the site's admin +//! directory, reachability from a ref, or the fact that neither could be +//! obtained. A `.pixi/envs/` is outside that account **by the installer's +//! own writing** — pixi writes a `.pixi/.gitignore` of `*` and `!config.toml`, +//! so nothing under `.pixi/envs/` is in any index, in any `status` output, or +//! reachable from any commit, in any clone, ever. +//! +//! The set of bytes the site's verdict is uncertain about and the set of bytes +//! under the tag are disjoint by construction, and the construction is a file +//! somebody else wrote. The shipped contradiction is what settles it: the dirt +//! probe already reports nothing about `.pixi/envs`, so a devlaunch that refused +//! this would print *this site holds work that exists nowhere else, 0 bytes* and +//! in the same breath refuse to reclaim 5 GB of it "because we could not prove +//! it safe" — two readings of one directory in one report. +//! +//! # The gate is a declaration, and it never reads a name +//! +//! [`declared_regenerable`] is the whole of what admits a directory: the first +//! 43 bytes of `/CACHEDIR.TAG` are the Cache Directory Tagging +//! Specification's published signature (), which +//! the program that created the directory wrote there to say the contents are +//! regenerable and belong outside a backup. +//! +//! Measured (devlaunch#468 §2, pixi 0.77.0 / uv 0.12.5 / npm 11.18.0): rattler, +//! cargo, uv and pytest all write one; `python -m venv` and npm write none, and +//! npm writes none anywhere beneath `node_modules` either. **The same directory +//! name lands on both sides** — a `.venv` is admitted or refused depending on +//! which program made it — which is why the predicate reads a file and compares +//! no directory name at all. `.pixi` and `.pixi/envs` appear nowhere in it. +//! +//! The walk **does not descend past a tag**: the outermost tagged directory is +//! the unit, because the outer declaration covers everything inside it and +//! because descending would double-count the same bytes under R3. It also stops +//! at every site the forest holds, so a nested worktree's own derivatives are +//! found once, by that site's own pass, and attributed to it. +//! +//! # What the tag does not promise, said out loud +//! +//! pixi does not defend its own declaration. Measured: a planted `my-notes.txt` +//! and a hand-written `site-packages/mypkg` both survived `pixi install +//! --frozen` unmentioned. So the tag is a claim about the directory's *purpose*, +//! not a proof about its current contents, and the argument for removal rests on +//! the disjoint-byte-sets reading above, corroborated by the declaration and by +//! the recipe being on disk — never on "everything in there was installed". +//! +//! The sharper-looking alternative is refused with numbers. Unioning every +//! `conda-meta/*.json` `files` array and calling the rest foreign looks perfect +//! on a throwaway environment (6650 recorded, 6678 walked) and fails on a real +//! one: 11002 recorded against 12210 walked, a 1208-file delta that is the pypi +//! half recorded in `.dist-info/RECORD` plus `__pycache__` trees no installer +//! records. Roughly 10% false positives, which stands every environment. +//! +//! # A recipe, or it stands +//! +//! The tag says *regenerable*; it does not say *by what*. So a tagged directory +//! is reclaimed only when a reader on this side answers with the thing that +//! re-derives it, and a tag no reader recognises stands and is named with its +//! bytes. That is principle 1 inside the rule rather than bolted onto it, and it +//! is why [`Derivative`] carries a [`Recipe`] rather than a flag. +//! +//! One reader is implemented and it reaches ~94.5 GB of the measured 104.5. The +//! measurements behind its four cases are on devlaunch#468 §3: a lock that names +//! the environment re-derives it offline (5507 of 5507 files in 0.52 s, with +//! every proxy variable pointed at a dead port); a **stale** lock still +//! re-derives what was there, because the environment on disk was itself +//! produced from that lock; an **absent** lock re-derives nothing; and an +//! environment the lock **no longer names** is reproducible from nothing on +//! disk, so it stands with `pixi clean -e ` as the pointer. +//! +//! # `manifest_path` is not a field of anything here +//! +//! `conda-meta/pixi` records the manifest as an absolute path, written by +//! whoever ran the install — so for every environment installed inside a +//! container it is `/workspaces//…` and does not resolve on the host. That +//! is the same trap as a container-path worktree registration, and devlaunch#445 +//! and devlaunch#446 answer it by never resolving a recorded path. The +//! constructive form of that answer is that the field does not exist to be +//! resolved: this module reads `environment_name`, which is a name, and finds +//! the lockfile by walking **up from the tag, inside the site**. + +use std::io::Read; +use std::path::{Path, PathBuf}; + +use super::{Blank, Inside, Place, Reason, Subject, Verdict, inside_the_clone}; +use crate::flows::disk_usage::{self, DiskUsage}; + +/// The Cache Directory Tagging Specification's signature, all 43 bytes of it. +/// +/// A file's first 43 bytes, compared as bytes. Not a prefix of a line, not a +/// trimmed string: the specification defines the signature as exactly this +/// sequence at offset 0, and anything looser admits a file that merely mentions +/// it. +const CACHEDIR_SIGNATURE: &[u8; 43] = b"Signature: 8a477f597d28d172789f06886806bc55"; + +/// The file the signature lives in, named by the specification. +const CACHEDIR_TAG: &str = "CACHEDIR.TAG"; + +/// What pixi writes into an installed environment, and the one field read from +/// it. See the module header for why `manifest_path` is not the other one. +const PIXI_RECORD: [&str; 2] = ["conda-meta", "pixi"]; +const PIXI_LOCK: &str = "pixi.lock"; + +/// Whether the program that created `directory` declared it regenerable. +/// +/// **The one expression of "what counts as a derivative."** Every site that asks +/// the question calls this, so the plan and the acting pass cannot come to +/// disagree about the same directory — a rule written twice is the defect this +/// module is most exposed to, since the answer decides whether gigabytes go. +/// +/// It reads a file and compares 43 bytes. It does not look at `directory`'s +/// name, its parent's name, or its depth, and nothing in this module supplies +/// one: the only string joined onto the path is [`CACHEDIR_TAG`], which the +/// specification fixes. +fn declared_regenerable(directory: &Path) -> bool { + let Ok(mut file) = std::fs::File::open(directory.join(CACHEDIR_TAG)) else { + return false; + }; + let mut head = [0u8; CACHEDIR_SIGNATURE.len()]; + file.read_exact(&mut head).is_ok() && &head == CACHEDIR_SIGNATURE +} + +/// What re-derives a tagged directory. One arm per implemented reader, matched +/// exhaustively everywhere, so a second reader is a compile error at every site +/// rather than a branch nobody notices. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Recipe { + /// A pixi environment, re-derived by `pixi install` from the lockfile at + /// `lock`. The environment is named because `pixi clean -e` and the lock's + /// own `environments:` map are both keyed on it. + PixiEnvironment { environment: String, lock: Inside }, +} + +impl Recipe { + /// What a plan line says re-derives it. + pub fn describe(&self) -> String { + match self { + Self::PixiEnvironment { environment, lock } => format!( + "a pixi environment, re-derived by `pixi install -e {environment}` from {}", + lock.as_str() + ), + } + } +} + +/// Why nothing on disk re-derives a tagged directory. +/// +/// Every arm stands the directory. They are separate because the words differ +/// and one of them has a pointer: an environment the lockfile no longer names is +/// `pixi clean -e `'s to remove, and nothing else's. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NoRecipe { + /// No reader on this side recognised the directory. A `rust/target`, a + /// `.pytest_cache`, a `uv venv` somebody `uv pip install`ed into: the tag + /// is a claim about purpose and devlaunch has nothing that re-derives it. + NoReaderRecognisedIt, + /// A reader recognised it and its lockfile is not there. Measured: with the + /// lock absent, `pixi install --frozen --offline` restores 0 files. + LockfileAbsent, + /// The lockfile is there and does not name this environment. Measured as a + /// real population — add an environment, install it, drop it from the + /// manifest and reinstall — the directory survives and pixi never mentions + /// it again. + LockfileDoesNotNameIt { environment: String }, + /// A record was there and would not read. + CouldNotRead(std::io::ErrorKind), +} + +impl NoRecipe { + /// The words the plan's standing line interpolates. + pub fn describe(&self) -> String { + match self { + Self::NoReaderRecognisedIt => { + "its creator declared it regenerable and devlaunch has no reader that \ + re-derives it" + .to_owned() + } + Self::LockfileAbsent => { + "there is no lockfile inside this worktree to re-derive it from".to_owned() + } + Self::LockfileDoesNotNameIt { environment } => format!( + "the lockfile no longer names the environment {environment}, so nothing on \ + disk re-derives it; `pixi clean -e {environment}` is what removes it" + ), + Self::CouldNotRead(kind) => { + format!("a record that would re-derive it could not be read ({kind})") + } + } + } +} + +/// A directory whose creator declared it regenerable and whose recipe is on +/// disk. +/// +/// Private fields, no `Default`, and the only constructor is a read that +/// answered — the same discipline [`Proof`](super::Proof) has, for the same +/// reason: *derivable* must not be the fallthrough of a filter. The +/// `public-api.rest.txt` snapshot is where the absent constructor is pinned, and +/// `nothing_but_a_read_mints_a_derivative` is the test that says so out loud. +/// +/// There is deliberately no `Option` anywhere. Its `None` would mean +/// both *nothing tagged here* and *tagged but not costable*, which is the +/// two-meanings-one-value shape [`Tagged`] exists to refuse. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Derivative { + at: Inside, + bytes: DiskUsage, + from: Recipe, +} + +impl Derivative { + /// Where inside the clone it sits. + pub fn at(&self) -> &Inside { + &self.at + } + + /// What removing it frees, through + /// [`exclusive_usage`](disk_usage::exclusive_usage) like every other figure + /// devlaunch prints. Measured: rattler copies out of the shared package + /// cache rather than hardlinking into the prefix, even where both are on one + /// filesystem, so every byte of an environment is billed to its own tree and + /// every byte comes back. + pub fn usage(&self) -> &DiskUsage { + &self.bytes + } + + /// What re-derives it. + pub fn recipe(&self) -> &Recipe { + &self.from + } +} + +/// A tagged directory that was read, and what devlaunch concluded about it. +/// +/// There is deliberately no arm for "no tag": an untagged directory is not a +/// `Tagged` at all and cannot be constructed as one. Positive space, not a +/// filtered-down negative one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Tagged { + /// A reader answered, and no claimant's reason reaches it. + Derivable(Derivative), + /// A reader could not cost it. Carries its bytes, so principle 2 is served + /// by visibility where it is not served by reclamation. + CouldNotCost { + at: Inside, + bytes: DiskUsage, + why: NoRecipe, + }, + /// A claimant's reason reaches it: somebody asserted a claim over the + /// directory and made no distinction between its parts. Named with its + /// bytes, never removed. See [`claims_in`]. + Claimed { + at: Inside, + bytes: DiskUsage, + by: Box, + }, +} + +impl Tagged { + /// Where inside the clone it sits, whichever arm it is. + pub fn at(&self) -> &Inside { + match self { + Self::Derivable(derivative) => derivative.at(), + Self::CouldNotCost { at, .. } | Self::Claimed { at, .. } => at, + } + } + + /// Its bytes, whichever arm it is. Every arm carries them, because an + /// artifact devlaunch will not reclaim is still an artifact somebody should + /// be told the size of. + pub fn usage(&self) -> &DiskUsage { + match self { + Self::Derivable(derivative) => derivative.usage(), + Self::CouldNotCost { bytes, .. } | Self::Claimed { bytes, .. } => bytes, + } + } + + /// Why it is staying, or nothing when it is going. + pub fn standing(&self) -> Option { + match self { + Self::Derivable(_) => None, + Self::CouldNotCost { why, .. } => Some(why.describe()), + Self::Claimed { by, .. } => Some(match by.as_ref() { + Reason::Holds { losses, .. } => losses.describe(), + Reason::CouldNotProve { blank, .. } => blank.describe(), + }), + } + } + + /// The one it names when it is derivable, so a caller cannot act on an arm + /// that was not one. + pub fn derivable(&self) -> Option<&Derivative> { + match self { + Self::Derivable(derivative) => Some(derivative), + Self::CouldNotCost { .. } | Self::Claimed { .. } => None, + } + } +} + +/// The first reason in `standing` a **claimant** asserts, which is the only kind +/// that reaches a subtree. +/// +/// **The one expression of "what counts as a claimant", and it is one call to +/// [`Reason::subject`].** That method is a wildcard-free match, so a new +/// [`Blank`](super::Blank) arm has to answer devlaunch#468 §6's question at the +/// point it is added rather than inheriting a default here — and there is no +/// second list of arms anywhere for it to drift against. Nothing else in this +/// module looks at a reason to decide whether it pins, and the fold's input is +/// therefore every reason in force rather than a pre-filtered set somebody else +/// filtered by another rule. +/// +/// The fold is derived rather than chosen. Ask of each reason: is this a +/// statement about git's account of the site's content, or a statement by a +/// claimant about the directory? `Holds { Uncommitted }`, `Holds { Unpushed }` +/// and every `CouldNotProve` whose blank is about git's account are the former, +/// and they do not reach the tagged subtree because the tagged subtree was never +/// in that account — it is gitignored by the installer's own writing. A +/// `git worktree lock` and a repository lock that could not be taken are the +/// latter: somebody asserted a claim over the directory and made no distinction +/// between its parts, and a lock may mean *running right now*. +fn first_claim(standing: &[Reason]) -> Option<&Reason> { + standing + .iter() + .find(|reason| reason.subject() == Subject::AClaim) +} + +/// Every standing reason in force over one site's subtree: its own, and every +/// ancestor's. +/// +/// Unfiltered on purpose. Which of them *pin* is [`first_claim`]'s question and +/// only its, asked where the answer is used; a list filtered here as well would +/// be the same rule written twice, in two places that can come to disagree +/// about the same directory. +/// +/// An ancestor's reason reaches down because a lock on a directory is a claim +/// over everything in it — the same reading that makes a lock stand the site +/// rather than only its top level. +pub(super) fn claims_over(inherited: &[Reason], own: &Verdict) -> Vec { + let mut claims = inherited.to_vec(); + if let Verdict::Stands(standing) = own { + claims.extend(standing.iter().cloned()); + } + claims +} + +/// Whether this pass costs the tagged derivatives inside the sites it stands. +/// +/// `--prune` asks; `dl --ls` does not, and that is not an optimisation to be +/// tidied away later. Costing one derivative is a full walk of a site's tree +/// plus an `exclusive_usage` over a 12000-file environment, and the listing is a +/// read-only command people run casually — the same reason `site_reasons` opens +/// with a `read_dir` that fails. +/// +/// The skipping arm yields no derivatives because none were asked for, and the +/// one caller that passes it discards the field rather than reading it as an +/// answer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum Derivatives { + Weighed, + NotAsked, +} + +/// Every tagged directory inside `site`, with what devlaunch concluded about it. +/// +/// The walk starts *below* `site` and never tests `site` itself: a site's own +/// verdict is the sweep's answer about that directory, and a tag on it would be +/// a second answer to a question already asked. It descends into everything else +/// except a tag (the unit is the outermost one) and except a place the forest +/// holds (that site's own pass covers it, and attributes it to the right path). +pub(super) fn tagged_in( + clone: &Path, + site: &Path, + claims: &[Reason], + forest: &[PathBuf], +) -> Vec { + let mut found = Vec::new(); + descend(clone, site, site, claims, forest, &mut found); + + found.sort_by(|left, right| left.at().cmp(right.at())); + found +} + +fn descend( + clone: &Path, + site: &Path, + at: &Path, + claims: &[Reason], + forest: &[PathBuf], + into: &mut Vec, +) { + let Ok(entries) = std::fs::read_dir(at) else { + return; + }; + let mut children: Vec = entries + .filter_map(Result::ok) + // Directories only, and a symlink is never followed: following one + // walks a removal out of the tree `--prune` is scoped to. + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) + .map(|entry| entry.path()) + .collect(); + children.sort(); + for child in children { + if forest.contains(&child) { + // A site. It answers for itself, with its own verdict and its own + // derivatives, attributed to its own path. + continue; + } + if !declared_regenerable(&child) { + descend(clone, site, &child, claims, forest, into); + continue; + } + let Some(place) = inside_the_clone(clone, &child) else { + continue; + }; + into.push(classify(clone, &child, site, place, claims, forest)); + // The walk does not descend past a tag: the outer declaration covers + // everything inside it, and descending would bill the same bytes twice. + } +} + +/// One tagged directory's verdict: the claimant fold first, then the recipe. +/// +/// The order is the argument's order. A claim is about the directory as a whole +/// and admits no distinction between its parts, so it settles the question +/// before any reader is asked; only where nothing claims it does *what +/// re-derives this* become the question. +fn classify( + clone: &Path, + tag: &Path, + site: &Path, + place: Inside, + claims: &[Reason], + forest: &[PathBuf], +) -> Tagged { + let bytes = disk_usage::exclusive_usage(tag); + // A site under the tag is a claim on the directory by something the tag does + // not speak for, and it needs no new arm: it lands in the claimant column + // with that site's own reason. The tag is never a candidate then, whatever + // the reader would have said. + if let Some(nested) = forest + .iter() + .find(|it| it.starts_with(tag)) + .and_then(|it| inside_the_clone(clone, it)) + { + return Tagged::Claimed { + at: place, + bytes, + by: Box::new(Reason::CouldNotProve { + at: Place::ASite(nested), + blank: Blank::ASiteSitsInside, + }), + }; + } + if let Some(claim) = first_claim(claims) { + return Tagged::Claimed { + at: place, + bytes, + by: Box::new(claim.clone()), + }; + } + match pixi_recipe(clone, tag, site) { + Ok(from) => Tagged::Derivable(Derivative { + at: place, + bytes, + from, + }), + Err(why) => Tagged::CouldNotCost { + at: place, + bytes, + why, + }, + } +} + +/// The pixi reader: three reads in order, any silence yielding a [`NoRecipe`]. +/// +/// 1. the tag, which the caller already has; +/// 2. `conda-meta/pixi` parses and yields `environment_name` — and nothing else; +/// 3. a `pixi.lock` found by walking **up from the tag, inside the site**, whose +/// `environments:` map names that environment. +/// +/// Read 3 is a walk and not a resolution of a recorded path, for the reason in +/// the module header. It stops at `site` because a lockfile above the site +/// belongs to a tree this pass is not deciding about. +fn pixi_recipe(clone: &Path, tag: &Path, site: &Path) -> Result { + let record = tag.join(PIXI_RECORD[0]).join(PIXI_RECORD[1]); + let content = match std::fs::read_to_string(&record) { + Ok(content) => content, + // Not there at all is not a failure to read: it is this reader saying + // the directory is not one of its own. + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(NoRecipe::NoReaderRecognisedIt); + } + Err(error) => return Err(NoRecipe::CouldNotRead(error.kind())), + }; + let Some(environment) = environment_name(&content) else { + return Err(NoRecipe::CouldNotRead(std::io::ErrorKind::InvalidData)); + }; + let Some(lock) = lockfile_above(tag, site) else { + return Err(NoRecipe::LockfileAbsent); + }; + let listed = match std::fs::read_to_string(&lock) { + Ok(listed) => listed, + Err(error) => return Err(NoRecipe::CouldNotRead(error.kind())), + }; + if !environments_in(&listed).iter().any(|it| it == &environment) { + return Err(NoRecipe::LockfileDoesNotNameIt { environment }); + } + let Some(at) = inside_the_clone(clone, &lock) else { + return Err(NoRecipe::CouldNotRead(std::io::ErrorKind::InvalidData)); + }; + Ok(Recipe::PixiEnvironment { + environment, + lock: at, + }) +} + +/// `environment_name` out of `conda-meta/pixi`, and nothing else out of it. +/// +/// Deliberately not a `serde` struct: a struct would have to name the fields it +/// ignores, and `manifest_path` is the one field this module must not be able to +/// carry. Reading one key by name is the constructive form of not having it. +fn environment_name(record: &str) -> Option { + let parsed: serde_json::Value = serde_json::from_str(record).ok()?; + let name = parsed.get("environment_name")?.as_str()?; + (!name.is_empty()).then(|| name.to_owned()) +} + +/// The nearest `pixi.lock` at or above `tag`, never above `site`. +fn lockfile_above(tag: &Path, site: &Path) -> Option { + let mut at = tag; + loop { + let candidate = at.join(PIXI_LOCK); + if candidate.is_file() { + return Some(candidate); + } + if at == site { + return None; + } + at = at.parent()?; + } +} + +/// The environment names a lockfile's top-level `environments:` map holds. +/// +/// A four-line scan rather than a YAML dependency, and the shape it reads is +/// pinned by tests over real lockfile text: `environments:` at column zero, one +/// key per environment at the block's own indent, the block ending at the next +/// line in column zero. What it cannot read reads as *not named*, which stands +/// the directory. +fn environments_in(lock: &str) -> Vec { + let mut names = Vec::new(); + let mut inside = false; + let mut depth: Option = None; + for line in lock.lines() { + if line.trim().is_empty() { + continue; + } + let indent = line.len() - line.trim_start().len(); + if indent == 0 { + inside = line.trim_end() == "environments:"; + depth = None; + continue; + } + if !inside { + continue; + } + if indent != *depth.get_or_insert(indent) { + continue; + } + let Some(name) = line.trim().strip_suffix(':') else { + continue; + }; + names.push(name.trim_matches(['"', '\'']).to_owned()); + } + names +} + +/// One derivative the acting pass reclaimed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReclaimedDerivative { + pub path: PathBuf, + /// The figure the plan measured, so what somebody is told they got back is + /// what they said yes to. + pub usage: DiskUsage, +} + +/// One derivative the plan named that the acting pass would not reclaim. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WithheldDerivative { + pub path: PathBuf, + pub because: NotDerivableNow, +} + +/// What the re-read said instead of *derivable*. +/// +/// Two arms rather than an `Option`: *the tag is gone* and *the tag is +/// there and something changed about it* are different facts, and the whole +/// discipline of this module is that they do not share a value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NotDerivableNow { + /// The re-read found no tag at that place at all — it was removed, or its + /// whole site was. + NoTagThere, + /// The re-read answered, and the answer was not derivable: a claim appeared, + /// or the lockfile stopped naming it. + Answered(Box), +} + +impl NotDerivableNow { + /// The words the report interpolates. + pub fn describe(&self) -> String { + match self { + Self::NoTagThere => { + "there is no longer a cache tag at that place, so nothing there declares \ + itself regenerable" + .to_owned() + } + Self::Answered(tagged) => tagged.standing().unwrap_or_else(|| { + // Unreachable: the acting pass only builds this arm from a + // re-read that was *not* derivable, and every other arm carries + // words. Total rather than reachable. + "it could not be shown to be derivable a second time".to_owned() + }), + } + } +} + +#[cfg(test)] +mod tests; diff --git a/rust/devlaunch-core/src/flows/agent_worktrees/derivatives/tests.rs b/rust/devlaunch-core/src/flows/agent_worktrees/derivatives/tests.rs new file mode 100644 index 00000000..55fbbec5 --- /dev/null +++ b/rust/devlaunch-core/src/flows/agent_worktrees/derivatives/tests.rs @@ -0,0 +1,559 @@ +//! The gate, the walk and the reader, at the seam each one is claimed at. +//! +//! **Every row here is a real directory tree.** The whole decision rests on +//! reading files that programs wrote, so a fixture that stubbed the read would +//! be testing the stub — and the one claim most worth breaking, that the +//! predicate never reads a directory's name, can only be shown by building +//! directories whose names would fool a name-matcher and watching them land on +//! the side their *contents* put them on. + +use super::*; + +/// The signature as a real writer emits it: 43 bytes, then the specification's +/// own explanatory comment. Nothing here trims or reflows it — the point is +/// that the first 43 bytes are compared as bytes. +const REAL_TAG: &str = "Signature: 8a477f597d28d172789f06886806bc55\n\ + # This file is a cache directory tag created by a build tool.\n\ + # For information about cache directory tags see https://bford.info/cachedir/\n"; + +fn dir(at: &Path) -> PathBuf { + std::fs::create_dir_all(at).expect("a directory"); + at.to_path_buf() +} + +/// A directory carrying the published tag, whatever it is called. +fn tagged(at: &Path) -> PathBuf { + dir(at); + std::fs::write(at.join("CACHEDIR.TAG"), REAL_TAG).expect("a cache tag"); + at.to_path_buf() +} + +/// What pixi writes into an installed environment. `manifest_path` is the +/// container path a real one carries on every host, present here precisely +/// because nothing may read it. +fn pixi_record(env: &Path, environment: &str) { + let meta = dir(&env.join("conda-meta")); + std::fs::write( + meta.join("pixi"), + serde_json::json!({ + "manifest_path": "/workspaces/devlaunch-container/pyproject.toml", + "environment_name": environment, + "pixi_version": "0.77.0", + "environment_lock_file_hash": "cb70a71a2c1df89c", + }) + .to_string(), + ) + .expect("pixi's own record"); +} + +/// A lockfile naming `environments`, in the shape pixi writes. +fn lock(at: &Path, environments: &[&str]) { + let mut text = String::from("version: 7\nplatforms:\n- name: linux-64\nenvironments:\n"); + for environment in environments { + text.push_str(&format!( + " {environment}:\n channels:\n - url: https://conda.anaconda.org/conda-forge/\n packages:\n linux-64:\n - conda: https://example.invalid/a.conda\n" + )); + } + text.push_str("packages:\n- conda: https://example.invalid/a.conda\n"); + std::fs::write(at.join("pixi.lock"), text).expect("a lockfile"); +} + +/// A clone with one site in it, which is the only shape `Inside` can spell. +struct World { + dir: tempfile::TempDir, +} + +impl World { + fn new() -> Self { + Self { + dir: tempfile::tempdir().expect("a scratch directory"), + } + } + + fn clone_root(&self) -> PathBuf { + self.dir.path().to_path_buf() + } + + fn site(&self, leaf: &str) -> PathBuf { + dir(&self + .clone_root() + .join(".claude") + .join("worktrees") + .join(leaf)) + } + + /// The walk, with nothing claiming anything and no other site in the forest. + fn walk(&self, site: &Path) -> Vec { + tagged_in(&self.clone_root(), site, &[], &[site.to_path_buf()]) + } +} + +fn places(found: &[Tagged]) -> Vec { + found.iter().map(|it| it.at().as_str().to_owned()).collect() +} + +// --------------------------------------------------------------------------- +// the gate +// --------------------------------------------------------------------------- + +#[test] +fn the_gate_is_the_published_signature_and_nothing_looser() { + let world = World::new(); + let root = world.clone_root(); + + assert!( + declared_regenerable(&tagged(&root.join("real"))), + "the published 43 bytes at offset 0 are the whole gate" + ); + assert!( + !declared_regenerable(&dir(&root.join("untagged"))), + "no CACHEDIR.TAG at all is not a declaration" + ); + + // Every one of these is a file that mentions the signature and is not one. + for (name, content) in [ + ("wrong-hex", "Signature: 8a477f597d28d172789f06886806bc54\n"), + ( + "leading-space", + " Signature: 8a477f597d28d172789f06886806bc55\n", + ), + ( + "not-at-offset-zero", + "# a comment first\nSignature: 8a477f597d28d172789f06886806bc55\n", + ), + ("truncated", "Signature: 8a477f597d28d172789f0688680"), + ("empty", ""), + ] { + let at = dir(&root.join(name)); + std::fs::write(at.join("CACHEDIR.TAG"), content).expect("a near miss"); + assert!( + !declared_regenerable(&at), + "{name} is not the specification's signature and must not read as one" + ); + } +} + +#[test] +fn the_gate_never_reads_a_directorys_name() { + // The measured population this row stands for (devlaunch#468 §2): rattler, + // cargo, uv and pytest write a tag; `python -m venv` and npm write none, + // and npm writes none anywhere beneath `node_modules` either. **The same + // name lands on both sides** — a `.venv` is admitted or refused depending + // on which program made it — so a predicate keyed on the name cannot + // express the rule at all. + // + // The claim under test is about the predicate, so the test ranges over the + // predicate: every one of these names is put to it, and the answer tracks + // the file and never the name. + let world = World::new(); + let root = world.clone_root(); + for name in [ + ".venv", + "node_modules", + ".pixi", + "envs", + "default", + "target", + "src", + "a directory nobody would ever call a cache", + ] { + let untagged = dir(&root.join("plain").join(name)); + let with_tag = tagged(&root.join("declared").join(name)); + assert!( + !declared_regenerable(&untagged), + "{name} carries no tag and must be refused whatever it is called" + ); + assert!( + declared_regenerable(&with_tag), + "{name} carries the tag and must be admitted whatever it is called" + ); + } +} + +#[test] +fn a_stdlib_venv_and_a_node_modules_are_not_found_by_the_walk() { + // The fixture row devlaunch#472 asks for, at the level the walk decides: + // both are shaped exactly like the population a name-matcher would take, + // and neither is a candidate, because neither program ever made the claim. + let world = World::new(); + let site = world.site("agent-one"); + let venv = dir(&site.join(".venv").join("lib").join("python3.12")); + std::fs::write(venv.join("os.py"), "stdlib\n").expect("a stdlib file"); + std::fs::write( + site.join(".venv").join("pyvenv.cfg"), + "home = /usr/bin\ninclude-system-site-packages = false\n", + ) + .expect("a pyvenv.cfg"); + let package = dir(&site.join("node_modules").join("lodash")); + std::fs::write(package.join("index.js"), "module.exports = {}\n").expect("a package"); + + assert!( + world.walk(&site).is_empty(), + "nothing in a stdlib venv or a node_modules declares itself regenerable" + ); +} + +// --------------------------------------------------------------------------- +// the walk +// --------------------------------------------------------------------------- + +#[test] +fn the_walk_does_not_descend_past_a_tag() { + let world = World::new(); + let site = world.site("agent-one"); + let outer = tagged(&site.join(".pixi").join("envs").join("default")); + // A second tag inside the first: cargo's own `target` is exactly this shape + // when somebody builds inside an environment. The outer declaration covers + // it, and reporting both would bill the same bytes twice under R3. + tagged(&outer.join("share").join("build")); + + assert_eq!( + places(&world.walk(&site)), + vec![".claude/worktrees/agent-one/.pixi/envs/default"], + "the outermost tag is the unit" + ); +} + +#[test] +fn the_walk_never_puts_the_question_to_the_site_itself() { + // A site's own verdict is the sweep's answer about that directory. A tag on + // it would be a second answer to a question already asked, and the second + // answer would be the one that deletes a registered worktree. + let world = World::new(); + let site = world.site("agent-one"); + tagged(&site); + + assert!( + world.walk(&site).is_empty(), + "the walk starts below the site and never tests the site" + ); +} + +#[test] +fn the_walk_stops_at_a_site_the_forest_already_holds() { + let world = World::new(); + let site = world.site("agent-one"); + let nested = dir(&site.join(".claude").join("worktrees").join("agent-two")); + tagged(&nested.join(".pixi").join("envs").join("default")); + + let found = tagged_in( + &world.clone_root(), + &site, + &[], + &[site.clone(), nested.clone()], + ); + + assert!( + found.is_empty(), + "a nested site's derivatives are that site's own pass to find, so they are \ + attributed to its path and counted once: {found:?}" + ); +} + +#[test] +fn a_symlink_is_never_followed_out_of_the_tree() { + let world = World::new(); + let site = world.site("agent-one"); + let outside = tagged(&world.clone_root().join("elsewhere")); + std::os::unix::fs::symlink(&outside, site.join("link")).expect("a symlink"); + + assert!( + world.walk(&site).is_empty(), + "following a link walks a removal out of the tree --prune is scoped to" + ); +} + +// --------------------------------------------------------------------------- +// the reader +// --------------------------------------------------------------------------- + +#[test] +fn a_lock_that_names_the_environment_re_derives_it() { + let world = World::new(); + let site = world.site("agent-one"); + let env = tagged(&site.join(".pixi").join("envs").join("default")); + pixi_record(&env, "default"); + lock(&site, &["default"]); + + let found = world.walk(&site); + + let [Tagged::Derivable(derivative)] = &found[..] else { + panic!("a tag with its recipe on disk is derivable: {found:?}"); + }; + assert_eq!( + derivative.recipe(), + &Recipe::PixiEnvironment { + environment: "default".to_owned(), + lock: crate::flows::agent_worktrees::inside_the_clone( + &world.clone_root(), + &site.join("pixi.lock") + ) + .expect("the lockfile's place"), + } + ); +} + +#[test] +fn the_reader_stores_no_recorded_path_at_all() { + // `conda-meta/pixi` records the manifest as an absolute path written by + // whoever ran the install, so on every environment installed in a container + // it is a `/workspaces//…` that does not resolve on the host — the same + // trap devlaunch#445 and devlaunch#446 answer by never resolving a recorded + // path. The constructive form is that the field does not exist to be + // resolved, and this is the test that says the value cannot carry it. + let world = World::new(); + let site = world.site("agent-one"); + let env = tagged(&site.join(".pixi").join("envs").join("default")); + pixi_record(&env, "default"); + lock(&site, &["default"]); + + let found = world.walk(&site); + + let rendered = format!("{found:?}"); + assert!( + !rendered.contains("/workspaces/devlaunch-container"), + "the recorded manifest path reached the value: {rendered}" + ); + assert!( + !rendered.contains("manifest"), + "nothing about a manifest is carried: {rendered}" + ); +} + +#[test] +fn a_lock_absent_stands_the_environment() { + // Measured: with the lock gone, `pixi install --frozen --offline` restores + // 0 files. Nothing on disk re-derives it, so it stands and is named. + let world = World::new(); + let site = world.site("agent-one"); + let env = tagged(&site.join(".pixi").join("envs").join("default")); + pixi_record(&env, "default"); + std::fs::write(env.join("big.so"), "a great many bytes\n").expect("env bytes"); + + let found = world.walk(&site); + + let [Tagged::CouldNotCost { why, bytes, .. }] = &found[..] else { + panic!("no lock means no recipe: {found:?}"); + }; + assert_eq!(why, &NoRecipe::LockfileAbsent); + assert!( + bytes.known_bytes() > 0, + "principle 2 is served by visibility where it is not served by reclamation" + ); +} + +#[test] +fn a_lock_that_no_longer_names_the_environment_stands_with_its_pointer() { + // Measured as a real population: add an `extra` environment, install it, + // drop it from the manifest and reinstall. The directory survives, the + // lockfile stops listing it, and pixi never mentions it again. + let world = World::new(); + let site = world.site("agent-one"); + let env = tagged(&site.join(".pixi").join("envs").join("extra")); + pixi_record(&env, "extra"); + lock(&site, &["default"]); + + let found = world.walk(&site); + + let [Tagged::CouldNotCost { why, .. }] = &found[..] else { + panic!("an environment nothing names is not derivable: {found:?}"); + }; + assert_eq!( + why, + &NoRecipe::LockfileDoesNotNameIt { + environment: "extra".to_owned() + } + ); + assert!( + why.describe().contains("pixi clean -e extra"), + "principle 2 is served by a pointer where it is not served by reclamation: {}", + why.describe() + ); +} + +#[test] +fn a_stale_lock_still_re_derives_what_was_there() { + // A stale *lock* is not a stale *environment*: the environment on disk was + // itself produced from that lock, so the lock reproduces exactly what is + // there. Measured: `--frozen --offline` against a lock whose manifest has + // moved on restored all 5507 files. The manifest is therefore never read, + // and this fixture puts one there that disagrees to prove it is not. + let world = World::new(); + let site = world.site("agent-one"); + let env = tagged(&site.join(".pixi").join("envs").join("default")); + pixi_record(&env, "default"); + lock(&site, &["default"]); + std::fs::write( + site.join("pixi.toml"), + "[dependencies]\nsomething-the-lock-has-never-heard-of = \"*\"\n", + ) + .expect("a manifest the lock does not match"); + + let found = world.walk(&site); + + assert!( + matches!(&found[..], [Tagged::Derivable(_)]), + "a lock that names the environment re-derives it whatever the manifest says: \ + {found:?}" + ); +} + +#[test] +fn a_tag_no_reader_recognises_stands_and_is_named() { + // cargo's `target`, pytest's cache, a `uv venv` somebody `uv pip install`ed + // into: the tag is a claim about purpose and devlaunch has nothing that + // re-derives them. One general gate, an open set of readers, exactly one + // implemented, and a tag with no reader stands. + let world = World::new(); + let site = world.site("agent-one"); + let target = tagged(&site.join("rust").join("target")); + std::fs::write(target.join("libthing.rlib"), "built bytes\n").expect("build output"); + lock(&site, &["default"]); + + let found = world.walk(&site); + + let [Tagged::CouldNotCost { why, .. }] = &found[..] else { + panic!("no reader recognises a cargo target: {found:?}"); + }; + assert_eq!(why, &NoRecipe::NoReaderRecognisedIt); +} + +#[test] +fn a_lockfile_above_the_site_is_not_this_sites_recipe() { + // The walk up stops at the site. A lockfile in the clone root belongs to a + // tree this pass is not deciding about, and reading it would be resolving + // one directory's record against another directory's contents. + let world = World::new(); + let site = world.site("agent-one"); + let env = tagged(&site.join(".pixi").join("envs").join("default")); + pixi_record(&env, "default"); + lock(&world.clone_root(), &["default"]); + + let found = world.walk(&site); + + assert!( + matches!( + &found[..], + [Tagged::CouldNotCost { + why: NoRecipe::LockfileAbsent, + .. + }] + ), + "the walk up stops at the site: {found:?}" + ); +} + +#[test] +fn the_environments_block_is_read_the_way_pixi_writes_it() { + // The four-line scan, against the shape a real lockfile has: a top-level + // `environments:` map, one key per environment, ending at the next line in + // column zero. `packages:` below it is not an environment. + let text = "version: 7\n\ + platforms:\n\ + - name: linux-64\n\ + environments:\n \ + default:\n \ + channels:\n \ + - url: https://conda.anaconda.org/conda-forge/\n \ + packages:\n \ + linux-64:\n \ + - conda: https://example.invalid/a.conda\n \ + py312:\n \ + channels: []\n\ + packages:\n\ + - conda: https://example.invalid/a.conda\n"; + + assert_eq!( + environments_in(text), + vec!["default".to_owned(), "py312".to_owned()] + ); +} + +#[test] +fn a_lockfile_with_no_environments_block_names_nothing() { + assert!(environments_in("version: 7\npackages: []\n").is_empty()); + assert!(environments_in("").is_empty()); +} + +// --------------------------------------------------------------------------- +// the claimant fold +// --------------------------------------------------------------------------- + +#[test] +fn a_claim_in_force_pins_the_derivative_and_an_account_of_content_does_not() { + // The fold, at the level it is decided: ask of each reason whether it is a + // claimant's or git's account of the site's content. The two rows below are + // the whole of devlaunch#468 §6 in one assertion, and they go through + // `Reason::subject` — the one expression of what a claimant is — rather + // than through a second list of arms here. + let world = World::new(); + let site = world.site("agent-one"); + let env = tagged(&site.join(".pixi").join("envs").join("default")); + pixi_record(&env, "default"); + lock(&site, &["default"]); + let place = Place::ASite( + crate::flows::agent_worktrees::inside_the_clone(&world.clone_root(), &site) + .expect("the site's place"), + ); + + let claimant = Reason::CouldNotProve { + at: place.clone(), + blank: Blank::ThirdPartyClaim(Some("a portable device".to_owned())), + }; + let content = Reason::CouldNotProve { + at: place, + blank: Blank::NothingToAskThrough, + }; + + let claimed = tagged_in( + &world.clone_root(), + &site, + std::slice::from_ref(&claimant), + std::slice::from_ref(&site), + ); + assert!( + matches!(&claimed[..], [Tagged::Claimed { by, .. }] if by.as_ref() == &claimant), + "a lock is a claim over the directory and admits no distinction between its \ + parts: {claimed:?}" + ); + + let derivable = tagged_in( + &world.clone_root(), + &site, + std::slice::from_ref(&content), + std::slice::from_ref(&site), + ); + assert!( + matches!(&derivable[..], [Tagged::Derivable(_)]), + "git's account of the site's content was never about these bytes: {derivable:?}" + ); +} + +#[test] +fn a_site_inside_a_tag_makes_it_a_claim_the_tag_does_not_speak_for() { + let world = World::new(); + let site = world.site("agent-one"); + // A tag planted above a nested site's place. Absurd in the wild and + // constructible in a minute, which is the only reason it needs a verdict. + let over = tagged(&site.join(".claude")); + let nested = dir(&over.join("worktrees").join("agent-two")); + + let found = tagged_in( + &world.clone_root(), + &site, + &[], + &[site.clone(), nested.clone()], + ); + + let [Tagged::Claimed { by, .. }] = &found[..] else { + panic!("a site under a tag is never a candidate: {found:?}"); + }; + assert_eq!(by.subject(), Subject::AClaim); + assert!(matches!( + by.as_ref(), + Reason::CouldNotProve { + blank: Blank::ASiteSitsInside, + .. + } + )); +} diff --git a/rust/devlaunch-core/src/flows/agent_worktrees/tests.rs b/rust/devlaunch-core/src/flows/agent_worktrees/tests.rs index a218d67f..914b82ba 100644 --- a/rust/devlaunch-core/src/flows/agent_worktrees/tests.rs +++ b/rust/devlaunch-core/src/flows/agent_worktrees/tests.rs @@ -273,6 +273,102 @@ fn commit(work: &Path, message: &str) { run_git(work, &["commit", "-m", message]); } +// --------------------------------------------------------------------------- +// the tagged derivatives (devlaunch#468) +// --------------------------------------------------------------------------- + +/// The Cache Directory Tagging Specification's file, as a real writer emits it. +fn cachedir_tag(at: &Path) { + std::fs::write( + at.join("CACHEDIR.TAG"), + "Signature: 8a477f597d28d172789f06886806bc55\n\ + # This file is a cache directory tag created by a build tool.\n", + ) + .expect("a cache tag"); +} + +/// An installed pixi environment inside `site`, with the lockfile that +/// re-derives it beside it — the shape 18 of the 72 directories on the +/// reference host carried, and the difference between 104 GB and about 10. +/// +/// `.pixi/config.toml` is here because it is the one path `.pixi/.gitignore` +/// un-ignores and the one human-writable file in there: the tag sits one level +/// below it, which is what makes the unit's edge pixi's own rather than +/// devlaunch's. +fn installed_env(site: &Path, environment: &str) -> PathBuf { + let pixi = site.join(".pixi"); + let env = pixi.join("envs").join(environment); + std::fs::create_dir_all(env.join("conda-meta")).expect("an installed environment"); + std::fs::write(pixi.join("config.toml"), "[repodata-config]\n").expect("pixi's own config"); + std::fs::write(pixi.join(".gitignore"), "*\n!config.toml\n").expect("pixi's own ignore"); + cachedir_tag(&env); + std::fs::write( + env.join("conda-meta").join("pixi"), + format!( + "{{\"manifest_path\": \"/workspaces/devlaunch-container/pyproject.toml\", \ + \"environment_name\": \"{environment}\"}}" + ), + ) + .expect("pixi's own record"); + std::fs::create_dir_all(env.join("lib")).expect("the environment's lib"); + std::fs::write(env.join("lib").join("libthing.so"), "a great many bytes\n") + .expect("environment bytes"); + std::fs::write( + site.join("pixi.lock"), + format!("version: 7\nenvironments:\n {environment}:\n channels: []\npackages: []\n"), + ) + .expect("the lockfile that re-derives it"); + env +} + +/// Commit and push everything in a site except the environment, and gitignore +/// the environment the way every pixi project gitignores it. +/// +/// The ordinary finished-task shape: the lockfile and the ignore file are on the +/// forge, so the only bytes in the site that exist nowhere else are the ones +/// under the tag — which is exactly the population devlaunch#468 is about. +fn commit_the_project(world: &Clone, worktree: &Path, branch: &str) { + std::fs::write(worktree.join(".gitignore"), ".pixi/\n").expect("an ignore file"); + commit(worktree, "the project and its lockfile"); + run_git(&world.clone, &["push", "origin", branch]); + world.fetch(); +} + +/// `git worktree lock` by hand, for a registration whose recorded path no longer +/// resolves — which is every registration a host sees, and which git's own +/// `worktree lock` will not take as an argument. +fn lock_by_hand(world: &Clone, leaf: &str) { + std::fs::write( + world + .clone + .join(".git") + .join("worktrees") + .join(leaf) + .join("locked"), + "", + ) + .expect("the lock git's own listing reads"); +} + +/// The derivatives a plan will reclaim, by the place each sits at. +fn reclaiming(found: &CloneWorktrees) -> Vec { + found + .derivatives() + .iter() + .filter(|it| it.derivable().is_some()) + .map(|it| it.at().as_str().to_owned()) + .collect() +} + +/// The derivatives a plan names and will not reclaim, with why. +fn standing_derivatives(found: &CloneWorktrees) -> Vec<(String, String)> { + found + .derivatives() + .iter() + .filter_map(|it| Some((it.at().as_str().to_owned(), it.standing()?))) + .collect() +} + /// The directories a plan removes, in the order it reports them. fn going_dirs(found: &CloneWorktrees) -> Vec { found @@ -1406,3 +1502,346 @@ fn a_clean_clone_with_collectable_worktrees_reads_as_nothing_to_lose() { assert_eq!(json, serde_json::json!({ "nothingToLose": true })); } + +// --------------------------------------------------------------------------- +// reclaiming the tagged derivative subtrees (devlaunch#468, devlaunch#472) +// --------------------------------------------------------------------------- + +#[test] +fn an_environment_inside_a_standing_worktree_is_reclaimed_and_the_worktree_stands() { + // The fixture devlaunch#468 asks for: a site that must stand, holding both + // a `.pixi/envs/default` and a file a human wrote. The first goes, the + // second stays, and the site still stands. + // + // What makes that legitimate rather than the thin end of the wedge: the + // site stands on `Holds { Uncommitted }`, which is git's account of its + // content, and nothing under `.pixi/envs/` has ever been in that account — + // pixi's own `.pixi/.gitignore` puts it outside every index, every status + // and every commit. The two sets of bytes are disjoint by construction, and + // the construction is a file the installer wrote. + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + let env = installed_env(&worktree, "default"); + commit_the_project(&world, &worktree, "agent-one"); + std::fs::write(worktree.join("NOTES.md"), "an afternoon nobody else has\n") + .expect("the human's own file"); + world.containerise(); + + let plan = world.plan(); + + assert!(going_dirs(&plan).is_empty(), "the site must stand"); + assert_eq!( + reclaiming(&plan), + vec![".claude/worktrees/agent-one/.pixi/envs/default"], + ); + + let (report, _) = world.act(&plan); + + assert_eq!( + report + .reclaimed + .iter() + .map(|it| it.path.clone()) + .collect::>(), + vec![env.clone()], + ); + assert!(!env.exists(), "the tagged directory is what goes"); + assert!( + worktree.join(".pixi").join("config.toml").is_file(), + "`.pixi` holds config.toml and is never what goes" + ); + assert!( + worktree.join("NOTES.md").is_file(), + "the site still stands, and so does what it holds" + ); + assert!(worktree.is_dir(), "the site still stands"); +} + +#[test] +fn reclaiming_an_environment_needs_no_flag_and_no_second_question() { + // devlaunch#459 refused one flag carrying two consents, and this is a + // removal with a proof rather than a force. It rides `--prune`'s own y/N: + // the plan says which directory and how big, and nothing else is typed. + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + installed_env(&worktree, "default"); + commit_the_project(&world, &worktree, "agent-one"); + std::fs::write(worktree.join("NOTES.md"), "unsaved\n").expect("the human's own file"); + world.containerise(); + + let plan = world + .sweep(Insistence::NotInsisted) + .expect("a sweep of a clone that has worktrees"); + + assert_eq!(reclaiming(&plan).len(), 1, "no flag was typed"); + assert!( + !plan.nothing_to_do(), + "a run with a derivative to reclaim has something to do, so the question is asked" + ); +} + +#[test] +fn the_plan_names_each_derivative_and_its_size_before_the_question() { + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + installed_env(&worktree, "default"); + commit_the_project(&world, &worktree, "agent-one"); + std::fs::write(worktree.join("NOTES.md"), "unsaved\n").expect("the human's own file"); + world.containerise(); + + let plan = world.plan(); + + let [tagged] = plan.derivatives() else { + panic!("one derivative: {:?}", plan.derivatives()); + }; + assert!( + tagged.usage().known_bytes() > 0, + "a plan line with no figure is a y/N answering a total nobody can decompose" + ); + let Some(derivative) = tagged.derivable() else { + panic!("derivable: {tagged:?}"); + }; + let Recipe::PixiEnvironment { environment, lock } = derivative.recipe(); + assert_eq!(environment, "default"); + assert_eq!(lock.as_str(), ".claude/worktrees/agent-one/pixi.lock"); +} + +#[test] +fn a_locked_worktrees_environment_is_named_and_never_reclaimed() { + // A lock is a claim over the directory by a party this pass cannot + // interrogate, and it makes no distinction between the directory's parts. + // It may also mean *running right now*. devlaunch#426 Ask 2 holds at the + // subtree level for the same reason it holds at the site. + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + installed_env(&worktree, "default"); + run_git( + &world.clone, + &["worktree", "lock", &worktree.display().to_string()], + ); + world.containerise(); + + let plan = world.plan(); + + assert!(reclaiming(&plan).is_empty(), "a claim pins the subtree"); + let [(at, why)] = &standing_derivatives(&plan)[..] else { + panic!("named with its bytes: {:?}", plan.derivatives()); + }; + assert_eq!(at, ".claude/worktrees/agent-one/.pixi/envs/default"); + assert!(why.contains("locked"), "{why}"); +} + +#[test] +fn an_environment_inside_a_worktree_that_is_going_is_not_reported_twice() { + // devlaunch#446 §6's two-recursions rule, extended one artifact over: the + // byte recursion stops at the outermost thing that goes, so a derivative + // inside a site that is itself going rides on the site's own figure and is + // never a unit of its own. Two lines for one set of bytes is the double + // count R3 forbids. + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + installed_env(&worktree, "default"); + commit_the_project(&world, &worktree, "agent-one"); + world.containerise(); + + let plan = world.plan(); + + assert_eq!(going_dirs(&plan), std::slice::from_ref(&worktree)); + assert!( + plan.derivatives().is_empty(), + "the site's own removal accounts for it: {:?}", + plan.derivatives() + ); +} + +#[test] +fn a_planted_file_inside_an_environment_goes_with_it_and_the_row_says_so() { + // The honest row, recorded rather than hidden. pixi does not defend its own + // declaration: a planted `my-notes.txt` and a hand-written + // `site-packages/mypkg` both survived `pixi install --frozen` unmentioned. + // So the tag is a claim about the directory's purpose, not a proof about + // its current contents, and the case for removal rests on the tagged + // subtree being outside what the site's verdict is about. + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + let env = installed_env(&worktree, "default"); + commit_the_project(&world, &worktree, "agent-one"); + std::fs::write(env.join("my-notes.txt"), "planted by hand\n").expect("a planted file"); + std::fs::write(worktree.join("NOTES.md"), "unsaved\n").expect("the human's own file"); + world.containerise(); + + let plan = world.plan(); + let (report, _) = world.act(&plan); + + assert_eq!(report.reclaimed.len(), 1); + assert!( + !env.join("my-notes.txt").exists(), + "everything under the tag goes, and the plan's own words are what warn about it" + ); +} + +#[test] +fn a_node_modules_and_a_stdlib_venv_beside_an_environment_are_untouched() { + // The row that proves the predicate never reads a name, at the level a + // whole run decides. Both of these are shaped exactly like the population a + // name-matcher would take — measured, npm writes no tag anywhere beneath + // `node_modules` and `python -m venv` writes none at all — and the tagged + // environment beside them goes. + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + let env = installed_env(&worktree, "default"); + commit_the_project(&world, &worktree, "agent-one"); + std::fs::write(worktree.join("NOTES.md"), "unsaved\n").expect("the human's own file"); + let modules = worktree.join("node_modules").join("lodash"); + std::fs::create_dir_all(&modules).expect("an npm install"); + std::fs::write(modules.join("index.js"), "module.exports = {}\n").expect("a package"); + let venv = worktree.join(".venv").join("lib"); + std::fs::create_dir_all(&venv).expect("a stdlib venv"); + std::fs::write( + worktree.join(".venv").join("pyvenv.cfg"), + "home = /usr/bin\n", + ) + .expect("a pyvenv.cfg"); + world.containerise(); + + let plan = world.plan(); + + assert_eq!( + reclaiming(&plan), + vec![".claude/worktrees/agent-one/.pixi/envs/default"], + "only the one that declared itself: {:?}", + plan.derivatives() + ); + + world.act(&plan); + + assert!(!env.exists()); + assert!( + modules.join("index.js").is_file(), + "npm never made the claim" + ); + assert!( + worktree.join(".venv").join("pyvenv.cfg").is_file(), + "the stdlib venv never made the claim" + ); +} + +#[test] +fn another_repositorys_environment_is_still_derivable() { + // Named explicitly on devlaunch#468 §6 because a reviewer will ask. A + // foreign worktree stands, and it stands on `NotThisClonesToAccountFor`, + // which is git's account of content being out of scope rather than a + // claimant's assertion. Whose repository the environment belongs to was + // never part of the argument: the tag and the lockfile beside it say what + // they say either way. + let world = Clone::new(); + let theirs = OtherRepository::new(world.tmp()); + let at = worktrees_dir(&world.clone).join("theirs"); + theirs.worktree_at(&at, "their-branch"); + installed_env(&at, "default"); + + let plan = world.plan(); + + assert!(going_dirs(&plan).is_empty(), "a foreign site always stands"); + assert_eq!( + reclaiming(&plan), + vec![".claude/worktrees/theirs/.pixi/envs/default"], + ); +} + +#[test] +fn an_environment_whose_lockfile_went_away_is_withheld_by_the_second_read() { + // The acting pass re-reads both records, and it re-reads them with the same + // weighing the plan ran, so the two passes cannot answer different + // questions. The approved set shrinks here and can never grow. + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + let env = installed_env(&worktree, "default"); + commit_the_project(&world, &worktree, "agent-one"); + std::fs::write(worktree.join("NOTES.md"), "unsaved\n").expect("the human's own file"); + world.containerise(); + + let plan = world.plan(); + assert_eq!(reclaiming(&plan).len(), 1); + + std::fs::remove_file(worktree.join("pixi.lock")).expect("the lockfile going away"); + let (report, _) = world.act(&plan); + + assert!(report.reclaimed.is_empty(), "nothing was re-derivable"); + let [withheld] = &report.withheld_derivatives[..] else { + panic!("one withheld: {report:?}"); + }; + assert_eq!(withheld.path, env); + assert!(env.is_dir(), "and it is still there"); + assert!( + withheld.because.describe().contains("lockfile"), + "{}", + withheld.because.describe() + ); +} + +#[test] +fn an_environment_claimed_between_the_plan_and_the_act_is_withheld() { + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + let env = installed_env(&worktree, "default"); + commit_the_project(&world, &worktree, "agent-one"); + std::fs::write(worktree.join("NOTES.md"), "unsaved\n").expect("the human's own file"); + world.containerise(); + + let plan = world.plan(); + assert_eq!(reclaiming(&plan).len(), 1); + + lock_by_hand(&world, "agent-one"); + let (report, _) = world.act(&plan); + + assert!(report.reclaimed.is_empty()); + assert_eq!(report.withheld_derivatives.len(), 1, "{report:?}"); + assert!(env.is_dir()); +} + +#[test] +fn the_listing_path_never_costs_a_derivative() { + // `dl --ls` is a read-only command people run casually, and costing one + // derivative is a full walk of a site plus an `exclusive_usage` over a + // 12000-file environment. The clone's verdict is the same either way — a + // derivative is not a reason a site stands — so the listing asks for none. + let world = Clone::new(); + let worktree = world.worktree("agent-one"); + installed_env(&worktree, "default"); + commit_the_project(&world, &worktree, "agent-one"); + std::fs::write(worktree.join("NOTES.md"), "unsaved\n").expect("the human's own file"); + world.containerise(); + + let runner = ProcessRunner::new(); + let git = Git::new(&runner); + let picture = ClonePicture::of(&git, &world.clone).expect("git's own listing"); + let asked = weigh_clone( + &git, + &world.clone, + Some(&world.bare), + &picture, + Derivatives::Weighed, + |_| Insistence::NotInsisted, + ); + let not_asked = weigh_clone( + &git, + &world.clone, + Some(&world.bare), + &picture, + Derivatives::NotAsked, + |_| Insistence::NotInsisted, + ); + + assert_eq!(asked.derivatives.len(), 1, "the prune path asks"); + assert!( + not_asked.derivatives.is_empty(), + "the listing path does not" + ); + assert_eq!( + asked.standing, not_asked.standing, + "and the verdict is the same either way: a derivative is never a reason a site \ + stands, so the listing loses nothing by not costing one" + ); + assert_eq!(asked.going, not_asked.going); +} diff --git a/rust/devlaunch-core/tests/public_api_snapshots.rs b/rust/devlaunch-core/tests/public_api_snapshots.rs index f77ba5fb..a64f0f40 100644 --- a/rust/devlaunch-core/tests/public_api_snapshots.rs +++ b/rust/devlaunch-core/tests/public_api_snapshots.rs @@ -195,3 +195,42 @@ fn each_file_is_anchored_on_a_row_every_generation_produces() { "the rest file does not contain the crate root's row" ); } + +#[test] +fn nothing_but_a_read_mints_a_derivative() { + // devlaunch#472's asserted **absence**, in the file that would show it + // arriving. `Derivative` says *this directory's creator declared it + // regenerable and its recipe is on disk*, and the only thing entitled to + // say that is a read that answered -- so the type has private fields, no + // `Default`, and no constructor at all. The failure this pins is not a + // wrong value, it is a value that can be built without asking, which is + // `Proof`'s discipline one artifact over and the reason `Option` + // is refused as well. + // + // `clone` is exempt and only `clone`: it needs one in hand to make another, + // so it mints nothing. + const TY: &str = "devlaunch_core::flows::agent_worktrees::Derivative"; + // Anchored first, because an assertion about the absence of rows passes + // loudest when the type is absent too -- which is what a rename, a moved + // module or a truncated regeneration all look like from in here. + assert!( + rows(REST).contains(&format!("pub struct {TY}").as_str()), + "the snapshot does not declare {TY}, so nothing below it is under test" + ); + let minting: Vec<&str> = rows(REST) + .into_iter() + .filter(|row| { + let hands_one_back = row.starts_with("pub fn ") + && row.ends_with(&format!("-> {TY}")) + && !row.contains(&format!("{TY}::clone(")); + let has_a_default = *row == format!("impl core::default::Default for {TY}"); + let has_a_public_field = row.starts_with(&format!("pub {TY}::")) && row.contains(": "); + hands_one_back || has_a_default || has_a_public_field + }) + .collect(); + assert!( + minting.is_empty(), + "a Derivative must be handed to you by a read that answered, and these would \ + build one without asking: {minting:#?}" + ); +} diff --git a/rust/dl/src/render.rs b/rust/dl/src/render.rs index 8b2ab8aa..3e638469 100644 --- a/rust/dl/src/render.rs +++ b/rust/dl/src/render.rs @@ -2079,11 +2079,30 @@ fn worktree_plan_lines(sweep: &WorktreeSweep) -> Vec { if sweep.nothing_to_say() { return Vec::new(); } + let derivable = sweep + .clones() + .iter() + .flat_map(|found| found.derivatives()) + .filter(|it| it.derivable().is_some()) + .count(); let mut lines = vec![ - format!( - "Agent git worktrees inside the clones above -- {}:", - describe_usage(&sweep.freed()) - ), + if derivable == 0 { + format!( + "Agent git worktrees inside the clones above -- {}:", + describe_usage(&sweep.freed()) + ) + } else { + // Two figures because they are two claims about two disjoint sets + // of directories, and each says which set it is about. One number + // covering both would describe neither, and an unlabelled pair + // reads as a total and a part of it. + format!( + "Agent git worktrees inside the clones above -- {} in worktrees that go, and \ + {} in regenerable subtrees inside the ones that stay:", + describe_usage(&sweep.freed()), + describe_usage(&sweep.derivatives_freed()) + ) + }, String::new(), ]; for found in sweep.clones() { @@ -2117,6 +2136,30 @@ fn worktree_plan_lines(sweep: &WorktreeSweep) -> Vec { standing.reasons().describe() )); } + // Each derivative by name and by size, before the question is asked. + // These sit inside worktrees the run has just said it is leaving, so + // the line has to say which directory it means and what it costs, or + // the y/N is answering a total nobody can decompose. + for tagged in found.derivatives() { + let at = found.clone_path().join(tagged.at().as_str()); + lines.push(match tagged.standing() { + None => format!( + " - reclaiming {} ({}): {}", + at.display(), + describe_usage(tagged.usage()), + tagged + .derivable() + .map(|it| it.recipe().describe()) + .unwrap_or_default() + ), + Some(why) => format!( + " - leaving {} ({}): {}", + at.display(), + describe_usage(tagged.usage()), + why + ), + }); + } } lines.push(String::new()); // Said once, rather than implied by every line above it. `--prune` is a @@ -2128,6 +2171,18 @@ fn worktree_plan_lines(sweep: &WorktreeSweep) -> Vec { repository cache; --prune does not fetch." .to_owned(), ); + if derivable > 0 { + // What is being consented to, said once. The bytes come back the moment + // a person runs the command in the line above their own directory, and + // the tag is the creating program's own declaration rather than + // anything dl inferred from a directory name. + lines.push( + "A regenerable subtree is one whose creator wrote a CACHEDIR.TAG into it and \ + whose lockfile is still beside it; putting one back is one command and no \ + network beyond the shared package cache." + .to_owned(), + ); + } lines.push(String::new()); lines } @@ -2172,6 +2227,20 @@ fn worktree_report_lines(report: &WorktreeReport) -> Vec { report.forgotten )); } + if !report.reclaimed.is_empty() { + lines.push(format!( + "Reclaimed {} regenerable subtree(s) inside the worktrees that stayed -- {}.", + report.reclaimed.len(), + describe_usage(&report.derivatives_freed()) + )); + } + for withheld in &report.withheld_derivatives { + lines.push(format!( + "Left {}: {}. That was not so when the plan above was printed.", + withheld.path.display(), + withheld.because.describe() + )); + } for withheld in &report.withheld { lines.push(format!( "Left {}: {} -- add --force-worktrees to remove it anyway. That was not so when \ From a884c7f1b1aae660fe1d74e6cc64e7bf48afb0d3 Mon Sep 17 00:00:00 2001 From: Austin Gregg-Smith Date: Sun, 30 Aug 2026 01:05:24 +0100 Subject: [PATCH 2/2] Take the order the generator renders the two withheld types in --- rust/devlaunch-core/public-api.rest.txt | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/rust/devlaunch-core/public-api.rest.txt b/rust/devlaunch-core/public-api.rest.txt index 4e0c6020..ea79b3ef 100644 --- a/rust/devlaunch-core/public-api.rest.txt +++ b/rust/devlaunch-core/public-api.rest.txt @@ -1010,17 +1010,6 @@ pub fn devlaunch_core::flows::agent_worktrees::StandingSite::eq(&self, &devlaunc impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::StandingSite pub fn devlaunch_core::flows::agent_worktrees::StandingSite::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::StandingSite -pub struct devlaunch_core::flows::agent_worktrees::WithheldWorktree -pub devlaunch_core::flows::agent_worktrees::WithheldWorktree::because: devlaunch_core::flows::agent_worktrees::Standing -pub devlaunch_core::flows::agent_worktrees::WithheldWorktree::path: std::path::PathBuf -impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::WithheldWorktree -pub fn devlaunch_core::flows::agent_worktrees::WithheldWorktree::clone(&self) -> devlaunch_core::flows::agent_worktrees::WithheldWorktree -impl core::cmp::Eq for devlaunch_core::flows::agent_worktrees::WithheldWorktree -impl core::cmp::PartialEq for devlaunch_core::flows::agent_worktrees::WithheldWorktree -pub fn devlaunch_core::flows::agent_worktrees::WithheldWorktree::eq(&self, &devlaunch_core::flows::agent_worktrees::WithheldWorktree) -> bool -impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::WithheldWorktree -pub fn devlaunch_core::flows::agent_worktrees::WithheldWorktree::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::WithheldWorktree pub struct devlaunch_core::flows::agent_worktrees::WithheldDerivative pub devlaunch_core::flows::agent_worktrees::WithheldDerivative::because: devlaunch_core::flows::agent_worktrees::NotDerivableNow pub devlaunch_core::flows::agent_worktrees::WithheldDerivative::path: std::path::PathBuf @@ -1032,6 +1021,17 @@ pub fn devlaunch_core::flows::agent_worktrees::WithheldDerivative::eq(&self, &de impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::WithheldDerivative pub fn devlaunch_core::flows::agent_worktrees::WithheldDerivative::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::WithheldDerivative +pub struct devlaunch_core::flows::agent_worktrees::WithheldWorktree +pub devlaunch_core::flows::agent_worktrees::WithheldWorktree::because: devlaunch_core::flows::agent_worktrees::Standing +pub devlaunch_core::flows::agent_worktrees::WithheldWorktree::path: std::path::PathBuf +impl core::clone::Clone for devlaunch_core::flows::agent_worktrees::WithheldWorktree +pub fn devlaunch_core::flows::agent_worktrees::WithheldWorktree::clone(&self) -> devlaunch_core::flows::agent_worktrees::WithheldWorktree +impl core::cmp::Eq for devlaunch_core::flows::agent_worktrees::WithheldWorktree +impl core::cmp::PartialEq for devlaunch_core::flows::agent_worktrees::WithheldWorktree +pub fn devlaunch_core::flows::agent_worktrees::WithheldWorktree::eq(&self, &devlaunch_core::flows::agent_worktrees::WithheldWorktree) -> bool +impl core::fmt::Debug for devlaunch_core::flows::agent_worktrees::WithheldWorktree +pub fn devlaunch_core::flows::agent_worktrees::WithheldWorktree::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for devlaunch_core::flows::agent_worktrees::WithheldWorktree pub struct devlaunch_core::flows::agent_worktrees::WorktreeReport pub devlaunch_core::flows::agent_worktrees::WorktreeReport::forget_refused: alloc::vec::Vec pub devlaunch_core::flows::agent_worktrees::WorktreeReport::forgotten: usize