perf: verify and cleanup hash over every core, and the race gate learns to notice

.github/workflows/ci.yml

@@ -575,9 +575,20 @@ jobs:
       - name: did anything concurrent change
         id: look
         # The three files concurrency is allowed to live in are not a guess.
-        # They are the map in internal/guard/concurrency_test.go, and a fourth
-        # file growing a goroutine turns that guard red before it gets here - so
-        # this list cannot quietly fall behind the tree.
+        # They are the map in internal/guard/concurrency_test.go.
+        #
+        # This used to say the map guard was enough to keep the two in step: "a
+        # fourth file growing a goroutine turns that guard red before it gets
+        # here, so this list cannot quietly fall behind the tree". That was not
+        # true, and adding the fourth file on 2026-09-05 is what showed it. The
+        # map guard is red only while the new file is NOT in the map. The moment
+        # somebody adds it there - which is what the guard's own message tells
+        # them to do - it goes green, and nothing at all asks whether this line
+        # was updated too. The list could then fall behind exactly when it
+        # mattered: concurrency living in a file the detector is not run for.
+        #
+        # TestTheRaceDetectorIsRunForEveryFileThatDeclaresConcurrency now reads
+        # this line and compares it against the map, so the two cannot drift.
         #
         # go.mod is watched as well. A toolchain or dependency change can alter
         # what the detector sees even when none of our own lines moved.
@@ -589,7 +600,7 @@ jobs:
         # in somebody else's file.
         run: |
           set -euo pipefail
-          watched='internal/format/registry.go cmd/tfg/main.go internal/gui/window/run.go go.mod'
+          watched='internal/format/registry.go cmd/tfg/main.go internal/gui/window/run.go internal/audit/parallel.go go.mod'
           # On a pull request there is no "before" - the field belongs to a push
           # - so this asked for something empty and every pull request answered
           # "touched". That quietly undid the decision of 2026-08-20, because

CHANGELOG.md

@@ -14,6 +14,21 @@ because it turns other people's test suites red.
 
 ## [Unreleased]
 
+### Changed
+
+- **`verify` and `cleanup` read the files over several threads, so checking a
+  large run is several times faster.** Nothing about what they report changes -
+  the same differences, in the same order, with the same exit codes.
+
+  Measured on 6.1 GB in 96 files: `tfg verify` went from 4.28-4.30 seconds to
+  0.54-0.56 seconds on an eight core machine. The gain follows the number of
+  cores you have, up to the point where the disk becomes the limit. On a set of
+  files too large to sit in memory, where every byte has to come off the disk,
+  it is about 1.6 times faster rather than nine.
+
+  Small runs are unaffected either way. A handful of files was already
+  instant and still is.
+
 ## [0.3.0-rc1] - 2026-09-03
 
 ### Breaking

internal/audit/audit.go

@@ -202,52 +202,37 @@ func Verify(ctx context.Context, dir string, m *manifest.Manifest, skip string)
 	folded := make(map[string]string, len(claimed))
 	for _, f := range claimed {
 		folded[core.FoldName(comparablePath(f.Path))] = f.Path
-	}
-
-	for _, f := range claimed {
-		if err := ctx.Err(); err != nil {
-			return diffs, err
-		}
 		seen[comparablePath(f.Path)] = true
+	}
 
-		// The raw path, never the compared one. Cleaning resolves a parent step
-		// against the text of the path rather than against the disk, and the
-		// two differ exactly where a link sits in the middle - which is the
-		// case core.Boundary exists for.
-		full, err := resolved(boundary, f)
-		if err != nil {
-			return nil, err
-		}
-		info, statErr := os.Stat(full)
-		if statErr != nil {
-			diffs = append(diffs, Difference{Kind: Missing, Path: f.Path})
-			continue
-		}
-
-		// Size first. It is free, it catches the common failure, and it names
-		// the cause more precisely than a hash mismatch would.
-		if info.Size() != f.Bytes {
-			diffs = append(diffs, Difference{
-				Kind: WrongSize, Path: f.Path,
-				Want: fmt.Sprintf("%d B", f.Bytes),
-				Got:  fmt.Sprintf("%d B", info.Size()),
-			})
-			continue
-		}
+	// Every path is resolved here, on one goroutine, in order, before a single
+	// byte is read - and that is a correctness requirement rather than a step
+	// that happens to come first.
+	//
+	// A path that leaves the directory refuses the WHOLE pass. If that refusal
+	// could arrive from a worker, stopping the others would mean a lower index
+	// never got asked, so the same manifest would name a different file on
+	// different days. Settled in order, the first one that refuses is the first
+	// one there is.
+	full, err := claimedPaths(boundary, claimed)
+	if err != nil {
+		return nil, err
+	}
 
-		sum, hashErr := hashFile(full)
-		if hashErr != nil {
-			diffs = append(diffs, Difference{Kind: Unreadable, Path: f.Path, Got: hashErr.Error()})
-			continue
-		}
-		if sum != f.Hashes.SHA256 {
-			diffs = append(diffs, Difference{
-				Kind: WrongHash, Path: f.Path,
-				Want: f.Hashes.SHA256,
-				Got:  sum,
-			})
+	found, stopped := inOrder(ctx, len(claimed), func(i int) Difference {
+		return compare(claimed[i], full[i])
+	})
+	for _, d := range found {
+		if d.Kind != "" {
+			diffs = append(diffs, d)
 		}
 	}
+	if stopped != nil {
+		// Left unsorted and the walk below left undone, which is what the
+		// sequential version did. A cancelled pass reports what it compared,
+		// and the files it never reached are not absent - nobody looked.
+		return diffs, stopped
+	}
 
 	// Not asked again here. This loop walks a slice already in memory, the walk
 	// above asks before it, and Verify ends on ctx.Err() - so a check would be a
@@ -378,6 +363,65 @@ func walk(ctx context.Context, dir string) ([]string, error) {
 	return out, nil
 }
 
+// claimedPaths is where each claimed file sits on the disk, in the order the
+// manifest gives them, refusing on the first one that leaves the directory.
+//
+// Shared by Verify and Inspect because both have to refuse the same manifest
+// for the same reason, and because both then hand the answers to workers that
+// must not be able to refuse anything themselves. See parallel.go.
+func claimedPaths(b core.Boundary, files []manifest.File) ([]string, error) {
+	full := make([]string, len(files))
+	for i, f := range files {
+		// The raw path, never the compared one. Cleaning resolves a parent step
+		// against the text of the path rather than against the disk, and the
+		// two differ exactly where a link sits in the middle - which is the
+		// case core.Boundary exists for.
+		p, err := resolved(b, f)
+		if err != nil {
+			return nil, err
+		}
+		full[i] = p
+	}
+	return full, nil
+}
+
+// compare is what one claimed file comes to: the difference it shows, or the
+// zero Difference when what is on the disk is what the manifest describes.
+//
+// Kind is a string, so its zero value is not one of the kinds and can mean
+// agreement on its own. No sentinel anybody has to remember, and no pointer
+// per file - a run of ten thousand would allocate ten thousand of them to say
+// "nothing to report" ten thousand times.
+func compare(f manifest.File, full string) Difference {
+	info, statErr := os.Stat(full)
+	if statErr != nil {
+		return Difference{Kind: Missing, Path: f.Path}
+	}
+
+	// Size first. It is free, it catches the common failure, and it names the
+	// cause more precisely than a hash mismatch would.
+	if info.Size() != f.Bytes {
+		return Difference{
+			Kind: WrongSize, Path: f.Path,
+			Want: fmt.Sprintf("%d B", f.Bytes),
+			Got:  fmt.Sprintf("%d B", info.Size()),
+		}
+	}
+
+	sum, hashErr := hashFile(full)
+	if hashErr != nil {
+		return Difference{Kind: Unreadable, Path: f.Path, Got: hashErr.Error()}
+	}
+	if sum != f.Hashes.SHA256 {
+		return Difference{
+			Kind: WrongHash, Path: f.Path,
+			Want: f.Hashes.SHA256,
+			Got:  sum,
+		}
+	}
+	return Difference{}
+}
+
 // hashFile streams the file rather than reading it in. A run of this tool
 // produces files measured in gigabytes, and verify has to survive its own
 // output.

internal/audit/cleanup.go

@@ -62,51 +62,54 @@ func (c Candidate) Removable(force bool) bool {
 // does, and CLI.md section 9 rules out asking them interactively.
 func Inspect(ctx context.Context, dir string, m *manifest.Manifest) ([]Candidate, error) {
 	boundary := core.NewBoundary(dir)
+	claimed := Claimed(m)
 
-	var out []Candidate
-	for _, f := range Claimed(m) {
-		if err := ctx.Err(); err != nil {
-			return out, err
-		}
-		full, err := resolved(boundary, f)
-		if err != nil {
-			// Nothing is inspected and nothing is offered. A list that points
-			// outside the directory is not a list this tool acts on, and the
-			// preview is where somebody decides - so it must not show the entry
-			// as something it would remove.
-			return nil, err
-		}
+	// Nothing is inspected and nothing is offered when one of these points
+	// outside the directory. A list that points outside is not a list this tool
+	// acts on, and the preview is where somebody decides - so it must not show
+	// the entry as something it would remove.
+	//
+	// Resolved before anything is read and on one goroutine, for the reason
+	// written out at claimedPaths and at parallel.go: a refusal that could
+	// arrive from a worker would name whichever file lost the race.
+	full, err := claimedPaths(boundary, claimed)
+	if err != nil {
+		return nil, err
+	}
 
-		info, err := os.Stat(full)
-		if errors.Is(err, fs.ErrNotExist) {
-			out = append(out, Candidate{Path: f.Path, Disposition: Absent})
-			continue
-		}
-		if err != nil {
-			out = append(out, Candidate{Path: f.Path, Disposition: Unreachable, Detail: err.Error()})
-			continue
-		}
+	// In order, because this list is what cleanup removes from and what it
+	// printed to a person beforehand.
+	return inOrder(ctx, len(claimed), func(i int) Candidate {
+		return look(claimed[i], full[i])
+	})
+}
 
-		// Size before hash, so a file of obviously the wrong length does not
-		// cost a read of however many gigabytes it is.
-		if info.Size() != f.Bytes {
-			out = append(out, Candidate{Path: f.Path, Disposition: Changed,
-				Detail: fmt.Sprintf("it is %d B and the manifest recorded %d B", info.Size(), f.Bytes)})
-			continue
-		}
-		sum, err := hashFile(full)
-		if err != nil {
-			out = append(out, Candidate{Path: f.Path, Disposition: Unreachable, Detail: err.Error()})
-			continue
-		}
-		if sum != f.Hashes.SHA256 {
-			out = append(out, Candidate{Path: f.Path, Disposition: Changed,
-				Detail: "its content is not the content this run wrote"})
-			continue
-		}
-		out = append(out, Candidate{Path: f.Path, Disposition: Ready})
+// look is what one claimed file comes to for cleanup: whether it may be
+// removed, and if not, why not in the words a person needs.
+func look(f manifest.File, full string) Candidate {
+	info, err := os.Stat(full)
+	if errors.Is(err, fs.ErrNotExist) {
+		return Candidate{Path: f.Path, Disposition: Absent}
 	}
-	return out, ctx.Err()
+	if err != nil {
+		return Candidate{Path: f.Path, Disposition: Unreachable, Detail: err.Error()}
+	}
+
+	// Size before hash, so a file of obviously the wrong length does not cost
+	// a read of however many gigabytes it is.
+	if info.Size() != f.Bytes {
+		return Candidate{Path: f.Path, Disposition: Changed,
+			Detail: fmt.Sprintf("it is %d B and the manifest recorded %d B", info.Size(), f.Bytes)}
+	}
+	sum, err := hashFile(full)
+	if err != nil {
+		return Candidate{Path: f.Path, Disposition: Unreachable, Detail: err.Error()}
+	}
+	if sum != f.Hashes.SHA256 {
+		return Candidate{Path: f.Path, Disposition: Changed,
+			Detail: "its content is not the content this run wrote"}
+	}
+	return Candidate{Path: f.Path, Disposition: Ready}
 }
 
 // Outcome is what happened to one file.