perf: one filler instead of thirteen, and eight bytes per draw instead of one

CHANGELOG.md

@@ -14,6 +14,29 @@ because it turns other people's test suites red.
 
 ## [Unreleased]
 
+### Breaking
+
+- **ZIP, TAR.GZ and WAV files have different bytes.** The padding these three
+  formats write is now drawn from the random generator eight bytes at a time
+  instead of one byte at a time. For a ZIP or a TAR.GZ that padding is almost
+  the whole file, so almost every byte changes.
+
+  Nothing a reader can see is different. The size is the same to the byte, the
+  structure is the same, the same tools open the same files. What changes is the
+  content of the padding, so a hash you recorded from an earlier version will
+  not match one you generate now.
+
+  This is what makes large archives faster to produce. Measured on 64 MB files,
+  runs interleaved: ZIP 2.8 times faster, TAR.GZ 2.7 to 3.3 times faster.
+
+  WAV is on this list for consistency rather than for speed. Its padding is at
+  most a few bytes per file, so only those bytes differ and the time to produce
+  one is unchanged.
+
+  Every other format keeps exactly the bytes it had. Twelve of them share the
+  same new code and were checked against their recorded hashes and across every
+  format at five sizes and two seeds.
+
 ### Changed
 
 - **`verify` and `cleanup` read the files over several threads, so checking a
@@ -29,6 +52,27 @@ because it turns other people's test suites red.
   Small runs are unaffected either way. A handful of files was already
   instant and still is.
 
+- **Starting a run of many files no longer spends most of its time deciding
+  whether it may start.** Before writing anything, `tfg` checks that it is not
+  about to write over something of yours. That check asked the filesystem two
+  questions for every file it planned. It reads the folder once instead.
+
+  Measured on a run of 100 000 files: `--dry-run` went from 15.7-19.7 seconds to
+  0.49-0.56 seconds. The check was 97% of it.
+
+- **A name held by a shortcut that points at nothing now counts as taken.** The
+  check followed the shortcut, found nothing at the other end, and replaced it
+  without a word. It asks what the folder holds now, so anything already sitting
+  under a name your run wants is refused, whatever it points at.
+
+### Fixed
+
+- **`verify` and `cleanup` read each file in larger pieces**, which takes about
+  a fifth off the time spent hashing.
+
+- **Every run used to pause twice to tidy memory**, however small it was. It
+  pauses once. Nothing about what a run produces changes.
+
 ## [0.3.0-rc1] - 2026-09-03
 
 ### Breaking

internal/audit/audit.go

@@ -219,8 +219,8 @@ func Verify(ctx context.Context, dir string, m *manifest.Manifest, skip string)
 		return nil, err
 	}
 
-	found, stopped := inOrder(ctx, len(claimed), func(i int) Difference {
-		return compare(claimed[i], full[i])
+	found, stopped := inOrder(ctx, len(claimed), func(i int, scratch []byte) Difference {
+		return compare(claimed[i], full[i], scratch)
 	})
 	for _, d := range found {
 		if d.Kind != "" {
@@ -392,7 +392,7 @@ func claimedPaths(b core.Boundary, files []manifest.File) ([]string, error) {
 // agreement on its own. No sentinel anybody has to remember, and no pointer
 // per file - a run of ten thousand would allocate ten thousand of them to say
 // "nothing to report" ten thousand times.
-func compare(f manifest.File, full string) Difference {
+func compare(f manifest.File, full string, scratch []byte) Difference {
 	info, statErr := os.Stat(full)
 	if statErr != nil {
 		return Difference{Kind: Missing, Path: f.Path}
@@ -408,7 +408,7 @@ func compare(f manifest.File, full string) Difference {
 		}
 	}
 
-	sum, hashErr := hashFile(full)
+	sum, hashErr := hashFile(full, scratch)
 	if hashErr != nil {
 		return Difference{Kind: Unreadable, Path: f.Path, Got: hashErr.Error()}
 	}
@@ -422,18 +422,48 @@ func compare(f manifest.File, full string) Difference {
 	return Difference{}
 }
 
+// hashScratch is how much of a file is read at a time.
+//
+// Measured 2026-09-05 on a 256 MB file, median of five interleaved runs:
+// io.Copy 199 ms, 64 KiB 182 ms, 256 KiB 161 ms, 1 MiB 162 ms, 4 MiB 160 ms.
+// The whole win is in by a quarter of a megabyte and nothing above it can be
+// told apart, so this is the size that buys it without asking every worker for
+// a megabyte.
+const hashScratch = 256 << 10
+
+// onlyRead hides everything but Read from io.CopyBuffer.
+//
+// This is not a wrapper for its own sake and without it the buffer below does
+// nothing at all. os.File implements io.WriterTo, so io.CopyBuffer hands the
+// whole job to the file and THROWS THE BUFFER AWAY - measured on 2026-09-05,
+// passing 128 KiB, 256 KiB or 1 MiB with a plain file on the other end all take
+// the same 167-172 ms that io.Copy takes, because all four are the same 32 KiB
+// path inside. Hidden behind this, a 256 KiB buffer takes 150-161 ms.
+//
+// The report that suggested the buffer said it would "cut the syscall count 32
+// fold" and would have changed nothing as written. That is the whole reason
+// this type exists rather than a comment saying the buffer is bigger.
+type onlyRead struct{ io.Reader }
+
 // hashFile streams the file rather than reading it in. A run of this tool
 // produces files measured in gigabytes, and verify has to survive its own
 // output.
-func hashFile(path string) (string, error) {
+//
+// scratch belongs to the caller and is reused across every file that caller
+// answers for. It may be nil, and then this asks for one of its own - which is
+// what a sequential caller wants and what every test that reaches here gets.
+func hashFile(path string, scratch []byte) (string, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return "", err
 	}
 	defer func() { _ = f.Close() }()
 
+	if scratch == nil {
+		scratch = make([]byte, hashScratch)
+	}
 	h := sha256.New()
-	if _, err := io.Copy(h, f); err != nil {
+	if _, err := io.CopyBuffer(h, onlyRead{f}, scratch); err != nil {
 		return "", err
 	}
 	return hex.EncodeToString(h.Sum(nil)), nil

internal/audit/cleanup.go

@@ -79,14 +79,14 @@ func Inspect(ctx context.Context, dir string, m *manifest.Manifest) ([]Candidate
 
 	// In order, because this list is what cleanup removes from and what it
 	// printed to a person beforehand.
-	return inOrder(ctx, len(claimed), func(i int) Candidate {
-		return look(claimed[i], full[i])
+	return inOrder(ctx, len(claimed), func(i int, scratch []byte) Candidate {
+		return look(claimed[i], full[i], scratch)
 	})
 }
 
 // look is what one claimed file comes to for cleanup: whether it may be
 // removed, and if not, why not in the words a person needs.
-func look(f manifest.File, full string) Candidate {
+func look(f manifest.File, full string, scratch []byte) Candidate {
 	info, err := os.Stat(full)
 	if errors.Is(err, fs.ErrNotExist) {
 		return Candidate{Path: f.Path, Disposition: Absent}
@@ -101,7 +101,7 @@ func look(f manifest.File, full string) Candidate {
 		return Candidate{Path: f.Path, Disposition: Changed,
 			Detail: fmt.Sprintf("it is %d B and the manifest recorded %d B", info.Size(), f.Bytes)}
 	}
-	sum, err := hashFile(full)
+	sum, err := hashFile(full, scratch)
 	if err != nil {
 		return Candidate{Path: f.Path, Disposition: Unreachable, Detail: err.Error()}
 	}

internal/audit/parallel.go

@@ -75,7 +75,7 @@ func widthFor(n int) int {
 //     own sake: if a refusal could arrive here, stopping the other goroutines
 //     would mean a LOWER index never got asked, and the same manifest would
 //     name a different file on different days.
-func inOrder[T any](ctx context.Context, n int, one func(i int) T) ([]T, error) {
+func inOrder[T any](ctx context.Context, n int, one func(i int, scratch []byte) T) ([]T, error) {
 	out := make([]T, n)
 	done := make([]bool, n)
 
@@ -111,7 +111,20 @@ func inOrder[T any](ctx context.Context, n int, one func(i int) T) ([]T, error)
 // out and done are written at indices no other goroutine will take, because
 // the counter hands each index out exactly once. They are the only things
 // written at all.
-func drain[T any](ctx context.Context, next *atomic.Int64, out []T, done []bool, one func(i int) T) {
+func drain[T any](ctx context.Context, next *atomic.Int64, out []T, done []bool, one func(i int, scratch []byte) T) {
+	// One buffer per goroutine, handed to every item that goroutine answers.
+	//
+	// Per worker rather than per item, because a run of a hundred thousand
+	// files would otherwise allocate a hundred thousand of these. And per
+	// worker rather than one shared, because two goroutines reading files into
+	// the same bytes is the data race this whole file is written to avoid.
+	//
+	// The size is measured rather than picked. Hashing a 256 MB file, median of
+	// five: io.Copy 199 ms, 64 KiB 182 ms, 256 KiB 161 ms, 1 MiB 162 ms, 4 MiB
+	// 160 ms. The win is all in by a quarter of a megabyte and nothing above it
+	// is distinguishable, so sixteen workers cost four megabytes rather than
+	// sixty four.
+	scratch := make([]byte, hashScratch)
 	for {
 		i := int(next.Add(1)) - 1
 		// Cancellation is asked per item rather than per pass, so a Ctrl+C
@@ -120,7 +133,7 @@ func drain[T any](ctx context.Context, next *atomic.Int64, out []T, done []bool,
 		if i >= len(out) || ctx.Err() != nil {
 			return
 		}
-		out[i] = one(i)
+		out[i] = one(i, scratch)
 		done[i] = true
 	}
 }

internal/core/fill.go

@@ -0,0 +1,59 @@
+package core
+
+import (
+	"encoding/binary"
+	// D11 promises the same bytes from the same seed, so a deliberate,
+	// reproducible generator is the product rather than a weakness. Nothing
+	// here ever makes a secret.
+	// nosemgrep: go.lang.security.audit.crypto.math_random.math-random-used
+	"math/rand/v2"
+)
+
+// Bulk random bytes are drawn eight at a time, and the two byte orders are two
+// functions rather than one function with a flag.
+//
+// Eight at a time rather than one is the whole point. Measured 2026-09-06 over
+// 64 MiB through a 32 KiB buffer, repetitions interleaved with the order
+// reversed: one draw per byte runs at 182 MB/s, eight bytes per draw at
+// 2499 MB/s. The write path of an archive is slower than the disk underneath it
+// when it draws a byte at a time (P2 in PERFORMANCE-REVIEW-2026-09-05.md).
+//
+// The orders stay separate because picking the wrong one is not a style
+// mistake - it silently rewrites every file a format has ever produced, which
+// is exactly what D11 forbids. A boolean argument would put that mistake one
+// typo away and would read the same in review either way. Two names cannot be
+// confused by accident, and the compiler cannot help with a flag.
+//
+// Neither function draws anything for an empty slice, and both spend exactly
+// one draw on a trailing group shorter than eight bytes. That is what the ten
+// call sites did before they were folded into here, so folding them in moved no
+// bytes except where the owner asked for them to move.
+
+// FillRandomBE fills b with random bytes, most significant byte of each draw
+// first. This is the order bmp, gif, ico, opc, png, tiff and - since 0.3.0 -
+// targz, wav and zip write.
+func FillRandomBE(b []byte, rng *rand.Rand) {
+	i := 0
+	for ; i+8 <= len(b); i += 8 {
+		binary.BigEndian.PutUint64(b[i:], rng.Uint64())
+	}
+	if i < len(b) {
+		var eight [8]byte
+		binary.BigEndian.PutUint64(eight[:], rng.Uint64())
+		copy(b[i:], eight[:])
+	}
+}
+
+// FillRandomLE fills b with random bytes, least significant byte of each draw
+// first. This is the order avif, jpg, jxl and webp write.
+func FillRandomLE(b []byte, rng *rand.Rand) {
+	i := 0
+	for ; i+8 <= len(b); i += 8 {
+		binary.LittleEndian.PutUint64(b[i:], rng.Uint64())
+	}
+	if i < len(b) {
+		var eight [8]byte
+		binary.LittleEndian.PutUint64(eight[:], rng.Uint64())
+		copy(b[i:], eight[:])
+	}
+}

internal/engine/engine.go

@@ -433,128 +433,6 @@ type Progress struct {
 	BytesTotal int64
 }
 
-// DefaultManifestName is where the manifest lands when nothing says otherwise.
-//
-// It lives here rather than in the caller because the engine has to know the
-// name to protect it from being written over, and two copies of a file name
-// are two things to keep in step.
-const DefaultManifestName = "manifest.json"
-
-// preflight answers whether this run may start, without writing anything.
-//
-// Both checks used to sit after the dry run had already returned, so
-// --dry-run reported success for runs that would refuse to start.
-func preflight(ctx context.Context, files []PlannedFile, opt Options) error {
-	// Free space first. Finding out at file five thousand of ten thousand
-	// leaves a half written set and a full disk on a machine somebody works on.
-	needed := TotalBytes(files)
-	if available, err := opt.availableBytes(opt.OutDir); err == nil {
-		if available < needed {
-			return &SpaceError{Needed: needed, Available: available, Path: opt.OutDir}
-		}
-	}
-	// A failure to read the free space is not a reason to refuse. A disk we
-	// cannot measure is not the same as a disk that is full.
-
-	// Pointing --out at a file rather than a directory is a mistake somebody
-	// makes, and it used to arrive as two messages about one fault, the first
-	// of them saying "there is nothing at that path" about a path that has
-	// something at it. The system reports ENOTDIR and our mapping only knew
-	// "missing", "no permission" and "already there".
-	if info, err := os.Stat(opt.OutDir); err == nil && !info.IsDir() {
-		return &RecipeError{Setting: SettingOutDir,
-			Detail: fmt.Sprintf("the output directory %s is a file, not a directory", opt.OutDir),
-			Remedy: "Point the output directory at a directory, or at one that does not exist yet and it will be created"}
-	}
-
-	// The manifest is checked with the files it would describe, and leaving it
-	// out cost exactly what it protects. A second run into the same directory
-	// wrote a fresh manifest over the old one, so every file the old one listed
-	// stopped being anybody's - cleanup reported nothing to remove and those
-	// files could never be cleaned up by this tool again. It happened on a
-	// successful run as readily as on a refused one, and on the successful one
-	// it happened in silence.
-	if path := ManifestPath(opt); exists(path) {
-		return &CollisionError{Path: path, Manifest: true}
-	}
-
-	// Nothing else is written over either. This tool runs in directories that
-	// belong to the user, so destroying their work is the one failure that
-	// cannot be undone by running again.
-	//
-	// The temporary name is checked as well as the final one. It used to be
-	// created with os.Create, which truncates, so a file already sitting under
-	// that name lost its contents without a word while the collision check
-	// looked only at the name the file ends up with. Asking the filesystem
-	// instead, with O_EXCL at the write, was tried on 2026-08-25 and taken
-	// back out - see writeOne for what it costs on Windows. So this check is
-	// the whole of the protection rather than the friendlier half of it.
-	//
-	// Which run leaves such a file behind is worth being accurate about,
-	// because the comment here used to get it wrong. The name carries the
-	// process id, so a run killed outright leaves one that no later run can
-	// meet: the next run builds a different name and walks past it. Nor is it
-	// lost - verify names it as one of ours rather than as something nobody
-	// asked for, and cleanup leaves it alone because untouchable rule 7 makes
-	// the manifest the whole authority over what may be deleted, and a file
-	// that never finished never reached one.
-	for _, f := range files {
-		// Two questions of the filesystem per planned file, so a large run on a
-		// slow share spends real time in here. Until 2026-08-26 that time could
-		// not be interrupted: preflight took no context, so a preview had
-		// nothing to cancel, the window offered no button for it, and closing
-		// the window waited for the whole loop on the interface thread. The
-		// same reasoning that put a context into the directory walk in
-		// internal/audit on 2026-08-25.
-		if err := ctx.Err(); err != nil {
-			return err
-		}
-		if path := filepath.Join(opt.OutDir, f.Name); exists(path) {
-			return &CollisionError{Path: path}
-		}
-		if path := tempPathFor(opt.OutDir, f.Name); exists(path) {
-			return &CollisionError{Path: path}
-		}
-	}
-	return nil
-}
-
-// tempPathFor is the name a file is written under before it is renamed into
-// place. One definition, used by the check above and by the write below, so
-// the two cannot disagree about what is being protected.
-func tempPathFor(outDir, name string) string {
-	return fmt.Sprintf("%s%s%d", filepath.Join(outDir, name), core.PartialMarker, os.Getpid())
-}
-
-func exists(path string) bool {
-	_, err := os.Stat(path)
-	return err == nil
-}
-
-// manifestNameOf is where this run's record lands. One definition, because the
-// preflight check, the claim and the save all have to mean the same file.
-func manifestNameOf(opt Options) string {
-	if opt.ManifestName == "" {
-		return DefaultManifestName
-	}
-	return opt.ManifestName
-}
-
-// ManifestPath is the file this run's record lands in, for the callers that
-// save it. The engine claims that name before the first file and refuses the
-// run if it is taken, so a saver joining the path its own way is answering a
-// question this package has already answered.
-//
-// Exported on 2026-08-27 because both savers did join it their own way. The
-// window's did not handle an empty name at all, so a caller that left it blank
-// would have asked the manifest to be written to the output directory itself -
-// a rename of a file onto a directory. Nothing reached that, because all three
-// screens fill the field in, and "nothing reaches it today" is the description
-// of a fault waiting for a fourth screen rather than of a safe piece of code.
-func ManifestPath(opt Options) string {
-	return filepath.Join(opt.OutDir, manifestNameOf(opt))
-}
-
 // Run writes a planned set of files.
 //
 // Each file is written under a temporary name and only then renamed, so the