format: archives can hold their files in directories

CHANGELOG.md

@@ -70,6 +70,30 @@ because it turns other people's test suites red.
 
 ### Added
 
+- **An archive can hold its files in directories.** `--set depth=3` puts every
+  file three levels down, and `--set directory_entries=true` also makes the
+  archive list the directories themselves. Both work on `zip` and on `targz`.
+
+  Two settings rather than one, because they are two questions. Depth is about
+  the paths inside. Directory entries are about whether the archive names the
+  directories at all - and extractors differ there: some create a directory
+  when they meet a path that needs one, and some create only what the archive
+  names. An archive is the one format where you can test both.
+
+  The default is flat, which is what archives from this tool have always been,
+  so **no existing file changes by a byte**. Asking for `directory_entries`
+  without a depth is refused rather than quietly ignored: a flat archive has no
+  directories to name, and the message says so and names both settings.
+
+  Depth goes up to 50. The limit is measured rather than picked: a `.tar.gz`
+  writes USTAR headers, which carry a path in a 155 byte prefix and a 100 byte
+  name split on a slash, and past a certain length no split works. Directories
+  cost 512 bytes each in a `.tar.gz` and about 76 plus the path in a `.zip`.
+  The size you order is still the size you get, to the byte.
+
+  The padding entry stays at the top of the archive rather than moving into the
+  directories, so you can always tell it apart from the files you asked for.
+
 - **A zip can be locked with ZipCrypto, the old scheme.** `--set encryption=zipcrypto`.
 
   It is here for what it does to a reader rather than for what it protects. Measured: .NET's own `ZipFile` opens one of these, reports the entry at its true length, hands back a stream and fills it with the ENCRYPTED bytes - and never says the entry was encrypted at all. An application built on that library processes noise and calls it data. AES fails loudly in the same library, which is the safer defect and the less interesting one.

internal/format/archive/archive.go

@@ -113,6 +113,19 @@ func mustSize(s string) int64 {
 // somebody kept them so by hand, and the comment saying so was the whole
 // mechanism.
 var axes = map[string]format.Property{
+	Depth: {
+		Name: Depth, Kind: format.PropertyInt,
+		Min: 0, Max: maxDepth,
+		Default: strconv.Itoa(defaultDepth),
+		Detail:  "How many directories deep the files inside sit. 0 puts them all at the top.",
+	},
+	DirectoryEntries: {
+		Name: DirectoryEntries, Kind: format.PropertyBool,
+		Default: "false",
+		Detail: "Whether the archive also lists the directories themselves. " +
+			"Most file browsers show the same folders either way, so the difference is in the entry list rather than on screen. " +
+			"It matters for readers that only create a directory the archive names.",
+	},
 	Entries: {
 		Name: Entries, Kind: format.PropertyInt,
 		Min: 0, Max: maxEntries,

internal/format/archive/layout.go

@@ -0,0 +1,186 @@
+package archive
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/donislawdev/TestingFilesGenerator/internal/format"
+)
+
+// Where the files inside an archive sit, and whether the archive lists the
+// directories themselves.
+//
+// Both containers read this one description rather than each growing its own,
+// for the reason the package exists: zip and tar held six constants with
+// identical values and two identical readers before it, and the only thing
+// keeping them equal was a comment.
+//
+// Two settings rather than one, and the pairing is deliberate. A tester asking
+// "does the tool under test cope with nested paths" wants depth. A tester
+// asking "does it cope with an archive that does NOT list its directories"
+// wants the other, because extractors differ: some create a directory when
+// they meet a path that needs one, and some only create what the archive
+// names. An archive is the one format where those are separate questions.
+const (
+	Depth            = "depth"
+	DirectoryEntries = "directory_entries"
+)
+
+const (
+	// defaultDepth is flat, and it has to stay flat. Every archive this tool
+	// has written so far holds its files at the top, so any other default
+	// would move the bytes of all of them - untouchable rule 3, and the reason
+	// this change needs no version bump at all.
+	defaultDepth = 0
+
+	// maxDepth is measured rather than chosen, and the measurement is about
+	// tar rather than about taste.
+	//
+	// targz pins tar.FormatUSTAR (size.go), which carries a path in two
+	// fields: a 155 byte prefix and a 100 byte name, split ON A SLASH. So a
+	// path is writable when some slash leaves at most 155 before it and at
+	// most 100 after it, which is a rule about where the slashes fall and not
+	// about length. With the segments below, slashes sit every 4 bytes, the
+	// last usable one is at 155, and the path therefore has to come to 256
+	// bytes or fewer: 4*depth + len(entry name) <= 256.
+	//
+	// Measured 2026-09-01 against Go's own archive/tar with USTAR pinned:
+	// depth 61 with a 12 byte name is taken at 256 bytes and depth 62 is
+	// REFUSED at 260. The size is flat the whole way - 1536 B at every depth
+	// up to the refusal, no hidden step - so the tar arithmetic needs no
+	// length term at all.
+	//
+	// 61 is therefore the ceiling for a 12 byte name and NOT the ceiling to
+	// declare, because the entry name is not always 12 bytes. The longest one
+	// this build can produce is targz_0001.tar.gz at 17, which lands the limit
+	// at 59. Fifty leaves room for a name of 56 bytes, which is far past
+	// anything the registry holds, and a guard proves it for every registered
+	// format rather than trusting this paragraph.
+	maxDepth = 50
+
+	// dirSegment numbers the levels so a path reads as what it is. Two digits
+	// because maxDepth is two digits, and a fixed width so every segment is
+	// the same size and the arithmetic above stays a multiplication.
+	dirSegment = "d%02d/"
+
+	// dirSegmentBytes is what one segment comes to once rendered - "d00/" is
+	// four bytes where the format string above is six. Written out rather than
+	// taken as len(dirSegment), which is the bug the depth guard caught the
+	// first time it ran: the arithmetic said every path was 2 bytes per level
+	// longer than it is, which would have understated the ceiling rather than
+	// overstating it, so nothing would have failed until somebody widened the
+	// segment. A guard compares this against a really rendered path.
+	dirSegmentBytes = 4
+)
+
+// Layout is what the two settings come to once read.
+type Layout struct {
+	// Depth is how many directories deep the files sit. Zero is flat.
+	Depth int
+	// DirEntries says whether the archive also names the directories.
+	DirEntries bool
+}
+
+// Path is where an entry called name sits under this layout.
+//
+// The empty name gives the directory chain itself with its trailing slash,
+// which is what both containers want a directory entry to be called.
+func (l Layout) Path(name string) string {
+	if l.Depth <= 0 {
+		return name
+	}
+	var b strings.Builder
+	b.Grow(l.Depth*len(dirSegment) + len(name))
+	for i := 0; i < l.Depth; i++ {
+		fmt.Fprintf(&b, dirSegment, i)
+	}
+	b.WriteString(name)
+	return b.String()
+}
+
+// Directories is every directory this layout creates, outermost first.
+//
+// Outermost first because that is the order an extractor wants to meet them
+// in: a reader that creates directories as it goes cannot make d00/d01 before
+// it has made d00. It is empty when nothing was asked for, so a caller can
+// range over it without asking whether the setting is on.
+func (l Layout) Directories() []string {
+	if !l.DirEntries || l.Depth <= 0 {
+		return nil
+	}
+	out := make([]string, 0, l.Depth)
+	for i := 1; i <= l.Depth; i++ {
+		out = append(out, Layout{Depth: i}.Path(""))
+	}
+	return out
+}
+
+// LongestPath is the longest path this layout can produce for an entry name of
+// the given length. It exists for the guard that proves maxDepth is safe.
+func LongestPath(depth, nameLen int) int {
+	return depth*dirSegmentBytes + nameLen
+}
+
+// MaxDepth is the deepest nesting this build offers, for the guard that checks
+// the declaration against what tar will actually take.
+func MaxDepth() int { return maxDepth }
+
+// ReadLayout works out where the files go, and refuses a pair that cannot mean
+// anything.
+//
+// directory_entries with a flat archive is the pair, and it is a refusal
+// rather than a setting quietly doing nothing. There are no directories in a
+// flat archive, so the answer would be the same whichever way it was set - and
+// rule 6 forbids exactly that silence. The message names BOTH halves, because
+// a reader who set one of them cannot tell from "directory_entries is not
+// allowed" which one to change.
+//
+// It is reachable from the window as well as from a recipe, which is why it
+// has to be a good message rather than an internal check: the control is a
+// checkbox, a checkbox always sends its value, and somebody can tick it while
+// depth is still nought.
+func ReadLayout(id string, r format.Request) (Layout, error) {
+	depth, err := intProperty(id, r.Properties, Depth, defaultDepth, 0, maxDepth)
+	if err != nil {
+		return Layout{}, err
+	}
+	dirs, err := boolProperty(id, r.Properties, DirectoryEntries, false)
+	if err != nil {
+		return Layout{}, err
+	}
+	if dirs && depth == 0 {
+		return Layout{}, &format.PropertyValueError{
+			Format: id,
+			Key:    DirectoryEntries,
+			Value:  "true",
+			Reason: "a flat archive has no directories to list, and " + Depth + " is 0",
+			Remedy: "Ask for " + Depth + " of 1 or more, or leave " + DirectoryEntries + " off.",
+		}
+	}
+	return Layout{Depth: depth, DirEntries: dirs}, nil
+}
+
+// boolProperty reads a true or false setting.
+//
+// The registry has already refused anything that is not true or false by the
+// time a generator runs, since the declaration says the kind. This repeats the
+// check for the same reason intProperty repeats its range: a caller reaching
+// the generator directly is not going through the registry.
+func boolProperty(id string, props map[string]string, key string, fallback bool) (bool, error) {
+	raw, ok := props[key]
+	if !ok || raw == "" {
+		return fallback, nil
+	}
+	v, err := strconv.ParseBool(strings.ToLower(raw))
+	if err != nil {
+		return false, &format.PropertyValueError{
+			Format: id,
+			Key:    key,
+			Value:  raw,
+			Reason: "it takes true or false",
+			Remedy: "Write " + key + ": true or " + key + ": false.",
+		}
+	}
+	return v, nil
+}

internal/format/targz/size.go

@@ -59,6 +59,16 @@ func roundUpBlock(n int64) int64 {
 // tarLength is the length of the tar stream before it reaches gzip.
 func tarLength(m memo) int64 {
 	total := int64(2 * tarBlock) // the end of archive marker
+	// A directory is a header and nothing else, so it costs exactly one block.
+	// Measured 2026-09-01 against archive/tar rather than read off the format:
+	// a tar holding one directory and nothing else comes to 1536 B, of which
+	// 1024 is the end of archive marker. A guard holds that number.
+	//
+	// The path length does NOT appear here, and that is measured too. USTAR
+	// splits a path across a 155 byte prefix and a 100 byte name, so a header
+	// is the same 512 bytes at every depth it accepts - flat all the way to
+	// the refusal, with no step. maxDepth is what keeps it on the near side.
+	total += int64(len(m.layout.Directories())) * tarBlock
 	for _, c := range m.children {
 		total += tarBlock + roundUpBlock(c.plan.Bytes)
 	}
@@ -279,6 +289,20 @@ func build(ctx context.Context, w io.Writer, m memo) error {
 	}
 	tw := tar.NewWriter(zw)
 
+	// Directories first, outermost first, and only when asked for. They are
+	// not in m.children on purpose: a child's seed is FileSeed(seed, index)
+	// over a running index, so a directory in that list would shift the seed
+	// of every file after it and rewrite its contents. That is untouchable
+	// rule 2 - an edit in one place moving the bytes in another.
+	for _, dir := range m.layout.Directories() {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+		if err := writeDirectory(tw, dir, m.own); err != nil {
+			return fmt.Errorf("targz: the directory %q could not be named: %w", dir, err)
+		}
+	}
+
 	for _, c := range m.children {
 		if err := ctx.Err(); err != nil {
 			return err
@@ -334,6 +358,34 @@ func writeEntry(ctx context.Context, tw *tar.Writer, e tarEntry, body func(io.Wr
 	return body(tw)
 }
 
+// writeDirectory names one directory in the tar.
+//
+// The mode is 0755 rather than own.Mode, and that is a decision rather than an
+// oversight. entry_mode is declared as "the permissions recorded for each file
+// inside", and a directory is not a file - recording 644 on one would produce
+// an archive that extracts into directories nothing can be written into, which
+// is a surprise nobody asked this setting for. The owner fields DO follow
+// entry_owner, so an archive that says everything belongs to root says it about
+// the directories too.
+//
+// It costs exactly one block and no more, which is the whole of what the
+// arithmetic above had to learn about it. Measured rather than read off the
+// format, and a guard holds the number.
+func writeDirectory(tw *tar.Writer, name string, own archive.Ownership) error {
+	return tw.WriteHeader(&tar.Header{
+		Name:     name,
+		Size:     0,
+		Mode:     0o755,
+		Uid:      own.Uid,
+		Gid:      own.Gid,
+		Uname:    own.Uname,
+		Gname:    own.Gname,
+		ModTime:  fixedTime,
+		Typeflag: tar.TypeDir,
+		Format:   tar.FormatUSTAR,
+	})
+}
+
 // tarEntry is one entry's header, as both the measuring pass and the
 // writing pass describe it.
 //

internal/format/targz/targz.go

@@ -116,6 +116,7 @@ func init() {
 		// package. Listed rather than received whole, so a format takes only
 		// the axes it can actually carry.
 		Properties: archive.Axes(archive.Entries, archive.EntryFormat, archive.EntrySize,
+			archive.Depth, archive.DirectoryEntries,
 			archive.EntryMode, archive.EntryOwner),
 
 		// Neither half of this format has anywhere to put a password, and
@@ -164,6 +165,11 @@ type memo struct {
 	// The zero value is not the default - ReadOwnership fills it, because
 	// the mode this format has always written is 644 rather than 0.
 	own archive.Ownership
+	// layout is where the files inside sit. It has to be here rather than an
+	// argument because tarLength counts from this struct and build writes
+	// from it: a layout the two disagreed about would promise one size and
+	// write another.
+	layout archive.Layout
 	// withExtra says whether the header carries a gzip extra field at all,
 	// and extraLen says how many bytes it holds. Two fields rather than one
 	// with a sentinel, matching withFiller beside them, because an EMPTY
@@ -185,8 +191,13 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
 		return format.Plan{}, err
 	}
 
-	m := memo{seed: r.Seed, own: own}
-	if m.children, err = planChildren(r, groups); err != nil {
+	layout, err := archive.ReadLayout("targz", r)
+	if err != nil {
+		return format.Plan{}, err
+	}
+
+	m := memo{seed: r.Seed, own: own, layout: layout}
+	if m.children, err = planChildren(r, groups, layout); err != nil {
 		return format.Plan{}, err
 	}
 
@@ -209,7 +220,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
 // Members are numbered across the whole archive rather than per group, so the
 // seed of a member does not move when a group above it changes count. That is
 // untouchable rule 2 applied one level down.
-func planChildren(r format.Request, groups []format.Content) ([]child, error) {
+func planChildren(r format.Request, groups []format.Content, layout archive.Layout) ([]child, error) {
 	var out []child
 	index := 0
 	// Numbering runs per format rather than per group, so two groups of the
@@ -231,7 +242,7 @@ func planChildren(r format.Request, groups []format.Content) ([]child, error) {
 			}
 			numbered[g.Format]++
 			out = append(out, child{
-				name: fmt.Sprintf("%s_%04d%s", g.Format, numbered[g.Format], desc.Extension),
+				name: layout.Path(fmt.Sprintf("%s_%04d%s", g.Format, numbered[g.Format], desc.Extension)),
 				desc: desc,
 				plan: cp,
 			})
@@ -290,7 +301,12 @@ func describe(target int64, label string, m memo, groups []format.Content) forma
 			// Stored rather than deflated, which is what makes the size exact
 			// in one pass. Stated here so a test can assert on it rather than
 			// infer it from how well the file compresses.
-			"compression":                "none",
+			"compression": "none",
+			// Where the files sit, and whether the directories are named -
+			// written every time rather than only when nested, so a harness
+			// never has to read a missing key as flat.
+			archive.Depth:                m.layout.Depth,
+			archive.DirectoryEntries:     m.layout.DirEntries,
 			format.PropertyLabelEmbedded: label != "",
 		},
 	}