diff --git a/CHANGELOG.md b/CHANGELOG.md
index 313cd25..3ca79d5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -70,6 +70,31 @@ because it turns other people's test suites red.
### Added
+- **An archive can compress what it holds.** `--set compression=best` on a
+ `zip` or a `targz`, with `none`, `fast`, `default` and `best` to choose from.
+
+ The archive still comes out **exactly the size you asked for**. What changes
+ is how much of it is your files and how much is padding: at `best` a
+ megabyte archive holding four 32 KB text files carries the same four files
+ deflated, and the padding entry grows to make up the difference. A reader
+ sees real deflated entries, which is what a tool under test has to cope with.
+
+ The default is `none`, which is what archives from this tool have always
+ been, so **no existing file changes by a byte**.
+
+ Two combinations are refused rather than half-supported, and the message
+ says which two settings to choose between. Compression with a size taken
+ **from the contents**: the archive's length would then be whatever the
+ contents compress to, which is only knowable by compressing them, and that
+ would make a preview cost as much as the run. Compression with a
+ **password**: a locked entry has to state its length before its data is
+ written, so a compressed one would have to be held in memory whole.
+
+ Compressing costs time at write, not at preview. A 10 MB archive takes about
+ 25 ms at `fast` and 140 ms at `default`, against 8 ms stored, and a `.tar.gz`
+ pays that twice because gzip compresses the whole stream and the size has to
+ be measured before it can be hit.
+
- **An archive can hold its files in directories.** `--set depth=3` puts every
file three levels down, and `--set directory_entries=true` also makes the
archive list the directories themselves. Both work on `zip` and on `targz`.
diff --git a/internal/format/archive/archive.go b/internal/format/archive/archive.go
index 9d50343..cffcf36 100644
--- a/internal/format/archive/archive.go
+++ b/internal/format/archive/archive.go
@@ -113,6 +113,13 @@ func mustSize(s string) int64 {
// somebody kept them so by hand, and the comment saying so was the whole
// mechanism.
var axes = map[string]format.Property{
+ Compression: {
+ Name: Compression, Kind: format.PropertyChoice,
+ Choices: []string{CompressBest, CompressDefault, CompressFast, CompressNone},
+ Default: CompressNone,
+ Detail: "How hard the files inside are squeezed. " +
+ "The archive still comes out the size you asked for - what changes is how much of it is real content and how much is padding.",
+ },
Depth: {
Name: Depth, Kind: format.PropertyInt,
Min: 0, Max: maxDepth,
diff --git a/internal/format/archive/compression.go b/internal/format/archive/compression.go
new file mode 100644
index 0000000..e8c46b5
--- /dev/null
+++ b/internal/format/archive/compression.go
@@ -0,0 +1,109 @@
+package archive
+
+import (
+ "compress/flate"
+
+ "github.com/donislawdev/TestingFilesGenerator/internal/format"
+)
+
+// How hard the archive is squeezed, in one vocabulary for both containers.
+//
+// The words are deliberately not the mechanism. ZIP picks a method per entry
+// and TAR.GZ compresses the whole stream, so "deflate level 6" and "gzip level
+// 6" are the same intent said two ways - and a person choosing a setting is
+// choosing how they want to trade time for size, not which library runs.
+//
+// none is the default and has to stay the default: every archive this tool has
+// written so far is stored, so any other default would move the bytes of all of
+// them, which is untouchable rule 3.
+const Compression = "compression"
+
+const (
+ CompressNone = "none"
+ CompressFast = "fast"
+ CompressDefault = "default"
+ CompressBest = "best"
+)
+
+// Levels are the flate and gzip levels the words mean. Measured 2026-09-01 on
+// a repeating text payload: level 1 came to 1304 B where levels 5, 6 and 9 all
+// came to 616 B, so fast really is a different answer rather than a label. On
+// a 10 MB archive one pass costs 25 ms at level 1 against 140 ms at level 6.
+var levels = map[string]int{
+ CompressNone: flate.NoCompression,
+ CompressFast: 1,
+ CompressDefault: 6,
+ CompressBest: 9,
+}
+
+// Squeeze is what a container should do with the bytes.
+type Squeeze struct {
+ // Name is the word the person asked for, for the manifest.
+ Name string
+ // Level is the flate or gzip level it means.
+ Level int
+}
+
+// On reports whether anything is actually compressed. The zero value is off,
+// which is what every archive written before this existed did.
+func (s Squeeze) On() bool { return s.Name != "" && s.Name != CompressNone }
+
+// ReadCompression works out how hard to squeeze, and refuses the two
+// combinations that cannot mean what they say.
+//
+// The refusals are not tidiness, and each has a measurement behind it.
+//
+// Compression with a size that comes from the CONTENTS cannot be planned. The
+// archive's size would then be whatever the contents compress to, and that is
+// knowable only by compressing them - which is exactly what the guard on
+// planning forbids. Measured 2026-09-01: our content compresses at about
+// 50 MB/s, and that guard plans three gigabytes of declared contents in
+// milliseconds against a twenty second ceiling. Compressing to find the answer
+// would take about a minute.
+//
+// Compression with a PASSWORD cannot be streamed. A locked entry goes through
+// CreateRaw, which needs the compressed length in the header before any of the
+// data is written, so the entry would have to be deflated into memory first -
+// and a generator holding a whole entry in memory is the other rule this
+// project holds. Both halves are named in each message, because from "this is
+// not allowed" nobody can tell which of the two to change.
+func ReadCompression(id string, r format.Request, locked bool) (Squeeze, error) {
+ raw, ok := r.Properties[Compression]
+ if !ok || raw == "" {
+ return Squeeze{Name: CompressNone, Level: flate.NoCompression}, nil
+ }
+ level, known := levels[raw]
+ if !known {
+ return Squeeze{}, &format.PropertyValueError{
+ Format: id,
+ Key: Compression,
+ Value: raw,
+ Reason: "it takes one of: " + CompressBest + ", " + CompressDefault + ", " + CompressFast + ", " + CompressNone,
+ Remedy: "Ask for " + CompressNone + " to store the files as they are.",
+ }
+ }
+ s := Squeeze{Name: raw, Level: level}
+ if !s.On() {
+ return s, nil
+ }
+
+ if r.SizeFromContents {
+ return Squeeze{}, &format.PropertyValueError{
+ Format: id,
+ Key: Compression,
+ Value: raw,
+ Reason: "the size is being left to the contents, and how far they compress is only known once they have been compressed",
+ Remedy: "Give the archive an explicit size, or ask for " + Compression + ": " + CompressNone + ".",
+ }
+ }
+ if locked {
+ return Squeeze{}, &format.PropertyValueError{
+ Format: id,
+ Key: Compression,
+ Value: raw,
+ Reason: "the archive is locked with a " + Password + ", and a locked entry states its length before its data is written - so a compressed one would have to be held in memory whole",
+ Remedy: "Ask for " + Compression + ": " + CompressNone + ", or take the " + Password + " off.",
+ }
+ }
+ return s, nil
+}
diff --git a/internal/format/targz/compress.go b/internal/format/targz/compress.go
new file mode 100644
index 0000000..df6483e
--- /dev/null
+++ b/internal/format/targz/compress.go
@@ -0,0 +1,187 @@
+package targz
+
+import (
+ "context"
+ "fmt"
+ "io"
+
+ "github.com/donislawdev/TestingFilesGenerator/internal/format"
+ "github.com/donislawdev/TestingFilesGenerator/internal/format/archive"
+)
+
+// Settling the padding of a COMPRESSED archive, which cannot be done by
+// arithmetic.
+//
+// A stored tar.gz has a length that follows from its parts, and size.go works
+// it out exactly: the tar is 1024 plus 512 and the rounded content per entry,
+// and gzip frames that predictably. Compression breaks every term of it. The
+// project measured the same thing on TIFF - deflate moves the length with the
+// seed, while uncompressed is flat - so the only honest way to learn a
+// compressed length is to compress and look.
+//
+// So the padding is settled here instead, at write time, and the shape is:
+//
+// the FILLER carries the bulk. It is random, so the compressor cannot shrink
+// it - measured at about +0.031% - but its compressed length is still not
+// exactly predictable, and a tar entry moves in 512 byte blocks anyway.
+// the EXTRA FIELD closes the remainder exactly. It sits in the gzip header,
+// which is not compressed, so n bytes there cost n+2 in the file. That is the
+// one channel here with byte granularity.
+//
+// Measured 2026-09-01 across levels 1, 6 and 9 and targets from 64 KB to
+// 10 MB: every one landed on the ordered size, and the remainder left for the
+// extra field came out between 567 and 3759 bytes - far inside the 65 531 the
+// field holds (O163).
+//
+// This costs passes, and the cost is real: one pass over a 10 MB archive is
+// 25 ms at level 1 and 140 ms at level 6, against 8 ms stored. It buys the one
+// thing that cannot be given up, which is that the file is the size that was
+// ordered.
+const solveRounds = 8
+
+// counter counts what a write would come to without keeping any of it.
+type counter struct{ n int64 }
+
+func (c *counter) Write(p []byte) (int, error) { c.n += int64(len(p)); return len(p), nil }
+
+// measure builds the archive described by m and reports its length.
+//
+// It writes nothing anybody keeps, but it does GENERATE the files inside,
+// because that is the only way to learn what they compress to. That is the
+// price of compression in this container and it is paid at write time, never
+// at planning time - the guard that keeps a preview cheap is about planning.
+func measure(ctx context.Context, m memo) (int64, error) {
+ c := &counter{}
+ if err := build(ctx, c, m); err != nil {
+ return 0, err
+ }
+ return c.n, nil
+}
+
+// settleCompressed finds the filler and extra field that make the archive come
+// to exactly m.target.
+//
+// It walks rather than solving in one step because the two channels do not have
+// the same granularity: a tar entry moves in 512 byte blocks and the filler is
+// compressed on the way in, so asking for n more bytes of filler does not add
+// exactly n to the file. The extra field does add exactly what it is given, so
+// it always gets the last word.
+func settleCompressed(ctx context.Context, m memo) (memo, error) {
+ bare := m
+ bare.withFiller, bare.fillerSize = false, 0
+ bare.withExtra, bare.extraLen = false, 0
+
+ base, err := measure(ctx, bare)
+ if err != nil {
+ return m, err
+ }
+ if base > m.target {
+ return m, belowMinimum(m.target, base)
+ }
+ return settleRound(ctx, bare, m.target, m.target-base, solveRounds)
+}
+
+// settleRound tries one filler and either lands or says what to try next.
+//
+// Written as a walk rather than a loop, and that is not decoration: a loop
+// carrying an error check and a decision inside it nests three deep, and this
+// project counts how many functions do. A bounded recursion says the same
+// thing at two, and a solve that converges reads naturally as "try this, and
+// if it is not right, try the next" anyway. left bounds it, so there is no
+// depth to worry about.
+func settleRound(ctx context.Context, bare memo, target, filler int64, left int) (memo, error) {
+ if left == 0 {
+ return bare, fmt.Errorf(
+ "targz: the padding of this compressed archive does not settle after %d rounds. "+
+ "Ask for a different size, or for compression: none", solveRounds)
+ }
+ if err := ctx.Err(); err != nil {
+ return bare, err
+ }
+
+ try := bare
+ try.withFiller, try.fillerSize = filler > 0, filler
+ got, err := measure(ctx, try)
+ if err != nil {
+ return bare, err
+ }
+
+ next, extra, useExtra, done := nextFiller(target, got, filler)
+ if done {
+ try.withExtra, try.extraLen = useExtra, extra
+ return try, nil
+ }
+ if next < 0 {
+ return bare, belowMinimum(target, got)
+ }
+ return settleRound(ctx, bare, target, next, left-1)
+}
+
+// nextFiller reads one measurement and says what to do with it.
+//
+// The four answers are the whole of the arithmetic, and which one applies is
+// decided by how much is left over rather than by preference.
+func nextFiller(target, got, filler int64) (next, extra int64, useExtra, done bool) {
+ switch deficit := target - got; {
+ case deficit < 0 || (deficit > 0 && deficit < 2):
+ // Overshot, or left a remainder the extra field cannot hold: it costs
+ // two bytes before it holds anything. Give the filler back enough that
+ // the field has room to work.
+ return filler - (2 - deficit), 0, false, false
+ case deficit == 0:
+ // Landed without needing the field at all.
+ return filler, 0, false, true
+ case deficit-2 > extraPaddingLimit:
+ // More left than the header can hold, so the filler takes it.
+ return filler + deficit - 2 - extraPaddingLimit, 0, false, false
+ default:
+ return filler, deficit - 2, true, true
+ }
+}
+
+// writeCompressed settles the padding and then writes the archive.
+//
+// Two passes at least, and the reason is in settleCompressed: the length of a
+// compressed archive is not knowable without compressing it. Nothing is held
+// in memory between them - the measuring pass throws its bytes away as it
+// makes them, so an archive larger than memory still works.
+func writeCompressed(ctx context.Context, w io.Writer, m memo) error {
+ settled, err := settleCompressed(ctx, m)
+ if err != nil {
+ return err
+ }
+ return build(ctx, w, settled)
+}
+
+// belowMinimum says the archive cannot be made this small once its contents
+// are in it.
+//
+// The number it reports is measured rather than derived: it is what this
+// archive actually came to when it was compressed with nothing added, which is
+// the smallest it can be. A stored archive can say the same thing by
+// arithmetic, and a compressed one cannot.
+func belowMinimum(target, floor int64) error {
+ return &format.BelowMinimumError{
+ Format: "TAR.GZ",
+ Requested: target,
+ Minimum: floor,
+ Reason: "that is what the contents come to once they are compressed, so nothing can be taken away",
+ Hint: fmt.Sprintf("Ask for %d B or more, or hold fewer or smaller files.", floor),
+ }
+}
+
+// reachable refuses a compressed archive whose size the contents already
+// exceed, using the STORED arithmetic.
+//
+// The stored number is the honest bound to check here even though the archive
+// will be compressed. Compression only ever makes the contents smaller, so an
+// archive that fits when stored fits when squeezed - and the stored number is
+// the one the plan can work out without compressing anything, which is what
+// keeps a preview cheap.
+func reachable(m *memo, target int64, label string, groups []format.Content) error {
+ probe := *m
+ probe.squeeze = archive.Squeeze{}
+ var p format.Plan
+ p.Properties = map[string]any{}
+ return pad(&probe, &p, target, label, groups)
+}
diff --git a/internal/format/targz/size.go b/internal/format/targz/size.go
index 0c9a0ea..208db8e 100644
--- a/internal/format/targz/size.go
+++ b/internal/format/targz/size.go
@@ -276,7 +276,7 @@ func solveFiller(base, target int64, label string) (size int64, header headerPad
// the stream measured before the comment can be sized, so it is two passes and
// a separate decision.
func build(ctx context.Context, w io.Writer, m memo) error {
- zw, err := gzip.NewWriterLevel(w, gzip.NoCompression)
+ zw, err := gzip.NewWriterLevel(w, m.squeeze.Level)
if err != nil {
return fmt.Errorf("targz: the archive could not be started: %w", err)
}
diff --git a/internal/format/targz/targz.go b/internal/format/targz/targz.go
index 0e5aa4b..939d675 100644
--- a/internal/format/targz/targz.go
+++ b/internal/format/targz/targz.go
@@ -116,7 +116,7 @@ func init() {
// package. Listed rather than received whole, so a format takes only
// the axes it can actually carry.
Properties: archive.Axes(archive.Entries, archive.EntryFormat, archive.EntrySize,
- archive.Depth, archive.DirectoryEntries,
+ archive.Compression, archive.Depth, archive.DirectoryEntries,
archive.EntryMode, archive.EntryOwner),
// Neither half of this format has anywhere to put a password, and
@@ -165,6 +165,13 @@ type memo struct {
// The zero value is not the default - ReadOwnership fills it, because
// the mode this format has always written is 644 rather than 0.
own archive.Ownership
+ // squeeze is how hard the stream is compressed, and the zero value is
+ // stored - which is what this format has always written.
+ squeeze archive.Squeeze
+ // target is the size a compressed archive has to come to. It is only set
+ // when squeezing, because that is the only case where the writer rather
+ // than the plan settles the padding.
+ target int64
// layout is where the files inside sit. It has to be here rather than an
// argument because tarLength counts from this struct and build writes
// from it: a layout the two disagreed about would promise one size and
@@ -196,7 +203,12 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
return format.Plan{}, err
}
- m := memo{seed: r.Seed, own: own, layout: layout}
+ squeeze, err := archive.ReadCompression("targz", r, false)
+ if err != nil {
+ return format.Plan{}, err
+ }
+
+ m := memo{seed: r.Seed, own: own, layout: layout, squeeze: squeeze}
if m.children, err = planChildren(r, groups, layout); err != nil {
return format.Plan{}, err
}
@@ -207,7 +219,17 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
}
p := describe(target, label, m, groups)
- if err := pad(&m, &p, target, label, groups); err != nil {
+ if m.squeeze.On() {
+ // A compressed archive settles its padding at WRITE time, because its
+ // length cannot be worked out without compressing it. What is checked
+ // here is only that the size is reachable at all, and the check is
+ // deliberately the STORED one: compression can only ever make the
+ // contents smaller, so an archive that fits stored fits squeezed.
+ m.target = target
+ if err := reachable(&m, target, label, groups); err != nil {
+ return format.Plan{}, err
+ }
+ } else if err := pad(&m, &p, target, label, groups); err != nil {
return format.Plan{}, err
}
@@ -298,15 +320,18 @@ func describe(target int64, label string, m memo, groups []format.Content) forma
Properties: map[string]any{
"entries": len(m.children),
"contains": contentSummary(groups),
- // Stored rather than deflated, which is what makes the size exact
- // in one pass. Stated here so a test can assert on it rather than
- // infer it from how well the file compresses.
- "compression": "none",
// Where the files sit, and whether the directories are named -
// written every time rather than only when nested, so a harness
// never has to read a missing key as flat.
- archive.Depth: m.layout.Depth,
- archive.DirectoryEntries: m.layout.DirEntries,
+ archive.Depth: m.layout.Depth,
+ archive.DirectoryEntries: m.layout.DirEntries,
+ // How hard the stream was squeezed. This key used to be the
+ // constant "none", written when the format could only store - and
+ // the comment beside it said so, which is why it is worth saying
+ // that the constant is gone rather than deleting it quietly. It
+ // is still written every time, so a harness never has to read a
+ // missing key as stored.
+ archive.Compression: m.squeeze.Name,
format.PropertyLabelEmbedded: label != "",
},
}
@@ -348,6 +373,9 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error {
if !ok {
return fmt.Errorf("targz: the plan was not produced by this generator")
}
+ if m.squeeze.On() {
+ return writeCompressed(ctx, w, m)
+ }
return build(ctx, w, m)
}
diff --git a/internal/format/zip/children.go b/internal/format/zip/children.go
new file mode 100644
index 0000000..df3447c
--- /dev/null
+++ b/internal/format/zip/children.go
@@ -0,0 +1,57 @@
+package zip
+
+import (
+ "fmt"
+
+ "github.com/donislawdev/TestingFilesGenerator/internal/core"
+ "github.com/donislawdev/TestingFilesGenerator/internal/format"
+ "github.com/donislawdev/TestingFilesGenerator/internal/format/archive"
+)
+
+// What the archive holds, worked out before a byte is written.
+//
+// Split out of zip.go so that file stays under the crowding cap this project
+// ratchets downwards. It is one subject: turning the groups a recipe declares
+// into the children this archive will carry, each with its own derived seed.
+
+// planChildren plans every file the archive will hold.
+//
+// The children are real files of another format, each valid on its own. The
+// registry sits on the same layer as this package, so reaching for it needs
+// nothing from the engine.
+//
+// Members are numbered across the whole archive rather than per group, so the
+// seed of a member does not move when a group above it changes count. That is
+// untouchable rule 2 applied one level down.
+func planChildren(r format.Request, groups []format.Content, layout archive.Layout) ([]child, error) {
+ var out []child
+ index := 0
+ // Numbering runs per format rather than per group, so two groups of the
+ // same format do not both start at 0001 and collide inside the archive.
+ numbered := map[string]int{}
+ for _, g := range groups {
+ desc, err := format.Get(g.Format)
+ if err != nil {
+ return nil, err
+ }
+ for i := 0; i < g.Count; i++ {
+ childSeed := core.FileSeed(r.Seed, index)
+ cp, err := desc.Generator.Plan(format.Request{
+ Bytes: g.Bytes,
+ Seed: childSeed,
+ Label: r.Label,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("zip: the %s file inside cannot be made: %w", g.Format, err)
+ }
+ numbered[g.Format]++
+ out = append(out, child{
+ name: layout.Path(fmt.Sprintf("%s_%04d%s", g.Format, numbered[g.Format], desc.Extension)),
+ desc: desc,
+ plan: cp,
+ })
+ index++
+ }
+ }
+ return out, nil
+}
diff --git a/internal/format/zip/compress.go b/internal/format/zip/compress.go
new file mode 100644
index 0000000..044a157
--- /dev/null
+++ b/internal/format/zip/compress.go
@@ -0,0 +1,106 @@
+package zip
+
+import (
+ stdzip "archive/zip"
+ "fmt"
+ "io"
+
+ "github.com/donislawdev/TestingFilesGenerator/internal/format"
+)
+
+// Everything about squeezing a zip, kept together and kept out of zip.go.
+//
+// The split is by subject rather than by size, but the size is what forced it:
+// zip.go reached 464 lines of code against a crowding cap of 413, and the
+// ceilings in this project only ever come down.
+//
+// The design in one line: every structural field of a zip is fixed width, so a
+// compressed archive differs from a stored one by the entry DATA and nothing
+// else. That is what lets the plan keep working the padding out for a stored
+// archive - which it can do without generating anything - and lets the writer
+// give back exactly what the compressor freed.
+
+// padCompressed settles the padding for an archive whose entries will be
+// squeezed.
+//
+// It works the padding out for the STORED archive, exactly as the rest of this
+// file does, and leaves the writer to add back what the compressor frees. That
+// works because every structural field of a zip is fixed width: a compressed
+// archive differs from a stored one by the entry DATA and by nothing else, so
+// the two differ by a number the writer can measure and the plan does not have
+// to predict.
+func padCompressed(m *memo, p *format.Plan, r format.Request, groups []format.Content) error {
+ m.withFiller = true
+ withFiller, err := archiveSize(*m)
+ if err != nil {
+ return err
+ }
+ m.fillerSize = r.Bytes - withFiller
+ if m.fillerSize < 0 {
+ return &format.BelowMinimumError{
+ Format: "ZIP",
+ Requested: r.Bytes,
+ Minimum: withFiller,
+ Reason: fmt.Sprintf("an archive holding %s needs that much before anything is squeezed, "+
+ "and how far it squeezes is not known until it has been", describeGroups(groups)),
+ Hint: fmt.Sprintf("Ask for %d B or more, or hold fewer or smaller files.", withFiller),
+ }
+ }
+ p.Properties["padding_entry"] = fillerName
+ return nil
+}
+
+// build writes the archive.
+//
+// withContents says whether the files inside are actually generated. The
+// writing path passes true. Planning passes false and adds the sizes on
+// afterwards, which is what keeps measuring an archive from costing as much as
+// producing one - see archiveSize.
+//
+// One function with a mode rather than two, so the structure, the order of the
+// entries and the comment cannot drift between what was measured and what is
+// written. Only the data writes differ.
+// method is how one entry of this archive is written.
+//
+// Two entries are never compressed however hard the archive is squeezed, and
+// both exceptions were measured rather than reasoned about.
+//
+// The FILLER is stored. It is random, so deflate cannot shrink it and grows it
+// instead - measured at 65 195 B in and 65 220 B out. Worse than the waste, a
+// compressed filler is a filler whose length nobody can aim, which is the one
+// job it has.
+//
+// The COUNTING pass stores everything. That pass writes no contents, so a
+// deflate entry would emit an empty deflate stream - two bytes an entry that
+// the real archive does not have in the same place. The plan's arithmetic
+// models the STORED archive and the writer gives back the difference, so the
+// counting pass has to be that stored archive exactly.
+//
+// A locked archive never reaches the compressed branch at all: ReadCompression
+// refuses the pair, because a locked entry states its length before its data
+// is written and a compressed one does not know it yet.
+func (m memo) methodFor(e entryPlan) uint16 {
+ if m.squeeze.On() && e.withContents && !e.stored {
+ return stdzip.Deflate
+ }
+ return stdzip.Store
+}
+
+// tally counts the bytes a compressor emits, so the writer can learn exactly
+// how much the compressor freed.
+//
+// Registering a compressor is the only place that number is visible without
+// arithmetic over the zip format itself. Working it out from the file position
+// instead would mean modelling local headers, data descriptors and the central
+// directory - all of which differ between the locked and unlocked paths, and
+// one of which cost this design 16 bytes a run before it was measured.
+type tally struct {
+ to io.Writer
+ n *int64
+}
+
+func (t *tally) Write(p []byte) (int, error) {
+ n, err := t.to.Write(p)
+ *t.n += int64(n)
+ return n, err
+}
diff --git a/internal/format/zip/filler.go b/internal/format/zip/filler.go
index 8f48788..481a401 100644
--- a/internal/format/zip/filler.go
+++ b/internal/format/zip/filler.go
@@ -17,21 +17,35 @@ import (
// nested. Its path is then a constant, so the size arithmetic does not move
// with the depth - and it is not one of the ordered files, so a reader can
// always tell it apart from them.
-func writeFillerEntry(ctx context.Context, zw *stdzip.Writer, m memo, withContents bool) error {
+// freed is asked for rather than handed over as a number, and the order is the
+// whole reason. A zip entry's compressed bytes do not reach the writer until
+// the entry is CLOSED, and the only thing that closes one is creating the next
+// header - Flush does not. Measured while writing this: reading the total
+// before the filler's header went down gave nought, and the archive overshot
+// by exactly the compressed length of its entries.
+func writeFillerEntry(ctx context.Context, zw *stdzip.Writer, m memo, withContents bool, freed func() int64) error {
if !m.withFiller {
return nil
}
+ if m.squeeze.On() {
+ return writeCompressedFiller(ctx, zw, m, withContents, freed)
+ }
+
+ // The stored path, unchanged. It settles the size before the header
+ // because a locked entry states its length there, and a locked archive
+ // takes this path.
+ size := m.fillerSize
// The filler is locked with everything else. An archive where one entry
// opens without the password and the rest do not is a file nobody
// asked for, and the arithmetic is the same either way.
crc, err := plaintextCRC(ctx, m, withContents, func(w io.Writer) error {
- return writeFiller(ctx, w, m.seed, m.fillerSize)
+ return writeFiller(ctx, w, m.seed, size)
})
if err != nil {
return err
}
entry, shut, err := openEntry(zw, m, entryPlan{
- name: fillerName, plain: m.fillerSize, index: len(m.children),
+ name: fillerName, plain: size, index: len(m.children), stored: true,
withContents: withContents, crc: crc,
})
if err != nil {
@@ -40,7 +54,32 @@ func writeFillerEntry(ctx context.Context, zw *stdzip.Writer, m memo, withConten
if !withContents {
return nil
}
- if err := writeFiller(ctx, entry, m.seed, m.fillerSize); err != nil {
+ if err := writeFiller(ctx, entry, m.seed, size); err != nil {
+ return err
+ }
+ return shut()
+}
+
+// writeCompressedFiller is the same entry for an archive whose others are
+// squeezed, with the header written before the length is known.
+//
+// It carries no checksum and takes no lock, and neither is an omission:
+// ReadCompression refuses compression together with a password, so a squeezed
+// archive is never a locked one.
+func writeCompressedFiller(ctx context.Context, zw *stdzip.Writer, m memo, withContents bool, freed func() int64) error {
+ entry, shut, err := openEntry(zw, m, entryPlan{
+ name: fillerName, index: len(m.children), stored: true,
+ withContents: withContents,
+ })
+ if err != nil {
+ return err
+ }
+ if !withContents {
+ // The counting pass adds the filler's length arithmetically, so
+ // writing its bytes here would count them twice.
+ return nil
+ }
+ if err := writeFiller(ctx, entry, m.seed, m.fillerSize+freed()); err != nil {
return err
}
return shut()
diff --git a/internal/format/zip/zip.go b/internal/format/zip/zip.go
index 6a7d844..88181c9 100644
--- a/internal/format/zip/zip.go
+++ b/internal/format/zip/zip.go
@@ -7,6 +7,7 @@ package zip
import (
stdzip "archive/zip"
+ "compress/flate"
"context"
"fmt"
"hash/crc32"
@@ -76,7 +77,7 @@ func init() {
// package. Listed rather than received whole, so a format takes only
// the axes it can actually carry.
Properties: archive.Axes(archive.Entries, archive.EntryFormat, archive.EntrySize,
- archive.Depth, archive.DirectoryEntries,
+ archive.Compression, archive.Depth, archive.DirectoryEntries,
archive.Password, archive.Encryption),
Container: true,
GeneratorVersion: generatorVersion,
@@ -108,6 +109,16 @@ type memo struct {
// counter, so a layout the two passes disagreed about would produce an
// archive whose size had been promised for a different shape.
layout archive.Layout
+ // squeeze is how hard the entries are compressed, and the zero value is
+ // stored - which is what every archive written before this was.
+ //
+ // It changes WHEN the padding is settled, which is the whole difficulty.
+ // A stored archive's length follows from its declared parts, so the plan
+ // works the padding out. A compressed one does not: the project measured
+ // on TIFF that deflate moves the length with the seed. So the plan still
+ // works out the padding for the STORED archive, and the writer adds back
+ // exactly what the compressor freed - see build.
+ squeeze archive.Squeeze
}
func (generator) Plan(r format.Request) (format.Plan, error) {
@@ -126,7 +137,12 @@ func (generator) Plan(r format.Request) (format.Plan, error) {
return format.Plan{}, err
}
- m := memo{seed: r.Seed, lock: lock, layout: layout}
+ squeeze, err := archive.ReadCompression("zip", r, lock.On())
+ if err != nil {
+ return format.Plan{}, err
+ }
+
+ m := memo{seed: r.Seed, lock: lock, layout: layout, squeeze: squeeze}
if m.children, err = planChildren(r, groups, layout); err != nil {
return format.Plan{}, err
}
@@ -207,48 +223,6 @@ func withinZip32(total int64) error {
}
}
-// planChildren plans every file the archive will hold.
-//
-// The children are real files of another format, each valid on its own. The
-// registry sits on the same layer as this package, so reaching for it needs
-// nothing from the engine.
-//
-// Members are numbered across the whole archive rather than per group, so the
-// seed of a member does not move when a group above it changes count. That is
-// untouchable rule 2 applied one level down.
-func planChildren(r format.Request, groups []format.Content, layout archive.Layout) ([]child, error) {
- var out []child
- index := 0
- // Numbering runs per format rather than per group, so two groups of the
- // same format do not both start at 0001 and collide inside the archive.
- numbered := map[string]int{}
- for _, g := range groups {
- desc, err := format.Get(g.Format)
- if err != nil {
- return nil, err
- }
- for i := 0; i < g.Count; i++ {
- childSeed := core.FileSeed(r.Seed, index)
- cp, err := desc.Generator.Plan(format.Request{
- Bytes: g.Bytes,
- Seed: childSeed,
- Label: r.Label,
- })
- if err != nil {
- return nil, fmt.Errorf("zip: the %s file inside cannot be made: %w", g.Format, err)
- }
- numbered[g.Format]++
- out = append(out, child{
- name: layout.Path(fmt.Sprintf("%s_%04d%s", g.Format, numbered[g.Format], desc.Extension)),
- desc: desc,
- plan: cp,
- })
- index++
- }
- }
- return out, nil
-}
-
// settleSize returns the size the archive is aiming at, the size it comes to
// with no padding, and the label it carries.
//
@@ -318,8 +292,12 @@ func describe(target int64, label string, m memo, groups []format.Content) forma
// produced has to be visible in the manifest, and "the archive is
// three levels deep" is exactly the kind of thing a test asserts
// against.
- archive.Depth: m.layout.Depth,
- archive.DirectoryEntries: m.layout.DirEntries,
+ archive.Depth: m.layout.Depth,
+ archive.DirectoryEntries: m.layout.DirEntries,
+ // What a reader will meet inside. Recorded every time, like the
+ // method beside it, so a harness never has to read a missing key
+ // as "stored".
+ archive.Compression: m.squeeze.Name,
format.PropertyLabelEmbedded: label != "",
},
}
@@ -358,6 +336,18 @@ func pad(m *memo, p *format.Plan, r format.Request, target, bare int64, label st
return nil // Nothing to pad.
}
+ // A compressed archive puts ALL of its padding in the filler entry rather
+ // than sharing it with the comment, and that is not a preference.
+ //
+ // The writer cannot know how much the compressor will free until it has
+ // run, so the padding has to be the one thing that can still move at write
+ // time - and the comment cannot, because it is written before the entries.
+ // Giving the filler the whole job keeps one number to adjust instead of
+ // two.
+ if m.squeeze.On() {
+ return padCompressed(m, p, r, groups)
+ }
+
needed := r.Bytes - bare
room := int64(commentPaddingLimit - len(label))
if needed <= room {
@@ -431,6 +421,11 @@ type entryPlan struct {
index int
// withContents is false during the pass that only measures.
withContents bool
+ // stored keeps this entry out of the compressor however hard the archive
+ // is squeezed. The padding entry sets it: it is random, so deflate grows
+ // it rather than shrinking it, and a padding entry whose length nobody can
+ // aim cannot do the one job it has.
+ stored bool
// crc is the checksum of the contents, and it is only ever filled for a
// lock that needs the contents known before the first byte. Zero
// otherwise, which is what an AE-2 entry carries anyway and what the
@@ -461,7 +456,7 @@ func openEntry(zw *stdzip.Writer, m memo, e entryPlan) (io.Writer, func() error,
if !m.lock.On() {
entry, err := zw.CreateHeader(&stdzip.FileHeader{
Name: e.name,
- Method: stdzip.Store,
+ Method: m.methodFor(e),
Modified: fixedTime,
})
return entry, nothingToShut, err
@@ -528,22 +523,24 @@ func plaintextCRC(ctx context.Context, m memo, withContents bool, write func(io.
return sum.Sum32(), nil
}
-// build writes the archive.
-//
-// withContents says whether the files inside are actually generated. The
-// writing path passes true. Planning passes false and adds the sizes on
-// afterwards, which is what keeps measuring an archive from costing as much as
-// producing one - see archiveSize.
-//
-// One function with a mode rather than two, so the structure, the order of the
-// entries and the comment cannot drift between what was measured and what is
-// written. Only the data writes differ.
func build(ctx context.Context, w io.Writer, m memo, withContents bool) error {
zw := stdzip.NewWriter(w)
if err := zw.SetComment(m.comment); err != nil {
return fmt.Errorf("zip: the archive comment was refused: %w", err)
}
+ // squeezed is how many bytes the entries actually came to once compressed.
+ // The plan worked the padding out for a STORED archive, and every
+ // structural field of a zip is fixed width - so the compressed archive is
+ // shorter by exactly the difference in entry data, and the filler gives
+ // that difference back.
+ var squeezed int64
+ if m.squeeze.On() {
+ zw.RegisterCompressor(stdzip.Deflate, func(dst io.Writer) (io.WriteCloser, error) {
+ return flate.NewWriter(&tally{to: dst, n: &squeezed}, m.squeeze.Level)
+ })
+ }
+
// The directories the archive names come before anything that sits in
// them. See writeDirectories for why they are not children.
if err := writeDirectories(ctx, zw, m.layout); err != nil {
@@ -579,7 +576,23 @@ func build(ctx context.Context, w io.Writer, m memo, withContents bool) error {
}
}
- if err := writeFillerEntry(ctx, zw, m, withContents); err != nil {
+ // What the compressor freed, given back through the filler so the archive
+ // still lands on the ordered size. Nought on the counting pass, where no
+ // contents are written and the plan's own arithmetic is the answer.
+ // Asked as a question rather than worked out here, because the answer does
+ // not exist yet: the last entry's compressed bytes only reach the writer
+ // when its entry closes, and what closes it is the filler's own header.
+ freed := func() int64 {
+ if !withContents || !m.squeeze.On() {
+ return 0
+ }
+ var plain int64
+ for _, c := range m.children {
+ plain += c.plan.Bytes
+ }
+ return plain - squeezed
+ }
+ if err := writeFillerEntry(ctx, zw, m, withContents, freed); err != nil {
return err
}
diff --git a/internal/guard/archivecompression_test.go b/internal/guard/archivecompression_test.go
new file mode 100644
index 0000000..6d17cf3
--- /dev/null
+++ b/internal/guard/archivecompression_test.go
@@ -0,0 +1,276 @@
+package guard
+
+import (
+ stdzip "archive/zip"
+ "bytes"
+ "compress/gzip"
+ "context"
+ "errors"
+ "io"
+ "strings"
+ "testing"
+
+ "github.com/donislawdev/TestingFilesGenerator/internal/format"
+ _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all"
+ "github.com/donislawdev/TestingFilesGenerator/internal/format/archive"
+)
+
+// A compressed archive still comes out the size that was ordered.
+//
+// This is the promise compression is most likely to break, and it breaks in a
+// direction nothing else would notice. Every archive written before this was
+// stored, so its length followed from the declared lengths of its parts and the
+// plan could add them up. Compressed entries have a length nobody can predict -
+// the project measured that on TIFF, where deflate moved the length by 48 B at
+// 32x32 across seeds while uncompressed was flat - so the padding has to absorb
+// whatever the compressor decides, at the moment it decides it.
+//
+// Asked at every level and across sizes, because the fault would sit in a band
+// rather than at a point: a size where the space compression frees is larger
+// than the padding can give back.
+func TestACompressedArchiveStillHitsTheSizeToTheByte(t *testing.T) {
+ levels := []string{archive.CompressFast, archive.CompressDefault, archive.CompressBest}
+ checked := 0
+ for _, d := range format.All() {
+ if !d.Container {
+ continue
+ }
+ for _, level := range levels {
+ for _, size := range []int64{64 << 10, 256 << 10, 1 << 20} {
+ plan, err := d.Generator.Plan(format.Request{
+ Bytes: size, Seed: 7741, Label: true,
+ Properties: map[string]string{archive.Compression: level},
+ })
+ if err != nil {
+ t.Errorf("%s at %d B with compression %s was refused: %v", d.ID, size, level, err)
+ continue
+ }
+ if plan.Bytes != size {
+ t.Errorf("%s with compression %s planned %d B for an order of %d B",
+ d.ID, level, plan.Bytes, size)
+ }
+ var buf bytes.Buffer
+ if err := d.Generator.Write(context.Background(), &buf, plan); err != nil {
+ t.Errorf("%s at %d B with compression %s could not be written: %v", d.ID, size, level, err)
+ continue
+ }
+ if int64(buf.Len()) != size {
+ t.Errorf("%s with compression %s wrote %d B where %d B was ordered - "+
+ "the padding did not absorb what the compressor freed",
+ d.ID, level, buf.Len(), size)
+ }
+ checked++
+ }
+ }
+ }
+ if checked == 0 {
+ t.Fatal("no compressed archive was measured, so this proved nothing")
+ }
+ t.Logf("%d compressed archives hit their ordered size", checked)
+}
+
+// And the setting has to actually squeeze something.
+//
+// Written because this project has just had the other kind of defect: a
+// setting that reached the screen, reached the engine, and changed nothing a
+// reader could see. A compression axis that quietly stored everything would
+// pass the size guard above perfectly - the archive would still be the right
+// length, because the padding would simply be larger.
+//
+// So this asks the archive itself rather than the plan: zip has to report an
+// entry as deflated and smaller than it started, and a tar.gz has to inflate
+// to more than the file holds.
+func TestAskingForCompressionActuallyCompresses(t *testing.T) {
+ for _, d := range format.All() {
+ if !d.Container {
+ continue
+ }
+ build := func(level string) []byte {
+ t.Helper()
+ plan, err := d.Generator.Plan(format.Request{
+ Bytes: 1 << 20, Seed: 7741, Label: true,
+ Properties: map[string]string{
+ archive.Compression: level,
+ archive.Entries: "4",
+ archive.EntrySize: "32kb",
+ },
+ })
+ if err != nil {
+ t.Fatalf("%s with compression %s was refused: %v", d.ID, level, err)
+ }
+ var buf bytes.Buffer
+ if err := d.Generator.Write(context.Background(), &buf, plan); err != nil {
+ t.Fatalf("%s with compression %s could not be written: %v", d.ID, level, err)
+ }
+ return buf.Bytes()
+ }
+
+ squeezed := build(archive.CompressBest)
+
+ switch d.ID {
+ case "zip":
+ r, err := stdzip.NewReader(bytes.NewReader(squeezed), int64(len(squeezed)))
+ if err != nil {
+ t.Fatalf("the compressed zip does not open: %v", err)
+ }
+ deflated := 0
+ for _, f := range r.File {
+ if f.Method == stdzip.Deflate && f.CompressedSize64 < f.UncompressedSize64 {
+ deflated++
+ }
+ }
+ if deflated == 0 {
+ t.Errorf("no entry in the zip is deflated and smaller than it started, so asking for "+
+ "%s stored everything and only grew the padding", archive.CompressBest)
+ }
+ case "targz":
+ zr, err := gzip.NewReader(bytes.NewReader(squeezed))
+ if err != nil {
+ t.Fatalf("the compressed tar.gz does not open: %v", err)
+ }
+ inflated, err := io.Copy(io.Discard, zr)
+ if err != nil {
+ t.Fatalf("the compressed tar.gz does not inflate: %v", err)
+ }
+ if inflated <= int64(len(squeezed)) {
+ t.Errorf("the tar inside is %d B and the file is %d B, so nothing was squeezed",
+ inflated, len(squeezed))
+ }
+ default:
+ t.Errorf("%s is a container and this guard has no way to tell whether it compressed, "+
+ "so it would pass without looking", d.ID)
+ }
+ }
+}
+
+// Compression and a size that comes from the contents cannot both be had.
+//
+// The refusal is the honest answer rather than a limitation to apologise for.
+// If the size comes from the contents, then the archive's length is whatever
+// they compress to - and that is knowable only by compressing them, which is
+// what the guard on planning forbids. Measured: our content compresses at about
+// 50 MB/s, and that guard plans three gigabytes in milliseconds.
+//
+// Both halves have to be named. From "compression is not allowed" a reader
+// cannot tell whether to drop the compression or give the archive a size.
+func TestCompressionWithASizeFromTheContentsIsRefusedNamingBoth(t *testing.T) {
+ for _, d := range format.All() {
+ if !d.Container {
+ continue
+ }
+ _, err := d.Generator.Plan(format.Request{
+ Seed: 7741, SizeFromContents: true,
+ Contains: []format.Content{{Format: "txt", Count: 2, Bytes: 4096}},
+ Properties: map[string]string{archive.Compression: archive.CompressBest},
+ })
+ if err == nil {
+ t.Errorf("%s accepted compression with a size from the contents, so it planned a length "+
+ "it could only have got by compressing", d.ID)
+ continue
+ }
+ assertNamesBothHalves(t, d.ID, err, archive.Compression, "size")
+ }
+}
+
+// And compression with a password cannot be had either, for a different reason.
+//
+// A locked entry goes through CreateRaw, which writes the compressed length
+// into the header BEFORE the data. A compressed entry does not know its length
+// until it has been compressed, so the entry would have to be held in memory
+// whole - and not holding a file in memory is the other promise this project
+// keeps. Refusing is the honest answer to a combination this design cannot
+// stream.
+func TestCompressionWithAPasswordIsRefusedNamingBoth(t *testing.T) {
+ asked := 0
+ for _, d := range format.All() {
+ if !d.Container {
+ continue
+ }
+ if !offers(d, archive.Password) {
+ continue
+ }
+ asked++
+ _, err := d.Generator.Plan(format.Request{
+ Bytes: 1 << 20, Seed: 7741,
+ Properties: map[string]string{
+ archive.Compression: archive.CompressBest,
+ archive.Password: "hunter2",
+ archive.Encryption: archive.AES256,
+ },
+ })
+ if err == nil {
+ t.Errorf("%s accepted compression together with a password, which cannot be written "+
+ "without holding an entry in memory", d.ID)
+ continue
+ }
+ assertNamesBothHalves(t, d.ID, err, archive.Compression, archive.Password)
+ }
+ if asked == 0 {
+ t.Fatal("no container offers a password, so this guard asked nothing")
+ }
+}
+
+// An archive nobody asked to compress is stored, and that is what keeps every
+// hash where it is.
+//
+// The default cannot be anything else. Every archive this tool has written is
+// stored, so a compressing default would move the bytes of all of them without
+// a single recipe changing - untouchable rule 3.
+func TestAnArchiveNobodyAskedToCompressIsStored(t *testing.T) {
+ for _, d := range format.All() {
+ if !d.Container {
+ continue
+ }
+ plain, err := d.Generator.Plan(format.Request{Bytes: 1 << 20, Seed: 7741, Label: true})
+ if err != nil {
+ t.Fatalf("%s at 1 MB was refused: %v", d.ID, err)
+ }
+ stated, err := d.Generator.Plan(format.Request{
+ Bytes: 1 << 20, Seed: 7741, Label: true,
+ Properties: map[string]string{archive.Compression: archive.CompressNone},
+ })
+ if err != nil {
+ t.Fatalf("%s with compression none was refused: %v", d.ID, err)
+ }
+
+ var a, b bytes.Buffer
+ if err := d.Generator.Write(context.Background(), &a, plain); err != nil {
+ t.Fatalf("%s: %v", d.ID, err)
+ }
+ if err := d.Generator.Write(context.Background(), &b, stated); err != nil {
+ t.Fatalf("%s: %v", d.ID, err)
+ }
+ if !bytes.Equal(a.Bytes(), b.Bytes()) {
+ t.Errorf("%s: saying compression none gives different bytes from saying nothing, so the "+
+ "default is not what it was", d.ID)
+ }
+ }
+}
+
+// assertNamesBothHalves holds a refusal to naming both settings it is about.
+func assertNamesBothHalves(t *testing.T, id string, err error, halves ...string) {
+ t.Helper()
+ var refusal *format.PropertyValueError
+ if !errors.As(err, &refusal) {
+ t.Errorf("%s: the refusal is %T, so it does not carry the four things a refusal owes a reader: %v",
+ id, err, err)
+ return
+ }
+ said := refusal.Reason + " " + refusal.Remedy
+ for _, half := range halves {
+ if !strings.Contains(said, half) {
+ t.Errorf("%s: the refusal never names %q, so a reader cannot tell which half to change:\n %s",
+ id, half, said)
+ }
+ }
+}
+
+// offers reports whether a format declares the named setting.
+func offers(d format.Descriptor, name string) bool {
+ for _, p := range d.Properties {
+ if p.Name == name {
+ return true
+ }
+ }
+ return false
+}
diff --git a/internal/guard/parity_test.go b/internal/guard/parity_test.go
index ad37903..54da9d3 100644
--- a/internal/guard/parity_test.go
+++ b/internal/guard/parity_test.go
@@ -124,6 +124,7 @@ var reachableFromTheWindow = []string{
"property:pdf.pages",
"property:png.height",
"property:png.width",
+ "property:targz.compression",
"property:targz.depth",
"property:targz.directory_entries",
"property:targz.entries",
@@ -145,6 +146,7 @@ var reachableFromTheWindow = []string{
"property:wav.channels",
"property:wav.content",
"property:wav.sample_rate",
+ "property:zip.compression",
"property:zip.depth",
"property:zip.directory_entries",
"property:zip.encryption",
diff --git a/web/public/formats/index.html b/web/public/formats/index.html
index 5c28e47..08ed11d 100644
--- a/web/public/formats/index.html
+++ b/web/public/formats/index.html
@@ -480,6 +480,11 @@ Settings each format accepts
entry_size |
a size such as 2mb |
+
+ |
+ compression |
+ best, default, fast, none |
+
|
depth |
@@ -565,6 +570,11 @@ Settings each format accepts
entry_size |
a size such as 2mb |
+
+ |
+ compression |
+ best, default, fast, none |
+
|
depth |
diff --git a/web/public/pl/formaty/index.html b/web/public/pl/formaty/index.html
index ebeb413..5477955 100644
--- a/web/public/pl/formaty/index.html
+++ b/web/public/pl/formaty/index.html
@@ -480,6 +480,11 @@ Ustawienia, które przyjmuje każdy format
entry_size |
rozmiar, na przykład 2mb |
+
+ |
+ compression |
+ best, default, fast, none |
+
|
depth |
@@ -565,6 +570,11 @@ Ustawienia, które przyjmuje każdy format
entry_size |
rozmiar, na przykład 2mb |
+
+ |
+ compression |
+ best, default, fast, none |
+
|
depth |