From 8c387a36612e08d3230055e0c784547a0aaf221d Mon Sep 17 00:00:00 2001 From: KnifeLemon Date: Sun, 15 Nov 2026 00:53:25 +0900 Subject: [PATCH] perf(lzma): faster LZMA decoder for 7z, xz, .lzma and NSIS Rewrite the decoder's inner loop; output, errors and limits are unchanged. - Decode into the window up to the end of the caller's buffer or of the window, then copy the block out, instead of writing every byte twice. - Literal runs and match runs are decoded by two methods that keep the range coder, input position and state in locals for the whole run. - All probabilities live in one array reached through bounded refs. - Literal and tree bits are decoded with masks instead of branches; literals read both children's probabilities before the bit is known. - While 64 or more input bytes are buffered (a symbol reads at most 48) bytes are read without checks; the rest of a chunk or stream is read by a checked copy of the same code. Stream input is topped up. - Matches copy 8 bytes at a time when they don't overlap closely. Same machine, median of 3, CPU MB/s old -> new (7-Zip 26.02 -mmt=1): text .xz 24.6 -> 36.8 (77), mixed binary .xz 35.4 -> 42.1 (82), Tesseract NSIS payload 41.3 -> 52.1 (93). Tests: LzmaDecoderTests (all lc/lp/pb, wrapping dictionaries, odd read and input sizes, truncation, 7-Zip LZMA/LZMA2 property variants). Co-Authored-By: Claude Opus 5.5 --- .../Archives/Lzma/LzmaDecoder.cs | 705 ++++++++++++++---- .../Archives/Lzma/RangeDecoder.cs | 218 ++---- tests/Filee.Engines.Tests/LzmaDecoderTests.cs | 213 ++++++ 3 files changed, 834 insertions(+), 302 deletions(-) create mode 100644 tests/Filee.Engines.Tests/LzmaDecoderTests.cs diff --git a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs index d7a50c9ed..8937503c5 100644 --- a/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs +++ b/src/Filee.Engines/Archives/Lzma/LzmaDecoder.cs @@ -7,6 +7,7 @@ // before anything is allocated (see Lzma2Reader and XzReader). using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace Filee.Engines.Archives.Lzma; @@ -28,7 +29,7 @@ internal readonly record struct LzmaProperties(int Lc, int Lp, int Pb) } } -/// The probabilities of a match or repeat length (2 … 273). +/// The probabilities of a match or repeat length (2 … 273), as the encoder keeps them. internal sealed class LengthModel { public const int LowBits = 3; @@ -50,16 +51,6 @@ public void Reset() RangeCoder.Reset(High); } - /// A length minus 2 (0 … 271). - public uint Decode(RangeDecoder rc, int posState) - { - if (rc.DecodeBit(ref Choice[0]) == 0) - return rc.DecodeTree(Low.AsSpan(posState << LowBits, LowSymbols), LowBits); - if (rc.DecodeBit(ref Choice[1]) == 0) - return LowSymbols + rc.DecodeTree(Mid.AsSpan(posState << MidBits, MidSymbols), MidBits); - return LowSymbols + MidSymbols + rc.DecodeTree(High, HighBits); - } - public void Encode(RangeEncoder rc, uint length, int posState) { if (length < LowSymbols) @@ -92,7 +83,10 @@ public int Price(uint length, int posState) } } -/// The adaptive model shared by the LZMA encoder and decoder: probabilities, state and the last four distances. +/// +/// The encoder's adaptive model: probabilities, state and the last four distances. The decoder keeps the same model +/// in one array of its own (); the constants here are shared. +/// internal class LzmaModel { public const int States = 12; @@ -169,10 +163,24 @@ protected int LiteralBase(long position, int previous) => } /// -/// Decodes LZMA symbols into a sliding window and the caller's buffer. LZMA2 drives it chunk by chunk; raw LZMA -/// (.lzma, lzip) through . +/// Decodes LZMA symbols into a sliding window and hands what it decoded to the caller's buffer in blocks. LZMA2 drives +/// it chunk by chunk; raw LZMA (.lzma, lzip, 7z, NSIS) through . /// -internal sealed class LzmaDecoder : LzmaModel +/// +/// Speed: +/// - Symbols are decoded into the window up to the end of the caller's buffer or of the window, and copied out in one +/// block. Runs of literals and runs of matches are decoded by two methods that keep the range coder, the input +/// position and the state in locals (registers) for the whole run and hand over when the IsMatch bit changes. +/// - Input: while 64 bytes or more are left (a symbol reads at most 48), bytes are read without checks; the last ones +/// of a chunk or of the stream are read with checks (a second copy of the same code, picked by a type parameter). +/// - Every probability lives in one array (offsets below), reached through refs whose offsets are bounded by +/// construction: state < 12, pos state < 16, tree nodes < 2^bits, literal contexts < 2^(lc + lp). +/// - The bits of literals and of the length, slot and distance trees are close to random, so they are decoded with +/// masks instead of a branch on the bit value; literals read both children's probabilities before the bit is known. +/// The output, the errors and the limits are those of the plain decoder in the LZMA specification. A decoder that +/// threw is not used again. +/// +internal sealed class LzmaDecoder { /// The smallest window LZMA uses, whatever the header says. public const uint MinDictionary = 1 << 12; @@ -182,6 +190,33 @@ internal sealed class LzmaDecoder : LzmaModel private const int FirstWindow = 1 << 16; + // The probabilities, in one array: the single-bit decisions, the slot and distance trees, the two length models + // and, last, the literal coders (0x300 per context, 2^(lc + lp) contexts). + private const int States = LzmaModel.States; + private const int PosStates = LzmaModel.MaxPosStates; + private const int IsMatchAt = 0; + private const int IsRepAt = IsMatchAt + States * PosStates; + private const int IsRepG0At = IsRepAt + States; + private const int IsRepG1At = IsRepG0At + States; + private const int IsRepG2At = IsRepG1At + States; + private const int IsRep0LongAt = IsRepG2At + States; + private const int PosSlotAt = IsRep0LongAt + States * PosStates; + private const int PosSpecialAt = PosSlotAt + (LzmaModel.LengthStates << LzmaModel.PosSlotBits); + private const int AlignAt = PosSpecialAt + 1 + LzmaModel.FullDistances - LzmaModel.EndPosModelIndex; + private const int MatchLengthAt = AlignAt + (1 << LzmaModel.AlignBits); + private const int RepLengthAt = MatchLengthAt + LengthSize; + private const int LiteralAt = RepLengthAt + LengthSize; + private const int LiteralSize = 0x300; + + // One length model: the two choice bits, then the low and mid trees (8 symbols per pos state) and the high tree. + private const int Choice2At = 1; + private const int LowAt = 2; + private const int MidAt = LowAt + (PosStates << LengthModel.LowBits); + private const int HighAt = MidAt + (PosStates << LengthModel.MidBits); + private const int LengthSize = HighAt + (1 << LengthModel.HighBits); + + private const uint Top = RangeCoder.TopValue; + private readonly uint _dictionary; private byte[] _window; private int _pos; @@ -189,6 +224,15 @@ internal sealed class LzmaDecoder : LzmaModel private long _total; private int _pending; // bytes of the current match not yet copied out + private ushort[] _probabilities = []; + private int _runStop; // where the current Run stops + private int _runTotalBase; // pos + this has the low bits of the total + private int _lc; + private int _lpMask; + private int _pbMask; + private int _state; + private uint _rep0, _rep1, _rep2, _rep3; + /// The window size from the header (already checked against the caller's limit). /// The output size when the header names it; the window never grows past it. public LzmaDecoder(uint dictionary, long expectedSize = -1) @@ -209,9 +253,27 @@ public LzmaDecoder(uint dictionary, long expectedSize = -1) /// Bytes decoded since the last dictionary reset. public long Total => _total; - public override void ResetState() + /// Sets lc / lp / pb (the literal coders are resized when lc + lp changes); call after. + public void SetProperties(LzmaProperties properties) + { + if (properties.Lc is < 0 or > 8 || properties.Lp is < 0 or > 4 || properties.Pb is < 0 or > 4) + throw new InvalidDataException("the LZMA properties are out of range"); + _lc = properties.Lc; + _lpMask = (1 << properties.Lp) - 1; + _pbMask = (1 << properties.Pb) - 1; + // LiteralSize more at the end: the literal loops read both children of a node before knowing the bit, and + // after the last bit of a matched literal those (unused) reads may reach past the last coder. + var size = LiteralAt + (LiteralSize << (properties.Lc + properties.Lp)) + LiteralSize; + if (_probabilities.Length != size) + _probabilities = new ushort[size]; + } + + /// Probabilities to one half, state 0, all four distances 1, no match pending. + public void ResetState() { - base.ResetState(); + RangeCoder.Reset(_probabilities); + _state = 0; + _rep0 = _rep1 = _rep2 = _rep3 = 0; _pending = 0; } @@ -227,8 +289,16 @@ public void ResetDictionary() /// Appends stored bytes (an LZMA2 uncompressed chunk) to the window. public void AppendUncompressed(ReadOnlySpan data) { - foreach (var b in data) - Put(b); + while (!data.IsEmpty) + { + var count = Math.Min(data.Length, _window.Length - _pos); + data[..count].CopyTo(_window.AsSpan(_pos)); + _pos += count; + _total += count; + data = data[count..]; + if (_pos == _window.Length) + WindowEnd(); + } } /// @@ -237,139 +307,377 @@ public void AppendUncompressed(ReadOnlySpan data) /// public int Decode(RangeDecoder rc, Span output, bool allowEndMarker) { - // Range and code live in locals here and go back to rc around the multi-bit decoders (which keep their own). - rc.Load(out var range, out var code); + // The literal coders come with the properties; without them the probabilities below would not be there. + if (_probabilities.Length <= LiteralAt) + throw new InvalidOperationException("SetProperties comes before Decode"); var n = 0; + while (n < output.Length) + { + // Decode into the window up to the end of the output or of the window, then copy that block out. + var start = _pos; + var stop = start + Math.Min(output.Length - n, _window.Length - start); + var ended = false; + while (_pos < stop && !ended) + { + // Symbols read at most 48 input bytes: with InputMargin of them at hand none is checked for; the last + // few bytes of a chunk or of the input are read with checks. + ended = rc.TopUp(InputMargin) >= InputMargin + ? Run(rc, stop, allowEndMarker) + : Run(rc, stop, allowEndMarker); + } + var count = _pos - start; + _window.AsSpan(start, count).CopyTo(output[n..]); + n += count; + if (_pos == _window.Length) + WindowEnd(); + if (ended) + { + EndMarkerSeen = true; + break; + } + } + return n; + } + + /// + /// Decodes symbols into the window from until (at most the window's + /// end); a match that doesn't fit stays pending. With it also stops before a symbol + /// when fewer than input bytes are left. Returns true at the end marker. + /// + /// + /// Runs of literals and runs of matches are decoded by two methods that each loop over their kind of symbol and + /// hand over when the IsMatch bit changes. Each keeps the range coder in registers for its whole run: one method + /// for both had more live values than the CPU has registers, and the JIT spilled the coder in the literal loop. + /// + private bool Run(RangeDecoder rc, int stop, bool allowEndMarker) where TInput : struct, IInputMode + { + var start = _pos; + _runStop = stop; + _runTotalBase = (int)_total - start; // pos + this has the low bits of the total, which is all contexts use if (_pending > 0) - n = CopyMatch(output, 0, _pending); + { + var count = Math.Min(_pending, stop - _pos); + _pending -= count; + _pos = Copy(_window, _pos, (int)_rep0 + 1, count); + } + var next = NextSymbol(rc); + while (next != Stopped) + { + next = next == LiteralNext ? LiteralRun(rc) : MatchRun(rc, allowEndMarker); + if (next == EndMarker) + break; + } + _total += _pos - start; + return next == EndMarker; + } - while (n < output.Length) + // What follows the symbols a run decoded. + private const int Stopped = 0; // the end of the block or of the safe input: no IsMatch bit read + private const int LiteralNext = 1; // an IsMatch bit of 0 was read + private const int MatchNext = 2; // an IsMatch bit of 1 was read + private const int EndMarker = 3; + + /// Reads the IsMatch bit of the next symbol, unless Run has to stop before it. + private int NextSymbol(RangeDecoder rc) where TInput : struct, IInputMode + { + var pos = _pos; + rc.Take(out var range, out var code, out var inPos); + var inEnd = rc.End; + if (pos >= _runStop || (!TInput.Checked && inPos > inEnd - InputMargin)) + return Stopped; + ref var input = ref MemoryMarshal.GetArrayDataReference(rc.Buffer); + var posState = (pos + _runTotalBase) & _pbMask; + var bit = Bit(ref _probabilities[IsMatchAt + (_state << 4) + posState], ref range, ref code, ref input, ref inPos, ref inEnd, rc); + rc.Put(range, code, inPos); + return bit == 0 ? LiteralNext : MatchNext; + } + + /// + /// Decodes literals (the first one's IsMatch bit was read) until a match comes () or Run + /// has to stop (). + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private int LiteralRun(RangeDecoder rc) where TInput : struct, IInputMode + { + ref var win = ref MemoryMarshal.GetArrayDataReference(_window); + var pos = _pos; + var stop = _runStop; + var totalBase = _runTotalBase; + ref var probs = ref MemoryMarshal.GetArrayDataReference(_probabilities); + var state = _state; + ref var input = ref MemoryMarshal.GetArrayDataReference(rc.Buffer); + var inEnd = rc.End; + rc.Take(out var range, out var code, out var inPos); + int next; + while (true) { - var posState = (int)_total & PbMask; - var state2 = (State << 4) + posState; + // A literal, coded with the previous byte (and lp bits of the position) as context. + uint previous = pos > 0 ? Unsafe.Add(ref win, pos - 1) : _full ? Unsafe.Add(ref win, _window.Length - 1) : 0u; + var lc = _lc; + ref var literal = ref Unsafe.Add(ref probs, LiteralAt + LiteralSize * ((((pos + totalBase) & _lpMask) << lc) + (int)(previous >> (8 - lc)))); + // Each step reads the probabilities of both children before the bit is known and keeps the right one, + // so the next step doesn't wait for a load that depends on this bit. + uint symbol = 1; + uint p; + if (state < LzmaModel.LiteralStates) + { + p = Unsafe.Add(ref literal, 1); + do + { + uint p0 = Unsafe.Add(ref literal, symbol * 2); + uint p1 = Unsafe.Add(ref literal, symbol * 2 + 1); + var one = BitMask(p, ref Unsafe.Add(ref literal, symbol), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + p = p0 ^ ((p0 ^ p1) & one); + symbol = (symbol << 1) - one; + } + while (symbol < 0x100); + } + else + { + // After a match, while the bits equal those of the byte at the last distance, the probabilities for + // that bit value are used; after the first difference, the plain ones. + uint match = _window[ByteAt(pos, _rep0, _window.Length)]; + var offset = 0x100u; + match <<= 1; + var index = offset + (match & offset) + symbol; + p = Unsafe.Add(ref literal, index); + do + { + // The next index for a 0 (the offset stays where the match bit is 0) and for a 1 (where it is 1). + var after = match << 1; + var offset0 = offset & ~match; + var offset1 = offset & match; + var index0 = offset0 + (after & offset0) + symbol * 2; + var index1 = offset1 + (after & offset1) + symbol * 2 + 1; + uint p0 = Unsafe.Add(ref literal, index0); + uint p1 = Unsafe.Add(ref literal, index1); + var one = BitMask(p, ref Unsafe.Add(ref literal, index), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + p = p0 ^ ((p0 ^ p1) & one); + index = index0 ^ ((index0 ^ index1) & one); + offset = offset0 ^ ((offset0 ^ offset1) & one); + symbol = (symbol << 1) - one; + match = after; + } + while (symbol < 0x100); + } + Unsafe.Add(ref win, pos++) = (byte)symbol; + state = state < 4 ? 0 : state < 10 ? state - 3 : state - 6; - if (RangeDecoder.Bit(ref range, ref code, ref IsMatch[state2], rc) == 0) + if (pos >= stop || (!TInput.Checked && inPos > inEnd - InputMargin)) + { + next = Stopped; + break; + } + var posState = (pos + totalBase) & _pbMask; + if (Bit(ref Unsafe.Add(ref probs, IsMatchAt + (state << 4) + posState), ref range, ref code, ref input, ref inPos, ref inEnd, rc) != 0) { - var previous = IsEmpty ? 0 : ByteAt(0); - var probs = Literal.AsSpan(LiteralBase(_total, previous), 0x300); - // After a match the byte at the last distance steers the first bits (see DecodeMatchedLiteral). - rc.Store(range, code); - var symbol = State >= LiteralStates ? rc.DecodeMatchedLiteral(probs, ByteAt(Rep0)) : rc.DecodeTree(probs, 8); - rc.Load(out range, out code); - Put((byte)symbol); - output[n++] = (byte)symbol; - State = AfterLiteral(State); - continue; + next = MatchNext; + break; } + } + _pos = pos; + _state = state; + rc.Put(range, code, inPos); + return next; + } - uint length; - if (RangeDecoder.Bit(ref range, ref code, ref IsRep[State], rc) != 0) + /// + /// Decodes matches and repeats (the first one's IsMatch bit was read), each copied to the window up to Run's stop + /// (the rest stays pending), until a literal comes (), Run has to stop + /// () or the end marker (). + /// + [MethodImpl(MethodImplOptions.NoInlining)] + private int MatchRun(RangeDecoder rc, bool allowEndMarker) where TInput : struct, IInputMode + { + var window = _window; + var pos = _pos; + var stop = _runStop; + var totalBase = _runTotalBase; + var pbMask = _pbMask; + ref var probs = ref MemoryMarshal.GetArrayDataReference(_probabilities); + var state = _state; + uint rep0 = _rep0, rep1 = _rep1, rep2 = _rep2, rep3 = _rep3; + ref var input = ref MemoryMarshal.GetArrayDataReference(rc.Buffer); + var inEnd = rc.End; + rc.Take(out var range, out var code, out var inPos); + int next; + while (true) + { + var posState = (pos + totalBase) & pbMask; + var shortRep = false; + uint length = 0; + int lengthModel; + if (Bit(ref Unsafe.Add(ref probs, IsRepAt + state), ref range, ref code, ref input, ref inPos, ref inEnd, rc) != 0) { - if (IsEmpty) + if (pos == 0 && !_full) throw new InvalidDataException("the LZMA data repeats bytes before any were written"); - if (RangeDecoder.Bit(ref range, ref code, ref IsRepG0[State], rc) == 0) + lengthModel = RepLengthAt; + if (Bit(ref Unsafe.Add(ref probs, IsRepG0At + state), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) { - if (RangeDecoder.Bit(ref range, ref code, ref IsRep0Long[state2], rc) == 0) - { - // Short repeat: one byte from the last distance. - State = AfterShortRep(State); - var b = ByteAt(Rep0); - Put(b); - output[n++] = b; - continue; - } + // Rep0Long 0: a short repeat, one byte from the last distance. + shortRep = Bit(ref Unsafe.Add(ref probs, IsRep0LongAt + (state << 4) + posState), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0; } else { uint distance; - if (RangeDecoder.Bit(ref range, ref code, ref IsRepG1[State], rc) == 0) + if (Bit(ref Unsafe.Add(ref probs, IsRepG1At + state), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) { - distance = Rep1; + distance = rep1; } else { - if (RangeDecoder.Bit(ref range, ref code, ref IsRepG2[State], rc) == 0) + if (Bit(ref Unsafe.Add(ref probs, IsRepG2At + state), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) { - distance = Rep2; + distance = rep2; } else { - distance = Rep3; - Rep3 = Rep2; + distance = rep3; + rep3 = rep2; } - Rep2 = Rep1; + rep2 = rep1; } - Rep1 = Rep0; - Rep0 = distance; + rep1 = rep0; + rep0 = distance; } - rc.Store(range, code); - length = RepLength.Decode(rc, posState); - rc.Load(out range, out code); - State = AfterRep(State); } else { - Rep3 = Rep2; - Rep2 = Rep1; - Rep1 = Rep0; - rc.Store(range, code); - length = MatchLength.Decode(rc, posState); - State = AfterMatch(State); - Rep0 = DecodeDistance(rc, length); - rc.Load(out range, out code); - if (Rep0 == 0xFFFFFFFF) + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + lengthModel = MatchLengthAt; + } + + if (shortRep) + { + state = state < LzmaModel.LiteralStates ? 9 : 11; + window[pos] = window[ByteAt(pos, rep0, window.Length)]; + pos++; + } + else + { + // The length, from the match or the repeat length model. + ref var model = ref Unsafe.Add(ref probs, lengthModel); + if (Bit(ref model, ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) + { + length = Tree(ref Unsafe.Add(ref model, LowAt + (posState << LengthModel.LowBits)), LengthModel.LowBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); + } + else if (Bit(ref Unsafe.Add(ref model, Choice2At), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) { - if (!allowEndMarker) - throw new InvalidDataException("the LZMA data has an end marker where none is allowed"); - EndMarkerSeen = true; - rc.Store(range, code); - return n; + length = LengthModel.LowSymbols + Tree(ref Unsafe.Add(ref model, MidAt + (posState << LengthModel.MidBits)), LengthModel.MidBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); } + else + { + length = LengthModel.LowSymbols + LengthModel.MidSymbols + Tree(ref Unsafe.Add(ref model, HighAt), LengthModel.HighBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); + } + + if (lengthModel == RepLengthAt) + { + state = state < LzmaModel.LiteralStates ? 8 : 11; + } + else + { + state = state < LzmaModel.LiteralStates ? 7 : 10; + rep0 = DecodeDistance(ref probs, length, ref range, ref code, ref input, ref inPos, ref inEnd, rc); + if (rep0 == 0xFFFFFFFF) + { + if (!allowEndMarker) + throw new InvalidDataException("the LZMA data has an end marker where none is allowed"); + next = EndMarker; + break; + } + } + if (rep0 >= (_full ? (uint)window.Length : (uint)pos)) + throw new InvalidDataException("the LZMA data refers to bytes before the start of the window"); + var matchLength = (int)length + LzmaModel.MatchMinLength; + var fits = Math.Min(matchLength, stop - pos); + _pending = matchLength - fits; + pos = Copy(window, pos, (int)rep0 + 1, fits); } - if (!IsValidDistance(Rep0)) - throw new InvalidDataException("the LZMA data refers to bytes before the start of the window"); - n = CopyMatch(output, n, (int)length + MatchMinLength); - } - rc.Store(range, code); - return n; - } - private uint DecodeDistance(RangeDecoder rc, uint length) - { - var lengthState = (int)Math.Min(length, LengthStates - 1); - var slot = rc.DecodeTree(PosSlot.AsSpan(lengthState << PosSlotBits, 1 << PosSlotBits), PosSlotBits); - if (slot < StartPosModelIndex) - return slot; - var footerBits = (int)(slot >> 1) - 1; - var distance = (2 | (slot & 1)) << footerBits; - if (slot < EndPosModelIndex) - return distance + rc.DecodeReverseTree(PosSpecial.AsSpan((int)(distance - slot)), footerBits); - distance += rc.DecodeDirectBits(footerBits - AlignBits) << AlignBits; - return distance + rc.DecodeReverseTree(Align, AlignBits); + if (pos >= stop || (!TInput.Checked && inPos > inEnd - InputMargin)) + { + next = Stopped; + break; + } + posState = (pos + totalBase) & pbMask; + if (Bit(ref Unsafe.Add(ref probs, IsMatchAt + (state << 4) + posState), ref range, ref code, ref input, ref inPos, ref inEnd, rc) == 0) + { + next = LiteralNext; + break; + } + } + _pos = pos; + _state = state; + _rep0 = rep0; + _rep1 = rep1; + _rep2 = rep2; + _rep3 = rep3; + rc.Put(range, code, inPos); + return next; } - private bool IsEmpty => _pos == 0 && !_full; - - private bool IsValidDistance(uint distance) => - distance < _dictionary && (_full ? distance < (uint)_window.Length : distance < (uint)_pos); - - /// The byte + 1 back. + /// The index of the byte + 1 before . [MethodImpl(MethodImplOptions.AggressiveInlining)] - private byte ByteAt(uint distance) + private static long ByteAt(int pos, uint distance, int windowLength) { - var index = _pos - (long)distance - 1; - if (index < 0) - index += _window.Length; - return _window[index]; + var index = pos - (long)distance - 1; + return index < 0 ? index + windowLength : index; } + /// + /// Copies bytes from back to (which + /// has room for them) and returns the new position. The distance was checked against the window, so the source + /// lies inside it; only the source may wrap around its end. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void Put(byte b) + private static int Copy(byte[] window, int pos, int distance, int count) { - _window[_pos++] = b; - _total++; - if (_pos == _window.Length) - WindowEnd(); + var windowLength = window.Length; + var source = pos - distance; + if (source < 0) + source += windowLength; + ref var win = ref MemoryMarshal.GetArrayDataReference(window); + if (source + count <= windowLength) + { + // A forward copy is what LZMA means. It equals a block move when source and target don't overlap or the + // source lies after the target; 8 bytes at a time are the same while each read lies 8 or more bytes behind + // the write (or after it). + var ahead = source > pos; + if (count >= 32 && (ahead || distance >= count)) + { + window.AsSpan(source, count).CopyTo(window.AsSpan(pos, count)); // a move: the source may overlap after it + } + else if (ahead || distance >= 8) + { + var i = 0; + for (; i + 8 <= count; i += 8) + Unsafe.WriteUnaligned(ref Unsafe.Add(ref win, pos + i), Unsafe.ReadUnaligned(ref Unsafe.Add(ref win, source + i))); + for (; i < count; i++) + Unsafe.Add(ref win, pos + i) = Unsafe.Add(ref win, source + i); + } + else if (distance == 1) + { + Unsafe.InitBlockUnaligned(ref Unsafe.Add(ref win, pos), Unsafe.Add(ref win, source), (uint)count); + } + else + { + for (var i = 0; i < count; i++) + Unsafe.Add(ref win, pos + i) = Unsafe.Add(ref win, source + i); + } + } + else + { + // The source runs over the end of the window (only once it has wrapped): byte by byte. + for (var i = 0; i < count; i++) + { + Unsafe.Add(ref win, pos + i) = Unsafe.Add(ref win, source); + if (++source == windowLength) + source = 0; + } + } + return pos + count; } /// The write position reached the end of the window: grow it while it is below the dictionary, else wrap. @@ -387,43 +695,158 @@ private void WindowEnd() } } + // ───────── Bits ───────── + // The helpers below take the coder's locals by ref and are inlined into the run methods, so the locals stay in + // registers. + + /// How input bytes are read: while enough are at hand, else checked. + private interface IInputMode + { + static abstract bool Checked { get; } + } + + private readonly struct UncheckedInput : IInputMode + { + public static bool Checked => false; + } + + private readonly struct CheckedInput : IInputMode + { + public static bool Checked => true; + } + + /// More than the most input bytes one symbol reads (48: one per bit, and a match has at most 48 bits). + private const int InputMargin = 64; + + /// The next input byte; checked reads refill an empty buffer (or, for a whole chunk, report damage). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint Next(ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) where TInput : struct, IInputMode + { + if (TInput.Checked && inPos >= inEnd) + { + inEnd = rc.Refill(); + inPos = 0; + } + // Unchecked: the symbol started at most End - InputMargin. Checked: inPos < inEnd. Either way inside the buffer. + return Unsafe.Add(ref input, inPos++); + } + + /// One adaptive bit, with a branch on its value: for the decisions that steer the decoder. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint Bit(ref ushort probability, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) + where TInput : struct, IInputMode + { + uint p = probability; + var bound = (range >> RangeCoder.ProbabilityBits) * p; + uint bit; + if (code < bound) + { + range = bound; + probability = (ushort)(p + ((RangeCoder.ProbabilityOne - p) >> RangeCoder.MoveBits)); + bit = 0; + } + else + { + range -= bound; + code -= bound; + probability = (ushort)(p - (p >> RangeCoder.MoveBits)); + bit = 1; + } + if (range < Top) + { + range <<= 8; + code = code << 8 | Next(ref input, ref inPos, ref inEnd, rc); + } + return bit; + } + /// - /// Copies up to bytes from distance Rep0; what doesn't fit stays pending. Runs that reach - /// neither end of the window are copied in one go: as a block move when source and target don't overlap (or the - /// source lies after the target, where a forward copy reads every byte before it is overwritten), byte by byte when - /// the match repeats bytes it is still writing. + /// One adaptive bit without a branch on its value: returns all ones for a 1 and 0 for a 0, so a tree index moves + /// on with (m << 1) − mask. The probability update is the same as in : + /// p − ((p − 2017) >> 5) with an arithmetic shift is p + ((2048 − p) >> 5) for every p in 0 … 2048. /// - private int CopyMatch(Span output, int n, int length) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint BitMask(ref ushort probability, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) + where TInput : struct, IInputMode => + BitMask(probability, ref probability, ref range, ref code, ref input, ref inPos, ref inEnd, rc); + + /// with the probability already read into . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint BitMask(uint p, ref ushort probability, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) + where TInput : struct, IInputMode { - var count = Math.Min(length, output.Length - n); - _pending = length - count; - var distance = (int)Rep0 + 1; // Rep0 is below the window size (IsValidDistance), which fits an int - while (count > 0) + var bound = (range >> RangeCoder.ProbabilityBits) * p; + var one = (uint)(((long)bound - code - 1) >> 63); // code >= bound + range = bound + ((range - 2 * bound) & one); // bound for a 0, range − bound for a 1 + code -= bound & one; + probability = (ushort)(p - (uint)((int)(p - (~one & (RangeCoder.ProbabilityOne - (1 << RangeCoder.MoveBits) + 1))) >> RangeCoder.MoveBits)); + if (range < Top) { - var window = _window; - var source = _pos - distance; - if (source < 0) - source += window.Length; - var chunk = Math.Min(count, Math.Min(window.Length - _pos, window.Length - source)); - if (chunk >= 16 && (source > _pos || distance >= chunk)) - { - window.AsSpan(source, chunk).CopyTo(window.AsSpan(_pos, chunk)); - window.AsSpan(_pos, chunk).CopyTo(output.Slice(n, chunk)); - } - else + range <<= 8; + code = code << 8 | Next(ref input, ref inPos, ref inEnd, rc); + } + return one; + } + + /// A -bit symbol from a binary tree of probabilities (index 1 is the root). + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint Tree(ref ushort probs, int bits, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) + where TInput : struct, IInputMode + { + uint m = 1; + for (var i = 0; i < bits; i++) + m = (m << 1) - BitMask(ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + return m - (1u << bits); + } + + /// Like , least significant bit first. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint ReverseTree(ref ushort probs, int bits, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) + where TInput : struct, IInputMode + { + uint m = 1; + uint symbol = 0; + for (var i = 0; i < bits; i++) + { + var bit = 0u - BitMask(ref Unsafe.Add(ref probs, m), ref range, ref code, ref input, ref inPos, ref inEnd, rc); + m = (m << 1) + bit; + symbol |= bit << i; + } + return symbol; + } + + /// A match distance minus 1 (0xFFFFFFFF is the end marker): slot, then reverse-coded or direct bits. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint DecodeDistance(ref ushort probs, uint length, ref uint range, ref uint code, ref byte input, ref int inPos, ref int inEnd, RangeDecoder rc) + where TInput : struct, IInputMode + { + var lengthState = (int)Math.Min(length, LzmaModel.LengthStates - 1); + var slot = Tree(ref Unsafe.Add(ref probs, PosSlotAt + (lengthState << LzmaModel.PosSlotBits)), LzmaModel.PosSlotBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); + if (slot < LzmaModel.StartPosModelIndex) + return slot; + var footerBits = (int)(slot >> 1) - 1; + var distance = (2 | (slot & 1)) << footerBits; + if (slot < LzmaModel.EndPosModelIndex) + return distance + ReverseTree(ref Unsafe.Add(ref probs, PosSpecialAt + (int)(distance - slot)), footerBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); + // Direct bits of probability one half, then the last four through the align tree. + uint direct = 0; + for (var count = footerBits - LzmaModel.AlignBits; count > 0; count--) + { + range >>= 1; + code -= range; + var t = 0u - (code >> 31); // all ones when the subtraction went below zero (bit 0) + code += range & t; + if (code == range) + throw new InvalidDataException("the LZMA data is damaged"); + if (range < Top) { - var target = output.Slice(n, chunk); - for (var i = 0; i < target.Length; i++) - target[i] = window[_pos + i] = window[source + i]; + range <<= 8; + code = code << 8 | Next(ref input, ref inPos, ref inEnd, rc); } - n += chunk; - count -= chunk; - _pos += chunk; - _total += chunk; - if (_pos == window.Length) - WindowEnd(); + direct = (direct << 1) + (t + 1); } - return n; + distance += direct << LzmaModel.AlignBits; + return distance + ReverseTree(ref Unsafe.Add(ref probs, AlignAt), LzmaModel.AlignBits, ref range, ref code, ref input, ref inPos, ref inEnd, rc); } } diff --git a/src/Filee.Engines/Archives/Lzma/RangeDecoder.cs b/src/Filee.Engines/Archives/Lzma/RangeDecoder.cs index ed07c21c7..86e362cfb 100644 --- a/src/Filee.Engines/Archives/Lzma/RangeDecoder.cs +++ b/src/Filee.Engines/Archives/Lzma/RangeDecoder.cs @@ -1,6 +1,9 @@ // The LZMA range decoder (lzma-specification.txt, "The range decoder"): a 32-bit range and code, adaptive 11-bit // probabilities that move by 1/32 of the distance to the bound after each bit, and "direct" bits of probability 1/2. // Input comes from a byte array: one whole LZMA2 chunk, or a buffer that is refilled from a stream for raw LZMA. +// +// LzmaDecoder doesn't call in here per bit: it takes range, code and the input position into locals for a whole run +// of symbols (Take / Put back) and decodes the bits itself; only a refill comes back here. BCJ2 uses DecodeBit. using System.Runtime.CompilerServices; @@ -26,6 +29,7 @@ internal sealed class RangeDecoder private int _position; private int _end; private Stream? _source; + private bool _sourceEnded; private uint _range; private uint _code; @@ -35,9 +39,16 @@ internal sealed class RangeDecoder /// True when the coder ended cleanly: the encoder's flush leaves the code at zero. public bool IsFinishedOk => _code == 0; + /// The input buffer; bytes before are valid (the buffer itself never changes on refill). + internal byte[] Buffer => _buffer; + + /// The end of the valid input in . + internal int End => _end; + /// Starts decoding bytes of ; reading past them is damage. public void Start(byte[] buffer, int length) { + ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)length, (uint)buffer.Length, nameof(length)); _buffer = buffer; _position = 0; _end = length; @@ -52,6 +63,7 @@ public void Start(Stream source, byte[] buffer) _position = 0; _end = 0; _source = source; + _sourceEnded = false; Init(); } @@ -69,80 +81,75 @@ private void Init() } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal uint NextByte() + private uint NextByte() { if (_position == _end) Refill(); return _buffer[_position++]; } - private void Refill() + /// + /// Reads the next block of input into from its start and returns the new + /// (the position goes back to 0); the input ending, or a whole chunk already used up, is damage. + /// + [MethodImpl(MethodImplOptions.NoInlining)] + internal int Refill() { - if (_source is null || _buffer.Length == 0) + if (_source is null || _sourceEnded || _buffer.Length == 0) throw new InvalidDataException("the LZMA data ends too early"); var read = _source.Read(_buffer, 0, _buffer.Length); if (read <= 0) throw new InvalidDataException("the LZMA data ends too early"); + // LzmaDecoder reads the buffer without bounds checks up to End: never trust a stream for more than was asked. + if (read > _buffer.Length) + throw new InvalidOperationException("a stream returned more bytes than were asked for"); _position = 0; _end = read; + return read; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void Normalize() + /// + /// Makes at least unread bytes available when the input is a stream that still has them: + /// the unread rest moves to the start of the buffer and the stream fills the space after it. Returns the unread + /// bytes now in the buffer (fewer at the end of the stream, or of a whole chunk). + /// + internal int TopUp(int wanted) { - if (_range < RangeCoder.TopValue) + var left = _end - _position; + if (left >= wanted || _source is null || _sourceEnded || _buffer.Length < wanted) + return left; + System.Buffer.BlockCopy(_buffer, _position, _buffer, 0, left); + _position = 0; + _end = left; + while (_end < wanted) { - _range <<= 8; - _code = _code << 8 | NextByte(); + var read = _source.Read(_buffer, _end, _buffer.Length - _end); + if (read <= 0) + { + _sourceEnded = true; + break; + } + if (read > _buffer.Length - _end) + throw new InvalidOperationException("a stream returned more bytes than were asked for"); + _end += read; } + return _end - _position; } - /// - /// Hands range and code to a caller that keeps them in locals over many bits (); it gives them back - /// with before calling any other method here. - /// - public void Load(out uint range, out uint code) + /// Hands range, code and the input position to a decoder that keeps them in locals over many symbols. + internal void Take(out uint range, out uint code, out int position) { range = _range; code = _code; + position = _position; } - /// Takes range and code back from . - public void Store(uint range, uint code) + /// Takes back what handed out. + internal void Put(uint range, uint code, int position) { _range = range; _code = code; - } - - /// - /// on range and code held in the caller's locals, which the JIT keeps in registers: the - /// decoder's single-bit decisions (match or literal, which repeat) run through here. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Bit(ref uint range, ref uint code, ref ushort probability, RangeDecoder rc) - { - var p = probability; - var bound = (range >> RangeCoder.ProbabilityBits) * p; - int bit; - if (code < bound) - { - range = bound; - probability = (ushort)(p + ((RangeCoder.ProbabilityOne - p) >> RangeCoder.MoveBits)); - bit = 0; - } - else - { - range -= bound; - code -= bound; - probability = (ushort)(p - (p >> RangeCoder.MoveBits)); - bit = 1; - } - if (range < RangeCoder.TopValue) - { - range <<= 8; - code = code << 8 | rc.NextByte(); - } - return bit; + _position = position; } /// One bit with the adaptive probability (of a zero). @@ -165,122 +172,11 @@ public int DecodeBit(ref ushort probability) probability = (ushort)(p - (p >> RangeCoder.MoveBits)); bit = 1; } - Normalize(); - return bit; - } - - /// bits of probability one half, most significant first. - public uint DecodeDirectBits(int count) - { - uint result = 0; - for (; count > 0; count--) - { - _range >>= 1; - _code -= _range; - var t = 0u - (_code >> 31); // all ones when the subtraction went below zero (bit 0) - _code += _range & t; - if (_code == _range) - throw new InvalidDataException("the LZMA data is damaged"); - Normalize(); - result = (result << 1) + (t + 1); - } - return result; - } - - /// - /// A -bit symbol from a binary tree of probabilities (index 1 is the root). Range and code - /// stay in locals for the whole symbol (the hot path of literals and lengths), not in the fields for every bit. - /// - public uint DecodeTree(Span probabilities, int bits) - { - var range = _range; - var code = _code; - uint m = 1; - var end = 1u << bits; - while (m < end) - { - ref var probability = ref probabilities[(int)m]; - var p = probability; - var bound = (range >> RangeCoder.ProbabilityBits) * p; - if (code < bound) - { - range = bound; - probability = (ushort)(p + ((RangeCoder.ProbabilityOne - p) >> RangeCoder.MoveBits)); - m <<= 1; - } - else - { - range -= bound; - code -= bound; - probability = (ushort)(p - (p >> RangeCoder.MoveBits)); - m = (m << 1) + 1; - } - if (range < RangeCoder.TopValue) - { - range <<= 8; - code = code << 8 | NextByte(); - } - } - _range = range; - _code = code; - return m - end; - } - - /// - /// A literal after a match (the LZMA spec's "matched literal"): while its bits equal those of - /// , the probabilities for that bit value are used; after the first difference, the - /// plain ones. are the 0x300 of the literal's context. - /// - public uint DecodeMatchedLiteral(Span probabilities, uint matchByte) - { - var range = _range; - var code = _code; - uint symbol = 1; - var offset = 0x100u; - while (symbol < 0x100) - { - matchByte <<= 1; - var index = offset + (matchByte & offset) + symbol; - ref var probability = ref probabilities[(int)index]; - var p = probability; - var bound = (range >> RangeCoder.ProbabilityBits) * p; - if (code < bound) - { - range = bound; - probability = (ushort)(p + ((RangeCoder.ProbabilityOne - p) >> RangeCoder.MoveBits)); - symbol <<= 1; - offset &= ~matchByte; // a 0 where the match byte has a 1: plain probabilities from here on - } - else - { - range -= bound; - code -= bound; - probability = (ushort)(p - (p >> RangeCoder.MoveBits)); - symbol = (symbol << 1) + 1; - offset &= matchByte; - } - if (range < RangeCoder.TopValue) - { - range <<= 8; - code = code << 8 | NextByte(); - } - } - _range = range; - _code = code; - return symbol - 0x100; - } - - /// Like , least significant bit first. - public uint DecodeReverseTree(Span probabilities, int bits) - { - uint m = 1; - uint symbol = 0; - for (var i = 0; i < bits; i++) + if (_range < RangeCoder.TopValue) { - var bit = (uint)DecodeBit(ref probabilities[(int)m]); - m = (m << 1) + bit; - symbol |= bit << i; + _range <<= 8; + _code = _code << 8 | NextByte(); } - return symbol; + return bit; } } diff --git a/tests/Filee.Engines.Tests/LzmaDecoderTests.cs b/tests/Filee.Engines.Tests/LzmaDecoderTests.cs new file mode 100644 index 000000000..8cc40910c --- /dev/null +++ b/tests/Filee.Engines.Tests/LzmaDecoderTests.cs @@ -0,0 +1,213 @@ +// Filee's LZMA decoder (Archives/Lzma/LzmaDecoder.cs) against inputs that reach its edges: every lc / lp / pb, small +// dictionaries whose window wraps many times, matches that overlap themselves or cross the end of the window, reads +// of every size (so matches stay pending across calls), and input streams that hand out a few bytes at a time (so the +// decoder's input margin runs out in the middle of symbols and the buffer is topped up or read with checks). The +// streams come from Filee's own encoder and, where the bundled 7-Zip is present, from 7-Zip. + +using Filee.Engines.Archives; +using Filee.Engines.Archives.Lzma; +using static Filee.Engines.Tests.ArchiveBuilders; + +namespace Filee.Engines.Tests; + +public class LzmaDecoderTests(EngineFixture fx) : IClassFixture +{ + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + /// Text, noise, byte runs, short-period repeats (overlapping matches) and far copies with a few changes. + private static byte[] Structured(int length, int seed) + { + var random = new Random(seed); + var bytes = new byte[length]; + var at = 0; + while (at < length) + { + var run = Math.Min(length - at, random.Next(1, 12_000)); + switch (random.Next(5)) + { + case 0: + random.NextBytes(bytes.AsSpan(at, run)); + break; + case 1: + Words(run).CopyTo(bytes, at); + break; + case 2: + bytes.AsSpan(at, run).Fill((byte)random.Next(256)); + break; + case 3: + var period = random.Next(1, 24); + for (var i = 0; i < run; i++) + bytes[at + i] = i < period ? (byte)random.Next(256) : bytes[at + i - period]; + break; + default: + var from = at == 0 ? 0 : random.Next(at); + for (var i = 0; i < run; i++) + bytes[at + i] = at == 0 ? (byte)i : bytes[from + i % (at - from)]; + for (var k = random.Next(4); k > 0; k--) + bytes[at + random.Next(run)] ^= (byte)random.Next(1, 256); + break; + } + at += run; + } + return bytes; + } + + /// Raw LZMA (no end marker) from Filee's encoder with the given properties and dictionary. + private static byte[] EncodeRaw(byte[] data, LzmaProperties properties, int dictionary, bool optimal) + { + var settings = optimal ? new LzmaEncoderSettings(32, 64, Optimal: true, Tree: true) : new LzmaEncoderSettings(8, 32, Optimal: false, Tree: false); + var finder = new MatchFinder(dictionary, data.Length + 1024, settings.Depth, settings.Nice, settings.Tree); + Assert.Equal(data.Length, finder.Append(data)); + var encoder = new LzmaEncoder(finder, properties, settings); + var rc = new RangeEncoder(data.Length + data.Length / 8 + 1024); + encoder.Encode(rc, data.Length, long.MaxValue); + rc.Flush(); + return rc.Output.ToArray(); + } + + /// Reads to its end in pieces of random size (1 byte … 200 KB). + private static byte[] ReadInPieces(Stream stream, int seed) + { + var random = new Random(seed); + var buffer = new byte[200_000]; + using var output = new MemoryStream(); + while (true) + { + var want = random.Next(4) switch + { + 0 => random.Next(1, 4), + 1 => random.Next(1, 300), + 2 => random.Next(1, 70_000), + _ => buffer.Length, + }; + var n = stream.Read(buffer, 0, want); + if (n == 0) + return output.ToArray(); + output.Write(buffer, 0, n); + } + } + + /// A stream that hands out at most most bytes per read. + private sealed class Trickle(byte[] data, int most) : Stream + { + private int _at; + + public override int Read(byte[] buffer, int offset, int count) + { + var n = Math.Min(Math.Min(count, most), data.Length - _at); + data.AsSpan(_at, n).CopyTo(buffer.AsSpan(offset)); + _at += n; + return n; + } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => _at; set => throw new NotSupportedException(); } + public override void Flush() { } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + } + + [Theory] + [InlineData(3, 0, 2)] + [InlineData(0, 0, 0)] + [InlineData(8, 0, 0)] + [InlineData(0, 4, 4)] + [InlineData(1, 3, 0)] + [InlineData(4, 0, 4)] + [InlineData(8, 4, 4)] + public void Raw_lzma_with_any_properties_decodes_through_any_read_and_input_size(int lc, int lp, int pb) + { + var properties = new LzmaProperties(lc, lp, pb); + var data = Structured(150_000, lc * 25 + lp * 5 + pb); + // 4 KiB: the window wraps some 35 times; 64 KiB: the window grows, then wraps; 1 MiB: it only grows. + foreach (var dictionary in new[] { 1 << 12, 1 << 16, 1 << 20 }) + { + var packed = EncodeRaw(data, properties, dictionary, optimal: dictionary == 1 << 16); + foreach (var most in new[] { int.MaxValue, 7, 65 }) + { + using var reader = new LzmaRawReader(most == int.MaxValue ? new MemoryStream(packed) : new Trickle(packed, most), + properties, (uint)dictionary, data.Length); + var decoded = ReadInPieces(reader, dictionary + most); + Assert.True(data.AsSpan().SequenceEqual(decoded), $"lc={lc} lp={lp} pb={pb} dictionary={dictionary} reads of {most}"); + } + } + } + + [Fact] + public void Lzma2_with_a_small_dictionary_decodes_through_any_read_size() + { + var data = Structured(600_000, 7); + foreach (var settings in new[] + { + new Lzma2Settings(1 << 12, new LzmaEncoderSettings(8, 32, Optimal: false, Tree: false), Store: false), + new Lzma2Settings(1 << 15, new LzmaEncoderSettings(32, 64, Optimal: true, Tree: true), Store: false), + }) + { + using var packed = new MemoryStream(); + var writer = new Lzma2Writer(packed, settings, data.Length, CancellationToken.None); + writer.Write(data); + writer.Finish(); + foreach (var expected in new long[] { -1, data.Length }) + { + var reader = new Lzma2Reader(new ByteInput(new MemoryStream(packed.ToArray())), (uint)settings.Dictionary, expected, Ct); + using var output = new MemoryStream(); + var random = new Random(settings.Dictionary); + var buffer = new byte[100_000]; + int n; + while ((n = reader.Read(buffer.AsSpan(0, random.Next(1, buffer.Length)))) > 0) + output.Write(buffer, 0, n); + Assert.True(data.AsSpan().SequenceEqual(output.ToArray()), $"dictionary {settings.Dictionary}, expected size {expected}"); + } + } + } + + [Fact] + public void A_stream_that_ends_inside_a_symbol_is_damaged_however_it_is_read() + { + var data = Structured(100_000, 3); + var packed = EncodeRaw(data, LzmaProperties.Default, 1 << 16, optimal: false); + foreach (var cut in new[] { 5, 60, 64, 65, packed.Length / 2, packed.Length - 70, packed.Length - 1 }) + { + foreach (var most in new[] { int.MaxValue, 3, 64 }) + { + var part = packed[..cut]; + Assert.Throws(() => + { + using var reader = new LzmaRawReader(most == int.MaxValue ? new MemoryStream(part) : new Trickle(part, most), + LzmaProperties.Default, 1 << 16, data.Length); + ReadInPieces(reader, cut); + }); + } + } + } + + [Fact] + public async Task Seven_zip_lzma_with_other_properties_and_small_dictionaries_decodes() + { + Assert.SkipUnless(SevenZipExe is not null, "7-Zip not found"); + var dir = fx.NewFolder(); + var source = Path.Combine(dir, "in"); + Directory.CreateDirectory(source); + var data = Structured(700_000, 11); + await File.WriteAllBytesAsync(Path.Combine(source, "data.bin"), data, Ct); + string[] methods = + [ + "-m0=LZMA:d=4k:lc=8:lp=0:pb=0", "-m0=LZMA:d=64k:lc=0:lp=4:pb=4", "-m0=LZMA:d=1m:lc=4:lp=2:pb=1:a=0", + "-m0=LZMA2:d=4k:lc=1:lp=3:pb=0", "-m0=LZMA2:d=256k:lc=0:lp=0:pb=4:fb=273", "-m0=LZMA:mf=hc4:d=32k", + ]; + foreach (var method in methods) + { + var archive = Path.Combine(dir, $"{methods.ToList().IndexOf(method)}.7z"); + await Run7zAsync("a", "-t7z", "-mmt=off", method, archive, Path.Combine(source, "data.bin")); + var target = Path.Combine(dir, "out-" + Path.GetFileNameWithoutExtension(archive)); + using (var file = File.OpenRead(archive)) + SevenZipReader.Extract(file, Path.GetFileName(archive), target, Ct); + var decoded = await File.ReadAllBytesAsync(Path.Combine(target, "data.bin"), Ct); + Assert.True(data.AsSpan().SequenceEqual(decoded), method); + } + } +}