perf(archives): faster CRC-32 (System.IO.Hashing) and LHA CRC-16

Directory.Build.props

@@ -47,5 +47,6 @@
     <PublishReadyToRunExclude Include="Microsoft.Extensions.Logging.Abstractions.dll;Microsoft.Extensions.Logging.dll" />
     <PublishReadyToRunExclude Include="Microsoft.Extensions.Options.dll;Microsoft.Extensions.Primitives.dll" />
     <PublishReadyToRunExclude Include="Microsoft.IO.RecyclableMemoryStream.dll;Microsoft.Maui.Graphics.dll;System.Security.Cryptography.Pkcs.dll" />
+    <PublishReadyToRunExclude Include="System.IO.Hashing.dll" />
   </ItemGroup>
 </Project>

Directory.Packages.props

@@ -51,6 +51,8 @@
     <PackageVersion Include="PDFtoImage" Version="5.2.1" />
     <!-- Reads the .NET binary serialization inside Paint.NET's .pdn files as data, never instantiating types. -->
     <PackageVersion Include="System.Formats.Nrbf" Version="10.0.14" />
+    <!-- CRC-32 for the archive readers and writers (Archives/Crc32.cs), with the CPU's CRC instructions where there are any. -->
+    <PackageVersion Include="System.IO.Hashing" Version="10.0.14" />
     <PackageVersion Include="Markdig" Version="1.5.1" />
     <PackageVersion Include="ExcelNumberFormat" Version="1.1.0" />
     <!-- 5.2+ requires SkiaSharp 4 (see PDFtoImage above); 5.1.1 uses SkiaSharp 3.119 and HarfBuzzSharp 8.3 like Avalonia. -->

THIRD-PARTY-NOTICES.md

@@ -42,6 +42,7 @@ Each component remains under its own license.
 | pypandoc-hwpx (Pandoc AST mapping and package layout in `Filee.Engines/Hwp/Hwpx`, incl. the `blank.hwpx` reference document) | MIT, Copyright (c) 2024 pypandoc-hwpx Contributors | https://github.com/msjang/pypandoc-hwpx |
 | Microsoft.Extensions.* | MIT | https://github.com/dotnet/runtime |
 | System.Formats.Nrbf (reads Paint.NET .pdn files) | MIT | https://github.com/dotnet/runtime |
+| System.IO.Hashing (CRC-32 of archive contents) | MIT | https://github.com/dotnet/runtime |
 | cu2qu (fontTools): the cubic-to-quadratic approach followed by `Filee.Engines/Fonts/Cff/CubicToQuadratic.cs` | Apache-2.0, Copyright 2016 Google Inc. | https://github.com/fonttools/fonttools |
 
 libuiohook (the native input hook inside SharpHook) is the fork https://github.com/TolikPylypchuk/libuiohook, licensed

src/Filee.Engines/Archives/Crc32.cs

@@ -1,42 +1,19 @@
 // CRC-32 (IEEE 802.3, as in ZIP, gzip, ALZ, EGG and lzip) for checking what Filee's own archive readers unpack.
+// System.IO.Hashing computes it with the CPU's carry-less multiply or CRC instructions where there are any (x64,
+// ARM64) and with lookup tables elsewhere; this class only keeps the small API the readers and writers use.
 
 namespace Filee.Engines.Archives;
 
 /// <summary>Incremental CRC-32 with the reflected polynomial 0xEDB88320.</summary>
 internal sealed class Crc32
 {
-    private static readonly uint[] Table = CreateTable();
-    private uint _crc = 0xFFFFFFFF;
+    private readonly System.IO.Hashing.Crc32 _crc = new();
 
     /// <summary>The checksum of everything passed to <see cref="Append"/> so far.</summary>
-    public uint Value => ~_crc;
+    public uint Value => _crc.GetCurrentHashAsUInt32();
 
-    public void Append(ReadOnlySpan<byte> data)
-    {
-        var crc = _crc;
-        foreach (var b in data)
-            crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8);
-        _crc = crc;
-    }
+    public void Append(ReadOnlySpan<byte> data) => _crc.Append(data);
 
     /// <summary>CRC-32 of <paramref name="data"/>.</summary>
-    public static uint Compute(ReadOnlySpan<byte> data)
-    {
-        var crc = new Crc32();
-        crc.Append(data);
-        return crc.Value;
-    }
-
-    private static uint[] CreateTable()
-    {
-        var table = new uint[256];
-        for (uint i = 0; i < 256; i++)
-        {
-            var c = i;
-            for (var k = 0; k < 8; k++)
-                c = (c & 1) != 0 ? 0xEDB88320 ^ (c >> 1) : c >> 1;
-            table[i] = c;
-        }
-        return table;
-    }
+    public static uint Compute(ReadOnlySpan<byte> data) => System.IO.Hashing.Crc32.HashToUInt32(data);
 }

src/Filee.Engines/Archives/LhaReader.cs

@@ -290,7 +290,9 @@ private static string SplitName(ReadOnlySpan<byte> bytes)
 /// <summary>Read-through stream that computes the CRC-16 (ARC, polynomial 0xA001) of what it passes on.</summary>
 internal sealed class Crc16Stream(Stream inner) : Stream
 {
-    private static readonly ushort[] Table = CreateTable();
+    // Sixteen tables of 256: entry i of table k is the CRC change of byte i followed by k zero bytes. Sixteen bytes
+    // then take sixteen lookups that do not wait for each other, instead of a chain of sixteen ("slicing by 16").
+    private static readonly ushort[] Tables = CreateTables();
     private ushort _crc;
 
     /// <summary>The checksum of everything read so far.</summary>
@@ -312,13 +314,34 @@ public override long Position
     public override int Read(Span<byte> buffer)
     {
         var read = inner.Read(buffer);
-        var crc = _crc;
-        foreach (var b in buffer[..read])
-            crc = (ushort)(Table[(crc ^ b) & 0xFF] ^ (crc >> 8));
-        _crc = crc;
+        _crc = Update(_crc, buffer[..read]);
         return read;
     }
 
+    /// <summary>The CRC-16 <paramref name="crc"/> continued over <paramref name="data"/>.</summary>
+    internal static ushort Update(ushort crc, ReadOnlySpan<byte> data)
+    {
+        var t = Tables;
+        uint c = crc;
+        while (data.Length >= 16)
+        {
+            var x = BinaryPrimitives.ReadUInt64LittleEndian(data) ^ c;
+            var y = BinaryPrimitives.ReadUInt64LittleEndian(data[8..]);
+            c = (uint)(t[0xF00 + (int)(x & 0xFF)] ^ t[0xE00 + (int)((x >> 8) & 0xFF)]
+                ^ t[0xD00 + (int)((x >> 16) & 0xFF)] ^ t[0xC00 + (int)((x >> 24) & 0xFF)]
+                ^ t[0xB00 + (int)((x >> 32) & 0xFF)] ^ t[0xA00 + (int)((x >> 40) & 0xFF)]
+                ^ t[0x900 + (int)((x >> 48) & 0xFF)] ^ t[0x800 + (int)(x >> 56)]
+                ^ t[0x700 + (int)(y & 0xFF)] ^ t[0x600 + (int)((y >> 8) & 0xFF)]
+                ^ t[0x500 + (int)((y >> 16) & 0xFF)] ^ t[0x400 + (int)((y >> 24) & 0xFF)]
+                ^ t[0x300 + (int)((y >> 32) & 0xFF)] ^ t[0x200 + (int)((y >> 40) & 0xFF)]
+                ^ t[0x100 + (int)((y >> 48) & 0xFF)] ^ t[(int)(y >> 56)]);
+            data = data[16..];
+        }
+        foreach (var b in data)
+            c = t[(int)((c ^ b) & 0xFF)] ^ (c >> 8);
+        return (ushort)c;
+    }
+
     public override void Flush() { }
     public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
     public override void SetLength(long value) => throw new NotSupportedException();
@@ -331,16 +354,21 @@ protected override void Dispose(bool disposing)
         base.Dispose(disposing);
     }
 
-    private static ushort[] CreateTable()
+    private static ushort[] CreateTables()
     {
-        var table = new ushort[256];
+        var table = new ushort[16 * 256];
         for (var i = 0; i < 256; i++)
         {
             var c = (uint)i;
             for (var k = 0; k < 8; k++)
                 c = (c & 1) != 0 ? 0xA001 ^ (c >> 1) : c >> 1;
             table[i] = (ushort)c;
         }
+        for (var i = 256; i < table.Length; i++)
+        {
+            var previous = table[i - 256];
+            table[i] = (ushort)(table[previous & 0xFF] ^ (previous >> 8));
+        }
         return table;
     }
 }

src/Filee.Engines/Filee.Engines.csproj

@@ -22,6 +22,7 @@
     <PackageReference Include="System.Security.Cryptography.Pkcs" />
     <PackageReference Include="PDFtoImage" />
     <PackageReference Include="System.Formats.Nrbf" />
+    <PackageReference Include="System.IO.Hashing" />
     <PackageReference Include="Markdig" />
     <PackageReference Include="ExcelNumberFormat" />
     <PackageReference Include="Svg.Skia" />

tests/Filee.Engines.Tests/ChecksumTests.cs

@@ -0,0 +1,160 @@
+// The archive checksums: CRC-32 (Archives/Crc32, System.IO.Hashing underneath) and LHA's CRC-16 (Crc16Stream) give
+// the same values as the one-table-lookup-per-byte loops they replaced (kept here as the reference), for random
+// lengths up to 1 MB, starts at any alignment and data fed in pieces of random size. The check values are those of
+// the CRC catalogues ("123456789").
+
+using System.Text;
+using Filee.Engines.Archives;
+using static Filee.Engines.Tests.ArchiveBuilders;
+
+namespace Filee.Engines.Tests;
+
+public class ChecksumTests
+{
+    private const int Rounds = 80;
+    private const int MaxLength = 1 << 20;
+
+    private static readonly uint[] ReferenceCrc32Table = ReflectedTable(0xEDB88320);
+    private static readonly uint[] ReferenceCrc16Table = ReflectedTable(0xA001);
+
+    [Fact]
+    public void Check_values_match_the_catalogues()
+    {
+        var check = Encoding.ASCII.GetBytes("123456789");
+        Assert.Equal(0xCBF43926u, Crc32.Compute(check));
+        Assert.Equal(0xBB3D, Crc16(check));
+        Assert.Equal(0u, Crc32.Compute([]));
+        Assert.Equal(0, Crc16([]));
+    }
+
+    [Fact]
+    public void Crc32_matches_the_byte_table_whole_and_in_pieces()
+    {
+        var random = new Random(20261115);
+        var pool = Noise(MaxLength + 64, 1);
+        foreach (var (data, pieces) in Cases(random, pool))
+        {
+            var expected = ~Reflected(ReferenceCrc32Table, 0xFFFFFFFF, data.Span);
+            Assert.Equal(expected, Crc32.Compute(data.Span));
+            var crc = new Crc32();
+            var reference = 0xFFFFFFFFu;
+            foreach (var piece in pieces)
+            {
+                var part = data.Span.Slice(piece.Start, piece.Length);
+                crc.Append(part);
+                reference = Reflected(ReferenceCrc32Table, reference, part);
+                Assert.Equal(~reference, crc.Value);
+            }
+            Assert.Equal(expected, crc.Value);
+        }
+    }
+
+    [Fact]
+    public void Crc16_matches_the_byte_table_whole_and_through_reads_of_any_size()
+    {
+        var random = new Random(20261116);
+        var pool = Noise(MaxLength + 64, 2);
+        foreach (var (data, pieces) in Cases(random, pool))
+        {
+            var expected = (ushort)Reflected(ReferenceCrc16Table, 0, data.Span);
+            Assert.Equal(expected, Crc16(data.Span));
+
+            ushort chained = 0;
+            foreach (var piece in pieces)
+                chained = Crc16Stream.Update(chained, data.Span.Slice(piece.Start, piece.Length));
+            Assert.Equal(expected, chained);
+
+            // As LhaReader uses it: reads at unaligned buffer offsets from a stream that returns short reads.
+            using var stream = new Crc16Stream(new ChunkedStream(data.ToArray(), random));
+            var buffer = new byte[64 * 1024 + 3];
+            int read;
+            do
+            {
+                var offset = random.Next(4);
+                read = stream.Read(buffer, offset, random.Next(1, buffer.Length - offset + 1));
+            }
+            while (read > 0);
+            Assert.Equal(expected, stream.Value);
+        }
+    }
+
+    /// <summary>
+    /// Data of random length (the first 40 are 0 to 39 bytes, around the 16-byte step) at a random offset into
+    /// <paramref name="pool"/>, with a random split into consecutive pieces.
+    /// </summary>
+    private static IEnumerable<(ReadOnlyMemory<byte> Data, List<(int Start, int Length)> Pieces)> Cases(Random random, byte[] pool)
+    {
+        for (var round = 0; round < Rounds; round++)
+        {
+            var length = round < 40 ? round : random.Next(MaxLength + 1);
+            var start = random.Next(64);
+            var pieces = new List<(int Start, int Length)>();
+            for (var at = 0; at < length;)
+            {
+                var piece = Math.Min(length - at, random.Next(4) switch
+                {
+                    0 => random.Next(1, 17),
+                    1 => random.Next(1, 200),
+                    2 => random.Next(1, 70_000),
+                    _ => random.Next(1, length + 1),
+                });
+                pieces.Add((at, piece));
+                at += piece;
+            }
+            yield return (pool.AsMemory(start, length), pieces);
+        }
+    }
+
+    private static ushort Crc16(ReadOnlySpan<byte> data) => Crc16Stream.Update(0, data);
+
+    private static uint Reflected(uint[] table, uint crc, ReadOnlySpan<byte> data)
+    {
+        foreach (var b in data)
+            crc = table[(crc ^ b) & 0xFF] ^ (crc >> 8);
+        return crc;
+    }
+
+    private static uint[] ReflectedTable(uint polynomial)
+    {
+        var table = new uint[256];
+        for (uint i = 0; i < 256; i++)
+        {
+            var c = i;
+            for (var k = 0; k < 8; k++)
+                c = (c & 1) != 0 ? polynomial ^ (c >> 1) : c >> 1;
+            table[i] = c;
+        }
+        return table;
+    }
+
+    /// <summary>A read-only stream that hands out fewer bytes than asked, a random number each time.</summary>
+    private sealed class ChunkedStream(byte[] data, Random random) : Stream
+    {
+        private int _position;
+
+        public override bool CanRead => true;
+        public override bool CanSeek => false;
+        public override bool CanWrite => false;
+        public override long Length => data.Length;
+
+        public override long Position
+        {
+            get => _position;
+            set => throw new NotSupportedException();
+        }
+
+        public override int Read(Span<byte> buffer)
+        {
+            var count = Math.Min(Math.Min(buffer.Length, random.Next(1, 70_000)), data.Length - _position);
+            data.AsSpan(_position, count).CopyTo(buffer);
+            _position += count;
+            return count;
+        }
+
+        public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
+        public override void Flush() { }
+        public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+        public override void SetLength(long value) => throw new NotSupportedException();
+        public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+    }
+}