diff --git a/Directory.Build.props b/Directory.Build.props
index 3e9cceb6c..80591a5dd 100644
--- a/Directory.Build.props
+++ b/Directory.Build.props
@@ -47,5 +47,6 @@
+
diff --git a/Directory.Packages.props b/Directory.Packages.props
index e394d744c..5d9dd0783 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -51,6 +51,8 @@
+
+
diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md
index 205da3dd3..63cfbb215 100644
--- a/THIRD-PARTY-NOTICES.md
+++ b/THIRD-PARTY-NOTICES.md
@@ -42,6 +42,7 @@ Each component remains under its own license.
| pypandoc-hwpx (Pandoc AST mapping and package layout in `Filee.Engines/Hwp/Hwpx`, incl. the `blank.hwpx` reference document) | MIT, Copyright (c) 2024 pypandoc-hwpx Contributors | https://github.com/msjang/pypandoc-hwpx |
| Microsoft.Extensions.* | MIT | https://github.com/dotnet/runtime |
| System.Formats.Nrbf (reads Paint.NET .pdn files) | MIT | https://github.com/dotnet/runtime |
+| System.IO.Hashing (CRC-32 of archive contents) | MIT | https://github.com/dotnet/runtime |
| cu2qu (fontTools): the cubic-to-quadratic approach followed by `Filee.Engines/Fonts/Cff/CubicToQuadratic.cs` | Apache-2.0, Copyright 2016 Google Inc. | https://github.com/fonttools/fonttools |
libuiohook (the native input hook inside SharpHook) is the fork https://github.com/TolikPylypchuk/libuiohook, licensed
diff --git a/src/Filee.Engines/Archives/Crc32.cs b/src/Filee.Engines/Archives/Crc32.cs
index 5590a3902..32b4c52db 100644
--- a/src/Filee.Engines/Archives/Crc32.cs
+++ b/src/Filee.Engines/Archives/Crc32.cs
@@ -1,42 +1,19 @@
// CRC-32 (IEEE 802.3, as in ZIP, gzip, ALZ, EGG and lzip) for checking what Filee's own archive readers unpack.
+// System.IO.Hashing computes it with the CPU's carry-less multiply or CRC instructions where there are any (x64,
+// ARM64) and with lookup tables elsewhere; this class only keeps the small API the readers and writers use.
namespace Filee.Engines.Archives;
/// Incremental CRC-32 with the reflected polynomial 0xEDB88320.
internal sealed class Crc32
{
- private static readonly uint[] Table = CreateTable();
- private uint _crc = 0xFFFFFFFF;
+ private readonly System.IO.Hashing.Crc32 _crc = new();
/// The checksum of everything passed to so far.
- public uint Value => ~_crc;
+ public uint Value => _crc.GetCurrentHashAsUInt32();
- public void Append(ReadOnlySpan data)
- {
- var crc = _crc;
- foreach (var b in data)
- crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8);
- _crc = crc;
- }
+ public void Append(ReadOnlySpan data) => _crc.Append(data);
/// CRC-32 of .
- public static uint Compute(ReadOnlySpan data)
- {
- var crc = new Crc32();
- crc.Append(data);
- return crc.Value;
- }
-
- private static uint[] CreateTable()
- {
- var table = new uint[256];
- for (uint i = 0; i < 256; i++)
- {
- var c = i;
- for (var k = 0; k < 8; k++)
- c = (c & 1) != 0 ? 0xEDB88320 ^ (c >> 1) : c >> 1;
- table[i] = c;
- }
- return table;
- }
+ public static uint Compute(ReadOnlySpan data) => System.IO.Hashing.Crc32.HashToUInt32(data);
}
diff --git a/src/Filee.Engines/Archives/LhaReader.cs b/src/Filee.Engines/Archives/LhaReader.cs
index 5e22c6a96..f11c01aa7 100644
--- a/src/Filee.Engines/Archives/LhaReader.cs
+++ b/src/Filee.Engines/Archives/LhaReader.cs
@@ -290,7 +290,9 @@ private static string SplitName(ReadOnlySpan bytes)
/// Read-through stream that computes the CRC-16 (ARC, polynomial 0xA001) of what it passes on.
internal sealed class Crc16Stream(Stream inner) : Stream
{
- private static readonly ushort[] Table = CreateTable();
+ // Sixteen tables of 256: entry i of table k is the CRC change of byte i followed by k zero bytes. Sixteen bytes
+ // then take sixteen lookups that do not wait for each other, instead of a chain of sixteen ("slicing by 16").
+ private static readonly ushort[] Tables = CreateTables();
private ushort _crc;
/// The checksum of everything read so far.
@@ -312,13 +314,34 @@ public override long Position
public override int Read(Span buffer)
{
var read = inner.Read(buffer);
- var crc = _crc;
- foreach (var b in buffer[..read])
- crc = (ushort)(Table[(crc ^ b) & 0xFF] ^ (crc >> 8));
- _crc = crc;
+ _crc = Update(_crc, buffer[..read]);
return read;
}
+ /// The CRC-16 continued over .
+ internal static ushort Update(ushort crc, ReadOnlySpan data)
+ {
+ var t = Tables;
+ uint c = crc;
+ while (data.Length >= 16)
+ {
+ var x = BinaryPrimitives.ReadUInt64LittleEndian(data) ^ c;
+ var y = BinaryPrimitives.ReadUInt64LittleEndian(data[8..]);
+ c = (uint)(t[0xF00 + (int)(x & 0xFF)] ^ t[0xE00 + (int)((x >> 8) & 0xFF)]
+ ^ t[0xD00 + (int)((x >> 16) & 0xFF)] ^ t[0xC00 + (int)((x >> 24) & 0xFF)]
+ ^ t[0xB00 + (int)((x >> 32) & 0xFF)] ^ t[0xA00 + (int)((x >> 40) & 0xFF)]
+ ^ t[0x900 + (int)((x >> 48) & 0xFF)] ^ t[0x800 + (int)(x >> 56)]
+ ^ t[0x700 + (int)(y & 0xFF)] ^ t[0x600 + (int)((y >> 8) & 0xFF)]
+ ^ t[0x500 + (int)((y >> 16) & 0xFF)] ^ t[0x400 + (int)((y >> 24) & 0xFF)]
+ ^ t[0x300 + (int)((y >> 32) & 0xFF)] ^ t[0x200 + (int)((y >> 40) & 0xFF)]
+ ^ t[0x100 + (int)((y >> 48) & 0xFF)] ^ t[(int)(y >> 56)]);
+ data = data[16..];
+ }
+ foreach (var b in data)
+ c = t[(int)((c ^ b) & 0xFF)] ^ (c >> 8);
+ return (ushort)c;
+ }
+
public override void Flush() { }
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
@@ -331,9 +354,9 @@ protected override void Dispose(bool disposing)
base.Dispose(disposing);
}
- private static ushort[] CreateTable()
+ private static ushort[] CreateTables()
{
- var table = new ushort[256];
+ var table = new ushort[16 * 256];
for (var i = 0; i < 256; i++)
{
var c = (uint)i;
@@ -341,6 +364,11 @@ private static ushort[] CreateTable()
c = (c & 1) != 0 ? 0xA001 ^ (c >> 1) : c >> 1;
table[i] = (ushort)c;
}
+ for (var i = 256; i < table.Length; i++)
+ {
+ var previous = table[i - 256];
+ table[i] = (ushort)(table[previous & 0xFF] ^ (previous >> 8));
+ }
return table;
}
}
diff --git a/src/Filee.Engines/Filee.Engines.csproj b/src/Filee.Engines/Filee.Engines.csproj
index 2b2223089..c5e6a915d 100644
--- a/src/Filee.Engines/Filee.Engines.csproj
+++ b/src/Filee.Engines/Filee.Engines.csproj
@@ -22,6 +22,7 @@
+
diff --git a/tests/Filee.Engines.Tests/ChecksumTests.cs b/tests/Filee.Engines.Tests/ChecksumTests.cs
new file mode 100644
index 000000000..badb09292
--- /dev/null
+++ b/tests/Filee.Engines.Tests/ChecksumTests.cs
@@ -0,0 +1,160 @@
+// The archive checksums: CRC-32 (Archives/Crc32, System.IO.Hashing underneath) and LHA's CRC-16 (Crc16Stream) give
+// the same values as the one-table-lookup-per-byte loops they replaced (kept here as the reference), for random
+// lengths up to 1 MB, starts at any alignment and data fed in pieces of random size. The check values are those of
+// the CRC catalogues ("123456789").
+
+using System.Text;
+using Filee.Engines.Archives;
+using static Filee.Engines.Tests.ArchiveBuilders;
+
+namespace Filee.Engines.Tests;
+
+public class ChecksumTests
+{
+ private const int Rounds = 80;
+ private const int MaxLength = 1 << 20;
+
+ private static readonly uint[] ReferenceCrc32Table = ReflectedTable(0xEDB88320);
+ private static readonly uint[] ReferenceCrc16Table = ReflectedTable(0xA001);
+
+ [Fact]
+ public void Check_values_match_the_catalogues()
+ {
+ var check = Encoding.ASCII.GetBytes("123456789");
+ Assert.Equal(0xCBF43926u, Crc32.Compute(check));
+ Assert.Equal(0xBB3D, Crc16(check));
+ Assert.Equal(0u, Crc32.Compute([]));
+ Assert.Equal(0, Crc16([]));
+ }
+
+ [Fact]
+ public void Crc32_matches_the_byte_table_whole_and_in_pieces()
+ {
+ var random = new Random(20261115);
+ var pool = Noise(MaxLength + 64, 1);
+ foreach (var (data, pieces) in Cases(random, pool))
+ {
+ var expected = ~Reflected(ReferenceCrc32Table, 0xFFFFFFFF, data.Span);
+ Assert.Equal(expected, Crc32.Compute(data.Span));
+ var crc = new Crc32();
+ var reference = 0xFFFFFFFFu;
+ foreach (var piece in pieces)
+ {
+ var part = data.Span.Slice(piece.Start, piece.Length);
+ crc.Append(part);
+ reference = Reflected(ReferenceCrc32Table, reference, part);
+ Assert.Equal(~reference, crc.Value);
+ }
+ Assert.Equal(expected, crc.Value);
+ }
+ }
+
+ [Fact]
+ public void Crc16_matches_the_byte_table_whole_and_through_reads_of_any_size()
+ {
+ var random = new Random(20261116);
+ var pool = Noise(MaxLength + 64, 2);
+ foreach (var (data, pieces) in Cases(random, pool))
+ {
+ var expected = (ushort)Reflected(ReferenceCrc16Table, 0, data.Span);
+ Assert.Equal(expected, Crc16(data.Span));
+
+ ushort chained = 0;
+ foreach (var piece in pieces)
+ chained = Crc16Stream.Update(chained, data.Span.Slice(piece.Start, piece.Length));
+ Assert.Equal(expected, chained);
+
+ // As LhaReader uses it: reads at unaligned buffer offsets from a stream that returns short reads.
+ using var stream = new Crc16Stream(new ChunkedStream(data.ToArray(), random));
+ var buffer = new byte[64 * 1024 + 3];
+ int read;
+ do
+ {
+ var offset = random.Next(4);
+ read = stream.Read(buffer, offset, random.Next(1, buffer.Length - offset + 1));
+ }
+ while (read > 0);
+ Assert.Equal(expected, stream.Value);
+ }
+ }
+
+ ///
+ /// Data of random length (the first 40 are 0 to 39 bytes, around the 16-byte step) at a random offset into
+ /// , with a random split into consecutive pieces.
+ ///
+ private static IEnumerable<(ReadOnlyMemory Data, List<(int Start, int Length)> Pieces)> Cases(Random random, byte[] pool)
+ {
+ for (var round = 0; round < Rounds; round++)
+ {
+ var length = round < 40 ? round : random.Next(MaxLength + 1);
+ var start = random.Next(64);
+ var pieces = new List<(int Start, int Length)>();
+ for (var at = 0; at < length;)
+ {
+ var piece = Math.Min(length - at, random.Next(4) switch
+ {
+ 0 => random.Next(1, 17),
+ 1 => random.Next(1, 200),
+ 2 => random.Next(1, 70_000),
+ _ => random.Next(1, length + 1),
+ });
+ pieces.Add((at, piece));
+ at += piece;
+ }
+ yield return (pool.AsMemory(start, length), pieces);
+ }
+ }
+
+ private static ushort Crc16(ReadOnlySpan data) => Crc16Stream.Update(0, data);
+
+ private static uint Reflected(uint[] table, uint crc, ReadOnlySpan data)
+ {
+ foreach (var b in data)
+ crc = table[(crc ^ b) & 0xFF] ^ (crc >> 8);
+ return crc;
+ }
+
+ private static uint[] ReflectedTable(uint polynomial)
+ {
+ var table = new uint[256];
+ for (uint i = 0; i < 256; i++)
+ {
+ var c = i;
+ for (var k = 0; k < 8; k++)
+ c = (c & 1) != 0 ? polynomial ^ (c >> 1) : c >> 1;
+ table[i] = c;
+ }
+ return table;
+ }
+
+ /// A read-only stream that hands out fewer bytes than asked, a random number each time.
+ private sealed class ChunkedStream(byte[] data, Random random) : Stream
+ {
+ private int _position;
+
+ public override bool CanRead => true;
+ public override bool CanSeek => false;
+ public override bool CanWrite => false;
+ public override long Length => data.Length;
+
+ public override long Position
+ {
+ get => _position;
+ set => throw new NotSupportedException();
+ }
+
+ public override int Read(Span buffer)
+ {
+ var count = Math.Min(Math.Min(buffer.Length, random.Next(1, 70_000)), data.Length - _position);
+ data.AsSpan(_position, count).CopyTo(buffer);
+ _position += count;
+ return count;
+ }
+
+ public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
+ public override void Flush() { }
+ public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+ public override void SetLength(long value) => throw new NotSupportedException();
+ public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+ }
+}