From ac9723c32a021e57efb0480954683cfbb20f2c69 Mon Sep 17 00:00:00 2001 From: Gin <> Date: Wed, 21 Oct 2026 20:28:15 -0700 Subject: [PATCH 1/2] Fix PackedBinaryMatrix::submatrix() at tile edges If the submatrix width is a multiple of 64, or its height a multiple of the tile height, submatrix() called clear_padding() with 0 bits and cleared the whole last column or row of tiles. The tile height depends on the CPU (4 to 64), so results were different on each machine. SparseBinaryMatrix::submatrix() got this fix in ecdc7fccb. This makes the packed version the same. Co-Authored-By: Claude Opus 5.5 --- .../Kernels_PackedBinaryMatrixCore.tpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/SerialPrograms/Source/Kernels/BinaryMatrix/Kernels_PackedBinaryMatrixCore.tpp b/SerialPrograms/Source/Kernels/BinaryMatrix/Kernels_PackedBinaryMatrixCore.tpp index acef198332..45c593eabf 100644 --- a/SerialPrograms/Source/Kernels/BinaryMatrix/Kernels_PackedBinaryMatrixCore.tpp +++ b/SerialPrograms/Source/Kernels/BinaryMatrix/Kernels_PackedBinaryMatrixCore.tpp @@ -300,13 +300,20 @@ PackedBinaryMatrixCore PackedBinaryMatrixCore::submatrix( } #if 1 + // Clear the bits past the right and bottom edges, which came from outside the + // box. Skip an edge that ends at a tile boundary: `clear_padding()` with 0 bits + // would clear the whole last column or row of tiles. size_t wbits = width % TILE_WIDTH; - for (size_t r = 0; r < tile_height; r++){ - ret.tile(tile_width - 1, r).clear_padding(wbits, TILE_HEIGHT); + if (wbits != 0){ + for (size_t r = 0; r < tile_height; r++){ + ret.tile(tile_width - 1, r).clear_padding(wbits, TILE_HEIGHT); + } } size_t hbits = height % TILE_HEIGHT; - for (size_t c = 0; c < tile_width; c++){ - ret.tile(c, tile_height - 1).clear_padding(TILE_WIDTH, hbits); + if (hbits != 0){ + for (size_t c = 0; c < tile_width; c++){ + ret.tile(c, tile_height - 1).clear_padding(TILE_WIDTH, hbits); + } } #endif From 3f9479c0becfe251323d8085649b3a1641a98f2d Mon Sep 17 00:00:00 2001 From: Gin <> Date: Wed, 21 Oct 2026 20:28:15 -0700 Subject: [PATCH 2/2] Test every binary matrix type on real images The old BinaryMatrix test was debug code: it was not registered, ran timing loops, and only checked the matrix type the CPU dispatch picks. Now each matrix type that the build has and the CPU supports gets a test on 4 CommandLineTests images. It checks compress_rgb32_to_binary_range(), the bitwise operations and submatrix() against the image pixels, and that the padding bits stay 0. The tile checks now also run on the Default tile. Co-Authored-By: Claude Opus 5.5 --- .../Kernels_BinaryMatrix_Tests.cpp | 806 ++++++++++-------- 1 file changed, 472 insertions(+), 334 deletions(-) diff --git a/SerialPrograms/Source/Kernels/BinaryMatrix/Kernels_BinaryMatrix_Tests.cpp b/SerialPrograms/Source/Kernels/BinaryMatrix/Kernels_BinaryMatrix_Tests.cpp index 9729ba729e..72c2ea836b 100644 --- a/SerialPrograms/Source/Kernels/BinaryMatrix/Kernels_BinaryMatrix_Tests.cpp +++ b/SerialPrograms/Source/Kernels/BinaryMatrix/Kernels_BinaryMatrix_Tests.cpp @@ -4,424 +4,562 @@ * */ -#include "Common/Cpp/Color.h" -#include "Common/Cpp/Time.h" +#include +#include +#include +#include +#include +#include "Common/Cpp/CancellableScope.h" +#include "Common/Cpp/CpuId/CpuId.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" -#include "CommonFramework/ImageTypes/ImageViewRGB32.h" -#include "Kernels/BinaryMatrix/Kernels_BinaryMatrix.h" +#include "Kernels/BinaryImageFilters/Kernels_BinaryImage_BasicFilters.h" +#include "Kernels_BinaryMatrixTile_64x4_Default.h" #ifdef PA_AutoDispatch_arm64_20_M1 -#include "Kernels/BinaryMatrix/Kernels_BinaryMatrixTile_64x8_arm64_NEON.h" #include "Kernels/PartialWordAccess/Kernels_PartialWordAccess_arm64_NEON.h" -#include "Kernels/Waterfill/Kernels_Waterfill_Core_64x8_arm64_NEON.h" +#include "Kernels_BinaryMatrixTile_64x8_arm64_NEON.h" #endif -//#include "Kernels/BinaryMatrix/Kernels_BinaryMatrix_Arch_64xH_Default.h" -#include "Kernels/BinaryImageFilters/Kernels_BinaryImage_BasicFilters.h" #include "Tests/GuardPageImage.h" #include "Kernels_BinaryMatrix_Tests.h" -#include -#include -using std::cout; -using std::cerr; -using std::endl; -using std::flush; - namespace PokemonAutomation{ namespace Kernels{ -// Additional tests on binary matrix tile implementation -template int test_binary_matrix_tile_t(){ - size_t num_iters = 100000; - size_t sum = 0; - WallClock time_start, time_end; - size_t ns = 0; - double ms = 0; +namespace{ + +// Tile tests: check one binary matrix tile class against `get_bit()`. Only the tiles +// that compile with this file's default compiler flags are tested here: Default, and NEON +// on arm64. The x86 tiles need AVX2 or AVX512 flags. They are tested through the matrix +// tests below and the Waterfill tests. + +template +Tile random_tile(std::mt19937& generator){ Tile tile; - // BinaryTile_64x8_arm64_NEON tile; - // BinaryTile_64xH_Default<8, BinaryMatrixType::arm64x8_x64_NEON> tile; - for (size_t height = 0; height < tile.HEIGHT; ++height){ - for (size_t width = 0; width < tile.WIDTH; ++width){ - tile.set_zero(); - tile.set_ones(width, height); - for (size_t y = 0; y < tile.HEIGHT; ++y){ - for (size_t x = 0; x < tile.WIDTH; ++x){ - bool gt = y < height && x < width; - if (gt != tile.get_bit(x, y)){ - cout << "Tile::set_ones(width = " << width << ", height = " << height << "), tile (x = " << - x << ", y = " << y << "), wrong bit. Should be " << gt << endl; - return 1; - } - } - } + for (size_t y = 0; y < Tile::HEIGHT; y++){ + for (size_t x = 0; x < Tile::WIDTH; x++){ + tile.set_bit(x, y, generator() & 1); } } + return tile; +} - sum = 0; - time_start = current_time(); - for (size_t i = 0; i < num_iters; ++i){ - tile.set_ones(32, 4); - sum += tile.row(0) + tile.row(1) + tile.row(2) + tile.row(3); +// Compare each bit of `tile` with `expected_bit(x, y)`. Return a description of the first +// wrong bit, or an empty string if all bits are right. +template +std::string find_wrong_tile_bit(const Tile& tile, ExpectedBit expected_bit){ + for (size_t y = 0; y < Tile::HEIGHT; y++){ + for (size_t x = 0; x < Tile::WIDTH; x++){ + const bool expected = expected_bit(x, y); + if (tile.get_bit(x, y) != expected){ + return std::format("tile bit ({}, {}) should be {}", x, y, (int)expected); + } + } } - time_end = current_time(); - ns = std::chrono::duration_cast(time_end - time_start).count(); - ms = ns / 1000000.; - cout << "Execution enforcer: " << sum << endl; - cout << "Running " << num_iters << " iters, Tile::set_ones() time: " << ms / num_iters << " ms" << endl; - - for (size_t height = 0; height < tile.HEIGHT; ++height){ - for (size_t width = 0; width < tile.WIDTH; ++width){ - tile.set_ones(); + return ""; +} + +// `set_ones(width, height)` sets the bits (x, y) with x < width and y < height, and clears +// the others. `clear_padding(width, height)` clears the other bits and keeps these. +template +std::string check_tile_masks(std::mt19937& generator){ + for (size_t height = 0; height <= Tile::HEIGHT; height++){ + for (size_t width = 0; width <= Tile::WIDTH; width++){ + auto inside = [=](size_t x, size_t y){ + return x < width && y < height; + }; + + Tile tile = random_tile(generator); + tile.set_ones(width, height); + std::string error = find_wrong_tile_bit(tile, inside); + if (!error.empty()){ + return std::format("set_ones({}, {}): {}", width, height, error); + } + + const Tile original = random_tile(generator); + tile = original; tile.clear_padding(width, height); - for (size_t y = 0; y < tile.HEIGHT; ++y){ - for (size_t x = 0; x < tile.WIDTH; ++x){ - bool gt = y < height && x < width; - if (gt != tile.get_bit(x, y)){ - cout << "Tile::clear_padding(width = " << width << ", height = " << height << "), tile (x = " << - x << ", y = " << y << "), wrong bit. Should be " << gt << endl; - return 1; - } - } + error = find_wrong_tile_bit(tile, [&](size_t x, size_t y){ + return inside(x, y) && original.get_bit(x, y); + }); + if (!error.empty()){ + return std::format("clear_padding({}, {}): {}", width, height, error); } } } - sum = 0; - time_start = current_time(); - for (size_t i = 0; i < num_iters; ++i){ - tile.clear_padding(32, 4); - sum += tile.row(0) + tile.row(1) + tile.row(2) + tile.row(3); - } - time_end = current_time(); - ns = std::chrono::duration_cast(time_end - time_start).count(); - ms = ns / 1000000.; - cout << "Execution enforcer: " << sum << endl; - cout << "Running " << num_iters << " iters, Tile::clear_padding() time: " << ms / num_iters << " ms" << endl; + return ""; +} - tile.set_ones(35, 3); +// `invert()`, `^=`, `|=`, `&=` and `andnot()` on random tiles. `a.andnot(b)` keeps the +// bits of `a` that are not set in `b`. +template +std::string check_tile_bitwise(std::mt19937& generator){ + const Tile a = random_tile(generator); + const Tile b = random_tile(generator); + + Tile tile = a; tile.invert(); - for (size_t y = 0; y < tile.HEIGHT; ++y){ - for (size_t x = 0; x < tile.WIDTH; ++x){ - bool gt = !(x < 35 && y < 3); - if (gt != tile.get_bit(x, y)){ - cout << "Tile::invert(), tile (x = " << - x << ", y = " << y << "), wrong bit. Should be " << gt << endl; - return 1; - } - } + std::string error = find_wrong_tile_bit(tile, [&](size_t x, size_t y){ return !a.get_bit(x, y); }); + if (!error.empty()){ + return "invert(): " + error; + } + tile = a; + tile ^= b; + error = find_wrong_tile_bit(tile, [&](size_t x, size_t y){ return a.get_bit(x, y) != b.get_bit(x, y); }); + if (!error.empty()){ + return "operator^=(): " + error; } + tile = a; + tile |= b; + error = find_wrong_tile_bit(tile, [&](size_t x, size_t y){ return a.get_bit(x, y) || b.get_bit(x, y); }); + if (!error.empty()){ + return "operator|=(): " + error; + } + tile = a; + tile &= b; + error = find_wrong_tile_bit(tile, [&](size_t x, size_t y){ return a.get_bit(x, y) && b.get_bit(x, y); }); + if (!error.empty()){ + return "operator&=(): " + error; + } + tile = a; + tile.andnot(b); + error = find_wrong_tile_bit(tile, [&](size_t x, size_t y){ return a.get_bit(x, y) && !b.get_bit(x, y); }); + if (!error.empty()){ + return "andnot(): " + error; + } + return ""; +} - auto tile2 = tile; - tile.set_ones(35, 3); - tile2.set_ones(13, 6); - tile.andnot(tile2); - for (size_t y = 0; y < tile.HEIGHT; ++y){ - for (size_t x = 0; x < tile.WIDTH; ++x){ - bool gt = !(x < 13 && y < 6) && (x < 35 && y < 3); - if (gt != tile.get_bit(x, y)){ - cout << "Tile::andnot(), tile (x = " << - x << ", y = " << y << "), wrong bit. Should be " << gt << endl; - return 1; - } - } +// `row(y)` is the 64-bit word of row y, with bit x of the word at (x, y). `top()` is row 0 +// and `bottom()` is the last row. `set_bit()` changes only one bit. +template +std::string check_tile_rows(std::mt19937& generator){ + std::vector words(Tile::HEIGHT); + Tile tile; + for (size_t y = 0; y < Tile::HEIGHT; y++){ + words[y] = ((uint64_t)generator() << 32) | generator(); + tile.row(y) = words[y]; + } + std::string error = find_wrong_tile_bit(tile, [&](size_t x, size_t y){ return (words[y] >> x) & 1; }); + if (!error.empty()){ + return "row(): " + error; + } + if (tile.top() != words[0] || tile.bottom() != words[Tile::HEIGHT - 1]){ + return "top() or bottom() is not the first or last row"; + } + tile.top() = words[0] = ~words[0]; + tile.bottom() = words[Tile::HEIGHT - 1] = ~words[Tile::HEIGHT - 1]; + error = find_wrong_tile_bit(tile, [&](size_t x, size_t y){ return (words[y] >> x) & 1; }); + if (!error.empty()){ + return "writing top() or bottom(): " + error; } - tile.set_ones(35, 3); - { - uint64_t top_row = tile.top(); - for (size_t x = 0; x < tile.WIDTH; ++x){ - uint64_t bit = ((top_row >> x) & 1); - uint64_t gt = uint64_t(x < 35); - if (bit != gt){ - cout << "Tile::top(), tile (x = " << - x << "), wrong bit. Should be " << gt << endl; - return 1; + for (size_t y = 0; y < Tile::HEIGHT; y++){ + for (size_t x = 0; x < Tile::WIDTH; x++){ + const Tile original = tile; + const bool value = !original.get_bit(x, y); + tile.set_bit(x, y, value); + error = find_wrong_tile_bit(tile, [&](size_t bit_x, size_t bit_y){ + return bit_x == x && bit_y == y ? value : original.get_bit(bit_x, bit_y); + }); + if (!error.empty()){ + return std::format("set_bit({}, {}, {}): {}", x, y, (int)value, error); } } } - { - tile.set_zero(); - uint64_t& top_row = tile.top(); - top_row = 0xFFFFFFFFFFFFFFFF; - uint64_t top = tile.top(); - for (size_t x = 0; x < tile.WIDTH; ++x){ - uint64_t bit = ((top >> x) & 1); - if (bit != 1){ - cout << "& Tile::top(), tile (x = " << - x << "), wrong bit. Should be 1" << endl; - return 1; - } - } + return ""; +} + +// One of the `copy_to_shift_*()` functions, which OR a shifted copy of a tile into another +// tile. They are used by `submatrix()` to build each output tile from the (up to 4) input +// tiles that it overlaps. The first letter of the suffix is for x and the second for y: +// "p" means that destination bit x gets source bit x + shift, and "n" means that +// destination bit x gets source bit x - shift. +template +struct CopyToShift{ + const char* name; + void (Tile::*function)(Tile& tile, size_t shift_x, size_t shift_y) const; + bool positive_x; + bool positive_y; + + // The source bit that lands on destination bit (x, y), or false if none does. Going + // below 0 wraps around to a huge size_t, which is out of the tile too. + bool source_bit(const Tile& source, size_t shift_x, size_t shift_y, size_t x, size_t y) const{ + const size_t source_x = positive_x ? x + shift_x : x - shift_x; + const size_t source_y = positive_y ? y + shift_y : y - shift_y; + return source_x < Tile::WIDTH && source_y < Tile::HEIGHT && source.get_bit(source_x, source_y); } - { - tile.set_zero(); - uint64_t& bottom_row = tile.bottom(); - bottom_row = 0xFFFFFFFFFFFFFFFF; - uint64_t bottom = tile.bottom(); - for (size_t x = 0; x < tile.WIDTH; ++x){ - uint64_t bit = ((bottom >> x) & 1); - if (bit != 1){ - cout << "& Tile::bottom(), tile (x = " << - x << "), wrong bit. Should be 1" << endl; - return 1; +}; + +template +std::string check_tile_shifts(std::mt19937& generator){ + const CopyToShift shifts[] = { + {"copy_to_shift_pp", &Tile::copy_to_shift_pp, true, true}, + {"copy_to_shift_np", &Tile::copy_to_shift_np, false, true}, + {"copy_to_shift_pn", &Tile::copy_to_shift_pn, true, false}, + {"copy_to_shift_nn", &Tile::copy_to_shift_nn, false, false}, + }; + const Tile source = random_tile(generator); + const Tile destination = random_tile(generator); + for (const CopyToShift& shift : shifts){ + for (size_t shift_y = 0; shift_y < Tile::HEIGHT; shift_y++){ + for (size_t shift_x = 0; shift_x < Tile::WIDTH; shift_x++){ + Tile tile = destination; + (source.*shift.function)(tile, shift_x, shift_y); + const std::string error = find_wrong_tile_bit(tile, [&](size_t x, size_t y){ + return destination.get_bit(x, y) || shift.source_bit(source, shift_x, shift_y, x, y); + }); + if (!error.empty()){ + return std::format("{}(shift_x = {}, shift_y = {}): {}", shift.name, shift_x, shift_y, error); + } } } } - tile.set_zero(); - for (size_t y = 0; y < tile.HEIGHT; ++y){ - for (size_t x = 0; x < tile.WIDTH; ++x){ - tile.set_bit(x, y); - if (1 != tile.get_bit(x, y)){ - cout << "Tile::set_bit(), tile (x = " << - x << ", y = " << y << "), wrong bit. Should be 1" << endl; - return 1; - } - tile.set_bit(x, y, 1); - if (1 != tile.get_bit(x, y)){ - cout << "Tile::set_bit(), tile (x = " << - x << ", y = " << y << "), wrong bit. Should be 1" << endl; - return 1; - } - tile.set_bit(x, y, 0); - if (0 != tile.get_bit(x, y)){ - cout << "Tile::set_bit(), tile (x = " << - x << ", y = " << y << "), wrong bit. Should be 0" << endl; - return 1; + return ""; +} + +template +class Test_BinaryTile : public UnitTest{ +public: + Test_BinaryTile(const std::string& tile_name) + : UnitTest("Kernels::BinaryMatrix - tile - " + tile_name) + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + std::mt19937 generator(0); + for (auto check : {check_tile_masks, check_tile_bitwise, check_tile_rows, check_tile_shifts}){ + const std::string error = check(generator); + if (!error.empty()){ + return "Error: " + error + "."; } } + return true; } +}; - std::srand(0); - auto src_tile = tile; - auto dst_tile = tile; - for (size_t num_tests = 0; num_tests < 30; ++num_tests){ - src_tile.set_zero(); - dst_tile.set_zero(); - for (size_t y = 0; y < tile.HEIGHT; ++y){ - for (size_t x = 0; x < tile.WIDTH; ++x){ - src_tile.set_bit(x, y, std::rand() % 2); - dst_tile.set_bit(x, y, std::rand() % 2); - } - } - auto test_copy_to_shift = [&](size_t shift_x, size_t shift_y, const std::string& function_name, - std::function gt_test, - std::function get_src_bit_test){ - // check against GT - for (size_t y = 0; y < tile.HEIGHT; ++y){ - for (size_t x = 0; x < tile.WIDTH; ++x){ - bool gt = false; - if (gt_test(shift_x, shift_y, x, y)){ - gt = dst_tile.get_bit(x, y) || get_src_bit_test(shift_x, shift_y, x, y); - // cout << "set the source bit with ||" << endl; - }else{ - // cout << "Use the original dest bit" << endl; - gt = dst_tile.get_bit(x, y); - } - if (tile.get_bit(x, y) != gt){ - cout << "Tile::" << function_name << "(shift_x=" << shift_x << ", shift_y=" << shift_y << "), tile (x = " << - x << ", y = " << y << "), wrong bit. Should be " << gt << endl; - cout << src_tile.dump() << endl; - cout << dst_tile.dump() << endl; - cout << tile.dump() << endl; - return 1; - } - } - } - return 0; - }; - for (size_t shift_y = 0; shift_y < tile.HEIGHT; ++shift_y){ - for (size_t shift_x = 0; shift_x < tile.WIDTH; ++shift_x){ - tile = dst_tile; - src_tile.copy_to_shift_pp(tile, shift_x, shift_y); - auto gt_test_pp = [&](size_t shift_x, size_t shift_y, size_t x, size_t y) -> bool{ - return x + shift_x < tile.WIDTH && y + shift_y < tile.HEIGHT; - }; - auto get_src_bit_test_pp = [&](size_t shift_x, size_t shift_y, size_t x, size_t y){ - return src_tile.get_bit(x + shift_x, y + shift_y); - }; - if (test_copy_to_shift(shift_x, shift_y, "copy_to_shift_pp", gt_test_pp, get_src_bit_test_pp) != 0){ - return 1; - } +#ifdef PA_AutoDispatch_arm64_20_M1 +// Check `PartialWordAccess_arm64_NEON`, which loads and stores only the first 0 to 16 +// bytes of a 16-byte NEON vector. NEON kernels use it for the leftover pixels at the end +// of a row. The test data sits in the middle of a buffer filled with `UNTOUCHED`: +// - Both loads must return the data bytes in the low lanes and 0 in the other lanes. +// - The store must write the data bytes and keep every other byte of the buffer. +class Test_PartialWordAccess_arm64_NEON : public UnitTest{ +public: + Test_PartialWordAccess_arm64_NEON() + : UnitTest("Kernels::PartialWordAccess - arm64_NEON") + {} - tile = dst_tile; - src_tile.copy_to_shift_np(tile, shift_x, shift_y); - auto gt_test_np = [&](size_t shift_x, size_t shift_y, size_t x, size_t y) -> bool{ - return x >= shift_x && y + shift_y < tile.HEIGHT; - }; - auto get_src_bit_test_np = [&](size_t shift_x, size_t shift_y, size_t x, size_t y){ - return src_tile.get_bit(x - shift_x, y + shift_y); - }; - if (test_copy_to_shift(shift_x, shift_y, "copy_to_shift_np", gt_test_np, get_src_bit_test_np) != 0){ - return 1; - } + virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ + constexpr uint8_t UNTOUCHED = 99; + constexpr size_t START = 16; + + for (size_t bytes = 0; bytes <= 16; bytes++){ + const PartialWordAccess_arm64_NEON access(bytes); - tile = dst_tile; - src_tile.copy_to_shift_pn(tile, shift_x, shift_y); - auto gt_test_pn = [&](size_t shift_x, size_t shift_y, size_t x, size_t y) -> bool{ - return x + shift_x < tile.WIDTH && y >= shift_y; - }; - auto get_src_bit_test_pn = [&](size_t shift_x, size_t shift_y, size_t x, size_t y){ - return src_tile.get_bit(x + shift_x, y - shift_y); - }; - if (test_copy_to_shift(shift_x, shift_y, "copy_to_shift_pn", gt_test_pn, get_src_bit_test_pn) != 0){ - return 1; + uint8_t buffer[48]; + for (size_t i = 0; i < 48; i++){ + buffer[i] = i < START || i >= START + bytes ? UNTOUCHED : (uint8_t)(100 + i); + } + const uint8x16_t loads[] = { + access.load_int_no_read_past_end(buffer + START), + access.load_int_no_read_before_ptr(buffer + START), + }; + for (const uint8x16_t& loaded : loads){ + uint8_t lanes[16]; + vst1q_u8(lanes, loaded); + for (size_t i = 0; i < 16; i++){ + const int expected = i < bytes ? buffer[START + i] : 0; + if (lanes[i] != expected){ + return std::format("Error: loading {} bytes: lane {} is {} but should be {}.", bytes, i, (int)lanes[i], expected); + } } + } - tile = dst_tile; - src_tile.copy_to_shift_nn(tile, shift_x, shift_y); - auto gt_test_nn = [&](size_t shift_x, size_t shift_y, size_t x, size_t y) -> bool{ - return x >= shift_x && y >= shift_y; - }; - auto get_src_bit_test_nn = [&](size_t shift_x, size_t shift_y, size_t x, size_t y){ - return src_tile.get_bit(x - shift_x, y - shift_y); - }; - if (test_copy_to_shift(shift_x, shift_y, "copy_to_shift_nn", gt_test_nn, get_src_bit_test_nn) != 0){ - return 1; + uint8_t stored[48]; + for (size_t i = 0; i < 48; i++){ + stored[i] = UNTOUCHED; + } + access.store_int_no_past_end(stored + START, vld1q_u8(buffer + START)); + for (size_t i = 0; i < 48; i++){ + if (stored[i] != buffer[i]){ + return std::format("Error: storing {} bytes: byte {} is {} but should be {}.", bytes, i, (int)stored[i], (int)buffer[i]); } } } + return true; } +}; +#endif - cout << "Tile tests passed" << endl; - return 0; -} -int test_binary_matrix_tile(){ -#ifdef PA_AutoDispatch_arm64_20_M1 - if (test_binary_matrix_tile_t() != 0){ - return 1; - } - for (size_t num_bytes = 0; num_bytes <= 16; ++num_bytes){ - PartialWordAccess_arm64_NEON partial(num_bytes); +// Matrix tests: check every matrix operation that has code per matrix type, through the +// `PackedBinaryMatrix_IB` interface, against the pixels of a real image. - uint8_t buffer[48]; - for (int i = 0; i < 48; i++){ - buffer[i] = 99; - } - for (uint8_t i = 0; i < 16; i++){ - buffer[i + 16] = i; - } +// An ARGB color range: a pixel is in range if each of its A, R, G and B bytes is within +// the bytes of `mins` and `maxs`. +struct ColorRange{ + uint32_t mins; + uint32_t maxs; +}; - // uint8x16_t x = partial.load(buffer+16); - uint8x16_t x = partial.load_int_no_read_past_end(buffer + 16); - for (size_t i = 0; i < num_bytes; ++i){ - if (x[i] != buffer[i + 16]){ - cout << "Error: PartialWordAccess_arm64_NEON(" << num_bytes << ")::load_int_no_read_past_end(), i = " << i << " is " << int(x[i]) - << ", but should be " << int(buffer[i + 16]) << endl; - return 1; - } +// The ranges to test. Each has pixels both in and out of range on every test image. +constexpr ColorRange RANGES[] = { + {0xff000000, 0xff8080ff}, + {0xff204060, 0xffe0c0ff}, + {0xff60607f, 0xffc8c8c8}, + {0xffa0a0a0, 0xffffffff}, +}; + +bool in_range(uint32_t pixel, const ColorRange& range){ + for (int shift = 0; shift < 32; shift += 8){ + const uint32_t value = (pixel >> shift) & 0xff; + if (value < ((range.mins >> shift) & 0xff) || value > ((range.maxs >> shift) & 0xff)){ + return false; } - x = partial.load_int_no_read_before_ptr(buffer + 16); - for (size_t i = 0; i < num_bytes; ++i){ - if (x[i] != buffer[i + 16]){ - cout << "Error: PartialWordAccess_arm64_NEON(" << num_bytes << ")::load_int_no_read_before_ptr(), i = " << i << " is " << int(x[i]) - << ", but should be " << int(buffer[i + 16]) << endl; - return 1; + } + return true; +} + +// Compare `matrix` with `expected_bit(x, y)`: its size must be `width` x `height`, each bit +// must match, and each padding bit must be 0. The padding bits are the bits of the edge +// tiles past the right and bottom of the matrix. They must stay 0, since Waterfill and +// other code read whole tiles. Return a description of the first problem, or an empty +// string if there is none. +template +std::string find_wrong_bit( + const PackedBinaryMatrix_IB& matrix, size_t width, size_t height, + ExpectedBit expected_bit +){ + if (matrix.width() != width || matrix.height() != height){ + return std::format("size is {} x {} but should be {} x {}", matrix.width(), matrix.height(), width, height); + } + for (size_t y = 0; y < height; y++){ + for (size_t x = 0; x < width; x++){ + const bool expected = expected_bit(x, y); + if (matrix.get(x, y) != expected){ + return std::format("bit ({}, {}) should be {}", x, y, (int)expected); } } - for (int i = 0; i < 48; i++){ - buffer[i] = 99; + } + + // `dump_tiles()` prints every bit of every tile, padding included, as rows of '0' and + // '1' that end with '\n'. + size_t x = 0, y = 0; + for (char bit : matrix.dump_tiles()){ + if (bit == '\n'){ + x = 0; + y++; + continue; } - partial.store_int_no_past_end(buffer + 16, x); - for (size_t i = 0; i < num_bytes; ++i){ - if (x[i] != buffer[i + 16]){ - cout << "Error: PartialWordAccess_arm64_NEON(" << num_bytes << ")::store_int_no_past_end(), i = " << i << " is " << int(buffer[i + 16]) - << ", but should be " << int(x[i]) << endl; - return 1; - } + if (bit == '1' && (x >= width || y >= height)){ + return std::format("padding bit ({}, {}) is 1", x, y); } + x++; } -#endif - return 0; + return ""; } +// A rectangle to cut out with `submatrix()`. +struct SubmatrixBox{ + size_t x; + size_t y; + size_t width; + size_t height; +}; + + +// Check one binary matrix type on a real image: +// 1. `compress_rgb32_to_binary_range()` with one range at a time, and with all of `RANGES` +// in one pass. Bit (x, y) must be 1 if pixel (x, y) is in range. +// 2. `invert()`, `^=`, `|=`, `&=`, `set_ones()` and `set_zero()` on those matrices. +// 3. `submatrix()` on boxes that start inside or outside a tile, that end at a tile +// edge or inside a tile, and that go past the edge of the matrix. +// The image is a guard-page image, so the test also crashes if a compress function reads +// past the last pixel. class Test_BinaryMatrix : public UnitTest{ public: - Test_BinaryMatrix( - const std::string& image - ) - : UnitTest("Kernels::BinaryMatrix - " + image) + Test_BinaryMatrix(const std::string& type_name, BinaryMatrixType type, const std::string& image) + : UnitTest("Kernels::BinaryMatrix - " + type_name + " - " + image) + , m_type(type) , m_image(UNIT_TEST_RESOURCE_PATH() + image) {} virtual UnitTestResult run(Logger& logger, CancellableScope& scope) const override{ - // Copy the test image into guard-page memory with packed rows, so a - // kernel that reads or writes past the last pixel faults. - ImageRGB32 file_image(m_image); - GuardPageImageRGB32 image(file_image); + const ImageRGB32 file_image(m_image); + const GuardPageImageRGB32 image(file_image); + const size_t width = image.width(); + const size_t height = image.height(); + auto bit_in_range = [&](size_t range_index){ + return [&, range_index](size_t x, size_t y){ + return in_range(image.pixel(x, y), RANGES[range_index]); + }; + }; - if (test_binary_matrix_tile() != 0){ - return false; + // 1. Compress, one range at a time. + std::vector> matrices; + for (size_t c = 0; c < std::size(RANGES); c++){ + matrices.emplace_back(make_PackedBinaryMatrix(m_type, width, height)); + compress_rgb32_to_binary_range( + image.data(), image.bytes_per_row(), *matrices.back(), + RANGES[c].mins, RANGES[c].maxs + ); + const std::string error = find_wrong_bit(*matrices.back(), width, height, bit_in_range(c)); + if (!error.empty()){ + return std::format("Error: compress range [{:08x}, {:08x}]: {}.", RANGES[c].mins, RANGES[c].maxs, error); + } } - const size_t width = image.width(), height = image.height(); - - const Color min_color(0, 0, 0), max_color(63, 63, 63); - const uint32_t mins = uint32_t(min_color), maxs = uint32_t(max_color); - - auto binary_matrix = make_PackedBinaryMatrix(get_BinaryMatrixType(), width, height); - - auto time_start = current_time(); - compress_rgb32_to_binary_range( - image.data(), image.bytes_per_row(), *binary_matrix, mins, maxs - ); - auto time_end = current_time(); - size_t ns = std::chrono::duration_cast(time_end - time_start).count(); - double ms = ns / 1000000.; - cout << "One binary matrix creation. time: " << ms << " ms" << endl; - - size_t error_count = 0; - for (size_t y = 0; y < height; y++){ - for (size_t x = 0; x < width; x++){ - const Color color(image.pixel(x, y)); - bool in_range = (min_color.alpha() <= color.alpha() && color.alpha() <= max_color.alpha()); - in_range = in_range && (min_color.red() <= color.red() && color.red() <= max_color.red()); - in_range = in_range && (min_color.green() <= color.green() && color.green() <= max_color.green()); - in_range = in_range && (min_color.blue() <= color.blue() && color.blue() <= max_color.blue()); - - const bool v_default = binary_matrix->get(x, y); - - if (error_count < 10){ - if (v_default != in_range){ - cout << "Error: matrix (" << x << ", " << y << ") got " - << v_default << " but GT is " << in_range << endl; - ++error_count; - } - } + // 1. Compress, all ranges in one pass. + std::vector> multi_matrices; + std::vector filters; + for (const ColorRange& range : RANGES){ + multi_matrices.emplace_back(make_PackedBinaryMatrix(m_type, width, height)); + filters.emplace_back(*multi_matrices.back(), range.mins, range.maxs); + } + compress_rgb32_to_binary_range(image.data(), image.bytes_per_row(), filters.data(), filters.size()); + for (size_t c = 0; c < std::size(RANGES); c++){ + const std::string error = find_wrong_bit(*multi_matrices[c], width, height, bit_in_range(c)); + if (!error.empty()){ + return std::format("Error: compress all ranges at once, range [{:08x}, {:08x}]: {}.", RANGES[c].mins, RANGES[c].maxs, error); } } - if (error_count){ - return false; + scope.throw_if_cancelled(); + + // 2. Operations on the matrices of the first two ranges. + const PackedBinaryMatrix_IB& a = *matrices[0]; + const PackedBinaryMatrix_IB& b = *matrices[1]; + auto bit_a = bit_in_range(0); + auto bit_b = bit_in_range(1); + + std::unique_ptr matrix = a.clone(); + matrix->invert(); + std::string error = find_wrong_bit(*matrix, width, height, [&](size_t x, size_t y){ return !bit_a(x, y); }); + if (!error.empty()){ + return "Error: invert(): " + error + "."; } - - - // We try to wait for three seconds: - const size_t num_iters = size_t(3000 / ms); - time_start = current_time(); - for (size_t i = 0; i < num_iters; i++){ - compress_rgb32_to_binary_range( - image.data(), image.bytes_per_row(), *binary_matrix, mins, maxs + matrix = a.clone(); + *matrix ^= b; + error = find_wrong_bit(*matrix, width, height, [&](size_t x, size_t y){ return bit_a(x, y) != bit_b(x, y); }); + if (!error.empty()){ + return "Error: operator^=(): " + error + "."; + } + matrix = a.clone(); + *matrix |= b; + error = find_wrong_bit(*matrix, width, height, [&](size_t x, size_t y){ return bit_a(x, y) || bit_b(x, y); }); + if (!error.empty()){ + return "Error: operator|=(): " + error + "."; + } + matrix = a.clone(); + *matrix &= b; + error = find_wrong_bit(*matrix, width, height, [&](size_t x, size_t y){ return bit_a(x, y) && bit_b(x, y); }); + if (!error.empty()){ + return "Error: operator&=(): " + error + "."; + } + matrix->set_ones(); + error = find_wrong_bit(*matrix, width, height, [](size_t, size_t){ return true; }); + if (!error.empty()){ + return "Error: set_ones(): " + error + "."; + } + matrix->set_zero(); + error = find_wrong_bit(*matrix, width, height, [](size_t, size_t){ return false; }); + if (!error.empty()){ + return "Error: set_zero(): " + error + "."; + } + scope.throw_if_cancelled(); + + // 3. Submatrices of the first range's matrix. The parts past the edge of the matrix + // must be 0. The 64 x 64 box ends at a tile edge for every matrix type. + const SubmatrixBox boxes[] = { + {0, 0, width, height}, + {1, 1, width - 1, height - 1}, + {5, 3, 64, 64}, + {37, 11, 40, 32}, + {70, 9, 100, 100}, + {width + 3, 0, 10, 10}, + }; + for (const SubmatrixBox& box : boxes){ + const std::unique_ptr sub = a.submatrix( + box.x, box.y, box.width, box.height ); + error = find_wrong_bit(*sub, box.width, box.height, [&](size_t x, size_t y){ + x += box.x; + y += box.y; + return x < width && y < height && bit_a(x, y); + }); + if (!error.empty()){ + return std::format( + "Error: submatrix({}, {}, {}, {}): {}.", + box.x, box.y, box.width, box.height, error + ); + } } - time_end = current_time(); - ms = (double)std::chrono::duration_cast(time_end - time_start).count(); - cout << "Running " << num_iters << " iters, average creation impl. time: " << ms / (double)num_iters << " ms" << endl; - - // cout << binary_matrix->dump() << flush; - return true; - }; + } private: + BinaryMatrixType m_type; std::string m_image; }; +} + + + +// Add the tile tests, and a matrix test for each image and each binary matrix type that +// this build has and this CPU supports, including Default. void add_tests_BinaryMatrix(UnitTestDatabase& database){ + database.add>("64x4_Default"); + struct MatrixType{ + std::string name; + BinaryMatrixType type; + }; + std::vector types{ + {"64x4_Default", BinaryMatrixType::i64x4_Default}, + }; +#ifdef PA_AutoDispatch_x64_08_Nehalem + if (CPU_CAPABILITY_CURRENT.OK_08_Nehalem){ + types.push_back({"64x8_x64_SSE42", BinaryMatrixType::i64x8_x64_SSE42}); + } +#endif +#ifdef PA_AutoDispatch_x64_13_Haswell + if (CPU_CAPABILITY_CURRENT.OK_13_Haswell){ + types.push_back({"64x16_x64_AVX2", BinaryMatrixType::i64x16_x64_AVX2}); + } +#endif +#ifdef PA_AutoDispatch_x64_17_Skylake + if (CPU_CAPABILITY_CURRENT.OK_17_Skylake){ + types.push_back({"64x32_x64_AVX512", BinaryMatrixType::i64x32_x64_AVX512}); + types.push_back({"64x64_x64_AVX512", BinaryMatrixType::i64x64_x64_AVX512}); + } +#endif +#ifdef PA_AutoDispatch_arm64_20_M1 + if (CPU_CAPABILITY_CURRENT.OK_M1){ + database.add>("64x8_arm64_NEON"); + database.add(); + types.push_back({"64x8_arm64_NEON", BinaryMatrixType::arm64x8_x64_NEON}); + } +#endif + + // Images in the CommandLineTests repo. No width or height is a multiple of 8, so the + // tiles at the right and bottom edges are only partly used. + const std::vector images{ + "NintendoSwitch/Switch2/ghostassassin/system_settings_display_dark.png", // 1916 x 1078 + "OCR/Lv.1-0.png", // 237 x 77 + "PokemonSwSh/SelectionArrowFinder/mac_neon_error_1.png", // 106 x 410 + "OCR/german-nature-sanft.png", // 511 x 91 + }; + + for (const MatrixType& type : types){ + for (const std::string& image : images){ + database.add(type.name, type.type, image); + } + } }