diff --git a/AGENTS.md b/AGENTS.md index 3fc0e4cd52..27506f6738 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -126,6 +126,11 @@ migrate a file to the wrappers when you're already touching it and it's straight Windows/Linux; `~/Library/Application Support/SerialPrograms/DownloadedResources` on macOS). - Headless Linux: `QT_QPA_PLATFORM=offscreen` and `TESSDATA_PREFIX=/Resources/Tesseract`. +- `SerialProgramsTests` (built by default next to `SerialPrograms`; `-DTEST_RUNNER_EXE=OFF` + turns it off) runs the same tests without the GUI, settings or device scans, so it can run + while SerialPrograms is open: `ninja SerialProgramsTests`, then + `SerialProgramsTests --filter ` (repeatable). Its exit code is the number + of failed tests. Same requirements as above. - Test framework: test classes derive `UnitTest` (`Common/Cpp/TestRunners/UnitTest.h`) and are collected via per-module `add_tests(UnitTestDatabase&)` — see `SerialPrograms/Source/Tests/CommandLineTests.h`. diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 35bb597e32..7923bcd16e 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -63,6 +63,9 @@ find_package(Threads REQUIRED) # Define a toggle option (ON by default) option(COMMAND_LINE_EXE "Build the optional executable" ON) +# Build SerialProgramsTests, which runs the unit tests without the GUI. +option(TEST_RUNNER_EXE "Build the unit test executable, SerialProgramsTests" ON) + # Build only the GUI-free core: CoreLib and SerialProgramsCommandLine. This needs # no Qt, OpenCV, ONNX Runtime, Tesseract, DPP or Discord SDK, so none of them are # searched for. @@ -336,6 +339,10 @@ if(NOT EXISTS "${DEST_DIR}/Scripts") ) endif() +if(TEST_RUNNER_EXE) + include(Source/Tests/TestRunnerExecutable.cmake) +endif() + if (WIN32) add_library(OpenCV_lib IMPORTED UNKNOWN) target_include_directories(SerialProgramsLib SYSTEM PRIVATE ../3rdParty/opencv-4.12.0/) diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 8448f23aad..28fed887f4 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -43,6 +43,23 @@ namespace PokemonAutomation{ namespace ComputerPrograms{ +namespace{ + +bool matches_any(std::string_view test_name, std::span filters){ + if (filters.empty()){ + return true; + } + for (std::string_view filter : filters){ + if (test_name.contains(filter)){ + return true; + } + } + return false; +} + +} // namespace + + @@ -84,6 +101,17 @@ const UnitTestDatabase& UNIT_TESTS_ALL(){ static UnitTestDatabase database = make_UNIT_TESTS_ALL(); return database; } + +std::vector unit_test_names(std::span filters){ + std::vector ret; + for (const auto& [test_name, test] : UNIT_TESTS_ALL()){ + if (matches_any(test_name, filters)){ + ret.emplace_back(test_name); + } + } + return ret; +} + StringSelectDatabase make_TEST_DATABASE(){ StringSelectDatabase database; for (auto& test : UNIT_TESTS_ALL()){ @@ -355,7 +383,7 @@ void UnitTestRunner::on_test_finished( -bool CommandLineUnitTestRunner::run(bool augment_color_distortions){ +size_t CommandLineUnitTestRunner::run(bool augment_color_distortions, std::span filters){ if (augment_color_distortions){ // Load the distortion data before any test runs. Each `ImageUnitTest` checks the global // manager when it runs, and only runs with distortions if it is loaded. @@ -363,7 +391,7 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions){ initialize_test_image_distortions(UNIT_TEST_RESOURCE_PATH()); }catch (Exception& e){ m_logger.log("Unable to load the test image distortions: " + e.to_str(), COLOR_RED); - return true; + return 1; } const TestImageDistortionManager& manager = *test_image_distortion_manager(); std::string labels; @@ -381,8 +409,10 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions){ GlobalThreadPools::unlimited_normal() ); runner.add_listener(*this); - for (const auto& test : UNIT_TESTS_ALL()){ - runner.add_test(test.second); + for (const auto& [test_name, test] : UNIT_TESTS_ALL()){ + if (matches_any(test_name, filters)){ + runner.add_test(test); + } } runner.run(); @@ -392,7 +422,7 @@ bool CommandLineUnitTestRunner::run(bool augment_color_distortions){ "\n Failed: " + tostr_u_commas(m_failed_tests.load(std::memory_order_acquire)) + "\n Skipped: " + tostr_u_commas(m_skipped_tests.load(std::memory_order_acquire)) ); - return m_failed_tests.load(std::memory_order_acquire) > 0; + return m_failed_tests.load(std::memory_order_acquire); } void CommandLineUnitTestRunner::on_test_finished( std::shared_ptr test, diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h index c3aeb94f07..10f69c8ab8 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.h @@ -8,6 +8,10 @@ #define PokemonAutomation_ComputerPrograms_UnitTestRunner_H #include +#include +#include +#include +#include #include "Common/Cpp/TestRunners/ParallelUnitTestRunner.h" #include "Common/Cpp/Options/StaticTextOption.h" #include "Common/Cpp/Options/EnumDropdownOption.h" @@ -89,12 +93,13 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List , m_failed_tests(0) {} - // Run all the unit tests. Return true if any test failed. + // Run the unit tests whose names contain any of `filters`, or all of them if `filters` is + // empty. Return the number of failed tests. // If `augment_color_distortions` is true, also run each image test under different color // distortion augmentation to widen the test coverage, using the distortion data in the // CaptureCardCalibration/ folder of the test repo. If that data can't be loaded, log the - // error and return true without running any test. - bool run(bool augment_color_distortions); + // error and return 1 without running any test. + size_t run(bool augment_color_distortions, std::span filters); private: @@ -112,6 +117,11 @@ class CommandLineUnitTestRunner : public PokemonAutomation::UnitTestRunner::List }; +// The names of the unit tests that contain any of `filters`, or of all the tests if `filters` +// is empty, in name order. +std::vector unit_test_names(std::span filters); + + } diff --git a/SerialPrograms/Source/Tests/CommandLineTests.cpp b/SerialPrograms/Source/Tests/CommandLineTests.cpp index bed7f8997e..4c73b74376 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.cpp +++ b/SerialPrograms/Source/Tests/CommandLineTests.cpp @@ -19,6 +19,10 @@ namespace PokemonAutomation{ int run_command_line_tests(bool augment_color_distortions){ + return run_unit_tests(augment_color_distortions, {}) == 0 ? 0 : 1; +} + +size_t run_unit_tests(bool augment_color_distortions, std::span filters){ install_crash_backtrace_handlers(); // Keep stdout line-buffered: when piped, stdio defaults to full // buffering, and the last few KB of test output - including the lines @@ -28,8 +32,9 @@ int run_command_line_tests(bool augment_color_distortions){ setvbuf(stdout, nullptr, _IOLBF, 8192); cout << "Running parallel unit tests..." << endl; ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); - if (runner.run(augment_color_distortions)){ - return 1; + size_t failed = runner.run(augment_color_distortions, filters); + if (failed > 0){ + return failed; } cout << "Running parallel unit tests... Done!" << endl; return 0; diff --git a/SerialPrograms/Source/Tests/CommandLineTests.h b/SerialPrograms/Source/Tests/CommandLineTests.h index 010de0c81f..7264fef290 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.h +++ b/SerialPrograms/Source/Tests/CommandLineTests.h @@ -5,7 +5,9 @@ * Run all the unit tests from the command line, without launching the GUI. * * How to run: - * Launch the program with the argument "--command-line-test-mode", or set + * Run the SerialProgramsTests executable (Tests/TestRunner_Main.cpp): it runs only the tests, + * with no GUI, no settings and no device scans, and takes "--filter ". + * Or launch the program with the argument "--command-line-test-mode", or set * "20-GlobalSettings": "COMMAND_LINE_TESTS": "RUN" to true in SerialPrograms-Settings.json. * * How the tests work: @@ -40,6 +42,9 @@ #ifndef PokemonAutomation_Tests_CommandLineTests_H #define PokemonAutomation_Tests_CommandLineTests_H +#include +#include +#include namespace PokemonAutomation{ @@ -51,6 +56,11 @@ namespace PokemonAutomation{ // Return 0 if all tests are passed, and 1 if any test failed or the distortion data can't be loaded. int run_command_line_tests(bool augment_color_distortions); +// Run the unit tests whose names contain any of `filters`, or all of them if `filters` is empty, +// and print the results to stdout. Return the number of failed tests, or 1 if +// `augment_color_distortions` is set and the distortion data can't be loaded. +size_t run_unit_tests(bool augment_color_distortions, std::span filters); + } diff --git a/SerialPrograms/Source/Tests/TestRunnerExecutable.cmake b/SerialPrograms/Source/Tests/TestRunnerExecutable.cmake new file mode 100644 index 0000000000..f57621abf3 --- /dev/null +++ b/SerialPrograms/Source/Tests/TestRunnerExecutable.cmake @@ -0,0 +1,19 @@ +# This cmake file is to be included by CMakeLists.txt to build SerialProgramsTests, an executable +# that runs the unit tests without the GUI (see Source/Tests/TestRunner_Main.cpp). + +add_executable(SerialProgramsTests Source/Tests/TestRunner_Main.cpp) + +# The tests are registered by explicit calls, not by static constructors, so the linker may +# leave out the parts of the library that the tests don't use. +target_link_libraries(SerialProgramsTests PRIVATE SerialProgramsLib) +apply_common_target_properties(SerialProgramsTests) + +# The tests read Resources/ from in or above the executable's folder, like SerialPrograms. +add_dependencies(SerialProgramsTests clone_packages) +if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/Resources") + add_custom_command( + TARGET SerialProgramsTests POST_BUILD + COMMAND ${CMAKE_COMMAND} -E echo "[Deploy] Copying resources ${RESOURCES_DIR} to $/Resources" + COMMAND ${CMAKE_COMMAND} -E copy_directory "${RESOURCES_DIR}" "$/Resources" + ) +endif() diff --git a/SerialPrograms/Source/Tests/TestRunner_Main.cpp b/SerialPrograms/Source/Tests/TestRunner_Main.cpp new file mode 100644 index 0000000000..7dbc85ec16 --- /dev/null +++ b/SerialPrograms/Source/Tests/TestRunner_Main.cpp @@ -0,0 +1,177 @@ +/* Unit Test Runner Main + * + * From: https://github.com/PokemonAutomation/ + * + * SerialProgramsTests: an executable that runs the unit tests and nothing else. It runs the + * same tests as "SerialPrograms --command-line-test-mode" (see Tests/CommandLineTests.h), but + * opens no window, reads and writes no settings, and does not look for serial ports or cameras. + * So it can run while SerialPrograms is open on the same computer. + * + * Usage: + * SerialProgramsTests [--filter ]... [--augment-color-distortions] + * + * --filter Select the tests whose names contain . Repeat it to + * select the tests that match any of them. Default: all tests. + * --augment-color-distortions Also run the image tests with color distortions. + * + * Exit code: the number of failed tests (at most 254), or 255 for a bad argument or when no + * test matches the filters. + * + * It finds Resources/ and the CommandLineTests repo the same way SerialPrograms does: in or + * above the folder that holds the executable. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Concurrency/Qt6.9ThreadBugWorkaround.h" +#include "Common/Cpp/Concurrency/FireForgetDispatcher.h" +#include "Common/Cpp/Concurrency/PeriodicRunner.h" +#include "Common/Cpp/Concurrency/Watchdog.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/Logging/FileLogger.h" +#include "Common/Cpp/Logging/GlobalLogger.h" +#include "Common/Cpp/Logging/LastLogTracker.h" +#include "Common/Cpp/Logging/MultiOutputLogger.h" +#include "Common/Qt/GlobalThreadPoolsQt.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/GlobalServices.h" +#include "CommonFramework/Globals.h" +#include "CommonFramework/Logging/Logger.h" +#include "CommonFramework/Tools/GlobalThreadPools.h" +#include "CommonTools/OCR/OCR_Routines.h" +#include "ComputerPrograms/ComputerPrograms_UnitTestRunner.h" +#include "Tests/CommandLineTests.h" + +namespace PokemonAutomation{ +namespace{ + +constexpr int EXIT_BAD_ARGUMENTS = 255; +constexpr size_t MAX_FAILURE_EXIT_CODE = 254; + +struct TestRunnerArguments{ + // Views of argv, which outlives every use. + std::vector filters; + bool augment_color_distortions = false; +}; + +// Return nothing if an argument is unknown or incomplete. +std::optional parse_arguments(int argc, char* argv[]){ + TestRunnerArguments arguments; + for (int i = 1; i < argc; i++){ + const std::string_view arg = argv[i]; + if (arg == "--filter" && i + 1 < argc){ + arguments.filters.emplace_back(argv[++i]); + }else if (arg == "--augment-color-distortions"){ + arguments.augment_color_distortions = true; + }else{ + std::cerr << "Unknown or incomplete argument: " << arg << "\n" + << "Usage: SerialProgramsTests [--filter ]... [--augment-color-distortions]" + << std::endl; + return std::nullopt; + } + } + return arguments; +} + +// The full test logs go to "SerialProgramsTests.log" in this executable's own user folder, +// not in the folder of SerialPrograms. +const std::string& test_log_path(){ + static const std::string path = + USER_FILE_PATH() + Filesystem::application_binary_name().stem().string() + ".log"; + return path; +} +FileLogger& test_file_logger(){ + static FileLogger logger( + GlobalThreadPools::unlimited_normal(), + FileLoggerConfig{ + .file_path = test_log_path(), + } + ); + return logger; +} + +int run_tests(const TestRunnerArguments& arguments){ + std::vector names = ComputerPrograms::unit_test_names(arguments.filters); + if (names.empty()){ + std::cerr << "No test matches the filters." << std::endl; + return EXIT_BAD_ARGUMENTS; + } + + Logger& logger = global_logger_command_line(); + logger.log("Resources folder: " + RESOURCE_PATH()); + logger.log("Unit test resources folder: " + UNIT_TEST_RESOURCE_PATH()); + logger.log("Log file: " + test_log_path()); + + size_t failed = run_unit_tests(arguments.augment_color_distortions, arguments.filters); + return (int)std::min(failed, MAX_FAILURE_EXIT_CODE); +} + +// Same order as the end of main() in CommonFramework/Main.cpp, for the services the tests use. +void stop_global_services(){ + OCR::clear_ocr_cache(); + global_periodic_runner().stop(); + global_watchdog().stop(); + test_file_logger().stop(); +} +void stop_thread_pools(){ + GlobalThreadPools::computation_realtime().stop(); + GlobalThreadPools::computation_normal().stop(); + GlobalThreadPools::unlimited_realtime().stop(); + GlobalThreadPools::unlimited_normal().stop(); + global_dispatcher.stop(); + + GlobalThreadPools::qt_worker_threadpool().stop(); + GlobalThreadPools::qt_event_threadpool().stop(); +} + +// The body of main(). `argc` is a reference because QCoreApplication keeps one. +int test_runner_main(int& argc, char* argv[]){ + std::optional arguments = parse_arguments(argc, argv); + if (!arguments){ + return EXIT_BAD_ARGUMENTS; + } + + // The tests use Qt (images, thread pools, events) but no window. + QCoreApplication application(argc, argv); + + { + MultiOutputLogger& logger = global_multi_logger(); + logger.add_listener(global_last_log_history()); + logger.add_listener(test_file_logger()); + } + + // Construct the Qt thread pools on the main thread, as SerialPrograms does. + GlobalThreadPools::qt_worker_threadpool(); + GlobalThreadPools::qt_event_threadpool(); + + int ret = run_tests(*arguments); + + stop_global_services(); +#ifdef PA_ENABLE_QT_ADOPTION_WORKAROUND + // See the Qt 6.9 workaround at the end of main() in CommonFramework/Main.cpp. + std::cout.flush(); + std::quick_exit(ret); +#endif + stop_thread_pools(); + return ret; +} + +} // namespace + + +bool use_qt_ui(){ + return false; +} + + +} // namespace PokemonAutomation + + +int main(int argc, char* argv[]){ + return PokemonAutomation::test_runner_main(argc, argv); +}