From c781960de2ea45f06d5870e4a4e39a16e22287df Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:47:46 +0000 Subject: [PATCH 1/6] ML: temporary [ORT-DIAG] stderr markers and ORT INFO logging While investigating the intermittent Mac Intel PaddleOCR segfault: - Print the ONNX Runtime version, hardware_concurrency() and the chosen intra/inter-op thread counts to stderr (the file logger is not in the CI log). - Run the global Ort::Env at ORT_LOGGING_LEVEL_INFO so ONNX Runtime's own session-creation progress (Session Options, thread pool params, graph transforms, BFCArena, ...) lands in the CI log. - Add [ORT-DIAG] stderr markers around create_session() and the first PaddleOCRPipeline::recognize_line() Run() calls, to pin down which stage the crash is in. All of this is temporary: revert when the crash is understood. --- .../Source/ML/Inference/ML_PaddleOCRPipeline.cpp | 5 +++++ .../Source/ML/Models/ML_ONNXRuntimeHelpers.cpp | 4 ++++ SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp | 10 +++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp index ab9ab35fb7..066d79e626 100644 --- a/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp +++ b/SerialPrograms/Source/ML/Inference/ML_PaddleOCRPipeline.cpp @@ -87,6 +87,7 @@ PaddleOCRPipeline::PaddleOCRPipeline( , m_det_path(std::move(det_path)) { load_dictionary(Filesystem::Path(dict_path)); + std::cerr << "[ORT-DIAG] PaddleOCRPipeline ctor finished: " << rec_path << std::endl; } PaddleOCRDetector& PaddleOCRPipeline::detector(){ @@ -381,6 +382,10 @@ std::string PaddleOCRPipeline::recognize_line(const cv::Mat& cropped_image){ const char* input_names[] = {m_input_name.c_str()}; const char* output_names[] = {m_output_name.c_str()}; + static int diag_run_count = 0; + if (++diag_run_count <= 20){ + std::cerr << "[ORT-DIAG] recognize_line: about to call session->Run() (call " << diag_run_count << ")" << std::endl; + } try{ if (debugging){ m_logger.log("[OCR-DEBUG] Calling m_rec_session.Run() now..."); diff --git a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp index d0dac9d2a8..620173c414 100644 --- a/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp +++ b/SerialPrograms/Source/ML/Models/ML_ONNXRuntimeHelpers.cpp @@ -167,9 +167,11 @@ Ort::Session create_session( const std::string& model_cache_path, bool try_gpu ){ + std::cerr << "[ORT-DIAG] create_session(" << model_path << ", try_gpu=" << (try_gpu ? 1 : 0) << ")" << std::endl; bool write_flag_file = true; std::string file_hash; std::tie(write_flag_file, file_hash) = clean_up_old_model_cache(model_cache_path, model_path); + std::cerr << "[ORT-DIAG] clean_up_old_model_cache done, write_flag_file=" << write_flag_file << std::endl; auto onnx_path = str_to_onnx_str(model_path); @@ -205,7 +207,9 @@ Ort::Session create_session( Ort::SessionOptions cpu_options = create_session_options(model_cache_path, false); cpu_options.DisablePerSessionThreads(); + std::cerr << "[ORT-DIAG] calling Ort::Session constructor (CPU, per-session threads disabled)..." << std::endl; Ort::Session session{global_ort_env(), onnx_path.c_str(), cpu_options}; + std::cerr << "[ORT-DIAG] Ort::Session constructor returned: " << model_path << std::endl; logger.log("Ort::Session created"); return session; } diff --git a/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp b/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp index c69ace4a39..e16195442e 100644 --- a/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp +++ b/SerialPrograms/Source/ML/Models/ML_OrtEnv.cpp @@ -4,6 +4,8 @@ * */ +#include +#include #include "Common/Cpp/Exceptions.h" #include "Common/Cpp/PrettyPrint.h" #include "CommonFramework/Logging/Logger.h" @@ -19,11 +21,15 @@ Ort::ThreadingOptions make_ort_threading_options(){ { int threads = (int)PerformanceOptions::instance().ONNX_OPTIONS.MAX_INTRA_OP_THREADS; global_logger_tagged().log("Setting ONNX Intra-Op Threads: " + tostr_u_commas(threads)); + std::cerr << "[ORT-DIAG] ONNX Runtime " << Ort::GetVersionString() + << ", hardware_concurrency() = " << std::thread::hardware_concurrency() + << ", SetGlobalIntraOpNumThreads(" << threads << ")" << std::endl; ret.SetGlobalIntraOpNumThreads(threads); } { int threads = (int)PerformanceOptions::instance().ONNX_OPTIONS.MAX_INTER_OP_THREADS; global_logger_tagged().log("Setting ONNX Inter-Op Threads: " + tostr_u_commas(threads)); + std::cerr << "[ORT-DIAG] SetGlobalInterOpNumThreads(" << threads << ")" << std::endl; ret.SetGlobalInterOpNumThreads(threads); } ret.SetGlobalSpinControl(0); @@ -31,6 +37,7 @@ Ort::ThreadingOptions make_ort_threading_options(){ // them, and they are very slow on Intel CPUs. ONNX Runtime only supports this on x86; // see Common/Cpp/Hardware/FloatingPointMode.h. ret.SetGlobalDenormalAsZero(); + std::cerr << "[ORT-DIAG] SetGlobalSpinControl(0), SetGlobalDenormalAsZero()" << std::endl; return ret; } @@ -42,9 +49,10 @@ Ort::Env& global_ort_env(){ #endif static Ort::Env env( make_ort_threading_options(), - ORT_LOGGING_LEVEL_WARNING, + ORT_LOGGING_LEVEL_INFO, // [ORT-DIAG] was WARNING "Global_ThreadPool_Env" ); + std::cerr << "[ORT-DIAG] Ort::Env created" << std::endl; return env; } From bcae0e96812f31297e26039229c1fe539f79c4f9 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 1 Nov 2026 07:18:57 +0000 Subject: [PATCH 2/6] Tests: [ORT-DIAG] dump a backtrace to stderr on a fatal signal in test mode The CI log lost the output right up to the segfault because stdout is fully buffered when piped: in the #1812 Mac Intel run the crash happened ~2.5 s after 'Running parallel unit tests...' and not a single line after that was captured. Two temporary diagnostics: - A SIGSEGV/SIGBUS/SIGABRT/SIGILL/SIGFPE handler (command-line test mode only) that writes a native backtrace of the crashing thread straight to fd 2 with backtrace_symbols_fd(), then restores the default disposition and re-raises so the OS crash report and exit code are unchanged. - setvbuf(stdout, _IOLBF) in test mode, so the test 'Starting:'/'Passed:' lines reach the log as they happen and the log shows the last thing the crashing process was doing. --- .../Source/Tests/CommandLineTests.cpp | 121 +++++++++++++----- 1 file changed, 88 insertions(+), 33 deletions(-) diff --git a/SerialPrograms/Source/Tests/CommandLineTests.cpp b/SerialPrograms/Source/Tests/CommandLineTests.cpp index a723fef213..898d5cf8db 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.cpp +++ b/SerialPrograms/Source/Tests/CommandLineTests.cpp @@ -1,33 +1,88 @@ -/* Command Line Tests - * - * From: https://github.com/PokemonAutomation/ - * - */ - - -#include -#include "CommonFramework/Logging/Logger.h" -#include "ComputerPrograms/ComputerPrograms_UnitTestRunner.h" - -#include "CommandLineTests.h" - -using std::cout; -using std::endl; - -namespace PokemonAutomation{ - - - -int run_command_line_tests(bool augment_color_distortions){ - cout << "Running parallel unit tests..." << endl; - ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); - if (runner.run(augment_color_distortions)){ - return 1; - } - cout << "Running parallel unit tests... Done!" << endl; - return 0; -} - - - -} +/* Command Line Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + + +#include +#include "CommonFramework/Logging/Logger.h" +#include "ComputerPrograms/ComputerPrograms_UnitTestRunner.h" + +#include "CommandLineTests.h" + +#if defined(__APPLE__) || defined(__linux__) +#include +#include +#include +#endif + +using std::cout; +using std::endl; + +namespace PokemonAutomation{ + + +#if defined(__APPLE__) || defined(__linux__) +// On a fatal signal, dump a native backtrace of the crashing thread straight to stderr +// (fd 2, unbuffered) so it survives in the CI log even though stdout's buffer is lost when +// the process segfaults. Then restore the default disposition and re-raise, so the OS still +// writes its crash report (.ips on macOS) and the exit code still reflects the crash. +static void crash_backtrace_handler(int sig){ + // Re-entrancy guard: if we crash while handling a signal, just die. + static volatile sig_atomic_t in_handler = 0; + if (in_handler){ + _exit(128 + sig); + } + in_handler = 1; + + const char msg[] = "\n[PA-CRASH] Fatal signal, backtrace of the crashing thread:\n"; + if (write(2, msg, sizeof(msg) - 1) < 0){ + // Nothing more we can do. + } + + void* frames[128]; + int n = backtrace(frames, 128); + // Writes one symbolized frame per line to fd 2. glibc declares it void; Apple's returns int. +#if defined(__linux__) + backtrace_symbols_fd(frames, n, 2); +#else + if (backtrace_symbols_fd(frames, n, 2) < 0){ + // Nothing more we can do. + } +#endif + + // Let the OS do its normal crash handling (crash report + exit code). + signal(sig, SIG_DFL); + raise(sig); +} + +static void install_crash_backtrace_handlers(){ + static const int sigs[] = { SIGSEGV, SIGBUS, SIGABRT, SIGILL, SIGFPE }; + for (int sig : sigs){ + signal(sig, crash_backtrace_handler); + } +} +#else +static void install_crash_backtrace_handlers(){} +#endif + + +int run_command_line_tests(bool augment_color_distortions){ + install_crash_backtrace_handlers(); + // [ORT-DIAG] Keep stdout line-buffered in CI: when piped, stdio defaults to full + // buffering, so the last few KB of test output (and where the crash happened) is lost + // in the buffer when the process dies. + setvbuf(stdout, nullptr, _IOLBF, 0); + cout << "Running parallel unit tests..." << endl; + ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); + if (runner.run(augment_color_distortions)){ + return 1; + } + cout << "Running parallel unit tests... Done!" << endl; + return 0; +} + + + +} From 7ce78b89be2a9dc2d04e3ebddc172f05a8f2f4bb Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 1 Nov 2026 08:08:24 +0000 Subject: [PATCH 3/6] Tests: [ORT-DIAG] fix backtrace_symbols_fd() on Apple, restore CRLF Apple's execinfo.h declares backtrace_symbols_fd() as void (glibc's returns int), so ignore its result on both. Also restore the CRLF line endings the previous commit accidentally rewrote to LF. --- .../Source/Tests/CommandLineTests.cpp | 169 +++++++++--------- 1 file changed, 81 insertions(+), 88 deletions(-) diff --git a/SerialPrograms/Source/Tests/CommandLineTests.cpp b/SerialPrograms/Source/Tests/CommandLineTests.cpp index 898d5cf8db..c673a074a8 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.cpp +++ b/SerialPrograms/Source/Tests/CommandLineTests.cpp @@ -1,88 +1,81 @@ -/* Command Line Tests - * - * From: https://github.com/PokemonAutomation/ - * - */ - - -#include -#include "CommonFramework/Logging/Logger.h" -#include "ComputerPrograms/ComputerPrograms_UnitTestRunner.h" - -#include "CommandLineTests.h" - -#if defined(__APPLE__) || defined(__linux__) -#include -#include -#include -#endif - -using std::cout; -using std::endl; - -namespace PokemonAutomation{ - - -#if defined(__APPLE__) || defined(__linux__) -// On a fatal signal, dump a native backtrace of the crashing thread straight to stderr -// (fd 2, unbuffered) so it survives in the CI log even though stdout's buffer is lost when -// the process segfaults. Then restore the default disposition and re-raise, so the OS still -// writes its crash report (.ips on macOS) and the exit code still reflects the crash. -static void crash_backtrace_handler(int sig){ - // Re-entrancy guard: if we crash while handling a signal, just die. - static volatile sig_atomic_t in_handler = 0; - if (in_handler){ - _exit(128 + sig); - } - in_handler = 1; - - const char msg[] = "\n[PA-CRASH] Fatal signal, backtrace of the crashing thread:\n"; - if (write(2, msg, sizeof(msg) - 1) < 0){ - // Nothing more we can do. - } - - void* frames[128]; - int n = backtrace(frames, 128); - // Writes one symbolized frame per line to fd 2. glibc declares it void; Apple's returns int. -#if defined(__linux__) - backtrace_symbols_fd(frames, n, 2); -#else - if (backtrace_symbols_fd(frames, n, 2) < 0){ - // Nothing more we can do. - } -#endif - - // Let the OS do its normal crash handling (crash report + exit code). - signal(sig, SIG_DFL); - raise(sig); -} - -static void install_crash_backtrace_handlers(){ - static const int sigs[] = { SIGSEGV, SIGBUS, SIGABRT, SIGILL, SIGFPE }; - for (int sig : sigs){ - signal(sig, crash_backtrace_handler); - } -} -#else -static void install_crash_backtrace_handlers(){} -#endif - - -int run_command_line_tests(bool augment_color_distortions){ - install_crash_backtrace_handlers(); - // [ORT-DIAG] Keep stdout line-buffered in CI: when piped, stdio defaults to full - // buffering, so the last few KB of test output (and where the crash happened) is lost - // in the buffer when the process dies. - setvbuf(stdout, nullptr, _IOLBF, 0); - cout << "Running parallel unit tests..." << endl; - ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); - if (runner.run(augment_color_distortions)){ - return 1; - } - cout << "Running parallel unit tests... Done!" << endl; - return 0; -} - - - -} +/* Command Line Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + + +#include +#include "CommonFramework/Logging/Logger.h" +#include "ComputerPrograms/ComputerPrograms_UnitTestRunner.h" + +#include "CommandLineTests.h" + +#if defined(__APPLE__) || defined(__linux__) +#include +#include +#include +#endif + +using std::cout; +using std::endl; + +namespace PokemonAutomation{ + + +#if defined(__APPLE__) || defined(__linux__) +// On a fatal signal, dump a native backtrace of the crashing thread straight to stderr +// (fd 2, unbuffered) so it survives in the CI log even though stdout's buffer is lost when +// the process segfaults. Then restore the default disposition and re-raise, so the OS still +// writes its crash report (.ips on macOS) and the exit code still reflects the crash. +static void crash_backtrace_handler(int sig){ + // Re-entrancy guard: if we crash while handling a signal, just die. + static volatile sig_atomic_t in_handler = 0; + if (in_handler){ + _exit(128 + sig); + } + in_handler = 1; + + const char msg[] = "\n[PA-CRASH] Fatal signal, backtrace of the crashing thread:\n"; + if (write(2, msg, sizeof(msg) - 1) < 0){ + // Nothing more we can do. + } + + void* frames[128]; + int n = backtrace(frames, 128); + backtrace_symbols_fd(frames, n, 2); // writes one symbolized frame per line to fd 2 + + // Let the OS do its normal crash handling (crash report + exit code). + signal(sig, SIG_DFL); + raise(sig); +} + +static void install_crash_backtrace_handlers(){ + static const int sigs[] = { SIGSEGV, SIGBUS, SIGABRT, SIGILL, SIGFPE }; + for (int sig : sigs){ + signal(sig, crash_backtrace_handler); + } +} +#else +static void install_crash_backtrace_handlers(){} +#endif + + +int run_command_line_tests(bool augment_color_distortions){ + install_crash_backtrace_handlers(); + // [ORT-DIAG] Keep stdout line-buffered in CI: when piped, stdio defaults to full + // buffering, so the last few KB of test output (and where the crash happened) is lost + // in the buffer when the process dies. + setvbuf(stdout, nullptr, _IOLBF, 0); + cout << "Running parallel unit tests..." << endl; + ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); + if (runner.run(augment_color_distortions)){ + return 1; + } + cout << "Running parallel unit tests... Done!" << endl; + return 0; +} + + + +} From a28c0e8f7f66d32e43f7d4990eadd65ddcd67953 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:14:40 +0000 Subject: [PATCH 4/6] Tests: [ORT-DIAG] give setvbuf() an explicit 8 KB buffer setvbuf(stdout, nullptr, _IOLBF, 0) crashed the Windows CI runner with 0xC0000409 (stack-buffer-overrun) ~0.7 s into the test run, on the first stdout write after the call: with size 0, MSVC's ucrt appears to install a zero-size stdio buffer for stdout. 8192 is the conventional default. --- SerialPrograms/Source/Tests/CommandLineTests.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/Tests/CommandLineTests.cpp b/SerialPrograms/Source/Tests/CommandLineTests.cpp index c673a074a8..62cc4323ec 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.cpp +++ b/SerialPrograms/Source/Tests/CommandLineTests.cpp @@ -66,7 +66,10 @@ int run_command_line_tests(bool augment_color_distortions){ // [ORT-DIAG] Keep stdout line-buffered in CI: when piped, stdio defaults to full // buffering, so the last few KB of test output (and where the crash happened) is lost // in the buffer when the process dies. - setvbuf(stdout, nullptr, _IOLBF, 0); + // 8192 bytes: 0 means "implementation-defined" and MSVC's ucrt can end up with a + // zero-size buffer, which corrupts the next stdout write (observed 0xC0000409 on the + // Windows CI runner). + setvbuf(stdout, nullptr, _IOLBF, 8192); cout << "Running parallel unit tests..." << endl; ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); if (runner.run(augment_color_distortions)){ From 792fee9c2df8ba2327d285db9cb6cedc64a8ce03 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 1 Nov 2026 09:40:47 +0000 Subject: [PATCH 5/6] Tests: [ORT-DIAG] run the suite 3x, re-creating the PaddleOCR sessions The Mac Intel crash is probabilistic (~60% of runs) and lands in the PaddleOCR ONNX session creation, which happens only once per process because the sessions are cached. Run the full suite three times and clear the PaddleOCR cache between runs so the sessions are re-created each time, raising the per-CI-run reproduction odds to ~93%. Stop and fail on the first reported test failure. --- .../Source/Tests/CommandLineTests.cpp | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/Tests/CommandLineTests.cpp b/SerialPrograms/Source/Tests/CommandLineTests.cpp index 62cc4323ec..ab6fe321f0 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.cpp +++ b/SerialPrograms/Source/Tests/CommandLineTests.cpp @@ -7,6 +7,7 @@ #include #include "CommonFramework/Logging/Logger.h" +#include "CommonTools/OCR/OCR_PaddleOCR.h" #include "ComputerPrograms/ComputerPrograms_UnitTestRunner.h" #include "CommandLineTests.h" @@ -71,10 +72,26 @@ int run_command_line_tests(bool augment_color_distortions){ // Windows CI runner). setvbuf(stdout, nullptr, _IOLBF, 8192); cout << "Running parallel unit tests..." << endl; - ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); - if (runner.run(augment_color_distortions)){ - return 1; + + // [ORT-DIAG] The Mac Intel crash is probabilistic (~60% of CI runs) and happens at + // PaddleOCR's ONNX session creation, which normally happens only once per process + // (the sessions are cached). To raise the odds of reproducing it in CI, run the full + // suite several times, clearing the PaddleOCR cache between runs so the sessions are + // re-created each time. + constexpr int k_repeat_runs = 3; + for (int run_index = 1; run_index <= k_repeat_runs; ++run_index){ + std::cerr << "[ORT-DIAG] Test suite run " << run_index << " of " << k_repeat_runs << std::endl; + if (run_index > 1){ + OCR::clear_paddle_ocr_cache(); // Force PaddleOCR to re-create its ONNX sessions. + } + ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); + if (runner.run(augment_color_distortions)){ + std::cerr << "[ORT-DIAG] Test suite run " << run_index << " reported failures; stopping." + << std::endl; + return 1; + } } + cout << "Running parallel unit tests... Done!" << endl; return 0; } From 9e6744d3f3fee1547175fc3af96954c53e3ccb33 Mon Sep 17 00:00:00 2001 From: PA Dev Bot <344001113+pa-devbot@users.noreply.github.com> Date: Sun, 1 Nov 2026 10:31:53 +0000 Subject: [PATCH 6/6] ci: re-trigger