diff --git a/Common/Cpp/Sockets/ClientSocket_Qt.cpp b/Common/Cpp/Sockets/ClientSocket_Qt.cpp index 62f933d034..a008d26fe9 100644 --- a/Common/Cpp/Sockets/ClientSocket_Qt.cpp +++ b/Common/Cpp/Sockets/ClientSocket_Qt.cpp @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -97,6 +98,11 @@ size_t ClientSocket_Qt::send(const void* data, size_t bytes){ QSharedPointer send_data = QSharedPointer::create(); send_data->packet.assign((const char*)data, bytes); + // [SOCK-DIAG] Temporary: how long did the event thread take to + // deliver the queued send? A wait that reaches the 5 s timeout + // means the event thread never ran it. Remove with the rest of + // the diagnostics. + const auto send_start = std::chrono::steady_clock::now(); emit internal_send(send_data); // Wait for the socket's event thread to finish writing the packet. @@ -114,6 +120,16 @@ size_t ClientSocket_Qt::send(const void* data, size_t bytes){ ); } + // [SOCK-DIAG] Temporary: how long the event thread took. Remove + // with the rest of the diagnostics. + { + const long long ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - send_start + ).count(); + fprintf(stderr, "[SOCK-DIAG] client send %zu bytes: event thread took %lld ms%s\n", + bytes, ms, send_data->done ? "" : " (TIMED OUT - event thread never ran it)"); + } + return send_data->bytes_sent; } diff --git a/Common/Cpp/Sockets/Sockets_Tests.cpp b/Common/Cpp/Sockets/Sockets_Tests.cpp index 264d80e6fa..317dfd7192 100644 --- a/Common/Cpp/Sockets/Sockets_Tests.cpp +++ b/Common/Cpp/Sockets/Sockets_Tests.cpp @@ -12,6 +12,7 @@ */ #include +#include #include #include #include @@ -427,6 +428,10 @@ class Test_Sockets_RapidReconnect : public TestSocketsconnect_client(client, client_listener, port, cycle_no)){ error = "The client did not finish connecting (cycle " + std::to_string(cycle_no) + ")"; break; diff --git a/SerialPrograms/Source/Tests/CommandLineTests.cpp b/SerialPrograms/Source/Tests/CommandLineTests.cpp index 79eddc61a9..5bef9aa927 100644 --- a/SerialPrograms/Source/Tests/CommandLineTests.cpp +++ b/SerialPrograms/Source/Tests/CommandLineTests.cpp @@ -12,8 +12,16 @@ #include "CommandLineTests.h" #if defined(__APPLE__) || defined(__linux__) +#include +#include +#include #include +#include #include +#include +#if defined(__linux__) && defined(__x86_64__) +#include +#endif #include #endif @@ -24,25 +32,63 @@ namespace PokemonAutomation{ #if defined(__APPLE__) || defined(__linux__) -// On a fatal signal, dump a native backtrace of the crashing thread straight -// to the stderr file descriptor (fd 2, unbuffered). When stdout/stderr are -// piped (as in CI) the stdio buffers are lost when the process dies, and the -// app redirects std::cerr into its logger, so writing to fd 2 directly is the -// only way the backtrace reliably reaches the CI log. +// On a fatal signal, dump the signal, the faulting address (where the +// signal carries one), the thread id, and a native backtrace of the +// crashing thread straight to the stderr file descriptor (fd 2, +// unbuffered). When stdout/stderr are piped (as in CI) the stdio buffers +// are lost when the process dies, and the app redirects std::cerr into +// its logger, so writing to fd 2 directly is the only way the output +// reliably reaches the CI log. // -// backtrace() and backtrace_symbols_fd() are not strictly async-signal-safe, -// so a re-entrancy guard stops a crash inside the handler from looping: if we -// re-enter, just die immediately. -static void crash_backtrace_handler(int sig){ +// backtrace() and backtrace_symbols_fd() are not strictly async-signal- +// safe, so a re-entrancy guard stops a crash inside the handler from +// looping: if we re-enter, just die immediately. +static void crash_backtrace_handler(int sig, siginfo_t* info, void* ucontext_raw){ static volatile sig_atomic_t in_handler = 0; if (in_handler){ _exit(128 + sig); } in_handler = 1; - const char msg[] = "\n[PA-CRASH] Fatal signal, backtrace of the crashing thread:\n"; - if (write(2, msg, sizeof(msg) - 1) < 0){ - // Nothing more we can do. + // The faulting program counter: the exact instruction that died. + uintptr_t pc = 0; + if (ucontext_raw != nullptr){ + ucontext_t* ucontext = (ucontext_t*)ucontext_raw; + #if defined(__APPLE__) + // On Apple, uc_mcontext is a pointer (struct __darwin_mcontext64 *). + #if defined(__x86_64__) + pc = ucontext->uc_mcontext->__ss.__rip; + #elif defined(__aarch64__) + pc = ucontext->uc_mcontext->__ss.__pc; + #endif + #elif defined(__linux__) + #if defined(__x86_64__) + pc = (uintptr_t)ucontext->uc_mcontext.gregs[REG_RIP]; + #elif defined(__aarch64__) + pc = (uintptr_t)ucontext->uc_mcontext.pc; + #endif + #endif + } + + // For SIGSEGV/SIGBUS, si_addr is the faulting virtual address: which + // region the bad pointer landed in (stack, heap, a library) narrows + // the corruption source a lot. For SIGABRT it is typically null. + char msg[160]; + int len = snprintf( + msg, + sizeof(msg), + "\n[PA-CRASH] Fatal signal %d, pc 0x%lx, fault address 0x%lx, thread 0x%lx; backtrace:\n", + sig, + (unsigned long)pc, + (sig == SIGSEGV || sig == SIGBUS) + ? (unsigned long)(uintptr_t)info->si_addr + : 0UL, + (unsigned long)(uintptr_t)pthread_self() + ); + if (len > 0){ + if (write(2, msg, (size_t)len) < 0){ + // Nothing more we can do. + } } void* frames[128]; @@ -59,7 +105,13 @@ static void crash_backtrace_handler(int sig){ static void install_crash_backtrace_handlers(){ static const int sigs[] = { SIGSEGV, SIGBUS, SIGABRT, SIGILL, SIGFPE }; for (int sig : sigs){ - signal(sig, crash_backtrace_handler); + // SA_SIGINFO so the handler gets the faulting address. + struct sigaction sa; + std::memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = crash_backtrace_handler; + sa.sa_flags = SA_SIGINFO; + sigemptyset(&sa.sa_mask); + sigaction(sig, &sa, nullptr); } } #else @@ -79,8 +131,14 @@ int run_command_line_tests(bool augment_color_distortions){ setvbuf(stdout, nullptr, _IOLBF, 8192); cout << "Running parallel unit tests..." << endl; ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line()); - if (runner.run(augment_color_distortions)){ - return 1; + // [SOCK-DIAG] Temporary: run the suite 3x to raise the odds of + // catching the Mac Intel crash in a single CI run. Remove with the + // rest of the diagnostics. + for (int run_no = 1; run_no <= 3; run_no++){ + fprintf(stderr, "\n[SOCK-DIAG] Test suite run %d of 3\n", run_no); + if (runner.run(augment_color_distortions)){ + return 1; + } } cout << "Running parallel unit tests... Done!" << endl; return 0;