[DO NOT MERGE] [SOCK-DIAG] instrument the socket tests for the Mac Intel crash

Common/Cpp/Sockets/ClientSocket_Qt.cpp

@@ -5,6 +5,7 @@
  */
 
 #include <chrono>
+#include <cstdio>
 #include <QSharedPointer>
 #include <QTimer>
 #include <QTcpSocket>
@@ -97,6 +98,11 @@ size_t ClientSocket_Qt::send(const void* data, size_t bytes){
     QSharedPointer<SendData> send_data = QSharedPointer<SendData>::create();
     send_data->packet.assign((const char*)data, bytes);
 
+    //  [SOCK-DIAG] Temporary: how long did the event thread take to
+    //  deliver the queued send? A wait that reaches the 5 s timeout
+    //  means the event thread never ran it. Remove with the rest of
+    //  the diagnostics.
+    const auto send_start = std::chrono::steady_clock::now();
     emit internal_send(send_data);
 
     //  Wait for the socket's event thread to finish writing the packet.
@@ -114,6 +120,16 @@ size_t ClientSocket_Qt::send(const void* data, size_t bytes){
         );
     }
 
+    //  [SOCK-DIAG] Temporary: how long the event thread took. Remove
+    //  with the rest of the diagnostics.
+    {
+        const long long ms = std::chrono::duration_cast<std::chrono::milliseconds>(
+            std::chrono::steady_clock::now() - send_start
+        ).count();
+        fprintf(stderr, "[SOCK-DIAG] client send %zu bytes: event thread took %lld ms%s\n",
+            bytes, ms, send_data->done ? "" : " (TIMED OUT - event thread never ran it)");
+    }
+
     return send_data->bytes_sent;
 }
 

Common/Cpp/Sockets/Sockets_Tests.cpp

@@ -12,6 +12,7 @@
  */
 
 #include <atomic>
+#include <cstdio>
 #include <chrono>
 #include <cstdint>
 #include <cstring>
@@ -427,6 +428,10 @@ class Test_Sockets_RapidReconnect : public TestSockets<ClientSocketType, ServerS
             for (int cycle = 0; cycle < CYCLES; cycle++){
                 int cycle_no = cycle + 1;
 
+                //  [SOCK-DIAG] Temporary: where the test was when the
+                //  process died. Remove with the rest of the diagnostics.
+                fprintf(stderr, "[SOCK-DIAG] rapid reconnect cycle %d/%d\n", cycle_no, CYCLES);
+
                 if (!this->connect_client(client, client_listener, port, cycle_no)){
                     error = "The client did not finish connecting (cycle " + std::to_string(cycle_no) + ")";
                     break;

SerialPrograms/Source/Tests/CommandLineTests.cpp

@@ -12,8 +12,16 @@
 #include "CommandLineTests.h"
 
 #if defined(__APPLE__) || defined(__linux__)
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
 #include <execinfo.h>
+#include <pthread.h>
 #include <signal.h>
+#include <sys/ucontext.h>
+#if defined(__linux__) && defined(__x86_64__)
+#include <sys/reg.h>
+#endif
 #include <unistd.h>
 #endif
 
@@ -24,25 +32,63 @@ namespace PokemonAutomation{
 
 
 #if defined(__APPLE__) || defined(__linux__)
-//  On a fatal signal, dump a native backtrace of the crashing thread straight
-//  to the stderr file descriptor (fd 2, unbuffered). When stdout/stderr are
-//  piped (as in CI) the stdio buffers are lost when the process dies, and the
-//  app redirects std::cerr into its logger, so writing to fd 2 directly is the
-//  only way the backtrace reliably reaches the CI log.
+//  On a fatal signal, dump the signal, the faulting address (where the
+//  signal carries one), the thread id, and a native backtrace of the
+//  crashing thread straight to the stderr file descriptor (fd 2,
+//  unbuffered). When stdout/stderr are piped (as in CI) the stdio buffers
+//  are lost when the process dies, and the app redirects std::cerr into
+//  its logger, so writing to fd 2 directly is the only way the output
+//  reliably reaches the CI log.
 //
-//  backtrace() and backtrace_symbols_fd() are not strictly async-signal-safe,
-//  so a re-entrancy guard stops a crash inside the handler from looping: if we
-//  re-enter, just die immediately.
-static void crash_backtrace_handler(int sig){
+//  backtrace() and backtrace_symbols_fd() are not strictly async-signal-
+//  safe, so a re-entrancy guard stops a crash inside the handler from
+//  looping: if we re-enter, just die immediately.
+static void crash_backtrace_handler(int sig, siginfo_t* info, void* ucontext_raw){
     static volatile sig_atomic_t in_handler = 0;
     if (in_handler){
         _exit(128 + sig);
     }
     in_handler = 1;
 
-    const char msg[] = "\n[PA-CRASH] Fatal signal, backtrace of the crashing thread:\n";
-    if (write(2, msg, sizeof(msg) - 1) < 0){
-        //  Nothing more we can do.
+    //  The faulting program counter: the exact instruction that died.
+    uintptr_t pc = 0;
+    if (ucontext_raw != nullptr){
+        ucontext_t* ucontext = (ucontext_t*)ucontext_raw;
+        #if defined(__APPLE__)
+        //  On Apple, uc_mcontext is a pointer (struct __darwin_mcontext64 *).
+        #if defined(__x86_64__)
+            pc = ucontext->uc_mcontext->__ss.__rip;
+        #elif defined(__aarch64__)
+            pc = ucontext->uc_mcontext->__ss.__pc;
+        #endif
+        #elif defined(__linux__)
+        #if defined(__x86_64__)
+            pc = (uintptr_t)ucontext->uc_mcontext.gregs[REG_RIP];
+        #elif defined(__aarch64__)
+            pc = (uintptr_t)ucontext->uc_mcontext.pc;
+        #endif
+        #endif
+    }
+
+    //  For SIGSEGV/SIGBUS, si_addr is the faulting virtual address: which
+    //  region the bad pointer landed in (stack, heap, a library) narrows
+    //  the corruption source a lot. For SIGABRT it is typically null.
+    char msg[160];
+    int len = snprintf(
+        msg,
+        sizeof(msg),
+        "\n[PA-CRASH] Fatal signal %d, pc 0x%lx, fault address 0x%lx, thread 0x%lx; backtrace:\n",
+        sig,
+        (unsigned long)pc,
+        (sig == SIGSEGV || sig == SIGBUS)
+            ? (unsigned long)(uintptr_t)info->si_addr
+            : 0UL,
+        (unsigned long)(uintptr_t)pthread_self()
+    );
+    if (len > 0){
+        if (write(2, msg, (size_t)len) < 0){
+            //  Nothing more we can do.
+        }
     }
 
     void* frames[128];
@@ -59,7 +105,13 @@ static void crash_backtrace_handler(int sig){
 static void install_crash_backtrace_handlers(){
     static const int sigs[] = { SIGSEGV, SIGBUS, SIGABRT, SIGILL, SIGFPE };
     for (int sig : sigs){
-        signal(sig, crash_backtrace_handler);
+        //  SA_SIGINFO so the handler gets the faulting address.
+        struct sigaction sa;
+        std::memset(&sa, 0, sizeof(sa));
+        sa.sa_sigaction = crash_backtrace_handler;
+        sa.sa_flags = SA_SIGINFO;
+        sigemptyset(&sa.sa_mask);
+        sigaction(sig, &sa, nullptr);
     }
 }
 #else
@@ -79,8 +131,14 @@ int run_command_line_tests(bool augment_color_distortions){
     setvbuf(stdout, nullptr, _IOLBF, 8192);
     cout << "Running parallel unit tests..." << endl;
     ComputerPrograms::CommandLineUnitTestRunner runner(global_logger_command_line());
-    if (runner.run(augment_color_distortions)){
-        return 1;
+    //  [SOCK-DIAG] Temporary: run the suite 3x to raise the odds of
+    //  catching the Mac Intel crash in a single CI run. Remove with the
+    //  rest of the diagnostics.
+    for (int run_no = 1; run_no <= 3; run_no++){
+        fprintf(stderr, "\n[SOCK-DIAG] Test suite run %d of 3\n", run_no);
+        if (runner.run(augment_color_distortions)){
+            return 1;
+        }
     }
     cout << "Running parallel unit tests... Done!" << endl;
     return 0;