[DO NOT MERGE] Test runner: hang watchdog that dumps state + all-thread stacks on zero progress

Common/Cpp/Concurrency/Backends/ThreadPool_Default.h

@@ -47,6 +47,15 @@ class ThreadPool_Default final : public ThreadPool{
     }
     virtual WallDuration cpu_time() const override;
 
+    virtual size_t queued_tasks() const override{
+        std::lock_guard<Mutex> lg(m_lock);
+        return m_queue.size();
+    }
+    virtual size_t busy_threads() const override{
+        std::lock_guard<Mutex> lg(m_lock);
+        return m_busy_count;
+    }
+
 
 public:
     [[nodiscard]] virtual AsyncTask dispatch(std::function<void()>&& func) override;

Common/Cpp/Concurrency/Backends/Thread_Qt.tpp

@@ -10,6 +10,7 @@
 #include "Common/Cpp/Containers/Pimpl.tpp"
 #include "Common/Cpp/Concurrency/Qt6.9ThreadBugWorkaround.h"
 #include "Common/Cpp/Concurrency/Thread.h"
+#include "Common/Cpp/TestRunners/HangWatchdog.h"
 
 //using std::cout;
 //using std::endl;
@@ -64,6 +65,8 @@ struct Thread::Data : public QThread{
 
     virtual void run() override{
 //        cout << "starting: " << this << endl;
+        //  Register this thread for the hang watchdog's stack dump.
+        HangWatchdog::register_thread();
         m_function();
 //        m_finished.store(true, std::memory_order_acquire);
 //        cout << "exiting: " << this << endl;

Common/Cpp/Concurrency/Backends/Thread_StdThread.tpp

@@ -9,6 +9,7 @@
 #include <thread>
 #include "Common/Cpp/Containers/Pimpl.tpp"
 #include "Common/Cpp/Concurrency/Thread.h"
+#include "Common/Cpp/TestRunners/HangWatchdog.h"
 
 namespace PokemonAutomation{
 
@@ -46,7 +47,11 @@ Thread::Thread() = default;
 // - Uses CONSTRUCT_TOKEN to construct Pimpl<Data> with Data constructor
 // - Thread starts running immediately (same as std::thread behavior)
 Thread::Thread(std::function<void()>&& function)
-    : m_data(CONSTRUCT_TOKEN, std::move(function))
+    : m_data(CONSTRUCT_TOKEN, [function = std::move(function)](){
+        //  Register the new thread for the hang watchdog's stack dump.
+        HangWatchdog::register_thread();
+        function();
+    })
 {}
 
 void Thread::join(){

Common/Cpp/Concurrency/ThreadPool.h

@@ -34,6 +34,15 @@ class ThreadPool{
     virtual size_t max_threads() const = 0;
     virtual WallDuration cpu_time() const = 0;
 
+    //  Diagnostics: how many tasks are queued up waiting for a thread, and
+    //  how many threads are currently executing a task.
+    virtual size_t queued_tasks() const{
+        return 0;
+    }
+    virtual size_t busy_threads() const{
+        return 0;
+    }
+
 public:
     //  As of this writing, tasks dispatched earlier are not allowed to block
     //  on tasks that are dispatched later as it may cause a deadlock.

Common/Cpp/Logging/FileLogger.cpp

@@ -4,7 +4,9 @@
  *
  */
 
+#include <chrono>
 #include <iostream>
+#include <thread>
 #include "Common/Cpp/PrettyPrint.h"
 #include "Common/Cpp/Filesystem/FilePath.h"
 #include "Common/Cpp/Filesystem/Filesystem.h"
@@ -37,6 +39,23 @@ FileLogger::FileLogger(ThreadPool& thread_pool, FileLoggerConfig config)
 FileLogger::~FileLogger(){
     stop();
 }
+size_t FileLogger::pending_count() const{
+    //  std::mutex has no try_lock_for, so poll try_lock() for up to ~500ms.
+    //  The flusher only ever holds the lock briefly (file I/O happens outside
+    //  of it), so this essentially never times out; the timeout is there so the
+    //  diagnostic itself can never deadlock behind a stuck flusher.
+    std::unique_lock<Mutex> lg(m_lock, std::defer_lock);
+    for (int i = 0; i < 50; i++){
+        if (lg.try_lock()){
+            return m_queue.size();
+        }
+        std::this_thread::sleep_for(std::chrono::milliseconds(10));
+    }
+    return (size_t)-1;
+}
+
+
+
 void FileLogger::stop() noexcept{
     if (!m_thread){
         return;

Common/Cpp/Logging/FileLogger.h

@@ -48,6 +48,13 @@ class FileLogger : public Logger{
     ~FileLogger();
     void stop() noexcept;
 
+    //  Diagnostics: # of log entries currently queued for writing, or
+    //  (size_t)-1 if the lock could not be taken in time.
+    size_t pending_count() const;
+    size_t max_queue_size() const{
+        return m_config.max_queue_size;
+    }
+
     // Logger interface implementation
     virtual void log(const std::string& msg, Color color = Color()) override;
     virtual void log(std::string&& msg, Color color = Color()) override;