ScreenMachine: a closed loop over known screens

SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp

@@ -0,0 +1,97 @@
+/*  Screen Machine
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <algorithm>
+#include "CommonFramework/ImageTools/ImageDiff.h"
+#include "ScreenMachine.h"
+
+namespace PokemonAutomation{
+
+using namespace std::chrono_literals;
+
+
+namespace{
+
+const ScreenState* classify(const std::vector<ScreenState>& states, const ImageViewRGB32& screen){
+    for (const ScreenState& state : states){
+        if (state.matches(screen)){
+            return &state;
+        }
+    }
+    return nullptr;
+}
+
+}
+
+
+const ScreenState& read_screen_state(
+    VideoStream& stream, CancellableScope& scope, StepTrace& trace,
+    const std::vector<ScreenState>& states, VideoSnapshot& frame
+){
+    const ScreenState* last = nullptr;
+    WallClock video_lost = WallClock::max();
+    for (size_t c = 0; c < 20; c++){
+        VideoSnapshot screen = stream.video().snapshot();
+        if (!screen){
+            //  The app resets a frozen capture by itself, which takes a second or two.
+            const WallClock now = current_time();
+            video_lost = std::min(video_lost, now);
+            if (now - video_lost > std::chrono::seconds(10)){
+                trace.fail("No video from the capture card for 10s.");
+            }
+            c--;
+            scope.wait_for(500ms);
+            continue;
+        }
+        video_lost = WallClock::max();
+        const ScreenState* now = classify(states, screen);
+        frame = std::move(screen);
+        if (now != nullptr && now == last){
+            return *now;
+        }
+        last = now;
+        scope.wait_for(150ms);
+    }
+    if (last == nullptr){
+        trace.fail("The screen matches none of the known states.");
+    }
+    return *last;   //  Still changing after 3s: go with the latest read.
+}
+
+
+void ScreenMachineChecks::check(const ScreenState& state, VideoSnapshot frame){
+    const bool same_size = m_last_frame && frame &&
+        m_last_frame->width() == frame->width() && m_last_frame->height() == frame->height();
+    if (same_size && ImageMatch::pixel_RMSD(m_last_frame, frame) < 2.0){
+        if (++m_unchanged >= MAX_UNCHANGED){
+            m_trace.fail("The screen stopped changing: " + std::to_string(MAX_UNCHANGED) +
+                " steps on \"" + state.name + "\" did nothing.");
+        }
+    }else{
+        m_unchanged = 0;
+    }
+    m_last_frame = std::move(frame);
+
+    if (m_previous != nullptr && m_previous != &state && !m_previous->next.empty() &&
+        std::find(m_previous->next.begin(), m_previous->next.end(), state.name) == m_previous->next.end()
+    ){
+        m_trace.fail("Unexpected screen after \"" + m_previous->name + "\": \"" + state.name + "\".");
+    }
+    m_in_a_row = m_previous == &state ? m_in_a_row + 1 : 1;
+    if (m_in_a_row > state.max_in_a_row){
+        m_trace.fail("Screen \"" + state.name + "\" came up " + std::to_string(m_in_a_row) +
+            " times in a row: the step for it is not getting anywhere.");
+    }
+    if (++m_visits[state.name] > state.max_visits){
+        m_trace.fail("Screen \"" + state.name + "\" came up more than " +
+            std::to_string(state.max_visits) + " times: the step for it is not working.");
+    }
+    m_previous = &state;
+}
+
+
+
+}

SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h

@@ -0,0 +1,105 @@
+/*  Screen Machine
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ *  A closed loop over known screens: read the screen, take the one step that screen
+ *  calls for, read again. Every step follows a read, a screen counts only once two
+ *  frames agree, and each state lists the screens it may lead to, so a dropped or
+ *  wrong input shows up as an unexpected screen or a state that keeps coming back.
+ *
+ */
+
+#ifndef PokemonAutomation_CommonTools_ScreenMachine_H
+#define PokemonAutomation_CommonTools_ScreenMachine_H
+
+#include <stdint.h>
+#include <functional>
+#include <map>
+#include <string>
+#include <vector>
+#include "Common/Cpp/Time.h"
+#include "Common/Cpp/CancellableScope.h"
+#include "CommonFramework/ImageTypes/ImageViewRGB32.h"
+#include "CommonFramework/Tools/VideoStream.h"
+#include "CommonFramework/VideoPipeline/VideoFeed.h"
+#include "StepTrace.h"
+
+namespace PokemonAutomation{
+
+
+struct ScreenState{
+    std::string name;
+    //  Whether this is the screen. States are tried in order; the first match wins, so
+    //  the last state should match anything.
+    std::function<bool(const ImageViewRGB32& screen)> matches;
+    //  The one step to take on this screen.
+    std::function<void()> act;
+    //  The screens the step may lead to besides staying here. Empty: any.
+    std::vector<std::string> next;
+    //  Fail once the screen has been seen this many times.
+    size_t max_visits;
+    //  End the run after this state's step.
+    bool terminal = false;
+    //  Fail once the screen has been seen this many times in a row.
+    size_t max_in_a_row = SIZE_MAX;
+};
+
+
+//  The state on screen, once two frames 150ms apart agree. Waits out a video
+//  gap of up to 10s. Fails through `trace` if no state matches.
+const ScreenState& read_screen_state(
+    VideoStream& stream, CancellableScope& scope, StepTrace& trace,
+    const std::vector<ScreenState>& states, VideoSnapshot& frame
+);
+
+//  The checks between reads: the screen must be one the previous state can lead
+//  to, no state may come up too often, and steps must keep changing the screen.
+class ScreenMachineChecks{
+public:
+    ScreenMachineChecks(StepTrace& trace)
+        : m_trace(trace)
+    {}
+    void check(const ScreenState& state, VideoSnapshot frame);
+
+private:
+    //  A step that changes nothing on screen is a dead end (e.g. a menu the states do
+    //  not know): fail after a few instead of running out the visit limit.
+    static constexpr size_t MAX_UNCHANGED = 8;
+
+    StepTrace& m_trace;
+    std::map<std::string, size_t> m_visits;
+    const ScreenState* m_previous = nullptr;
+    VideoSnapshot m_last_frame;
+    size_t m_unchanged = 0;
+    size_t m_in_a_row = 0;
+};
+
+
+//  Runs the states until a terminal one has acted. Returns its name. Fails through
+//  `trace` on an unexpected screen, a state seen too often, a stall or the timeout.
+template <typename ControllerContext>
+std::string run_screen_machine(
+    VideoStream& stream, ControllerContext& context, StepTrace& trace,
+    const std::vector<ScreenState>& states, Milliseconds timeout
+){
+    ScreenMachineChecks checks(trace);
+    const WallClock deadline = current_time() + timeout;
+    while (current_time() < deadline){
+        context.wait_for_all_requests();
+        VideoSnapshot frame;
+        const ScreenState& state = read_screen_state(stream, context, trace, states, frame);
+        trace.screen(state.name);
+        checks.check(state, std::move(frame));
+        state.act();
+        if (state.terminal){
+            context.wait_for_all_requests();
+            return state.name;
+        }
+    }
+    trace.fail("Did not finish within " + std::to_string(timeout.count() / 1000) + "s.");
+}
+
+
+
+}
+#endif

SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp

@@ -0,0 +1,74 @@
+/*  Step Trace
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <cstdio>
+#include "Common/Cpp/PrettyPrint.h"
+#include "Common/Cpp/Filesystem/Filesystem.h"
+#include "CommonFramework/GlobalAutoPaths.h"
+#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h"
+#include "CommonFramework/VideoPipeline/VideoFeed.h"
+#include "StepTrace.h"
+
+namespace PokemonAutomation{
+
+
+StepTrace::StepTrace(VideoStream& stream, std::string label)
+    : m_stream(stream)
+    , m_label(std::move(label))
+    , m_folder(SCREENSHOTS_PATH() + "StepTrace/" + now_to_filestring() + "-" + m_label + "/")
+    , m_start(current_time())
+{
+    Filesystem::create_directories(Filesystem::Path(m_folder));
+    m_stream.log("[" + m_label + "] Trace screenshots: " + m_folder);
+}
+
+void StepTrace::screen(const std::string& name){
+    if (name == m_last){
+        return;
+    }
+    m_last = name;
+    m_recent.push_back(name);
+    if (m_recent.size() > 12){
+        m_recent.pop_front();
+    }
+    m_count++;
+    const double seconds = std::chrono::duration<double>(current_time() - m_start).count();
+    char prefix[16];
+    std::snprintf(prefix, sizeof(prefix), "%03zu-", m_count);
+    std::string path = m_folder + prefix + name + ".png";
+    VideoSnapshot screen = m_stream.video().snapshot();
+    if (screen){
+        screen.frame->save(path);
+    }else{
+        path = "(no video frame)";
+    }
+    m_stream.log("[" + m_label + "] screen " + std::to_string(m_count) + ": " + name +
+        " at " + tostr_fixed(seconds, 1) + "s, " + path);
+}
+
+void StepTrace::action(const std::string& what){
+    m_stream.log("[" + m_label + "] " + what);
+}
+
+std::string StepTrace::recent_screens() const{
+    std::string text;
+    for (const std::string& name : m_recent){
+        text += (text.empty() ? "" : " -> ") + name;
+    }
+    return text.empty() ? "(none)" : text;
+}
+
+void StepTrace::fail(const std::string& message) const{
+    OperationFailedExceptionWithScreenshot::fire(
+        ErrorReportMode::SEND_ERROR_REPORT,
+        m_label + ": " + message + " Recent screens: " + recent_screens() + ". Trace: " + m_folder,
+        m_stream
+    );
+}
+
+
+
+}

SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h

@@ -0,0 +1,53 @@
+/*  Step Trace
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ *  A step log for closed-loop routines: every change of recognized screen is
+ *  logged and saved as a numbered screenshot, every input is logged, and a
+ *  failure reports the screens that led up to it.
+ *
+ */
+
+#ifndef PokemonAutomation_CommonTools_StepTrace_H
+#define PokemonAutomation_CommonTools_StepTrace_H
+
+#include <deque>
+#include <string>
+#include "Common/Cpp/Time.h"
+#include "CommonFramework/Tools/VideoStream.h"
+
+namespace PokemonAutomation{
+
+
+class StepTrace{
+public:
+    //  Screenshots go to <Screenshots>/StepTrace/<time>-<label>/.
+    StepTrace(VideoStream& stream, std::string label);
+
+    //  Records the screen the routine recognized. Logs and saves a screenshot only
+    //  when it differs from the previous one.
+    void screen(const std::string& name);
+
+    //  Records an input or decision.
+    void action(const std::string& what);
+
+    //  The last few screens, oldest first, e.g. "dialog -> keyboard -> dialog".
+    std::string recent_screens() const;
+
+    //  Throws OperationFailedExceptionWithScreenshot with the recent screens appended.
+    [[noreturn]] void fail(const std::string& message) const;
+
+private:
+    VideoStream& m_stream;
+    std::string m_label;
+    std::string m_folder;
+    WallClock m_start;
+    size_t m_count = 0;
+    std::string m_last;
+    std::deque<std::string> m_recent;
+};
+
+
+
+}
+#endif

SerialPrograms/cmake/SourceFiles.cmake

@@ -883,6 +883,10 @@ file(GLOB LIBRARY_SOURCES
     Source/CommonTools/Options/TrainOCRModeOption.h
     Source/CommonTools/Resources/SpriteDatabase.cpp
     Source/CommonTools/Resources/SpriteDatabase.h
+    Source/CommonTools/ScreenMachine/ScreenMachine.cpp
+    Source/CommonTools/ScreenMachine/ScreenMachine.h
+    Source/CommonTools/ScreenMachine/StepTrace.cpp
+    Source/CommonTools/ScreenMachine/StepTrace.h
     Source/CommonTools/StartupChecks/StartProgramChecks.cpp
     Source/CommonTools/StartupChecks/StartProgramChecks.h
     Source/CommonTools/StartupChecks/VideoResolutionCheck.cpp