diff --git a/SerialPrograms/CMakeLists.txt b/SerialPrograms/CMakeLists.txt index 35bb597e32..d0966df32f 100644 --- a/SerialPrograms/CMakeLists.txt +++ b/SerialPrograms/CMakeLists.txt @@ -193,6 +193,11 @@ include(cmake/CoreLib.cmake) # Create the GUI library add_library(SerialProgramsLib STATIC ${CORE_LIBRARY_SOURCES} ${LIBRARY_SOURCES}) +# Data files built into the program (see cmake/EmbedTextFiles.cmake). +include(cmake/EmbedTextFiles.cmake) +pa_embed_text_files(SerialProgramsLib Source/Pokemon/Navigation/TestMaps + Pokemon/Navigation/Pokemon_Navigation_Tests.h PokemonAutomation::Pokemon navigation_test_map_json) + # Create binary target if (APPLE) diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp new file mode 100644 index 0000000000..8d528c4eaa --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp @@ -0,0 +1,97 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "ScreenMachine.h" + +namespace PokemonAutomation{ + +using namespace std::chrono_literals; + + +namespace{ + +const ScreenState* classify(const std::vector& states, const ImageViewRGB32& screen){ + for (const ScreenState& state : states){ + if (state.matches(screen)){ + return &state; + } + } + return nullptr; +} + +} + + +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, StepTrace& trace, + const std::vector& states, VideoSnapshot& frame +){ + const ScreenState* last = nullptr; + WallClock video_lost = WallClock::max(); + for (size_t c = 0; c < 20; c++){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + // The app resets a frozen capture by itself, which takes a second or two. + const WallClock now = current_time(); + video_lost = std::min(video_lost, now); + if (now - video_lost > std::chrono::seconds(10)){ + trace.fail("No video from the capture card for 10s."); + } + c--; + scope.wait_for(500ms); + continue; + } + video_lost = WallClock::max(); + const ScreenState* now = classify(states, screen); + frame = std::move(screen); + if (now != nullptr && now == last){ + return *now; + } + last = now; + scope.wait_for(150ms); + } + if (last == nullptr){ + trace.fail("The screen matches none of the known states."); + } + return *last; // Still changing after 3s: go with the latest read. +} + + +void ScreenMachineChecks::check(const ScreenState& state, VideoSnapshot frame){ + const bool same_size = m_last_frame && frame && + m_last_frame->width() == frame->width() && m_last_frame->height() == frame->height(); + if (same_size && ImageMatch::pixel_RMSD(m_last_frame, frame) < 2.0){ + if (++m_unchanged >= MAX_UNCHANGED){ + m_trace.fail("The screen stopped changing: " + std::to_string(MAX_UNCHANGED) + + " steps on \"" + state.name + "\" did nothing."); + } + }else{ + m_unchanged = 0; + } + m_last_frame = std::move(frame); + + if (m_previous != nullptr && m_previous != &state && !m_previous->next.empty() && + std::find(m_previous->next.begin(), m_previous->next.end(), state.name) == m_previous->next.end() + ){ + m_trace.fail("Unexpected screen after \"" + m_previous->name + "\": \"" + state.name + "\"."); + } + m_in_a_row = m_previous == &state ? m_in_a_row + 1 : 1; + if (m_in_a_row > state.max_in_a_row){ + m_trace.fail("Screen \"" + state.name + "\" came up " + std::to_string(m_in_a_row) + + " times in a row: the step for it is not getting anywhere."); + } + if (++m_visits[state.name] > state.max_visits){ + m_trace.fail("Screen \"" + state.name + "\" came up more than " + + std::to_string(state.max_visits) + " times: the step for it is not working."); + } + m_previous = &state; +} + + + +} diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h new file mode 100644 index 0000000000..f878ff709c --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h @@ -0,0 +1,105 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + * A closed loop over known screens: read the screen, take the one step that screen + * calls for, read again. Every step follows a read, a screen counts only once two + * frames agree, and each state lists the screens it may lead to, so a dropped or + * wrong input shows up as an unexpected screen or a state that keeps coming back. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_H +#define PokemonAutomation_CommonTools_ScreenMachine_H + +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/CancellableScope.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +struct ScreenState{ + std::string name; + // Whether this is the screen. States are tried in order; the first match wins, so + // the last state should match anything. + std::function matches; + // The one step to take on this screen. + std::function act; + // The screens the step may lead to besides staying here. Empty: any. + std::vector next; + // Fail once the screen has been seen this many times. + size_t max_visits; + // End the run after this state's step. + bool terminal = false; + // Fail once the screen has been seen this many times in a row. + size_t max_in_a_row = SIZE_MAX; +}; + + +// The state on screen, once two frames 150ms apart agree. Waits out a video +// gap of up to 10s. Fails through `trace` if no state matches. +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, StepTrace& trace, + const std::vector& states, VideoSnapshot& frame +); + +// The checks between reads: the screen must be one the previous state can lead +// to, no state may come up too often, and steps must keep changing the screen. +class ScreenMachineChecks{ +public: + ScreenMachineChecks(StepTrace& trace) + : m_trace(trace) + {} + void check(const ScreenState& state, VideoSnapshot frame); + +private: + // A step that changes nothing on screen is a dead end (e.g. a menu the states do + // not know): fail after a few instead of running out the visit limit. + static constexpr size_t MAX_UNCHANGED = 8; + + StepTrace& m_trace; + std::map m_visits; + const ScreenState* m_previous = nullptr; + VideoSnapshot m_last_frame; + size_t m_unchanged = 0; + size_t m_in_a_row = 0; +}; + + +// Runs the states until a terminal one has acted. Returns its name. Fails through +// `trace` on an unexpected screen, a state seen too often, a stall or the timeout. +template +std::string run_screen_machine( + VideoStream& stream, ControllerContext& context, StepTrace& trace, + const std::vector& states, Milliseconds timeout +){ + ScreenMachineChecks checks(trace); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + context.wait_for_all_requests(); + VideoSnapshot frame; + const ScreenState& state = read_screen_state(stream, context, trace, states, frame); + trace.screen(state.name); + checks.check(state, std::move(frame)); + state.act(); + if (state.terminal){ + context.wait_for_all_requests(); + return state.name; + } + } + trace.fail("Did not finish within " + std::to_string(timeout.count() / 1000) + "s."); +} + + + +} +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp new file mode 100644 index 0000000000..246bac04a0 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp @@ -0,0 +1,74 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/PrettyPrint.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +StepTrace::StepTrace(VideoStream& stream, std::string label) + : m_stream(stream) + , m_label(std::move(label)) + , m_folder(SCREENSHOTS_PATH() + "StepTrace/" + now_to_filestring() + "-" + m_label + "/") + , m_start(current_time()) +{ + Filesystem::create_directories(Filesystem::Path(m_folder)); + m_stream.log("[" + m_label + "] Trace screenshots: " + m_folder); +} + +void StepTrace::screen(const std::string& name){ + if (name == m_last){ + return; + } + m_last = name; + m_recent.push_back(name); + if (m_recent.size() > 12){ + m_recent.pop_front(); + } + m_count++; + const double seconds = std::chrono::duration(current_time() - m_start).count(); + char prefix[16]; + std::snprintf(prefix, sizeof(prefix), "%03zu-", m_count); + std::string path = m_folder + prefix + name + ".png"; + VideoSnapshot screen = m_stream.video().snapshot(); + if (screen){ + screen.frame->save(path); + }else{ + path = "(no video frame)"; + } + m_stream.log("[" + m_label + "] screen " + std::to_string(m_count) + ": " + name + + " at " + tostr_fixed(seconds, 1) + "s, " + path); +} + +void StepTrace::action(const std::string& what){ + m_stream.log("[" + m_label + "] " + what); +} + +std::string StepTrace::recent_screens() const{ + std::string text; + for (const std::string& name : m_recent){ + text += (text.empty() ? "" : " -> ") + name; + } + return text.empty() ? "(none)" : text; +} + +void StepTrace::fail(const std::string& message) const{ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + m_label + ": " + message + " Recent screens: " + recent_screens() + ". Trace: " + m_folder, + m_stream + ); +} + + + +} diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h new file mode 100644 index 0000000000..e0c4db09d0 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h @@ -0,0 +1,53 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + * A step log for closed-loop routines: every change of recognized screen is + * logged and saved as a numbered screenshot, every input is logged, and a + * failure reports the screens that led up to it. + * + */ + +#ifndef PokemonAutomation_CommonTools_StepTrace_H +#define PokemonAutomation_CommonTools_StepTrace_H + +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/Tools/VideoStream.h" + +namespace PokemonAutomation{ + + +class StepTrace{ +public: + // Screenshots go to /StepTrace/