From fc0c85b5780bd4bf75c937f8c395131b1dbd223e Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 13:59:18 -0800 Subject: [PATCH 01/23] BDSP AutoStory: the game actions the story steps use, with their tests What the BDSP story table's steps do in the game, without the story itself: saving and the X menu (Actions), the Bag and Potions (Bag), the Pokemon Center by its lit doorway (Center), trainer battles with the best move (Battle), the Starly battle, catching and running from wild battles (WildBattle), the Options the story needs (Settings), and a new game's intro to the bedroom (NewGame), with the detectors they read: the battle menu and move cursor, the X menu's cursor, the name list and the Switch's keyboard. Tests: the intro's decisions, and the lit doorway, Pokemon Center, save card and School lamp screenshots. Adapted in part from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../NintendoSwitch_SystemKeyboardDetector.cpp | 156 +++++ .../NintendoSwitch_SystemKeyboardDetector.h | 67 ++ .../PokemonBDSP_NameListDetector.cpp | 96 +++ .../Inference/PokemonBDSP_NameListDetector.h | 41 ++ .../Source/PokemonBDSP/PokemonBDSP_Tests.cpp | 2 + ...PokemonBDSP_AutoStory_BattleMenuReader.cpp | 150 +++++ .../PokemonBDSP_AutoStory_BattleMenuReader.h | 54 ++ ...kemonBDSP_AutoStory_MenuCursorDetector.cpp | 292 +++++++++ ...PokemonBDSP_AutoStory_MenuCursorDetector.h | 146 +++++ .../PokemonBDSP_AutoStory_Actions.cpp | 340 ++++++++++ .../AutoStory/PokemonBDSP_AutoStory_Actions.h | 63 ++ .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 271 ++++++++ .../PokemonBDSP_AutoStory_Actions_Tests.h | 25 + .../AutoStory/PokemonBDSP_AutoStory_Bag.cpp | 403 ++++++++++++ .../AutoStory/PokemonBDSP_AutoStory_Bag.h | 91 +++ .../PokemonBDSP_AutoStory_Battle.cpp | 602 ++++++++++++++++++ .../AutoStory/PokemonBDSP_AutoStory_Battle.h | 55 ++ .../PokemonBDSP_AutoStory_Center.cpp | 445 +++++++++++++ .../AutoStory/PokemonBDSP_AutoStory_Center.h | 118 ++++ .../PokemonBDSP_AutoStory_NewGame.cpp | 286 +++++++++ .../AutoStory/PokemonBDSP_AutoStory_NewGame.h | 71 +++ .../PokemonBDSP_AutoStory_Settings.cpp | 444 +++++++++++++ .../PokemonBDSP_AutoStory_Settings.h | 31 + .../PokemonBDSP_AutoStory_WildBattle.cpp | 444 +++++++++++++ .../PokemonBDSP_AutoStory_WildBattle.h | 56 ++ SerialPrograms/cmake/SourceFiles.cmake | 24 + 26 files changed, 4773 insertions(+) create mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp create mode 100644 SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp new file mode 100644 index 0000000000..765baba05f --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -0,0 +1,156 @@ +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch_SystemKeyboardDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + +using namespace std::chrono_literals; + + +// Key grid of the QWERTY layout and where each key sits: rows "1".."z", columns "1".."-". +static const char* const KEY_ROWS[] = {"1234567890-", "qwertyuiop/", "asdfghjkl:'", "zxcvbnm,.?!"}; +static constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770}; +static constexpr double KEY_COL_X0 = 0.077; +static constexpr double KEY_COL_STEP = 0.075; + + +SystemKeyboardDetector::SystemKeyboardDetector(Color color) + : m_color(color) + , m_key_panel(0.15, 0.555, 0.65, 0.020) + , m_bottom_bar(0.10, 0.935, 0.35, 0.030) + , m_ok_button(0.875, 0.760, 0.045, 0.020) +{} +void SystemKeyboardDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_key_panel); + items.add(m_color, m_bottom_bar); + items.add(m_color, m_ok_button); +} +bool SystemKeyboardDetector::detect(const ImageViewRGB32& screen){ + if (!is_white(extract_box_reference(screen, m_key_panel)) || + !is_white(extract_box_reference(screen, m_bottom_bar)) + ){ + return false; + } + // OK is blue, or flat grey while the field is empty (disabled). + const ImageStats ok = image_stats(extract_box_reference(screen, m_ok_button)); + return is_solid(ok, {0.172, 0.276, 0.552}) || is_grey(ok, 600, 690); +} + + +bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col){ + // The cursor is a cyan border around one key. Neighbouring keys overlap that border, + // so take the centroid of all cyan pixels and map it to the nearest key. + const ImageFloatBox area(0.02, 0.44, 0.84, 0.38); + ImageViewRGB32 keys = extract_box_reference(screen, area); + double count = 0, sum_x = 0, sum_y = 0; + for (size_t y = 0; y < keys.height(); y++){ + for (size_t x = 0; x < keys.width(); x++){ + const uint32_t pixel = keys.pixel(x, y); + const int red = (pixel >> 16) & 0xff; + const int green = (pixel >> 8) & 0xff; + const int blue = pixel & 0xff; + if (red < 150 && green > 200 && blue > 190){ + count++; + sum_x += (double)x; + sum_y += (double)y; + } + } + } + // The border is about 0.15% of the screen; a few stray pixels are not a cursor. + if (count < 0.0005 * screen.width() * screen.height()){ + return false; + } + const double cx = area.x + sum_x / count / keys.width() * area.width; + const double cy = area.y + sum_y / count / keys.height() * area.height; + col = (int)std::lround((cx - KEY_COL_X0) / KEY_COL_STEP); + row = 0; + for (int r = 1; r < 4; r++){ + if (std::abs(KEY_ROW_Y[r] - cy) < std::abs(KEY_ROW_Y[row] - cy)){ + row = r; + } + } + return 0 <= col && col < 11; +} + +bool system_keyboard_field_has_text(const ImageViewRGB32& screen){ + // Left part of the field, after the caret's position on an empty field. + const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.11, 0.235, 0.19, 0.030))); + return stats.stddev.sum() > 30; +} + + +void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text){ + for (char ch : text){ + const char key = (char)std::tolower((unsigned char)ch); + int target_row = -1; + int target_col = -1; + for (int r = 0; r < 4 && target_row < 0; r++){ + const char* p = std::strchr(KEY_ROWS[r], key); + if (p != nullptr && key != '\0'){ + target_row = r; + target_col = (int)(p - KEY_ROWS[r]); + } + } + if (target_row < 0){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::string("System keyboard: no key for '") + ch + "'.", + stream + ); + } + + // A button pressed while the keyboard slides in is ignored, so read the cursor + // again after each move until it is on the target. + bool arrived = false; + for (size_t attempt = 0; attempt < 10 && !arrived; attempt++){ + context.wait_for_all_requests(); + int row = -1; + int col = -1; + VideoSnapshot screen = stream.video().snapshot(); + if (!screen || !find_system_keyboard_cursor(screen, row, col)){ + pbf_wait(context, 300ms); + continue; + } + if (row == target_row && col == target_col){ + arrived = true; + break; + } + for (; row < target_row; row++) pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms); + for (; row > target_row; row--) pbf_press_dpad(context, DPAD_UP, 100ms, 200ms); + for (; col < target_col; col++) pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms); + for (; col > target_col; col--) pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms); + pbf_wait(context, 300ms); + } + if (!arrived){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::string("System keyboard: the cursor did not reach '") + key + "'.", + stream + ); + } + if (std::isupper((unsigned char)ch)){ + pbf_press_button(context, BUTTON_LCLICK, 100ms, 250ms); + } + pbf_press_button(context, BUTTON_A, 80ms, 300ms); + } + context.wait_for_all_requests(); +} + + + +} +} diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h new file mode 100644 index 0000000000..aa23e7ec81 --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h @@ -0,0 +1,67 @@ +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The Switch's own software keyboard, which games open for names (e.g. the + * player's name in BDSP): detect it, find the key under its cursor, and type + * a string by moving the cursor and checking where it landed. + * + * Measured on 1920x1080 captures of the QWERTY layout. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H +#define PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H + +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonTools/VisualDetector.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// White key panel and bottom bar, and the blue OK button. +class SystemKeyboardDetector : public StaticScreenDetector{ +public: + SystemKeyboardDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_key_panel; + ImageFloatBox m_bottom_bar; + ImageFloatBox m_ok_button; +}; +class SystemKeyboardWatcher : public DetectorToFinder{ +public: + SystemKeyboardWatcher(Color color = COLOR_CYAN) + : DetectorToFinder("SystemKeyboardWatcher", std::chrono::milliseconds(250), color) + {} +}; + + +// The key under the cursor, as row (0: digits .. 3: "zxcv") and column (0..10). +// The keyboard remembers the cursor between uses, so read it, don't assume it. +// Returns false if no cursor border is visible. +bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col); + +// Whether the text field holds any text. Pressing B on an empty field closes the +// keyboard, so delete only while this is true. +bool system_keyboard_field_has_text(const ImageViewRGB32& screen); + +// Types `text` (letters, digits and the symbols on the QWERTY grid). For each character, +// reads the cursor, moves it with the D-pad and checks it arrived before pressing A; an +// uppercase letter is typed with Shift (left stick click). Throws, with a screenshot, if +// the cursor can't be read or doesn't arrive. +void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text); + + + +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp new file mode 100644 index 0000000000..456e914374 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp @@ -0,0 +1,96 @@ +/* Name List Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Kernels/Waterfill/Kernels_Waterfill.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/Images/BinaryImage_FilterRgb32.h" +#include "CommonTools/ImageMatch/ExactImageMatcher.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "PokemonBDSP_NameListDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ +// Row centers, as a fraction of the screen height (1080p: 307, 370, 433, ...). +const double FIRST_ROW_Y = 0.284; +const double ROW_STEP_Y = 0.0587; +const int ROWS = 5; + +// The menu's selection arrows in `image`, as PokemonBDSP_SelectionArrow.cpp +// finds them: that file keeps its finder out of its header, so the same +// waterfill and template match are repeated here rather than changing it. +const ImageMatch::ExactImageMatcher& name_list_arrow(){ + static ImageMatch::ExactImageMatcher matcher(RESOURCE_PATH() + "PokemonBDSP/SelectionArrow.png"); + return matcher; +} +std::vector find_name_list_arrows(const ImageViewRGB32& image){ + using namespace Kernels; + using namespace Kernels::Waterfill; + PackedBinaryMatrix matrix = compress_rgb32_to_binary_max(image, 200, 200, 200); + std::vector objects = find_objects_inplace(matrix, 200); + std::vector ret; + for (const WaterfillObject& object : objects){ + if (object.width() > object.height() || object.height() > 3 * object.width()){ + continue; + } + if (name_list_arrow().rmsd(extract_box_reference(image, object)) <= 90){ + ret.emplace_back(object.min_x, object.min_y, object.max_x, object.max_y); + } + } + return ret; +} +} + + +NameListDetector::NameListDetector(Color color) + : m_color(color) + , m_arrows(0.137, 0.245, 0.030, 0.310) + , m_panel(0.140, 0.245, 0.140, 0.300) +{} +void NameListDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_arrows); + items.add(m_color, m_panel); +} +bool NameListDetector::detect(const ImageViewRGB32& screen){ + return selected_row(screen) >= 0; +} +int NameListDetector::selected_row(const ImageViewRGB32& screen) const{ + // The list is a white panel with a little dark text. A thin margin strip is not + // enough: the border's antialiasing reaches into it on some frames. + const ImageViewRGB32 panel = extract_box_reference(screen, m_panel); + size_t white = 0; + for (size_t y = 0; y < panel.height(); y += 2){ + for (size_t x = 0; x < panel.width(); x += 2){ + const uint32_t p = panel.pixel(x, y); + white += ((p >> 16) & 0xff) > 220 && ((p >> 8) & 0xff) > 220 && (p & 0xff) > 220; + } + } + const size_t samples = ((panel.width() + 1) / 2) * ((panel.height() + 1) / 2); + if (samples == 0 || white < samples * 6 / 10){ + return -1; + } + ImageViewRGB32 area = extract_box_reference(screen, m_arrows); + std::vector arrows = find_name_list_arrows(area); + if (arrows.size() != 1){ + return -1; + } + const double center_y = m_arrows.y + + m_arrows.height * (arrows[0].min_y + arrows[0].max_y) / 2.0 / (double)area.height(); + const int row = (int)std::lround((center_y - FIRST_ROW_Y) / ROW_STEP_Y); + return row >= 0 && row < ROWS ? row : -1; +} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h new file mode 100644 index 0000000000..59fa1efbeb --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h @@ -0,0 +1,41 @@ +/* Name List Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The list of preset names shown for the rival: "I'll tell you...", then four names. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_NameListDetector_H +#define PokemonAutomation_PokemonBDSP_NameListDetector_H + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +class NameListDetector : public StaticScreenDetector{ +public: + NameListDetector(Color color = COLOR_MAGENTA); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + + // The row the arrow is on: 0 is "I'll tell you...", 1 the first preset name. + // -1 if the list is not showing. + int selected_row(const ImageViewRGB32& screen) const; + +private: + Color m_color; + ImageFloatBox m_arrows; + ImageFloatBox m_panel; +}; + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp index 60409b4a26..7e6310b213 100644 --- a/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp @@ -5,6 +5,7 @@ */ #include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h" #include "Programs/AutoStory/PokemonBDSP_AutoStory_Maps_Tests.h" #include "Programs/AutoStory/PokemonBDSP_AutoStory_RoutePlanner_Tests.h" #include "Programs/AutoStory/PokemonBDSP_AutoStory_Segment00_Tests.h" @@ -20,6 +21,7 @@ namespace PokemonBDSP{ void add_tests(UnitTestDatabase& database){ + add_tests_AutoStory_Actions(database); add_tests_AutoStory_Maps(database); add_tests_AutoStory_StoryWorld(database); add_tests_AutoStory_RoutePlanner(database); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp new file mode 100644 index 0000000000..a54eadcc5e --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp @@ -0,0 +1,150 @@ +/* BDSP AutoStory - Battle Menu Reader + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonBDSP_AutoStory_BattleMenuReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +BattleMenuOption battle_menu_selected_option(BattleMenuDetector& menu, const ImageViewRGB32& screen){ + if (!screen || !menu.detect(screen)){ + return BattleMenuOption::UNKNOWN; + } + int selected = -1; + for (int row = 0; row < 4; row++){ + const double offset = row * 0.1075; + bool brackets = false; + // The corner brackets wiggle; all four must agree on one small offset. + const double offsets[] = {0.0, -0.003, 0.003, -0.006, 0.006}; + for (double dy : offsets){ + for (double dx : offsets){ + const ImageFloatBox corners[] = { + {0.793 + dx, 0.562 + offset + dy, 0.018, 0.023}, + {0.976 + dx, 0.562 + offset + dy, 0.018, 0.023}, + {0.793 + dx, 0.652 + offset + dy, 0.018, 0.023}, + {0.976 + dx, 0.652 + offset + dy, 0.018, 0.023}, + }; + bool candidate = true; + for (const ImageFloatBox& box : corners){ + const ImageViewRGB32 patch = extract_box_reference(screen, box); + size_t white = 0; + const size_t total = patch.width() * patch.height(); + for (size_t y = 0; y < patch.height(); y++){ + for (size_t x = 0; x < patch.width(); x++){ + const uint32_t pixel = patch.pixel(x, y); + const unsigned r = (pixel >> 16) & 255; + const unsigned g = (pixel >> 8) & 255; + const unsigned b = pixel & 255; + const unsigned low = std::min(r, std::min(g, b)); + const unsigned high = std::max(r, std::max(g, b)); + white += low > 210 && high - low < 35; + } + } + if (total == 0 || static_cast(white) / total < 0.15){ + candidate = false; + break; + } + } + if (candidate){ + brackets = true; + break; + } + } + if (brackets){ + break; + } + } + if (brackets){ + if (selected >= 0){ + return BattleMenuOption::UNKNOWN; + } + selected = row; + } + } + return static_cast(selected); +} + +BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen){ + BattleMoveMenuState result; + if (!screen) return result; + // Every populated move has a white PP panel, irrespective of move type or + // selected row. Find connected panels rather than assume a 4-row layout. + const ImageFloatBox search{0.90, 0.48, 0.075, 0.505}; + const ImageViewRGB32 patch = extract_box_reference(screen, search); + cv::Mat white((int)patch.height(), (int)patch.width(), CV_8UC1); + for (size_t y = 0; y < patch.height(); y++) for (size_t x = 0; x < patch.width(); x++) { + uint32_t p = patch.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + white.at((int)y, (int)x) = std::min(r, std::min(g, b)) > 225 && + std::max(r, std::max(g, b)) - std::min(r, std::min(g, b)) < 30 ? 255 : 0; + } + cv::Mat labels, stats, centers; + int components = cv::connectedComponentsWithStats(white, labels, stats, centers); + std::vector panels; + for (int component = 1; component < components; component++) { + cv::Rect box(stats.at(component, cv::CC_STAT_LEFT), stats.at(component, cv::CC_STAT_TOP), + stats.at(component, cv::CC_STAT_WIDTH), stats.at(component, cv::CC_STAT_HEIGHT)); + if (box.width < patch.width() * 0.60 || box.height < screen.height() * 0.055 || + box.height > screen.height() * 0.13) continue; + double area = (double)stats.at(component, cv::CC_STAT_AREA); + if (area / box.area() < 0.55) continue; + panels.push_back(box); + } + if (panels.empty() || panels.size() > 4) return result; + std::sort(panels.begin(), panels.end(), [](const cv::Rect& a, const cv::Rect& b){ return a.y < b.y; }); + // Reject inconsistent/overlapping geometry rather than count noise as moves. + for (size_t i = 1; i < panels.size(); i++) { + if (panels[i].y <= panels[i - 1].y + panels[i - 1].height || + std::abs(panels[i].height - panels[0].height) > screen.height() * 0.025) return result; + } + result.move_count = panels.size(); + auto is_corner = [&](double x, double y){ + if (x < 0 || y < 0 || x + 0.018 > 1 || y + 0.023 > 1) return false; + ImageViewRGB32 corner = extract_box_reference(screen, ImageFloatBox{x, y, 0.018, 0.023}); + size_t white_count = 0, total = corner.width() * corner.height(); + for (size_t py = 0; py < corner.height(); py++) for (size_t px = 0; px < corner.width(); px++) { + uint32_t p = corner.pixel(px, py); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + white_count += std::min(r, std::min(g, b)) > 210 && + std::max(r, std::max(g, b)) - std::min(r, std::min(g, b)) < 35; + } + return total && (double)white_count / total >= 0.15; + }; + for (size_t slot = 0; slot < panels.size(); slot++) { + double top = search.y + (double)panels[slot].y / screen.height(); + double bottom = top + (double)panels[slot].height / screen.height(); + result.buttons[slot] = ImageFloatBox{0.665, std::max(0.0, top - 0.01), 0.333, + std::min(1.0, bottom + 0.01) - std::max(0.0, top - 0.01)}; + bool selected = false; + for (double dy : {0.0, -0.003, 0.003, -0.006, 0.006, -0.010, 0.010}) { + for (double dx : {0.0, -0.003, 0.003}) { + if (is_corner(0.663 + dx, top - 0.010 + dy) && + is_corner(0.976 + dx, top - 0.010 + dy) && + is_corner(0.663 + dx, bottom - 0.014 + dy) && + is_corner(0.976 + dx, bottom - 0.014 + dy)) { selected = true; break; } + } + if (selected) break; + } + if (selected) { + if (result.selected_slot) { result.selected_slot = 0; return result; } + result.selected_slot = slot + 1; + } + } + return result; +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h new file mode 100644 index 0000000000..07d5c68eab --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h @@ -0,0 +1,54 @@ +/* BDSP AutoStory - Battle Menu Reader + * + * From: https://github.com/PokemonAutomation/ + * + * Which battle command and which move the cursor is on. Adapted from + * PokemonAutomation/Arduino-Source#1717 by Developer-Butters, which added + * them to PokemonBDSP_BattleMenuDetector; here they sit beside it, so the + * upstream detector stays as it is. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_BattleMenuReader_H +#define PokemonAutomation_PokemonBDSP_AutoStory_BattleMenuReader_H + +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Vertical command order in the BDSP battle menu. +enum class BattleMenuOption{ + UNKNOWN = -1, + BATTLE = 0, + POKEMON = 1, + BAG = 2, + RUN = 3, +}; + +// The command under the cursor: `menu` must see its battle menu, and white +// selection brackets must sit at all four corners of exactly one command. +// Missing or ambiguous brackets read UNKNOWN; the cursor is never guessed. +BattleMenuOption battle_menu_selected_option(BattleMenuDetector& menu, const ImageViewRGB32& screen); + +// Logical move slots are ordered top to bottom among the populated move +// buttons. `selected_slot` is 1 to 4, or 0 when missing or ambiguous. +struct BattleMoveMenuState{ + size_t move_count = 0; + size_t selected_slot = 0; + std::array buttons; +}; +BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen); + + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp new file mode 100644 index 0000000000..9cf2086703 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp @@ -0,0 +1,292 @@ +/* BDSP AutoStory - Menu Cursor Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include + +#include "CommonFramework/VideoPipeline/VideoFeed.h" + +#include "CommonTools/Images/ImageFilter.h" + +#include "PokemonBDSP_AutoStory_MenuCursorDetector.h" + +namespace PokemonAutomation { +namespace NintendoSwitch { +namespace PokemonBDSP { + + +std::string menu_cursor_to_string(MenuCursorPosition pos){ + switch (pos){ + case MenuCursorPosition::POKEDEX: + return "POKEDEX"; + case MenuCursorPosition::POKEMON: + return "POKEMON"; + case MenuCursorPosition::BAG: + return "BAG"; + case MenuCursorPosition::CARD: + return "CARD"; + case MenuCursorPosition::OPTIONS: + return "OPTIONS"; + case MenuCursorPosition::MAP: + return "MAP"; + case MenuCursorPosition::CAPSULES: + return "CAPSULES"; + case MenuCursorPosition::MYSTERYGIFT: + return "MYSTERYGIFT"; + default: + return "UNKNOWN"; + } +} + + +MenuCursorDetector::MenuCursorDetector(Color color) : + m_color(color) + // ------------------------------------------------------------------------- + // Top row — left to right + // ------------------------------------------------------------------------- + // TODO: replace each ImageFloatBox(0,0,0,0) with the real normalized + // (x, y, width, height) for a small region that gains dark pixels when + // that icon is hovered. Use the test program overlays to tune coordinates. + // ------------------------------------------------------------------------- + , m_pokedex_box{ + ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), + }, + m_pokemon_box{ + ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), + }, + m_bag_box{ + ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), + }, + m_card_box{ + ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), + } // ------------------------------------------------------------------------- + // Bottom row — left to right + // 5-icon layout: OPTIONS is the only bottom icon (far-left position). + // 8-icon layout: MAP, CAPSULES, OPTIONS, MYSTERYGIFT from left to right. + // OPTIONS sits at a different x-position in each layout — use a separate + // MenuCursorDetector instance if you need to detect OPTIONS in both. + // ------------------------------------------------------------------------- + , m_options_box{ + ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), + }, + m_map_box{ + ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), + }, + m_capsules_box{ + ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), + }, + m_mysterygift_box{ + ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), + } // ------------------------------------------------------------------------- + // Approach 2 search boxes — one per icon, same area as the dark-pixel box. + // The template PNG is slid across this area at its native size. + // ------------------------------------------------------------------------- + , m_icon_match_box{ + ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), // 0 POKEDEX + ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), // 1 POKEMON + ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), // 2 BAG + ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), // 3 CARD + ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), // 4 OPTIONS (8-icon layout) + ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), // 5 MAP + ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), // 6 CAPSULES + ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), // 7 MYSTERYGIFT + } { +} + + +void MenuCursorDetector::make_overlays(VideoOverlaySet& items) const { + for (const ImageFloatBox* icon_boxes : { + m_pokedex_box, + m_pokemon_box, + m_bag_box, + m_card_box, + m_options_box, + m_map_box, + m_capsules_box, + m_mysterygift_box, + }){ + for (int i = 0; i < BOXES_PER_ICON; i++){ + if (icon_boxes[i].width > 0.0 && icon_boxes[i].height > 0.0){ + items.add(m_color, icon_boxes[i]); + } + } + } + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + if (m_icon_match_box[i].width > 0.0 && m_icon_match_box[i].height > 0.0){ + items.add(m_color, m_icon_match_box[i]); + } + } +} + + +namespace { + +// Count how many of `n` boxes contain >= MIN_DARK_PIXEL_COUNT pixels with +// R, G, B all <= BLACK_CHANNEL_MAX. Boxes with zero width are skipped. +int count_dark_boxes(const ImageViewRGB32& screen, const ImageFloatBox* boxes, int n){ + const uint32_t ch = uint32_t(MenuCursorDetector::BLACK_CHANNEL_MAX); + const uint32_t maxs = 0xff000000u | (ch << 16) | (ch << 8) | ch; + + int count = 0; + for (int i = 0; i < n; i++){ + if (boxes[i].width == 0.0){ + continue; + } + size_t dark_pixels = 0; + filter_rgb32_range(dark_pixels, + extract_box_reference(screen, boxes[i]), + 0xff000000u, + maxs, + Color(0xffffffff), + false); + if (dark_pixels >= MenuCursorDetector::MIN_DARK_PIXEL_COUNT){ + count++; + } + } + return count; +} + +// Pixel-wise RMSD between two same-size images over RGB channels only. +double compute_rmsd(const ImageViewRGB32& a, const ImageViewRGB32& b){ + size_t n = a.width() * a.height(); + if (n == 0){ + return 0.0; + } + double sum_sq = 0.0; + for (size_t y = 0; y < a.height(); y++){ + for (size_t x = 0; x < a.width(); x++){ + uint32_t pa = a.pixel(x, y); + uint32_t pb = b.pixel(x, y); + int dr = int((pa >> 16) & 0xff) - int((pb >> 16) & 0xff); + int dg = int((pa >> 8) & 0xff) - int((pb >> 8) & 0xff); + int db = int(pa & 0xff) - int(pb & 0xff); + sum_sq += double(dr * dr + dg * dg + db * db); + } + } + return std::sqrt(sum_sq / (3.0 * n)); +} + +// Slide `tmpl` (at its native pixel size) across the region of `screen` +// defined by `box`. Returns the minimum RMSD found over all window positions. +double template_match_rmsd(const ImageViewRGB32& screen, const ImageFloatBox& box, const ImageViewRGB32& tmpl){ + ImageViewRGB32 region = extract_box_reference(screen, box); + size_t rw = tmpl.width(), rh = tmpl.height(); + size_t lw = region.width(), lh = region.height(); + if (rw == 0 || rh == 0 || rw > lw || rh > lh){ + return 0.0; + } + + double min_rmsd = std::numeric_limits::max(); + for (size_t dy = 0; dy <= lh - rh; dy++){ + for (size_t dx = 0; dx <= lw - rw; dx++){ + double r = compute_rmsd(region.sub_image(dx, dy, rw, rh), tmpl); + if (r < min_rmsd){ + min_rmsd = r; + } + } + } + return min_rmsd; +} + +} // anonymous namespace + + +MenuCursorPosition MenuCursorDetector::detect_dark_pixels(const ImageViewRGB32& screen) const { + struct IconEntry { + MenuCursorPosition position; + const ImageFloatBox* boxes; + }; + const IconEntry icons[] = { + {MenuCursorPosition::POKEDEX, m_pokedex_box}, + {MenuCursorPosition::POKEMON, m_pokemon_box}, + {MenuCursorPosition::BAG, m_bag_box}, + {MenuCursorPosition::CARD, m_card_box}, + {MenuCursorPosition::OPTIONS, m_options_box}, + {MenuCursorPosition::MAP, m_map_box}, + {MenuCursorPosition::CAPSULES, m_capsules_box}, + {MenuCursorPosition::MYSTERYGIFT, m_mysterygift_box}, + }; + + MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; + int best_count = 0; + + for (const auto& entry : icons){ + int count = count_dark_boxes(screen, entry.boxes, BOXES_PER_ICON); + if (count > best_count){ + best_count = count; + best_pos = entry.position; + } + } + + return (best_count >= REQUIRED_VOTES) ? best_pos : MenuCursorPosition::UNKNOWN; +} + + +MenuCursorPosition MenuCursorDetector::detect_image_match(const ImageViewRGB32& screen) const { + MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; + double best_rmsd = MATCH_RMSD_THRESHOLD; // must exceed this + + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + if (!m_icon_references[i].has_value()){ + continue; + } + if (m_icon_match_box[i].width == 0.0){ + continue; + } + double r = template_match_rmsd(screen, m_icon_match_box[i], *m_icon_references[i]); + if (r > best_rmsd){ + best_rmsd = r; + best_pos = static_cast(i); + } + } + return best_pos; +} + + +MenuCursorPosition MenuCursorDetector::detect(const ImageViewRGB32& screen) const { + MenuCursorPosition p1 = detect_dark_pixels(screen); + MenuCursorPosition p2 = detect_image_match(screen); + + if (p1 == p2){ + return p1; // both agree (or both UNKNOWN) + } + if (p1 == MenuCursorPosition::UNKNOWN){ + return p2; // only approach 2 fired + } + if (p2 == MenuCursorPosition::UNKNOWN){ + return p1; // only approach 1 fired + } + return p1; // conflict: prefer approach 1 +} + + +void MenuCursorDetector::set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference){ + m_icon_references[static_cast(icon)].emplace(std::move(reference)); +} + + +double MenuCursorDetector::icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const { + int idx = static_cast(icon); + if (!m_icon_references[idx].has_value() || m_icon_match_box[idx].width == 0.0){ + return -1.0; + } + return template_match_rmsd(screen, m_icon_match_box[idx], *m_icon_references[idx]); +} + + +MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& /*context*/){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + return MenuCursorPosition::UNKNOWN; + } + MenuCursorDetector detector; + return detector.detect(screen); +} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h new file mode 100644 index 0000000000..e7090deafb --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h @@ -0,0 +1,146 @@ +/* BDSP AutoStory - Menu Cursor Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Detects which icon the cursor is hovering over in the X-menu. + * + * Two independent detection approaches are provided for testing: + * + * Approach 1 — dark pixel count (detect_dark_pixels): + * Each icon has BOXES_PER_ICON small boxes placed over a region that gains + * dark pixels when hovered. A box fires if >= MIN_DARK_PIXEL_COUNT pixels + * in it have R, G, B all <= BLACK_CHANNEL_MAX. An icon is selected when its + * fire count reaches REQUIRED_VOTES. + * Tune BLACK_CHANNEL_MAX upward if real dark pixels are being missed. + * Tune it downward (or raise MIN_DARK_PIXEL_COUNT) to reduce false positives. + * + * Approach 2 — template matching (detect_image_match): + * Each icon has one larger search box and a hovered-state reference PNG loaded + * at runtime via set_icon_reference(). The reference PNG is slid (at its native + * pixel size, without resizing) across the search box and the minimum RMSD over + * all window positions is computed. The icon with the lowest min-RMSD that is + * also <= MATCH_RMSD_THRESHOLD is considered selected. + * Tune MATCH_RMSD_THRESHOLD downward to require a closer match. + * + * detect() is the production method and currently delegates to approach 1. + * It will become a combined vote once both approaches are validated. + * + * Menu layouts: + * 5-icon (early game): POKEDEX POKEMON BAG CARD / OPTIONS + * 8-icon (late game): POKEDEX POKEMON BAG CARD / MAP CAPSULES OPTIONS MYSTERYGIFT + * + * Note: OPTIONS sits at a different screen position in each layout. + * Use separate MenuCursorDetector instances with different box coordinates + * if you need to detect OPTIONS reliably in both layouts. + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H + +#include +#include +#include + +#include "Common/Cpp/Color.h" + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" + +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation { +namespace NintendoSwitch { +namespace PokemonBDSP { + + +enum class MenuCursorPosition { + UNKNOWN = -1, + POKEDEX = 0, + POKEMON = 1, + BAG = 2, + CARD = 3, + OPTIONS = 4, + MAP = 5, + CAPSULES = 6, + MYSTERYGIFT = 7, +}; + +static constexpr int MENU_CURSOR_NUM_ICONS = 8; + +std::string menu_cursor_to_string(MenuCursorPosition pos); + + +class MenuCursorDetector { +public: + // ── Approach 1: dark pixel detection ───────────────────────────────────── + static constexpr int BOXES_PER_ICON = 3; + static constexpr int REQUIRED_VOTES = 1; // out of BOXES_PER_ICON + static constexpr uint8_t BLACK_CHANNEL_MAX = 50; // R,G,B <= this → dark pixel + static constexpr size_t MIN_DARK_PIXEL_COUNT = 1; // dark pixels per box to fire + + // ── Approach 2: template match ─────────────────────────────────────────── + // Reference PNGs are the UN-hovered state of each icon. When hovered, the + // icon gains dark pixels and diverges from the reference, so RMSD increases. + // detect_image_match() selects the icon with the HIGHEST RMSD >= threshold. + // Tune upward to reduce false positives; downward if the correct icon is missed. + static constexpr double MATCH_RMSD_THRESHOLD = 53.0; + + MenuCursorDetector(Color color = COLOR_RED); + + void make_overlays(VideoOverlaySet& items) const; + + // Approach 1: votes across dark-pixel boxes, returns icon with highest + // vote count >= REQUIRED_VOTES, or UNKNOWN. + MenuCursorPosition detect_dark_pixels(const ImageViewRGB32& screen) const; + + // Approach 2: returns the icon with the highest RMSD against its reference, + // provided that RMSD >= MATCH_RMSD_THRESHOLD (high RMSD = hovered). + // Returns UNKNOWN if no references are loaded or nothing exceeds threshold. + MenuCursorPosition detect_image_match(const ImageViewRGB32& screen) const; + + // Production method — currently delegates to detect_dark_pixels(). + MenuCursorPosition detect(const ImageViewRGB32& screen) const; + + // Load a hovered-state reference image for one icon (enables approach 2 for + // that icon). The match box for that icon must already be set in the + // constructor (m_icon_match_box); this method just attaches the reference. + void set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference); + + // Returns the raw RMSD of `icon` against its reference on the given frame. + // Returns -1.0 if no reference is loaded for that icon. + double icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const; + +private: + Color m_color; + + // ── Approach 1 boxes ───────────────────────────────────────────────────── + // 3 boxes per icon targeting regions that gain dark pixels when hovered. + // All initialized to (0,0,0,0) — replace with real normalized coordinates. + ImageFloatBox m_pokedex_box[BOXES_PER_ICON]; + ImageFloatBox m_pokemon_box[BOXES_PER_ICON]; + ImageFloatBox m_bag_box[BOXES_PER_ICON]; + ImageFloatBox m_card_box[BOXES_PER_ICON]; + ImageFloatBox m_options_box[BOXES_PER_ICON]; + ImageFloatBox m_map_box[BOXES_PER_ICON]; + ImageFloatBox m_capsules_box[BOXES_PER_ICON]; + ImageFloatBox m_mysterygift_box[BOXES_PER_ICON]; + + // ── Approach 2 search boxes + references ──────────────────────────────── + // One search box per icon (the area to slide the template across). + // References are populated via set_icon_reference() at runtime. + ImageFloatBox m_icon_match_box[MENU_CURSOR_NUM_ICONS]; + std::array, MENU_CURSOR_NUM_ICONS> m_icon_references; +}; + + +// Captures the current frame from `stream` and runs detect() on it. +MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& context); + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp new file mode 100644 index 0000000000..2e8826bbef --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp @@ -0,0 +1,340 @@ +/* BDSP AutoStory - Actions + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Globals.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Programs/PokemonBDSP_GameNavigation.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_Actions.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +namespace{ + +// Opens the save prompt from the overworld and keeps a screenshot of it: it +// shows the play time and badges, a record of where each checkpoint save +// stands. Returns the map its card names. +std::string open_save_prompt(VideoStream& stream, ProControllerContext& context){ + overworld_to_menu(stream, context); + pbf_press_button(context, BUTTON_R, 80ms, 2000ms); + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + return ""; + } + const std::string folder = SCREENSHOTS_PATH() + "AutoStory/saves/"; + Filesystem::create_directories(Filesystem::Path(folder)); + const std::string path = folder + now_to_filestring() + ".png"; + screen.frame->save(path); + stream.log("Save prompt screenshot: " + path); + const std::string location = read_save_screen_location(*screen.frame); + stream.log("Save screen location: \"" + location + "\"."); + return location; +} + +// Backs out of the save prompt and its menu to the overworld. +void leave_save_prompt(VideoStream& stream, ProControllerContext& context){ + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + OverworldWatcher overworld; + if (wait_until(stream, context, std::chrono::seconds(15), {overworld}) < 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Failed to detect the overworld after leaving the save prompt.", stream); + } +} + +struct IconGrid { + int row, col; +}; + +IconGrid icon_to_grid(MenuCursorPosition pos, int num_icons){ + switch (pos){ + case MenuCursorPosition::POKEDEX: + return {0, 0}; + case MenuCursorPosition::POKEMON: + return {0, 1}; + case MenuCursorPosition::BAG: + return {0, 2}; + case MenuCursorPosition::CARD: + return {0, 3}; + case MenuCursorPosition::OPTIONS: + return {1, num_icons <= 5 ? 0 : 2}; + case MenuCursorPosition::MAP: + return {1, 0}; + case MenuCursorPosition::CAPSULES: + return {1, 1}; + case MenuCursorPosition::MYSTERYGIFT: + return {1, 3}; + default: + return {-1, -1}; + } +} + +} + +std::string read_location_from_save_screen(VideoStream& stream, ProControllerContext& context){ + const std::string location = open_save_prompt(stream, context); + leave_save_prompt(stream, context); + return location; +} + +bool save_game( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& expected_location +) { + static constexpr size_t MAX_RETRIES = 3; + + std::string wrong_place; + for (size_t attempt = 0; attempt < MAX_RETRIES && wrong_place.empty(); attempt++){ + try { + env.console.log("Checkpoint save attempt: " + std::to_string(attempt + 1)); + + // Only mash B on retries - no need to on first attempt + // if already cleanly in the overworld. + if (attempt > 0){ + pbf_mash_button(context, BUTTON_B, 1000ms); + } + + // The feedback-based save of save_game(VideoStream&, ...), split at the + // save prompt to check where it stands. + const std::string location = open_save_prompt(env.console, context); + if (!expected_location.empty() && !location_banner_matches(location, expected_location)){ + wrong_place = "About to save in \"" + location + "\", expected " + expected_location + "; not saving."; + env.console.log(wrong_place, COLOR_RED); + leave_save_prompt(env.console, context); + break; + } + pbf_press_button(context, BUTTON_ZL, 80ms, 5000ms); + + // Return to overworld cleanly. + pbf_mash_button(context, BUTTON_B, 1000ms); + + // Drain R/confirm/B before detection; an earlier overworld frame + // must not report success while saving commands are still queued. + context.wait_for_all_requests(); + + // Verify overworld. Generous timeout: save animation + fade can be slow. + OverworldWatcher overworld; + int ret = wait_until(env.console, context, std::chrono::seconds(15), {overworld}); + if (ret < 0){ + throw OperationFailedExceptionWithScreenshot(ErrorReportMode::SEND_ERROR_REPORT, + "Failed to detect overworld after save.", + env.console); + } + + env.console.log("Checkpoint save verified."); + return true; + + } catch (OperationFailedException& e){ + env.console.log("Checkpoint save failed: " + e.message(), COLOR_RED); + context.wait_for(1000ms); + } + } + // No blind fallback for the wrong place: it would save there. + if (!wrong_place.empty()){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, wrong_place, env.console); + } + + // Blind fallback: reuse the non-feedback save_game(context) from GameNavigation + // rather than duplicating the button sequence manually. + env.console.log("All save retries failed. Using blind fallback.", COLOR_ORANGE); + pbf_mash_button(context, BUTTON_B, 1000ms); + save_game(context); // PokemonBDSP_GameNavigation.h - non-feedback overload + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + + return false; +} + +void wait_for_dialogue(VideoStream& stream, + ProControllerContext& context, + const std::string& label, + Milliseconds timeout){ + auto log_detected = [&] { + if (!label.empty()){ + stream.log(label + " dialogue detected."); + } + }; + + ShortDialogWatcher watcher_cyan(COLOR_CYAN); + int ret = run_until(stream, + context, + [timeout](ProControllerContext& context){ + context.wait_for(timeout); + }, + {{watcher_cyan}}); + if (ret == 0){ + log_detected(); + return; + } + + ShortDialogWatcher watcher_red(COLOR_RED); + ret = run_until(stream, + context, + [timeout](ProControllerContext& context){ + context.wait_for(timeout); + }, + {{watcher_red}}); + if (ret == 0){ + log_detected(); + return; + } + + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + (label.empty() ? "wait_for_dialogue" : label) + ": dialogue not detected within timeout.", + stream); +} + +void open_menu(VideoStream& stream, ProControllerContext& context, MenuCursorPosition target, int num_icons){ + using namespace std::chrono_literals; + + // Previous single opening attempt: + // context.wait_for_all_requests(); + // overworld_to_menu(stream, context); + // pbf_wait(context, 1000ms); + // context.wait_for_all_requests(); + context.wait_for_all_requests(); + + MenuCursorDetector detector; + { + static const char* ICON_NAMES[MENU_CURSOR_NUM_ICONS] = { + "POKEDEX", + "POKEMON", + "BAG", + "CARD", + "OPTIONS", + "MAP", + "CAPSULES", + "MYSTERYGIFT", + }; + // Previous optional template directory: PokemonBDSP/AutoStory. + std::string ref_dir = RESOURCE_PATH() + "PokemonBDSP/Autostory"; + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + std::string path = ref_dir + "/" + ICON_NAMES[i] + ".png"; + // These templates are optional. The dark-pixel cursor detector + // remains available when a template is not packaged. + if (!Filesystem::is_regular_file(path)){ + continue; + } + // Previously constructed the image before checking its availability. + ImageRGB32 img(path); + if (img){ + detector.set_icon_reference(static_cast(i), std::move(img)); + } + } + } + // Previous single detection attempt retained for comparison. + // MenuCursorPosition cur = MenuCursorPosition::UNKNOWN; + // for (int i = 0; i < 10 && cur == MenuCursorPosition::UNKNOWN; i++){ + // context.wait_for(50ms); + // VideoSnapshot snap = stream.video().snapshot(); + // if (snap) cur = detector.detect(snap); + // } + // if (cur == MenuCursorPosition::UNKNOWN){ + // OperationFailedExceptionWithScreenshot::fire( + // ErrorReportMode::SEND_ERROR_REPORT, + // "Menu cursor not detected after opening menu.", + // stream + // ); + // } + + MenuCursorPosition cur = MenuCursorPosition::UNKNOWN; + bool menu_confirmed_open = false; + for (size_t attempt = 1; attempt <= 3; attempt++){ + stream.log("[Open menu] Open and detect attempt " + std::to_string(attempt) + "/3."); + try { + // A late-opening menu may already be visible after a timeout. + // Check before pressing X again, since X would close that menu. + if (!menu_confirmed_open){ + MenuWatcher already_open; + if (wait_until(stream, context, 300ms, {{already_open}}) < 0){ + overworld_to_menu(stream, context); + }else{ + stream.log("[Open menu] Menu already visible; skipping X."); + } + menu_confirmed_open = true; + }else{ + // Cursor detection retries send no input that could close the + // confirmed menu. Keep observing it instead of toggling X. + stream.log("[Open menu] Retrying cursor detection in the open menu."); + } + pbf_wait(context, 1000ms); + context.wait_for_all_requests(); + for (int frame = 0; frame < 10 && cur == MenuCursorPosition::UNKNOWN; frame++){ + context.wait_for(50ms); + const VideoSnapshot snapshot = stream.video().snapshot(); + if (snapshot){ + cur = detector.detect(snapshot); + } + } + if (cur != MenuCursorPosition::UNKNOWN){ + stream.log("[Open menu] Menu and cursor detected on attempt " + std::to_string(attempt) + ".", + COLOR_GREEN); + break; + } + stream.log("[Open menu] Menu cursor not detected on attempt " + std::to_string(attempt) + ".", + COLOR_ORANGE); + } catch (OperationFailedException& error){ + stream.log("[Open menu] Attempt " + std::to_string(attempt) + " failed: " + error.message(), COLOR_ORANGE); + } + } + if (cur == MenuCursorPosition::UNKNOWN){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Menu or cursor detection failed after 3 opening attempts.", + stream); + } + + IconGrid cur_g = icon_to_grid(cur, num_icons); + IconGrid tgt_g = icon_to_grid(target, num_icons); + if (tgt_g.row == -1){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Unknown target menu icon.", + stream); + } + + if (cur_g.row != tgt_g.row){ + DpadPosition vert = (cur_g.row < tgt_g.row) ? DPAD_DOWN : DPAD_UP; + pbf_press_dpad(context, vert, 80ms, 300ms); + context.wait_for_all_requests(); + } + + int col_diff = tgt_g.col - cur_g.col; + if (col_diff != 0){ + DpadPosition horiz = (col_diff > 0) ? DPAD_RIGHT : DPAD_LEFT; + for (int i = 0; i < std::abs(col_diff); i++){ + pbf_press_dpad(context, horiz, 80ms, 300ms); + context.wait_for_all_requests(); + } + } + + pbf_press_button(context, BUTTON_A, 80ms, 200ms); +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h new file mode 100644 index 0000000000..a2fa75786e --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h @@ -0,0 +1,63 @@ +/* BDSP AutoStory - Actions + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * The game's menus as the story uses them: saving, opening the X menu, and + * waiting for a dialogue. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Actions_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Actions_H + +#include +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Save the game from the overworld with retries. Returns true if the save was +// verified via overworld detection, false if the blind fallback was used. +// With `expected_location`, the map named on the save screen's card must be +// it (compared as a location banner is), or the save is backed out of and +// the function fails with a screenshot: r124a saved in Sandgem Town after a +// walk that was sure it stood on Route 202. +bool save_game( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& expected_location = "" +); + +// Opens the save screen from the overworld, reads where the game says we +// are (read_save_screen_location() in the Location module), and backs out +// without saving. +std::string read_location_from_save_screen(VideoStream& stream, ProControllerContext& context); + +// Wait (without pressing any buttons) until a short dialogue box appears. +// Logs `label + " dialogue detected."` when found (omit label to skip logging). +// Throws OperationFailedException if no dialogue appears within `timeout`. +void wait_for_dialogue( + VideoStream& stream, + ProControllerContext& context, + const std::string& label = "", + Milliseconds timeout = std::chrono::seconds(60) +); + +// Opens the X-menu from the overworld, navigates the cursor to `target`, and presses A. +// num_icons: 5 (early-game layout) or 8 (late-game layout). +void open_menu(VideoStream& stream, ProControllerContext& context, MenuCursorPosition target, int num_icons); + + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp new file mode 100644 index 0000000000..6abbf23dde --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -0,0 +1,271 @@ +/* BDSP AutoStory - Actions Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The new game's intro decisions, and the screenshot checks behind the + * Pokemon Center, the save card and the Jubilife School's lamp. Screenshots + * are in the unit test resources (the CommandLineTests checkout); a test + * whose screenshot is missing is skipped. + * + */ + +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "Data/PokemonBDSP_AutoStory_StoryMaps.h" +#include "Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_Center.h" +#include "PokemonBDSP_AutoStory_NewGame.h" +#include "PokemonBDSP_AutoStory_Actions_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + + +class Test_IntroDecisions : public UnitTest{ +public: + Test_IntroDecisions() + : UnitTest("PokemonBDSP::AutoStory - new-game intro decisions") + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + struct Case{ + const char* screen; + IntroProgress progress; + IntroStep expected; + }; + const IntroProgress start{false, false}; + const IntroProgress named{true, false}; + const IntroProgress done{true, true}; + const Case cases[] = { + {"user-select", start, IntroStep::SELECT_USER}, + {"home", done, IntroStep::FAIL_NOT_IN_GAME}, + {"keyboard", start, IntroStep::ENTER_PLAYER_NAME}, + {"keyboard", named, IntroStep::CANCEL_RIVAL_KEYBOARD}, + {"rival-list-tell", named, IntroStep::MOVE_TO_PRESET}, + {"rival-list-preset", named, IntroStep::PICK_PRESET}, + {"menu", done, IntroStep::CLOSE_MENU_AND_FINISH}, + // The appearance grid has its prompt in a dialogue box. + {"dialog", start, IntroStep::ADVANCE}, + {"dialog", done, IntroStep::ADVANCE}, + // Language and the appearance preview: no X before the bedroom is possible. + {"other", start, IntroStep::ADVANCE}, + {"other", named, IntroStep::ADVANCE}, + {"other", done, IntroStep::PROBE_FOR_MENU}, + }; + for (const Case& c : cases){ + const IntroStep actual = decide_intro_step(c.screen, c.progress); + if (actual != c.expected){ + return std::string("Screen \"") + c.screen + "\" (player named " + + (c.progress.player_named ? "yes" : "no") + ", rival named " + + (c.progress.rival_named ? "yes" : "no") + "): step " + + std::to_string((int)actual) + ", expected " + std::to_string((int)c.expected) + "."; + } + } + if (advance_after_probe(true)){ + return "A after X changed the screen would pick from whatever X opened."; + } + if (!advance_after_probe(false)){ + return "A must advance a screen X left unchanged."; + } + return true; + } +}; + + +// The Mart's doorway on the failure screenshots of r121a and r122a, kept in +// the unit test resources (the CommandLineTests checkout): where it is +// relative to the player, who is in the middle of the screen. +class Test_LitDoorwayScreenshots : public UnitTest{ +public: + Test_LitDoorwayScreenshots() + : UnitTest("PokemonBDSP::AutoStory - lit doorway screenshots") + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + struct Case{ + const char* file; + double tiles_across_min; + double tiles_across_max; + double rows_up_min; + double rows_up_max; + }; + const Case cases[] = { + // r121a: below-left of the door, which is part of a tile east, straight ahead. + {"PokemonBDSP/AutoStory/SandgemMartDoorUpRight-20261030-023443.png", 0.2, 0.7, 0.5, 1.5}, + // r122a: a tile west of the door and three or four rows south of its front. + {"PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", -1.3, -0.5, 2.5, 4.5}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const std::optional doorway = find_lit_doorway(screen); + if (!doorway){ + return std::string(c.file) + ": no lit doorway found."; + } + logger.log(std::string(c.file) + ": doorway at " + std::to_string(doorway->x) + ", " + + std::to_string(doorway->y) + ": " + std::to_string(doorway->tiles_across) + " tiles across, " + + std::to_string(doorway->rows_up) + " rows up."); + if (doorway->tiles_across < c.tiles_across_min || doorway->tiles_across > c.tiles_across_max){ + return std::string(c.file) + ": doorway " + std::to_string(doorway->tiles_across) + + " tiles across, expected " + std::to_string(c.tiles_across_min) + " to " + + std::to_string(c.tiles_across_max) + "."; + } + if (doorway->rows_up < c.rows_up_min || doorway->rows_up > c.rows_up_max){ + return std::string(c.file) + ": doorway " + std::to_string(doorway->rows_up) + + " rows up, expected " + std::to_string(c.rows_up_min) + " to " + + std::to_string(c.rows_up_max) + "."; + } + } + return true; + } +}; + + +// The save prompt r124a took beside Sandgem's Poke Mart, sure it stood on +// Route 202 (in CommandLineTests under PokemonBDSP/AutoStory): its card +// names Sandgem Town, which the save must read before confirming. +class Test_SaveScreenLocation : public UnitTest{ +public: + Test_SaveScreenLocation() + : UnitTest("PokemonBDSP::AutoStory - save screen location") + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + const std::string path = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/SaveScreenSandgemTown-20261030-034444.png"; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const std::string read = read_save_screen_location(ImageRGB32(path)); + logger.log("Save screen location read: \"" + read + "\"."); + if (!location_banner_matches(read, MAP_SANDGEM_TOWN)){ + return "Save screen location \"" + read + "\" should match Sandgem Town."; + } + if (location_banner_matches(read, MAP_ROUTE_202)){ + return "Save screen location \"" + read + "\" should not match Route 202."; + } + return true; + } +}; + + +// The two shots around r126's door walk: below the Center's sign, outside, +// and on the entrance mat inside, where the lit floor read as a doorway and +// the walk kept lining up with it (runs 2 and 3). +class Test_PokemonCenterInside : public UnitTest{ +public: + Test_PokemonCenterInside() + : UnitTest("PokemonBDSP::AutoStory - Pokemon Center inside") + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + struct Case{ + const char* file; + bool inside; + bool doorway; + }; + const Case cases[] = { + {.file = "PokemonBDSP/AutoStory/SandgemCenterInside-20261030-053555.png", .inside = true, .doorway = false}, + // r127c: walked in by the turn north at the door, facing north. + {.file = "PokemonBDSP/AutoStory/SandgemCenterInsideFacingNorth-r127c.png", .inside = true, .doorway = false}, + {.file = "PokemonBDSP/AutoStory/SandgemCenterBelowSign-20261030-053527.png", .inside = false, .doorway = true}, + {.file = "PokemonBDSP/AutoStory/SandgemMartDoorUpRight-20261030-023443.png", .inside = false, .doorway = true}, + {.file = "PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", .inside = false, .doorway = true}, + // Jubilife (2026-10-30): inside its Center twice, at its door, and + // on the doorstep after walking out (the banner up). + {.file = "PokemonBDSP/AutoStory/JubilifeCenterInside-20261030-c6.png", .inside = true, .doorway = false}, + {.file = "PokemonBDSP/AutoStory/JubilifeCenterInside-20261030-c4.png", .inside = true, .doorway = false}, + {.file = "PokemonBDSP/AutoStory/JubilifeCenterFrontDoor-20261030-q2.png", .inside = false, .doorway = true}, + {.file = "PokemonBDSP/AutoStory/JubilifeCenterDoorstep-20261030-c5.png", .inside = false, .doorway = true}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const bool inside = pokemon_center_inside(screen); + const std::optional doorway = find_lit_doorway(screen); + logger.log(std::string(c.file) + ": inside " + std::to_string(inside) + ", doorway " + + (doorway ? std::to_string(doorway->tiles_across) + " across, " + std::to_string(doorway->rows_up) + " rows up" : "none")); + if (inside != c.inside){ + return std::string(c.file) + ": expected " + (c.inside ? "inside" : "outside") + + " a Pokemon Center, read " + (inside ? "inside" : "outside") + "."; + } + if (doorway.has_value() != c.doorway){ + return std::string(c.file) + ": expected " + (c.doorway ? "a doorway" : "no doorway") + + ", read " + (doorway ? "one" : "none") + "."; + } + } + return true; + } +}; + + +// The School's lamp at night, from its doorsteps' row (2026-10-30): where +// the door took us in, the lamp reads within half a tile of the door; a +// tile either side, more. +class Test_SchoolLamp : public UnitTest{ +public: + Test_SchoolLamp() + : UnitTest("PokemonBDSP::AutoStory - Jubilife School lamp") + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + const ImageFloatBox box(0.55, 0.20, 0.30, 0.10); + struct Case{ const char* file; double min; double max; }; + const Case cases[] = { + {"PokemonBDSP/AutoStory/JubilifeSchoolRightOfDoor-20261030.png", -1.5, -0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolOneTileShort-20261030.png", -1.5, -0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolAtDoor-20261030.png", -0.5, 0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolAtDoor2-20261030.png", -0.5, 0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolDoorstep-20261030.png", -0.5, 0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolOneTilePast-20261030.png", 0.5, 1.5}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const std::optional lamp = find_lit_lamp_x(screen, box); + if (!lamp){ + return std::string(c.file) + ": no lit lamp found."; + } + const double across = tiles_across_at(*lamp) - tiles_across_at(0.67); + logger.log(std::string(c.file) + ": lamp at " + std::to_string(*lamp) + ", door " + std::to_string(across) + " tiles across."); + if (across < c.min || across > c.max){ + return std::string(c.file) + ": door " + std::to_string(across) + " tiles across, expected " + + std::to_string(c.min) + " to " + std::to_string(c.max) + "."; + } + } + return true; + } +}; + + +} + + +void add_tests_AutoStory_Actions(UnitTestDatabase& database){ + database.add(); + database.add(); + database.add(); + database.add(); + database.add(); +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h new file mode 100644 index 0000000000..b6b09c7d32 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h @@ -0,0 +1,25 @@ +/* BDSP AutoStory - Actions Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Actions_Tests_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Actions_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Tests that need no console: the new game's intro decisions, and the screenshot checks of the Pokemon Center, the save card and the School's lamp. +void add_tests_AutoStory_Actions(UnitTestDatabase& database); + + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp new file mode 100644 index 0000000000..70a24e5350 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp @@ -0,0 +1,403 @@ +/* BDSP AutoStory - Bag + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/OCR/OCR_RawTesseractOCR.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_Actions.h" +#include "PokemonBDSP_AutoStory_Bag.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +namespace{ + +ImageFloatBox bag_list_icon_box(size_t row){ + return ImageFloatBox(0.449, 0.212 + (row - 1) * 0.075, 0.033, 0.050); +} + +ImageFloatBox bag_list_name_box(size_t row){ + // Previous height included the faint lower selection border in rows 2/6. + // return ImageFloatBox(0.492, 0.213 + (row - 1) * 0.075, 0.335, 0.052); + return ImageFloatBox(0.492, 0.213 + (row - 1) * 0.075, 0.335, 0.050); +} + +int bag_confirm_pocket(VideoStream& stream, ProControllerContext& context){ + int previous = 0; + size_t matches = 0; + for (size_t check = 0; check < 10; check++){ + context.wait_for_all_requests(); + const auto frame = stream.video().snapshot(); + const int pocket = frame ? bag_pocket_position(frame) : 0; + matches = pocket != 0 && pocket == previous ? matches + 1 : pocket != 0 ? 1 : 0; + previous = pocket; + if (matches >= 3){ + return pocket; + } + pbf_wait(context, 100ms); + } + return 0; +} + +bool bag_navigate_to_pocket(VideoStream& stream, ProControllerContext& context, int target){ + int current = bag_confirm_pocket(stream, context); + for (size_t attempt = 0; attempt < 3; attempt++){ + const auto plan = plan_bag_pocket_move(current, target); + if (plan.action == BagPocketAction::CONFIRM){ + return true; + } + if (plan.action == BagPocketAction::UNKNOWN){ + stream.log("[Bag search] Selected pocket not detected.", COLOR_ORANGE); + return false; + } + stream.log("[Bag search] Pocket " + std::to_string(current) + " to " + std::to_string(target) + ": " + + (plan.action == BagPocketAction::RIGHT ? "right " : "left ") + std::to_string(plan.presses) + + " presses, attempt " + std::to_string(attempt + 1) + "/3."); + for (size_t press = 0; press < plan.presses; press++){ + pbf_press_dpad(context, plan.action == BagPocketAction::RIGHT ? DPAD_RIGHT : DPAD_LEFT, 80ms, 150ms); + } + pbf_wait(context, 350ms); + current = bag_confirm_pocket(stream, context); + stream.log( + "[Bag search] Final pocket=" + std::to_string(current) + ", expected=" + std::to_string(target) + "."); + if (current == target){ + return true; + } + } + stream.log("[Bag search] Target pocket not reached after 3 movement batches.", COLOR_ORANGE); + return false; +} + +// Slot 1's colored HP bar in the Bag. Read fill position rather than OCR. +double potion_hp_fraction(const ImageViewRGB32& frame){ + if (bag_pocket_position(frame) == 0){ + return -1; + } + const ImageViewRGB32 bar = extract_box_reference(frame, ImageFloatBox{0.137, 0.160, 0.202, 0.016}); + size_t filled = 0; + for (size_t x = 0; x < bar.width(); x++){ + size_t color = 0; + for (size_t y = 0; y < bar.height(); y++){ + uint32_t p = bar.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + const bool green = g > r + 30 && g > b + 20; + const bool yellow = r > 150 && g > 120 && b < 100; + const bool red = r > 150 && r > g + 50 && r > b + 50; + color += green || yellow || red; + } + filled += color > bar.height() / 4; + } + return bar.width() ? (double)filled / bar.width() : -1; +} + +double confirmed_potion_hp(VideoStream& stream, ProControllerContext& context){ + double previous = -1; + size_t stable = 0; + for (size_t check = 0; check < 20; check++){ + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + double hp = frame ? potion_hp_fraction(frame) : -1; + if (hp >= 0 && previous >= 0 && std::abs(hp - previous) <= 0.015){ + if (++stable >= 3){ + return hp; + } + }else{ + stable = 0; + } + previous = hp; + pbf_wait(context, 150ms); + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Slot 1 HP bar could not be confirmed.", + stream); +} + +} + +int bag_pocket_position(const ImageViewRGB32& frame){ + if (!frame){ + return 0; + } + int selected = 0; + for (int pocket = 1; pocket <= 8; pocket++){ + // The lower left edge of each tab is below the red NEW badge. + // This strip includes both the normal outline and enlarged selection. + const auto edge = + extract_box_reference(frame, ImageFloatBox(0.570 + (pocket - 1) * 0.0421, 0.098, 0.0045, 0.021)); + size_t colored = 0; + for (size_t y = 0; y < edge.height(); y++){ + for (size_t x = 0; x < edge.width(); x++){ + const uint32_t pixel = edge.pixel(x, y); + const int r = (pixel >> 16) & 255; + const int g = (pixel >> 8) & 255; + const int b = pixel & 255; + // Red when focused, blue when the item list has focus. + colored += (r > 160 && r > g + 60 && r > b + 60) || (b > 160 && b > r + 40 && b > g + 20); + } + } + const double pixels = (double)edge.width() * edge.height(); + if (pixels > 0 && colored / pixels >= 0.18){ + if (selected != 0){ + return 0; + } + selected = pocket; + } + } + return selected; +} + +// One narrow strip on the left side of the red selection border. Divide its +// height into the seven row bands; the item's art and NEW badge are outside it. +int bag_cursor_position(const ImageViewRGB32& frame){ + if (!frame){ + return 0; + } + const ImageViewRGB32 strip = extract_box_reference(frame, ImageFloatBox(0.431, 0.229, 0.005, 0.470)); + double best = 0.0; + double second = 0.0; + int position = 0; + for (int row = 0; row < 7; row++){ + const size_t top = static_cast(strip.height() * (row * 0.075 / 0.470)); + const size_t bottom = static_cast(strip.height() * ((row * 0.075 + 0.020) / 0.470)); + size_t red = 0; + size_t total = 0; + for (size_t y = top; y < bottom && y < strip.height(); y++){ + for (size_t x = 0; x < strip.width(); x++){ + const uint32_t pixel = strip.pixel(x, y); + const unsigned r = (pixel >> 16) & 255; + const unsigned g = (pixel >> 8) & 255; + const unsigned b = pixel & 255; + red += r > 180 && g < 120 && b < 120; + total++; + } + } + const double score = total == 0 ? 0.0 : static_cast(red) / total; + if (score > best){ + second = best; + best = score; + position = row + 1; + }else if (score > second){ + second = score; + } + } + return best >= 0.15 && best - second >= 0.10 ? position : 0; +} + +bool bag_list_is_empty(const ImageViewRGB32& frame){ + if (!frame || bag_pocket_position(frame) == 0 || bag_cursor_position(frame) != 0){ + return false; + } + // Empty pockets have no item icons. The central Empty label is outside + // this column; counting dark pixels in the name area included that label. + const auto icons = extract_box_reference(frame, ImageFloatBox(0.449, 0.213, 0.032, 0.500)); + const auto stats = image_stats(icons); + return stats.average.r > 220 && stats.average.g > 220 && stats.average.b > 220 && stats.stddev.sum() < 20; +} + +BagPocketMovePlan plan_bag_pocket_move(int current, int target){ + if (current < 1 || current > 8 || target < 1 || target > 8){ + return {}; + } + const int right = (target - current + 8) % 8; + const int left = (current - target + 8) % 8; + if (right == 0){ + return {BagPocketAction::CONFIRM, 0}; + } + return right <= left ? BagPocketMovePlan{BagPocketAction::RIGHT, static_cast(right)} + : BagPocketMovePlan{BagPocketAction::LEFT, static_cast(left)}; +} + +bool find_bag_item_by_name( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket, + int num_icons +){ + open_menu(stream, context, MenuCursorPosition::BAG, num_icons); + pbf_wait(context, 1000ms); + return find_bag_item_in_open_bag(stream, context, name, target_pocket); +} + +bool find_bag_item_in_open_bag( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket +){ + context.wait_for_all_requests(); + if (!bag_navigate_to_pocket(stream, context, target_pocket)) return false; + + // No reader for the game's language: the cursor's row, the pocket's + // first, if the pocket has one. Early in the story the pocket holds only + // what was bought or given (Potions in Medicine). + if (!story_reads_text()){ + const VideoSnapshot frame = stream.video().snapshot(); + const bool any = frame && !bag_list_is_empty(*frame.frame) && bag_cursor_position(*frame.frame) != 0; + stream.log("[Bag search] " + name + ": no text reader for the game's language; " + + (any ? "taking the pocket's cursor row." : "the pocket is empty."), COLOR_ORANGE); + return any; + } + + // Walk down the list reading the cursor row. The list does not wrap, so + // turn around once the cursor stops moving. + DpadPosition direction = DPAD_DOWN; + std::string previous; + size_t same = 0; + size_t confirmed = 0; + size_t unknown_cursor = 0; + for (size_t scan = 0; scan < 128; scan++){ + context.wait_for_all_requests(); + const VideoSnapshot frame = stream.video().snapshot(); + if (!frame || bag_list_is_empty(*frame.frame)){ + stream.log("[Bag search] " + name + ": pocket is empty.", COLOR_ORANGE); + return false; + } + const int cursor = bag_cursor_position(*frame.frame); + if (cursor == 0){ + if (++unknown_cursor >= 3){ + stream.log("[Bag search] " + name + ": cursor not found.", COLOR_ORANGE); + return false; + } + pbf_wait(context, 200ms); + continue; + } + unknown_cursor = 0; + const std::string text = OCR::tesseract_ocr_read(Language::English, + extract_box_reference(*frame.frame, bag_list_name_box(cursor))); + std::string normalized; + for (unsigned char ch : text){ + if (std::isalpha(ch)) normalized.push_back((char)std::tolower(ch)); + } + stream.log("[Bag search] Row " + std::to_string(cursor) + " reads \"" + normalized + "\"."); + if (normalized == name){ + if (++confirmed >= 2){ + stream.log("[Bag search] " + name + " confirmed in row " + std::to_string(cursor) + ".", COLOR_GREEN); + return true; + } + pbf_wait(context, 150ms); + continue; + } + confirmed = 0; + same = normalized == previous ? same + 1 : 0; + previous = normalized; + if (same >= 2){ + if (direction == DPAD_UP) break; + direction = DPAD_UP; + same = 0; + } + pbf_press_dpad(context, direction, 80ms, 300ms); + } + stream.log("[Bag search] " + name + " not found.", COLOR_ORANGE); + return false; +} + +bool bag_has_item(VideoStream& stream, ProControllerContext& context, const std::string& name, int target_pocket) { + const bool found = find_bag_item_by_name(stream, context, name, target_pocket, 5); + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + OverworldWatcher overworld; + if (wait_until(stream, context, std::chrono::seconds(10), {{overworld}}) != 0) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Bag: Did not return to the overworld after looking for " + name + ".", stream); + } + return found; +} + +void use_potion_first_pokemon(VideoStream& stream, ProControllerContext& context, double heal_below) { + OverlayBoxScope hp_overlay(stream.overlay(), COLOR_GREEN, {0.137, 0.160, 0.202, 0.016}, ""); + bool healed = false; + for (size_t dose = 0; dose < 3; dose++){ + stream.log("[Potion] Finding Bag/Medicine/Potion; dose " + std::to_string(dose + 1) + "/3."); + if (!find_bag_item_by_name(stream, context, "potion", 1, 5)) { + if (dose == 0 && heal_below < 1.0) { + // Only a check: without Potions there is nothing to do here. + stream.log("[Potion] No Potion in the Bag; skipping the HP check.", COLOR_ORANGE); + healed = true; + break; + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Could not verify Potion in the Medicine pocket.", + stream); + } + const double before = confirmed_potion_hp(stream, context); + stream.log("[Potion] Slot 1 HP bar before use=" + std::to_string(before * 100) + "%."); + if (before >= 0.98) { healed = true; break; } + if (dose == 0 && before >= heal_below) { + stream.log("[Potion] Slot 1 HP is at least " + std::to_string(heal_below * 100) + "%; no Potion needed."); + healed = true; + break; + } + if (before <= 0.005) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Slot 1 appears fainted; a Potion cannot revive it.", + stream); + } + pbf_press_button(context, BUTTON_A, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + OverworldWatcher overworld; + if (wait_until(stream, context, std::chrono::seconds(10), {{overworld}}) != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Did not return to the overworld after use.", + stream); + } + // Reopen Bag to verify the result, including when the last Potion was used. + open_menu(stream, context, MenuCursorPosition::BAG, 5); + pbf_wait(context, 750ms); + context.wait_for_all_requests(); + const double after = confirmed_potion_hp(stream, context); + stream.log("[Potion] Slot 1 HP bar after use=" + std::to_string(after * 100) + "%."); + if (after <= before + 0.005){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Slot 1 HP did not increase; refusing to continue movement.", + stream); + } + if (after >= 0.98){ + healed = true; + break; + } + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + } + if (!healed){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Full HP not confirmed after three uses.", + stream); + } + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + OverworldWatcher overworld; + if (wait_until(stream, context, std::chrono::seconds(10), {{overworld}}) != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Final overworld return not confirmed.", + stream); + } + stream.log("[Potion] Slot 1 HP bar full and overworld confirmed."); +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h new file mode 100644 index 0000000000..8e0e73695e --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h @@ -0,0 +1,91 @@ +/* BDSP AutoStory - Bag + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Finding an item in the Bag, and Potions on the lead. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Bag_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Bag_H + +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Pocket positions 1-8: Medicine, Balls, Battle Items, Berries, Other Items, +// TMs, Treasures, Key Items. Zero means the selected tab is uncertain. +int bag_pocket_position(const ImageViewRGB32& frame); +// Whether the open pocket shows no item. +bool bag_list_is_empty(const ImageViewRGB32& frame); +// The cursor's row, 1-7, or 0 if unknown. +int bag_cursor_position(const ImageViewRGB32& frame); + +enum class BagPocketAction{ + UNKNOWN, + CONFIRM, + LEFT, + RIGHT, +}; +struct BagPocketMovePlan{ + BagPocketAction action = BagPocketAction::UNKNOWN; + size_t presses = 0; +}; +// Circular eight-pocket route; ties prefer Right. Invalid positions send no input. +BagPocketMovePlan plan_bag_pocket_move(int current, int target); + +// Starts in the overworld, opens the Bag on `target_pocket`, and moves the cursor +// onto the item whose name reads `name` (letters only, lower case, e.g. "potion"). +// Reads the cursor row's name with OCR, so it does not depend on the capture +// card's colors or sharpness the way a template match does. +// Returns false if the pocket is empty or the name is not found. +bool find_bag_item_by_name( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket, + int num_icons = 8 +); +// The same search in a Bag that is already open, such as the one a battle's +// Bag command opens. +bool find_bag_item_in_open_bag( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket +); + +// Open the Bag from the overworld, look for `name` in the pocket, and close +// it again. True if the item is there. +bool bag_has_item( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket +); + +// Read the lead's HP bar in the Bag and, if it is below `heal_below` (a +// fraction of full), use Potions until it is full. heal_below = 1 always +// heals; 0.5 only checks and heals when the lead is under half, and is +// skipped when the Bag has no Potion. +void use_potion_first_pokemon( + VideoStream& stream, + ProControllerContext& context, + double heal_below = 1.0 +); + + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp new file mode 100644 index 0000000000..8635622af2 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp @@ -0,0 +1,602 @@ +/* BDSP AutoStory - Battle + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/OCR/OCR_RawTesseractOCR.h" +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_EndBattleDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_SelectionArrow.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_Bag.h" +#include "PokemonBDSP_AutoStory_Battle.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +BattleDialogWatcher::BattleDialogWatcher(Color color) + : DetectorToFinder( + "BattleDialogWatcher", + std::chrono::milliseconds(250), + color + ) +{ +} + +namespace{ + +// Early moves the starters and their foes have, as type, power and +// most PP. A status move has power 0. +struct KnownMove { const char* name; const char* type; int power; int pp; }; +const KnownMove KNOWN_MOVES[] = { + {"pound", "normal", 40, 35}, {"tackle", "normal", 40, 35}, {"scratch", "normal", 40, 35}, + {"quickattack", "normal", 40, 30}, {"headbutt", "normal", 70, 15}, + {"watergun", "water", 40, 25}, {"bubble", "water", 40, 30}, {"bubblebeam", "water", 65, 20}, + {"ember", "fire", 40, 25}, {"flamewheel", "fire", 60, 25}, {"machpunch", "fighting", 40, 30}, + {"absorb", "grass", 20, 25}, {"razorleaf", "grass", 55, 25}, {"bite", "dark", 60, 25}, + {"peck", "flying", 35, 35}, {"wingattack", "flying", 60, 35}, {"metalclaw", "steel", 50, 35}, + {"growl", "normal", 0, 40}, {"leer", "normal", 0, 30}, {"tailwhip", "normal", 0, 30}, + {"withdraw", "water", 0, 40}, {"taunt", "dark", 0, 20}, +}; + +// A move button's PP left, "8/25" at its right end; -1 if it cannot be +// read. The text is dark, or yellow to red as PP runs low, so text is +// whatever differs from the button's own color. `max_pp` helps when +// the OCR reads the slash as a digit. +int read_move_pp(const ImageViewRGB32& screen, const ImageFloatBox& button, int max_pp) { + const ImageViewRGB32 box = extract_box_reference(screen, ImageFloatBox{ + button.x + button.width * 0.75, button.y + button.height * 0.10, + button.width * 0.21, button.height * 0.80}); + if (box.width() == 0 || box.height() == 0) return -1; + std::vector pixels; + for (size_t y = 0; y < box.height(); y++) for (size_t x = 0; x < box.width(); x++) pixels.push_back(box.pixel(x, y)); + auto brightness = [](uint32_t p) { return ((p >> 16) & 255) + ((p >> 8) & 255) + (p & 255); }; + std::vector sorted = pixels; + std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end(), + [&](uint32_t a, uint32_t b) { return brightness(a) < brightness(b); }); + const uint32_t background = sorted[sorted.size() / 2]; + const size_t pad = 8; + ImageRGB32 text(box.width() + 2 * pad, box.height() + 2 * pad); + text.fill(0xffffffff); + for (size_t y = 0; y < box.height(); y++) { + for (size_t x = 0; x < box.width(); x++) { + const uint32_t p = box.pixel(x, y); + int distance = 0; + for (int shift : {16, 8, 0}) distance += std::abs((int)((p >> shift) & 255) - (int)((background >> shift) & 255)); + if (distance > 120) text.pixel(x + pad, y + pad) = 0xff000000; + } + } + const std::string read = OCR::tesseract_ocr_read(Language::English, text); + const size_t slash = read.find('/'); + std::string digits; + for (size_t i = 0; i < read.size() && i < slash; i++) if (std::isdigit((unsigned char)read[i])) digits.push_back(read[i]); + if (slash == std::string::npos && max_pp > 0) { + // "29135" for 29/35: drop the most PP and the misread slash. + const std::string max = std::to_string(max_pp); + if (digits.size() > max.size() + 1 && digits.compare(digits.size() - max.size(), max.size(), max) == 0) { + digits.resize(digits.size() - max.size() - 1); + } else { + return -1; + } + } + if (digits.empty() || digits.size() > 2) return -1; + return std::stoi(digits); +} + +// The best move on the move menu: the most powerful one we know, and +// on a tie one that is not Normal (an early starter's own type). +// Reads the names with OCR; 1 if none is known. +size_t pick_best_move(VideoStream& stream, const ImageViewRGB32& screen, + const BattleMoveMenuState& moves, const std::string& label) { + // Keep the menu as it looked, to check the reads against. + { + const std::string folder = SCREENSHOTS_PATH() + "AutoStory/moves/"; + Filesystem::create_directories(Filesystem::Path(folder)); + screen.save(folder + now_to_filestring() + "-" + label + ".png"); + } + size_t best = 1; + int best_score = -1; + bool any_attack = false; + size_t last_not_status = 1; + for (size_t i = 0; i < moves.move_count; i++) { + // The name is dark text on a light button (grey, or light blue under the + // cursor). Take the box right of the type icon and left of the PP, tall + // enough for a status move's name, which sits lower: it has no "Effective" + // line under it. + const ImageFloatBox& button = moves.buttons[i]; + const ImageFloatBox name_box{button.x + button.width * 0.17, button.y + button.height * 0.08, + button.width * 0.53, button.height * 0.80}; + const ImageViewRGB32 box = extract_box_reference(screen, name_box); + // Text is darker than halfway between the background and the darkest pixel; + // a fixed cut thins the strokes on the blue button. + std::vector gray(box.width() * box.height()); + for (size_t y = 0; y < box.height(); y++) { + for (size_t x = 0; x < box.width(); x++) { + const uint32_t pixel = box.pixel(x, y); + gray[y * box.width() + x] = (int)(((pixel >> 16) & 0xff) + ((pixel >> 8) & 0xff) + (pixel & 0xff)) / 3; + } + } + std::vector sorted = gray; + std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end()); + const int background = sorted[sorted.size() / 2]; + const int darkest = *std::min_element(gray.begin(), gray.end()); + const int threshold = (background + darkest) / 2; + // The name is the tallest band of text rows; "Effective" is smaller. + size_t band_top = 0, band_bottom = 0; + for (size_t y = 0; y < box.height();) { + auto has_text = [&](size_t row) { + for (size_t x = 0; x < box.width(); x++) if (gray[row * box.width() + x] < threshold) return true; + return false; + }; + if (!has_text(y)) { y++; continue; } + size_t end = y; + while (end < box.height() && has_text(end)) end++; + if (end - y > band_bottom - band_top) { band_top = y; band_bottom = end; } + y = end; + } + // Black text on white, with a margin, for the OCR. + const size_t pad = 6; + const size_t top = band_top >= 4 ? band_top - 4 : 0; + const size_t bottom = std::min(box.height(), band_bottom + 4); + ImageRGB32 text(box.width() + 2 * pad, bottom - top + 2 * pad); + text.fill(0xffffffff); + for (size_t y = top; y < bottom; y++) { + for (size_t x = 0; x < box.width(); x++) { + if (gray[y * box.width() + x] < threshold) text.pixel(x + pad, y - top + pad) = 0xff000000; + } + } + std::string name; + // Move names only where there is a reader for the game's + // language; an unread move counts as a plain attack. + for (unsigned char ch : story_reads_text() ? OCR::tesseract_ocr_read(g_story_language, text) : std::string()) { + if (std::isalpha(ch)) name.push_back((char)std::tolower(ch)); + } + // A slot read well once keeps its name when a later read fails. + static std::array last_read; + const KnownMove* known = nullptr; + for (int pass = 0; pass < 2 && !known; pass++) { + if (pass == 1) name = last_read[i]; + for (const KnownMove& move : KNOWN_MOVES) { + // The longest name that appears wins: "bubblebeam" over "bubble". + if (name.find(move.name) != std::string::npos && + (!known || std::string(move.name).size() > std::string(known->name).size())) { + known = &move; + } + } + } + if (known) last_read[i] = known->name; + // A move with no PP left cannot be chosen; the game says so and + // keeps the menu up. + const int pp = read_move_pp(screen, button, known ? known->pp : 0); + stream.log("[" + label + "] Move " + std::to_string(i + 1) + " PP left: " + (pp < 0 ? std::string("unread") : std::to_string(pp)) + "."); + if (pp == 0) { + continue; + } + // An unreadable move counts as a plain attack, ahead of known + // status moves. + const int score = known ? known->power * 2 + (std::string(known->type) != "normal" ? 1 : 0) : 40 * 2 - 1; + stream.log("[" + label + "] Move " + std::to_string(i + 1) + " reads \"" + name + "\"" + + (known ? std::string(" = ") + known->name + " (" + known->type + ", " + std::to_string(known->power) + ")" : " (unknown)") + "."); + if (score > best_score) { best_score = score; best = i + 1; } + if (known && known->power > 0) any_attack = true; + if (!known || known->power > 0) last_not_status = i + 1; + } + // No attack read: the newest move is the starter's own-type attack + // (Water Gun, Ember, Absorb), unless it is a status move. + if (!any_attack) best = last_not_status; + stream.log("[" + label + "] Best move: " + std::to_string(best) + "."); + return best; +} + +void select_battle_move_slot(VideoStream& stream, ProControllerContext& context, + BattleType battle_type, size_t target, const std::string& label, + bool confirm_default_target = false, size_t current_move = 1) { + BattleMenuDetector commands(battle_type); + VideoOverlaySet overlays(stream.overlay()); + commands.make_overlays(overlays); + overlays.add(COLOR_GREEN, {0.90, 0.48, 0.075, 0.505}); + // Reenter Fight from an already-open list without assuming + // that this resets the remembered move cursor. + if (confirm_default_target) { + context.wait_for_all_requests(); + const auto frame=stream.video().snapshot(); + if (frame && detect_battle_move_menu(frame).move_count) { + pbf_press_button(context,BUTTON_B,80ms,500ms); + } + } + bool fight = false; + size_t confirmed = 0; + for (size_t check = 0; check < 30; check++) { + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + BattleMenuOption selected = frame ? battle_menu_selected_option(commands, frame) : BattleMenuOption::UNKNOWN; + stream.log("[" + label + "] Command cursor=" + std::to_string((int)selected + 1) + "; target Battle=1."); + if (selected == BattleMenuOption::BATTLE) { + if (++confirmed >= 3) { fight = true; break; } + pbf_wait(context, 100ms); + } else { + confirmed = 0; + if (selected == BattleMenuOption::UNKNOWN) pbf_wait(context, 150ms); + else pbf_press_dpad(context, DPAD_UP, 80ms, 250ms); + } + } + if (!fight) OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Could not confirm Battle command; A not pressed.", stream); + pbf_press_button(context,BUTTON_A,80ms,confirm_default_target ? 1000ms : 500ms); + if (confirm_default_target) { + // Reuse the farming move/target confirmation sequence, but + // navigate relative to this battle's retained move cursor. + // Reopening Fight does not reset it to move 1. + size_t available=0,stable=0; + for (size_t check=0;check<20 && stable<3;check++) { + context.wait_for_all_requests(); + const auto frame=stream.video().snapshot(); + const auto moves=frame ? detect_battle_move_menu(frame) : BattleMoveMenuState{}; + if (moves.move_count && moves.move_count==available) stable++; + else {available=moves.move_count;stable=moves.move_count ? 1:0;} + if (stable<3) pbf_wait(context,100ms); + } + if (stable<3 || available "+std::to_string(target)+"."); + for (size_t slot=current_move;slottarget;slot--) pbf_press_dpad(context,DPAD_UP,80ms,400ms); + pbf_press_button(context,BUTTON_A,80ms,1000ms); + pbf_press_button(context,BUTTON_A,80ms,3000ms); + return; + } + confirmed = 0; + size_t previous_move_count = 0, stable_move_count = 0; + for (size_t check = 0; check < 30; check++) { + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + const BattleMoveMenuState moves = frame ? detect_battle_move_menu(frame) : BattleMoveMenuState{}; + const int slot = (int)moves.selected_slot; + VideoOverlaySet move_overlays(stream.overlay()); + for (size_t i = 0; i < moves.move_count; i++) + move_overlays.add(moves.selected_slot == i + 1 ? COLOR_GREEN : COLOR_YELLOW, moves.buttons[i]); + stream.log("[" + label + "] Available moves=" + std::to_string(moves.move_count) + + "; selected slot=" + std::to_string(slot) + "/4; target=" + std::to_string(target) + "."); + if (moves.move_count == previous_move_count && moves.move_count > 0) stable_move_count++; + else { previous_move_count = moves.move_count; stable_move_count = 1; confirmed = 0; } + // Target 0: pick the best move once the menu has settled. + if (target == 0 && stable_move_count >= 3 && frame) { + target = pick_best_move(stream, *frame.frame, moves, label); + } + if (moves.move_count > 0 && moves.move_count < target && stable_move_count >= 3) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Move slot " + std::to_string(target) + " is absent; no move confirmed.", stream); + } + if (slot == (int)target && stable_move_count >= 3) { + if (++confirmed >= 3) { + stream.log("[" + label + "] Move " + std::to_string(target) + " confirmed; using it."); + pbf_press_button(context, BUTTON_A, 80ms, 3000ms); + return; + } + pbf_wait(context, 100ms); + } else { + confirmed = 0; + if (slot == 0 || slot == (int)target) pbf_wait(context, 150ms); + else pbf_press_dpad(context, slot < (int)target ? DPAD_DOWN : DPAD_UP, 80ms, 250ms); + } + context.wait_for_all_requests(); + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Could not confirm move " + std::to_string(target) + "; A not pressed.", stream); +} + +// Our HP bar in the battle's bottom-left panel, as a fraction; -1 if +// no bar is there. Measured on 1920x1080 frames: the bar runs from +// x 88 to 466 on rows 961 to 965. +double battle_player_hp(const ImageViewRGB32& frame) { + const ImageViewRGB32 bar = extract_box_reference(frame, ImageFloatBox{0.046, 0.890, 0.197, 0.004}); + size_t filled = 0; + for (size_t x = 0; x < bar.width(); x++) { + size_t color = 0; + for (size_t y = 0; y < bar.height(); y++) { + const uint32_t p = bar.pixel(x, y); + const int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + const bool green = g > r + 30 && g > b + 20; + const bool yellow = r > 150 && g > 120 && b < 100; + const bool red = r > 150 && r > g + 50 && r > b + 50; + color += green || yellow || red; + } + filled += color > bar.height() / 2; + } + return bar.width() && filled ? (double)filled / bar.width() : -1; +} + +enum class BattleHeal { NOT_NEEDED, USED, NONE_LEFT }; + +// At the battle menu: if our HP is under `heal_below`, use a Potion +// from the Bag on the lead, which takes the turn. Otherwise the +// battle menu is left up. +BattleHeal battle_heal_if_low(VideoStream& stream, ProControllerContext& context, + BattleType battle_type, double heal_below, const std::string& label) { + context.wait_for_all_requests(); + double hp = -1; + for (size_t check = 0; check < 5 && hp < 0; check++) { + const auto frame = stream.video().snapshot(); + if (frame) hp = battle_player_hp(*frame.frame); + if (hp < 0) pbf_wait(context, 150ms); + context.wait_for_all_requests(); + } + stream.log("[" + label + "] Our HP bar reads " + std::to_string((int)(hp * 100)) + "%."); + if (hp < 0 || hp >= heal_below) return BattleHeal::NOT_NEEDED; + BattleMenuDetector commands(battle_type); + bool on_bag = false; + for (size_t check = 0; check < 30 && !on_bag; check++) { + context.wait_for_all_requests(); + const auto frame = stream.video().snapshot(); + const BattleMenuOption selected = frame ? battle_menu_selected_option(commands, frame) : BattleMenuOption::UNKNOWN; + if (selected == BattleMenuOption::BAG) on_bag = true; + else if (selected == BattleMenuOption::UNKNOWN) pbf_wait(context, 150ms); + else pbf_press_dpad(context, (int)selected < (int)BattleMenuOption::BAG ? DPAD_DOWN : DPAD_UP, 80ms, 300ms); + } + if (!on_bag) { + stream.log("[" + label + "] Could not reach the Bag command; fighting on.", COLOR_ORANGE); + return BattleHeal::NONE_LEFT; + } + pbf_press_button(context, BUTTON_A, 80ms, 2000ms); + if (!find_bag_item_in_open_bag(stream, context, "potion", 1)) { + stream.log("[" + label + "] No Potion left; fighting on.", COLOR_ORANGE); + pbf_press_button(context, BUTTON_B, 80ms, 1500ms); + pbf_press_button(context, BUTTON_B, 80ms, 1500ms); + context.wait_for_all_requests(); + return BattleHeal::NONE_LEFT; + } + // Potion, then the lead in the party list, then Use. + stream.log("[" + label + "] Using a Potion on the lead.", COLOR_BLUE); + pbf_press_button(context, BUTTON_A, 80ms, 1500ms); + pbf_press_button(context, BUTTON_A, 80ms, 1500ms); + pbf_press_button(context, BUTTON_A, 80ms, 2500ms); + context.wait_for_all_requests(); + return BattleHeal::USED; +} + +void fight_using_move_list( + VideoStream& stream, ProControllerContext& context, + BattleType battle_type, const std::vector& move_slots, + const std::string& label, bool confirm_default_target, bool post_battle_dialogue_cleanup, + bool doubles = false, bool switch_on_faint = false +) { + if (move_slots.empty() || std::any_of(move_slots.begin(),move_slots.end(), + [](size_t slot) { return slot>4; })) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label+": Supply a nonempty move list containing only positions 0 (best) through 4.",stream); + } + size_t move_index=0; + // The first Fight menu starts at move 1. Later turns retain + // this Pokemon's previous selection; a new battle resets it. + size_t current_move=1; + stream.log( + "[AutoStory] Starting " + label + " battle...", + COLOR_BLUE + ); + + // Scripted battles wait on the menu detector below, so no + // extra fixed delay is needed after a caller confirms it. + // Preserve the existing early-game transition delay. + if (!confirm_default_target) pbf_wait(context,5000ms); + + bool battle_menu_seen = false; + bool out_of_potions = false; + size_t potions_used = 0; + size_t turns = 0; + size_t faint_switches = 0; + const bool watch_learn_move = confirm_default_target || doubles || switch_on_faint; + EndBattleWatcher end_battle; + + while (true) { + + context.wait_for_all_requests(); + + BattleMenuWatcher battle_menu(battle_type); + // Reuse the farming handlers' move-learning prompt recovery. + SelectionArrowFinder learn_move(stream.overlay(),{0.50,0.62,0.40,0.18},COLOR_YELLOW); + + int ret = run_until( + stream, + context, + [&](ProControllerContext& context) { + // Clear narration with B while watching for commands. + // A here can open Fight before selection begins. + // With faint switching, a quiet 30 s means the party + // screen after a faint, which B cannot leave. + pbf_mash_button(context,BUTTON_B,switch_on_faint ? 30000ms : 120000ms); + }, + { + {battle_menu}, + battle_menu_seen + ? PeriodicInferenceCallback{end_battle} + : PeriodicInferenceCallback{}, + watch_learn_move ? PeriodicInferenceCallback{learn_move} : PeriodicInferenceCallback{}, + } + ); + + switch (ret) { + + // ------------------------------------------------- + // Battle menu detected. + // ------------------------------------------------- + case 0: + battle_menu_seen = true; + // Early battles take a few turns; this many means the chosen + // move is not being used (no PP, a misread menu). + if (++turns > 25) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": still at the battle menu after 25 turns.", stream); + } + + stream.log( + "[AutoStory] Battle menu detected.", + COLOR_BLUE + ); + + // A trainer battle we may lose: a Potion when HP is low. + if (battle_type == BattleType::TRAINER && !out_of_potions && potions_used < 4) { + const BattleHeal heal = battle_heal_if_low(stream, context, battle_type, 0.35, label); + if (heal == BattleHeal::USED) { + potions_used++; + break; + } + out_of_potions = heal == BattleHeal::NONE_LEFT; + } + stream.log("["+label+"] Move list position "+std::to_string(move_index+1)+ + ": use move "+std::to_string(move_slots[move_index])+"."); + select_battle_move_slot(stream, context, battle_type, move_slots[move_index], label, + confirm_default_target,current_move); + current_move=move_slots[move_index]; + // A double battle asks for the target next; B would + // back out of it. The default target is an opponent. + if (doubles) { + pbf_wait(context,1200ms); + pbf_press_button(context,BUTTON_A,80ms,1500ms); + } + // Advance once per move selection; retain the final entry. + if (move_index+1(stream, context, + [](ProControllerContext& ctx) { + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, {{overworld}}); + if (cleanup_ret != 0) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Post-battle cleanup timed out.", stream); + } + context.wait_for_all_requests(); + confirm_battle_overworld(stream, context, label + " cleanup"); + return; + } + + case 2: + stream.log("["+label+"] Declining move learning; preserving scripted move positions."); + pbf_move_right_joystick(context,{0,-1},160ms,840ms); + pbf_press_button(context,BUTTON_ZL,160ms,840ms); + break; + + default: + // After a faint the game shows the party: Down to the + // next Pokemon, A, then A on "Send out" (by hand, + // 2026-10-29 against Roark). + if (switch_on_faint && faint_switches < 5) { + faint_switches++; + stream.log("[" + label + "] No battle menu for 30 s: a faint; sending out the next Pokemon.", COLOR_ORANGE); + pbf_press_dpad(context, DPAD_DOWN, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 1200ms); + pbf_press_button(context, BUTTON_A, 80ms, 2000ms); + break; + } + stream.log( + "[AutoStory] " + label + " battle timeout.", + COLOR_RED + ); + + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "[AutoStory] " + label + " battle timeout.", stream); + } + } +} + +// Preserve the existing early-game callers and their B text cleanup. +void fight_using_move_slot(VideoStream& stream, ProControllerContext& context, + BattleType battle_type, size_t move_slot, const std::string& label) { + fight_using_move_list(stream,context,battle_type,{move_slot},label,false,true); +} + +} + +void confirm_battle_overworld(VideoStream& stream, ProControllerContext& context, + const std::string& label) { + OverworldDetector overworld; + ShortDialogDetector dialogue(COLOR_YELLOW); + BlackScreenDetector black; + VideoOverlaySet overlays(stream.overlay()); + overworld.make_overlays(overlays); + dialogue.make_overlays(overlays); + black.make_overlays(overlays); + const auto deadline = std::chrono::steady_clock::now() + 180s; + auto quiet_since = std::chrono::steady_clock::now(); + bool finished = false; + while (std::chrono::steady_clock::now() < deadline) { + const auto frame = stream.video().snapshot(); + if (!frame) OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": No video frame.", stream); + const auto now = std::chrono::steady_clock::now(); + if (dialogue.detect(frame) || black.detect(frame) || !overworld.detect(frame)) { + quiet_since = now; + } else if (now - quiet_since >= 4s) { + finished = true; + break; + } + pbf_mash_button(context, BUTTON_B, 100ms); + context.wait_for_all_requests(); + } + if (!finished) OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Dialogue-free overworld not confirmed for four seconds.", stream); + context.wait_for_all_requests(); +} + +void fight_story_battle(VideoStream& stream, ProControllerContext& context, const std::string& label, bool doubles) { + // The best move each turn; the next Pokemon after a faint; no new moves. + fight_using_move_list(stream, context, BattleType::TRAINER, {0}, label, false, true, doubles, true); +} + +void fight_trainer(VideoStream& stream, ProControllerContext& context, const std::string& label) { + // Move 0: the best move on the menu each turn, whatever the starter. + fight_using_move_slot(stream, context, BattleType::TRAINER, 0, label); +} + +void fight_wild_battle(VideoStream& stream, ProControllerContext& context, const std::string& label){ + fight_using_move_slot(stream, context, BattleType::STANDARD, 0, label); +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h new file mode 100644 index 0000000000..efc7e47697 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h @@ -0,0 +1,55 @@ +/* BDSP AutoStory - Battle + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Fighting a battle with the best move on the menu: the move names are read + * and scored by power, so it works whatever the starter knows. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Battle_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Battle_H + +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonTools/VisualDetector.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// A battle's text box, as a watcher. +class BattleDialogWatcher : public DetectorToFinder{ +public: + BattleDialogWatcher(Color color = COLOR_RED); +}; + +// Mashes B until the overworld has shown with no dialogue for four seconds. +// Fails with a screenshot after three minutes. +void confirm_battle_overworld(VideoStream& stream, ProControllerContext& context, const std::string& label); + +// A story battle that may go badly: the best move each turn, Potions +// under 35 %, the next Pokemon when one faints, and "Keep old moves" +// on level-up prompts. `doubles` confirms the default target each +// turn (tag and twin battles). Returns once the battle has ended, +// won or lost; after a loss we stand in the last Pokemon Center. +void fight_story_battle(VideoStream& stream, ProControllerContext& context, const std::string& label, bool doubles = false); + +// Any trainer battle, whatever the starter: the best move on the menu each +// turn until the battle ends, then the text after it. +void fight_trainer(VideoStream& stream, ProControllerContext& context, const std::string& label); + +// A wild battle at its battle menu, fought with the best move until it ends. +void fight_wild_battle(VideoStream& stream, ProControllerContext& context, const std::string& label); + + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp new file mode 100644 index 0000000000..35ed89c090 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp @@ -0,0 +1,445 @@ +/* BDSP AutoStory - Pokemon Center + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_RouteWalk.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h" +#include "PokemonBDSP_AutoStory_WildBattle.h" +#include "PokemonBDSP_AutoStory_Center.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; +using namespace Pokemon; + + +namespace{ + +// How many tiles the 1080p view spans across: a tile is about 125 px. +constexpr double SCREEN_WIDTH_TILES = 1920.0 / 125; +// The camera keeps the player in the middle of the screen across, and the +// centre of his tile a little above the middle down (r121a and r122a: his +// feet at y = 590 of 1080). +constexpr double PLAYER_X = 0.5; +constexpr double PLAYER_Y = 0.505; +// How much of the screen's height one row of tiles takes near the player: +// the doorway straight ahead of him (r121a) is centred this far above his +// tile's centre. Rows further north look shorter, so this undercounts them. +constexpr double SCREEN_ROW_HEIGHT = 0.12; +// Doorways are looked for down to here, 0 to 1: a door the player can walk +// into is north of him. +constexpr double DOORWAY_SEARCH_BOTTOM = 0.67; +// A doorway found within this many rows of straight ahead is at the front +// row; further, the walk to the door ended short. +constexpr double DOORWAY_FRONT_ROWS = 1.5; + +// The doorway is looked for on a view this size: 4 px of 1080p per pixel. +constexpr size_t SMALL_WIDTH = 480; +constexpr size_t SMALL_HEIGHT = 270; +// A doorway's lit columns run at least this many small rows (the Mart's 12, +// the Center's 15); the buildings' lit roof bands 3 to 5. +constexpr size_t DOORWAY_COLUMN_ROWS = 8; +// A doorway has this many lit small pixels (the Mart's 260 to 350); a roof +// band's run 60 to 75. +constexpr double DOORWAY_MIN_MASS = 80; + +// Whether a pixel is the pale yellow-white of a lit doorway: the Mart's is +// (235, 243, 205), the Center's (255, 255, 185). The buildings' white roof +// bands are bluer, their orange bands and the light they throw on the +// ground outside are darker. +bool lit_doorway_pixel(uint32_t pixel){ + const int r = (pixel >> 16) & 0xff; + const int g = (pixel >> 8) & 0xff; + const int b = pixel & 0xff; + return r >= 215 && g >= 215 && b >= 150 && b <= g - 15 && r >= g - 25; +} + +// Each column of the small view: how many lit doorway pixels it has in the +// search band, and the sum of their rows. +struct LitColumns{ + std::vector count; + std::vector row_sum; +}; +LitColumns lit_columns(const ImageViewRGB32& screen){ + const ImageRGB32 small = screen.scale_to(SMALL_WIDTH, SMALL_HEIGHT); + LitColumns columns{std::vector(small.width(), 0), std::vector(small.width(), 0)}; + const size_t bottom = (size_t)(DOORWAY_SEARCH_BOTTOM * (double)small.height()); + for (size_t y = 0; y < bottom; y++){ + for (size_t x = 0; x < small.width(); x++){ + if (lit_doorway_pixel(small.pixel(x, y))){ + columns.count[x]++; + columns.row_sum[x] += (double)y; + } + } + } + return columns; +} + +// A run of lit columns: its lit pixels, and the sums of their columns and rows. +struct LitRun{ + double mass = 0; + double column_sum = 0; + double row_sum = 0; +}; +// The run of columns with at least DOORWAY_COLUMN_ROWS lit that starts at +// `x`, allowing gaps of up to 3 columns (the door's frame). Moves `x` past it. +LitRun lit_run(const LitColumns& columns, size_t& x){ + LitRun run; + size_t gap = 0; + for (; x < columns.count.size() && gap <= 3; x++){ + if (columns.count[x] < DOORWAY_COLUMN_ROWS){ + gap++; + continue; + } + run.mass += (double)columns.count[x]; + run.column_sum += (double)columns.count[x] * (double)x; + run.row_sum += columns.row_sum[x]; + gap = 0; + } + return run; +} + +// Where a building's lit doorway is across and down the screen, 0 to 1: the +// biggest run of lit columns within 0.35 of the middle (a second building's +// door may show further off). Nothing if no doorway shows. +std::optional> lit_doorway_position(const ImageViewRGB32& screen){ + const LitColumns columns = lit_columns(screen); + const double width = (double)columns.count.size(); + std::optional> best; + double best_mass = 0; + for (size_t x = 0; x < columns.count.size(); ){ + if (columns.count[x] < DOORWAY_COLUMN_ROWS){ + x++; + continue; + } + const LitRun run = lit_run(columns, x); + const double center = run.column_sum / run.mass / width; + if (run.mass < DOORWAY_MIN_MASS || std::abs(center - PLAYER_X) >= 0.35 || run.mass <= best_mass){ + continue; + } + best = {center, run.row_sum / run.mass / (double)SMALL_HEIGHT}; + best_mass = run.mass; + } + return best; +} + +// Whether the whole view is a different scene from "before". +bool scene_changed_since(VideoStream& stream, ProControllerContext& context, const VideoSnapshot& before){ + context.wait_for_all_requests(); + VideoSnapshot now = stream.video().snapshot(); + return before && now && walk_screen_change(*before.frame, *now.frame) > WALK_SCENE_CHANGE; +} + +// Moves across by "tiles", east if positive, west if negative. +void step_across(VideoStream& stream, ProControllerContext& context, double tiles){ + const Direction side = tiles > 0 ? Direction::EAST : Direction::WEST; + route_turn(stream, context, side); + const Milliseconds hold = std::max(Milliseconds(40), + Milliseconds((int64_t)(std::abs(tiles) * ROUTE_MS_PER_TILE_ACROSS.count()))); + pbf_move_left_joystick(context, {tiles > 0 ? 1.0 : -1.0, 0}, hold, 300ms); +} + +// Whether the screen now shows a Pokemon Center's inside. +bool center_inside_now(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + VideoSnapshot now = stream.video().snapshot(); + return now && pokemon_center_inside(*now.frame); +} + +// Walks north into the door straight ahead, `doorway` away. Returns whether +// we are inside. +bool walk_into_door( + VideoStream& stream, ProControllerContext& context, + const VideoSnapshot& outside, const LitDoorway& doorway, const std::string& label +){ + // Time for the rows to the door (none when it is straight ahead), the + // door opening, and a start delay. + const double rows = std::max(0.0, doorway.rows_up - 1); + const Milliseconds limit = 1500ms + Milliseconds((int64_t)(rows * ROUTE_MS_PER_TILE_UP_DOWN.count())); + // The Center's fade is too light to read as black (r126 runs 2 and 3: a + // screen change of 49 at 400 ms), so its inside, once it shows, stops + // the walk too. + const WalkStop stop = walk_until(stream, context, run_from_wild_battle, 0, 1, limit, label + ": into the door", + [](const ImageViewRGB32& screen){ return pokemon_center_inside(screen); }).stop; + if (stop == WalkStop::TRANSITION || stop == WalkStop::ARRIVED){ + pbf_wait(context, 2000ms); + return true; + } + // The door slides open before it takes us in, and its fade is not always + // black enough to read as a transition, and may finish after the walk; a + // whole new scene says we are in. Not after a walk of several rows, which + // scrolls the view: the next look decides. + pbf_wait(context, 2000ms); + if (center_inside_now(stream, context)){ + stream.log("[Route] " + label + ": a Pokemon Center's inside shows; inside."); + return true; + } + if (doorway.rows_up > DOORWAY_FRONT_ROWS || !scene_changed_since(stream, context, outside)){ + return false; + } + stream.log("[Route] " + label + ": the scene changed; inside."); + return true; +} + +// Where the doorway is on screen, in the log. +void log_doorway(VideoStream& stream, const std::string& label, const LitDoorway& doorway){ + stream.log("[Route] " + label + ": doorway " + std::to_string(doorway.tiles_across) + + " tiles across, " + std::to_string(doorway.rows_up) + " rows up (at " + + std::to_string(doorway.x) + ", " + std::to_string(doorway.y) + " of the screen)."); +} + +} + + +bool pokemon_center_inside(const ImageViewRGB32& screen){ + // The orange back wall along the top, in two of three places: a plant + // or a bookcase can stand in one (Jubilife's top left), and at night + // it is darker (r 160..200 in Jubilife, 254 in Sandgem by day). + size_t wall = 0; + for (double x : {0.02, 0.25, 0.65}){ + const ImageStats box = image_stats(extract_box_reference(screen, ImageFloatBox(x, 0.03, 0.08, 0.06))); + const double r = box.average.r, g = box.average.g, b = box.average.b; + wall += r >= 140 && r > g + 40 && g > b + 40; + } + if (wall < 2){ + return false; + } + // Black below the floor, either side of the step out of the door. + for (const ImageFloatBox& box : {ImageFloatBox(0.02, 0.95, 0.30, 0.04), ImageFloatBox(0.65, 0.95, 0.33, 0.04)}){ + if (image_stats(extract_box_reference(screen, box)).average.sum() > 90){ + return false; + } + } + // And the floor lit between them: not a fade (349 in Jubilife at night). + return image_stats(extract_box_reference(screen, ImageFloatBox(0.30, 0.55, 0.40, 0.15))).average.sum() > 250; +} + +void fail_if_blacked_out(VideoStream& stream, ProControllerContext& context, const std::string& label){ + for (size_t look = 0; look < 10; look++){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + if (screen && indoors(*screen.frame)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": the party fainted; the game took us home (a room shows after the battle).", stream); + } + // The blackout's text and fade are dark; the grass is not. + if (screen && image_stats(*screen.frame).average.sum() > 150){ + return; + } + pbf_wait(context, 1000ms); + } +} + +std::optional find_lit_lamp_x(const ImageViewRGB32& screen, const ImageFloatBox& box){ + const size_t x0 = (size_t)(box.x * screen.width()), x1 = (size_t)((box.x + box.width) * screen.width()); + const size_t y0 = (size_t)(box.y * screen.height()), y1 = (size_t)((box.y + box.height) * screen.height()); + double sum = 0; + size_t count = 0; + for (size_t y = y0; y < y1 && y < screen.height(); y++){ + for (size_t x = x0; x < x1 && x < screen.width(); x++){ + const Color pixel(screen.pixel(x, y)); + if (pixel.red() > 235 && pixel.green() > 220 && pixel.blue() > 150){ + sum += x; + count++; + } + } + } + // Jubilife's lamps at night: 50 to 100 such pixels at 480 px across. + if (count * 480 * 480 < 30 * screen.width() * screen.width()){ + return std::nullopt; + } + return sum / count / screen.width(); +} + +double tiles_across_at(double screen_x){ + return (screen_x - PLAYER_X) * SCREEN_WIDTH_TILES; +} + +std::optional find_lit_doorway(const ImageViewRGB32& screen){ + // A Center's lit floor has the doorway's colour. + if (pokemon_center_inside(screen)){ + return std::nullopt; + } + const std::optional> position = lit_doorway_position(screen); + if (!position){ + return std::nullopt; + } + return LitDoorway{ + position->first, + position->second, + (position->first - PLAYER_X) * SCREEN_WIDTH_TILES, + (PLAYER_Y - position->second) / SCREEN_ROW_HEIGHT, + }; +} + +std::vector tiles_short_of_door( + VideoStream& stream, ProControllerContext& context, + const StoryWorld& grid, const TilePos& door, const std::string& label +){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + const std::optional doorway = screen ? find_lit_doorway(*screen.frame) : std::nullopt; + if (!doorway){ + stream.log("[Route] " + label + ": no lit doorway on screen; not walking again."); + return {}; + } + log_doorway(stream, label, *doorway); + if (doorway->rows_up <= DOORWAY_FRONT_ROWS){ + return {}; + } + return tiles_by_lit_doorway(*doorway, grid, door); +} + +std::vector tiles_by_lit_doorway(const LitDoorway& doorway, const StoryWorld& grid, const TilePos& door){ + const TilePos guess{door.x - (int)std::lround(doorway.tiles_across), door.y + (int)std::lround(doorway.rows_up)}; + std::vector tiles; + for (int y = guess.y - 1; y <= guess.y + 1; y++){ + for (int x = guess.x - 1; x <= guess.x + 1; x++){ + if (grid.standable(TilePos{x, y})){ + tiles.push_back(TilePos{x, y}); + } + } + } + return tiles; +} + +bool line_up_with_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + route_status(stream, label + ": lining up with the door."); + for (size_t tries = 0; tries < 6; tries++){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + if (screen && pokemon_center_inside(*screen.frame)){ + stream.log("[Route] " + label + ": a Pokemon Center's inside shows; not in front of the door."); + return false; + } + const std::optional doorway = screen ? find_lit_doorway(*screen.frame) : std::nullopt; + if (!doorway){ + stream.log("[Route] " + label + ": no lit doorway on screen."); + return false; + } + log_doorway(stream, label, *doorway); + // Steps overshoot: within a quarter tile, r128 measured 0.29, -0.28, + // 0.48, -0.67 and 0.48 tiles in turn and never settled. The door + // still takes us in from 0.4 (enter_lit_door()). + if (std::abs(doorway->tiles_across) <= 0.4){ + return true; + } + step_across(stream, context, doorway->tiles_across); + // Let a door that took us in show its inside before looking again. + pbf_wait(context, 1000ms); + } + stream.log("[Route] " + label + ": not lined up after 6 steps."); + return false; +} + +void enter_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + // Movement is not locked to tiles, and the door takes us in only when we + // are lined up with it. Its doorway is lit: find it on screen, where the + // camera keeps the player in the middle, move across by its offset, and + // walk in once it is straight ahead, looking again after every move. + route_status(stream, label + ": lining up with the door."); + VideoSnapshot last_outside; + for (size_t tries = 0; tries < 12; tries++){ + context.wait_for_all_requests(); + VideoSnapshot outside = stream.video().snapshot(); + if (!outside){ + continue; + } + // Inside a Center, its lit floor would read as another doorway. + if (pokemon_center_inside(*outside.frame)){ + stream.log("[Route] " + label + ": a Pokemon Center's inside shows; inside."); + return; + } + const std::optional doorway = find_lit_doorway(*outside.frame); + if (!doorway){ + // The door may have taken us in after all, more slowly than we watched. + pbf_wait(context, 1500ms); + if (!scene_changed_since(stream, context, last_outside)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": no lit doorway on screen.", stream); + } + stream.log("[Route] " + label + ": the scene changed; inside."); + pbf_wait(context, 1500ms); + return; + } + last_outside = outside; + log_doorway(stream, label, *doorway); + // We stand on tiles, so within half a tile is the door's column; + // a step moves about 0.7 of a reading, and at 0.25 r144a stepped + // between 0.3 and -0.4 twelve times. Try the door there, and step + // toward it if it does not take us. + if (std::abs(doorway->tiles_across) > 0.5){ + step_across(stream, context, doorway->tiles_across); + continue; + } + if (walk_into_door(stream, context, outside, *doorway, label)){ + pbf_wait(context, 1500ms); + return; + } + step_across(stream, context, doorway->tiles_across); + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": not through the door after 12 tries.", stream); +} + +void heal_at_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + enter_lit_door(stream, context, label); + + // The nurse's counter is straight ahead of the door. + route_status(stream, label + ": talking to the nurse."); + if (walk_to_talk(stream, context, run_from_wild_battle, 0, 1, 6000ms, label + ": to the nurse").stop != WalkStop::DIALOG){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": nurse not reached.", stream); + } + // A through her lines; the cursor starts on "Yes". Stop once no dialogue + // has shown for a while, so A does not start the talk over. + size_t quiet = 0; + for (size_t presses = 0; presses < 30 && quiet < 2; presses++){ + ShortDialogWatcher dialog(COLOR_YELLOW); + if (wait_until(stream, context, 1500ms, {{dialog}}) == 0){ + quiet = 0; + pbf_press_button(context, BUTTON_A, 80ms, 700ms); + }else{ + quiet++; + } + } + context.wait_for_all_requests(); + + route_status(stream, label + ": healed; walking out."); + leave_door_south(stream, context, run_from_wild_battle, label + ": out the door"); +} + + + +void move_across(VideoStream& stream, ProControllerContext& context, double tiles){ + step_across(stream, context, tiles); +} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h new file mode 100644 index 0000000000..f5e3577de5 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h @@ -0,0 +1,118 @@ +/* BDSP AutoStory - Pokemon Center + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Center_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Center_H + +#include +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryWorld.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Indoors.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// A building's lit doorway seen on the overworld, relative to the player, +// whom the camera keeps in the middle of the screen. +struct LitDoorway{ + double x; // centre across the screen, 0 to 1 + double y; // centre down the screen, 0 to 1 + double tiles_across; // east of the player if positive, west if negative + double rows_up; // rows north of the player's tile, roughly: 1 straight ahead +}; + +// Whether `screen` is the inside of a Pokemon Center seen from its door: +// the orange back wall at the top left, and the black below the entrance +// mat along the bottom edge. Its lit floor reads as a doorway +// (find_lit_doorway()), so this is what says we are in (r126 runs 2, 3). +bool pokemon_center_inside(const ImageViewRGB32& screen); + +// Finds a lit doorway (a Pokemon Center's or a Poke Mart's) in the upper +// two thirds of `screen`, the pale yellow-white of its open door. Nothing if +// none shows there, or inside a Pokemon Center, whose floor is that colour. +std::optional find_lit_doorway(const ImageViewRGB32& screen); + +// After a wild battle out of doors: fails with a screenshot if the screen +// turns into a room within 10 s, which means the party fainted and the +// game took us home (r258 lost its third training battle in Route 202's +// grass, woke in Twinleaf, and walked the Sandgem plan there). Returns +// once the outside shows, not black and not a room. +void fail_if_blacked_out(VideoStream& stream, ProControllerContext& context, const std::string& label); + +// A lit street lamp's head inside `box` (0 to 1 of the screen): the mean +// x of its white core, or nothing if fewer than a lamp's worth of white +// pixels show there (by day the lamps are not lit). +std::optional find_lit_lamp_x(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// How many tiles a thing seen at screen x is across from the player +// (positive: east). +double tiles_across_at(double screen_x); + +// Moves across by `tiles`, east if positive, west if negative, facing that +// way first. +void move_across(VideoStream& stream, ProControllerContext& context, double tiles); + +// The tiles a player who sees `doorway` (the door on world tile `door`) may +// stand on: the one its offset says, give or take one each way, since the +// rows are a rough count. Only standable tiles of `grid`. +std::vector tiles_by_lit_doorway( + const LitDoorway& doorway, const StoryWorld& grid, const Pokemon::TilePos& door +); + +// For a player who walked to the row below the door on world tile `door`: +// if its lit doorway shows further up the screen than straight ahead, the +// walk ended short (someone in the way stops an open leg, and the walk +// cannot tell), and the tiles he may stand on instead, read off the +// doorway's offset, are returned to walk from again. Empty when he is at +// the front row, or no doorway shows. +std::vector tiles_short_of_door( + VideoStream& stream, ProControllerContext& context, + const StoryWorld& grid, const Pokemon::TilePos& door, const std::string& label +); + +// For a player on the row below a building's door, its lit doorway on +// screen: steps across by the doorway's offset, looking again after each +// step, until it is straight ahead. Does not go in, and does not turn north +// to face it: a turn there walked us in (r127c). Returns false if no +// doorway shows, a Center's inside shows, or after 6 steps. +bool line_up_with_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + +// Goes in through a building's door (a Pokemon Center's or a Poke Mart's). +// Starts with its lit doorway on screen, within a few tiles: lines up with +// the doorway by sight and walks in, looking again after every move. Whole- +// tile steps sideways miss it, since movement is not locked to tiles, and +// its fade may not read as black: a new scene also counts as inside. Fails +// with a screenshot when no doorway shows, or after 12 moves. +void enter_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + +// Heals the party at a Pokemon Center. Starts in front of the Center with its +// lit doorway on screen, within a few tiles of it: lines up with the doorway +// by sight, walks in, talks to the nurse, accepts, and walks back out. Ends +// on the tile below the door. +void heal_at_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp new file mode 100644 index 0000000000..ac311dcba9 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -0,0 +1,286 @@ +/* BDSP AutoStory - New Game + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/ScreenMachine/ScreenMachine.h" +#include "CommonTools/ScreenMachine/StepTrace.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" +#include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" +#include "PokemonBDSP_AutoStory_NewGame.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ + // The Switch's own screens come first: BDSP's dialogue detector also fires on them. + if (console != nullptr && StartGameUserSelectDetector(*console).detect(screen)){ + return "user-select"; + } + if (console != nullptr && HomeMenuDetector(*console, COLOR_RED).detect(screen)){ + return "home"; + } + if (SystemKeyboardDetector().detect(screen)){ + return "keyboard"; + } + switch (NameListDetector().selected_row(screen)){ + case -1: + break; + case 0: + return "rival-list-tell"; + default: + return "rival-list-preset"; + } + if (MenuDetector().detect(screen)){ + return "menu"; + } + // Includes the appearance grid, whose prompt sits in a dialogue box. + if (ShortDialogDetector().detect(screen)){ + return "dialog"; + } + // Language, cutscenes, fades, the bedroom with no menu open. + return "other"; +} + +IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ + if (screen == "user-select") return IntroStep::SELECT_USER; + if (screen == "home") return IntroStep::FAIL_NOT_IN_GAME; + if (screen == "keyboard"){ + return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; + } + if (screen == "rival-list-tell") return IntroStep::MOVE_TO_PRESET; + if (screen == "rival-list-preset") return IntroStep::PICK_PRESET; + if (screen == "menu") return IntroStep::CLOSE_MENU_AND_FINISH; + if (screen == "dialog") return IntroStep::ADVANCE; + // The bedroom comes only after the rival is named. Before that, every unknown + // screen (language, the appearance preview, cutscenes) takes A. + return progress.rival_named ? IntroStep::PROBE_FOR_MENU : IntroStep::ADVANCE; +} +bool advance_after_probe(bool screen_changed){ + return !screen_changed; +} + +size_t run_intro_screen_tests(Logger& logger, const std::string& folder){ + size_t passed = 0; + size_t failed = 0; + for (const Filesystem::Path& dir : Filesystem::list_directory(Filesystem::Path(folder))){ + if (!Filesystem::is_directory(dir)){ + continue; + } + const std::string expected = dir.filename().string(); + for (const Filesystem::Path& file : Filesystem::list_directory(dir)){ + if (file.extension().string() != ".png"){ + continue; + } + const ImageRGB32 image(file.string()); + // 1080p and 720p. At 960x540 the keyboard's and the name list's checks + // do not hold (blurred key text, an arrow below the object size limit). + for (size_t width : {size_t(1920), size_t(1280)}){ + const std::string actual = classify_intro_screen(nullptr, image.scale_to(width, width * 9 / 16)); + if (actual == expected){ + passed++; + continue; + } + failed++; + logger.log("[Intro tests] FAIL " + expected + "/" + file.filename().string() + + " at " + std::to_string(width) + "px: read as " + actual, COLOR_RED); + } + } + } + logger.log("[Intro tests] " + std::to_string(passed) + " passed, " + std::to_string(failed) + " failed.", + failed == 0 ? COLOR_BLUE : COLOR_RED); + return failed; +} + + +namespace{ + +// Waits until the keyboard is gone (or, with `gone` false, has appeared). +bool wait_for_keyboard(VideoStream& stream, ProControllerContext& context, bool gone, Milliseconds timeout){ + context.wait_for_all_requests(); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + VideoSnapshot screen = stream.video().snapshot(); + if (screen && SystemKeyboardDetector().detect(screen) != gone){ + return true; + } + context.wait_for(100ms); + } + return false; +} + +void enter_player_name(VideoStream& stream, ProControllerContext& context, StepTrace& trace, const std::string& name){ + // Let the keyboard finish sliding in: inputs during that are dropped. + pbf_wait(context, 800ms); + context.wait_for_all_requests(); + // B deletes a character, and closes the keyboard on an empty field, so delete only + // while there is text (e.g. after a retry). + for (size_t c = 0; c < 14; c++){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen || !system_keyboard_field_has_text(screen)){ + break; + } + trace.action("Name field holds text; deleting a character."); + pbf_press_button(context, BUTTON_B, 60ms, 250ms); + context.wait_for_all_requests(); + } + trace.action("Typing the player's name: " + name); + type_on_system_keyboard(stream, context, name); + pbf_press_button(context, BUTTON_PLUS, 80ms, 0ms); + if (!wait_for_keyboard(stream, context, true, 5s)){ + trace.fail("The keyboard stayed open after PLUS (OK)."); + } + trace.action("Name confirmed."); +} + +} + + + +void play_new_game_intro( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& player_name +){ + VideoStream& stream = env.console; + StepTrace trace(stream, "Intro"); + + // Only close and reopen the game: the states below take it from the user select + // on. reset_game_from_home() waits for a loaded save's black screen to end, and a + // new game's first screen is a dialog on black, so it would sit there until its + // timeout. + trace.action("Restarting the game."); + go_home(env.console, context); + from_home_close_and_reopen_game(env.console, context, true); + + IntroProgress progress; + size_t x_without_menu = 0; + auto press = [&](Button button){ + pbf_press_button(context, button, 80ms, 0ms); + }; + + // Carries out what decide_intro_step() chose for `screen`. + auto act = [&](const char* screen){ + switch (decide_intro_step(screen, progress)){ + case IntroStep::SELECT_USER: + // The cursor starts on the user that launched the game. + press(BUTTON_A); + pbf_wait(context, 1000ms); + return; + case IntroStep::FAIL_NOT_IN_GAME: + trace.fail("The Switch HOME menu is showing: the game is not running in front."); + case IntroStep::ENTER_PLAYER_NAME: + enter_player_name(stream, context, trace, player_name); + progress.player_named = true; + return; + case IntroStep::CANCEL_RIVAL_KEYBOARD: + // Opened from "I'll tell you..."; inputs during the slide-in are dropped. + pbf_wait(context, 800ms); + press(BUTTON_X); + return; + case IntroStep::MOVE_TO_PRESET: + pbf_press_dpad(context, DPAD_DOWN, 100ms, 300ms); + return; + case IntroStep::PICK_PRESET: + press(BUTTON_A); + pbf_wait(context, 500ms); + progress.rival_named = true; + return; + case IntroStep::CLOSE_MENU_AND_FINISH:{ + trace.action("Main menu is open: the player is in control in the bedroom. Closing it."); + press(BUTTON_B); + pbf_wait(context, 1000ms); + context.wait_for_all_requests(); + VideoSnapshot screen_now = stream.video().snapshot(); + if (!screen_now || MenuDetector().detect(screen_now)){ + trace.fail("The main menu did not close."); + } + return; + } + case IntroStep::ADVANCE: + pbf_press_button(context, BUTTON_A, 60ms, 400ms); + return; + case IntroStep::PROBE_FOR_MENU:{ + context.wait_for_all_requests(); + VideoSnapshot before = stream.video().snapshot(); + press(BUTTON_X); + context.wait_for_all_requests(); + MenuWatcher menu; + if (wait_until(stream, context, 3s, {{menu}}) == 0){ + return; // The next read sees the menu and finishes. + } + VideoSnapshot after = stream.video().snapshot(); + const bool changed = !before || !after || ImageMatch::pixel_RMSD(before, after) > 10; + if (!advance_after_probe(changed)){ + trace.action("X changed the screen but no known menu opened; not pressing A."); + if (++x_without_menu >= 10){ + trace.fail("X keeps changing the screen without a known menu opening."); + } + return; + } + x_without_menu = 0; + pbf_press_button(context, BUTTON_A, 60ms, 400ms); + return; + } + } + }; + + auto screen_is = [&](const ImageViewRGB32& screen, const char* name){ + return classify_intro_screen(&env.console, screen) == name; + }; + // The machine's states: the screens classify_intro_screen() tells apart, with the + // keyboard split by whose name it asks for. Each state's step is act(). + const std::vector states{ + {"user-select", + [&](const ImageViewRGB32& s){ return screen_is(s, "user-select"); }, + [&]{ act("user-select"); }, {}, 3}, + {"home", + [&](const ImageViewRGB32& s){ return screen_is(s, "home"); }, + [&]{ act("home"); }, {}, 1}, + {"player-keyboard", + [&](const ImageViewRGB32& s){ return !progress.player_named && screen_is(s, "keyboard"); }, + [&]{ act("keyboard"); }, {"dialog", "other"}, 1}, + {"rival-keyboard", + [&](const ImageViewRGB32& s){ return progress.player_named && screen_is(s, "keyboard"); }, + [&]{ act("keyboard"); }, {"rival-list-tell", "dialog", "other"}, 3}, + {"rival-list-tell", + [&](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, + [&]{ act("rival-list-tell"); }, {"rival-list-preset", "other"}, 6}, + {"rival-list-preset", + [&](const ImageViewRGB32& s){ return screen_is(s, "rival-list-preset"); }, + [&]{ act("rival-list-preset"); }, {"dialog", "other"}, 3}, + {"menu", + [&](const ImageViewRGB32& s){ return screen_is(s, "menu"); }, + [&]{ act("menu"); }, {}, 2, true}, + {"dialog", + [&](const ImageViewRGB32& s){ return screen_is(s, "dialog"); }, + [&]{ act("dialog"); }, {}, 400}, + {"other", + [](const ImageViewRGB32&){ return true; }, + [&]{ act("other"); }, {}, 300, false, 60}, + }; + run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); + trace.action("Intro done."); +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h new file mode 100644 index 0000000000..7ab9e4fb5c --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h @@ -0,0 +1,71 @@ +/* BDSP AutoStory - New Game + * + * From: https://github.com/PokemonAutomation/ + * + * Plays a new game's intro (language, Professor Rowan, appearance, names) up to + * the bedroom with the player in control, which is where the Tutorial starts. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H +#define PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Which intro screen `screen` shows, as the intro's steps tell them apart: +// "user-select", "home", "keyboard", "rival-list-tell" (the rival's name list on +// "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or +// "other". The Switch's own screens need `console`; without it they read as +// whatever else they look like. +std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen); + +// What the intro does on a screen. Decided from the screen and how far the intro has +// got only, so it is tested without a console. +enum class IntroStep{ + SELECT_USER, // A on the Switch's user select + FAIL_NOT_IN_GAME, // HOME is showing + ENTER_PLAYER_NAME, // type the name on the first keyboard + CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list + MOVE_TO_PRESET, // Down: off "I'll tell you..." + PICK_PRESET, // A on a preset name + CLOSE_MENU_AND_FINISH, // B: the menu opened in the bedroom, the intro is over + ADVANCE, // A: dialogue, language, appearance, cutscenes + PROBE_FOR_MENU, // X, to see whether the player is in control +}; +struct IntroProgress{ + bool player_named = false; + bool rival_named = false; +}; +IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); + +// After PROBE_FOR_MENU found no menu: whether A may follow. Only when X changed +// nothing, since A on something X opened would pick from it (the Bag). +bool advance_after_probe(bool screen_changed); + +// Checks classify_intro_screen() on //*.png at 1920 and +// 1280 px wide. Logs each mismatch; returns the number of them. +size_t run_intro_screen_tests(Logger& logger, const std::string& folder); + +// From the title screen or any intro screen of a game with no save. Restarts the +// game first, so it always starts from the title. Names the player `player_name` +// and picks the rival's default name (Barry) from the list. Each step reads the +// screen and waits for the expected next one; an unexpected screen fails with a +// screenshot and the recent screens. +void play_new_game_intro( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& player_name +); + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp new file mode 100644 index 0000000000..ac9a427ccd --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp @@ -0,0 +1,444 @@ +/* BDSP AutoStory - Settings + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP_AutoStory_Bag.h" +#include "PokemonBDSP_AutoStory_Settings.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +namespace{ + +// Options chrome and position detectors are independent of setting values. +class TutorialOptionsDetector { +public: + bool detect(const ImageViewRGB32& screen) const { + ImageViewRGB32 panel = extract_box_reference(screen, ImageFloatBox{0.04, 0.065, 0.93, 0.70}); + size_t white = 0; + for (size_t y = 0; y < panel.height(); y += 4) + for (size_t x = 0; x < panel.width(); x += 4) { + uint32_t p = panel.pixel(x, y); + if (((p >> 16) & 255) > 225 && ((p >> 8) & 255) > 225 && (p & 255) > 225) white++; + } + double samples = (double)(((panel.width() + 3) / 4) * ((panel.height() + 3) / 4)); + // Distinguish Options from the icon menu and bag using its + // blue heading and full-width white settings panel. + ImageViewRGB32 heading = extract_box_reference(screen, ImageFloatBox{0.01, 0.005, 0.20, 0.025}); + size_t blue = 0, total = 0; + for (size_t y = 0; y < heading.height(); y++) + for (size_t x = 0; x < heading.width(); x++) { + uint32_t p = heading.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + blue += b > r + 20 && g > r + 10; + total++; + } + return samples > 0 && white / samples > 0.70 && total > 0 && blue > total / 4; + } +}; + +// Options fades in after A on its icon: poll instead of judging one frame. +bool wait_for_options_screen(VideoStream& stream, ProControllerContext& context, + const TutorialOptionsDetector& options_screen, Milliseconds timeout) { + context.wait_for_all_requests(); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline) { + auto frame = stream.video().snapshot(); + if (frame && options_screen.detect(frame)) return true; + context.wait_for(100ms); + } + return false; +} + +class TutorialOptionPositionDetector { +public: + // Locate the two long red horizontal lines surrounding the + // selected row. Scrolling changes their position automatically. + double selected_row(const ImageViewRGB32& screen) const { + int first = -1, last = -1; + for (size_t y = (size_t)(screen.height() * 0.08); y < screen.height() * 0.75; y++) { + size_t red = 0, total = 0; + for (size_t x = (size_t)(screen.width() * 0.08); x < screen.width() * 0.90; x += 2) { + uint32_t p = screen.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + red += r > 150 && r > g + 40 && r > b + 40; + total++; + } + if (total && red > total * 0.70) { + if (first < 0) first = (int)y; + last = (int)y; + } + } + if (first < 0 || last - first < screen.height() * 0.04 || last - first > screen.height() * 0.12) return -1; + return (first + last) / (2.0 * screen.height()); + } + // Locate selected red value text separately from the row border. + bool selected_value(const ImageViewRGB32& screen, double y) const { + ImageViewRGB32 value = extract_box_reference(screen, ImageFloatBox{0.33, y - 0.025, 0.54, 0.05}); + size_t red = 0; + for (size_t py = 0; py < value.height(); py++) + for (size_t px = 0; px < value.width(); px++) { + uint32_t p = value.pixel(px, py); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + red += r > g + 20 && r > b + 20; + } + return red > value.width() * value.height() * 0.0005; + } +}; + + +// Compare binary visual shapes with small alignment tolerance. +// No text recognition is needed for fixed English option labels. +double option_image_score(const ImageViewRGB32& screen, + const ImageFloatBox& box, const ImageRGB32& reference, bool red_only) { + ImageRGB32 actual = extract_box_reference(screen, box).scale_to(384, 32); + ImageRGB32 expected = reference.scale_to(384, 32); + auto ink = [red_only](uint32_t p){ + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + return red_only ? r > g + 20 && r > b + 20 : r + g + b < 500; + }; + double best = 0; + for (int dy = -2; dy <= 2; dy++) for (int dx = -2; dx <= 2; dx++) { + size_t overlap = 0, count = 0; + for (int y = 2; y < 30; y++) for (int x = 2; x < 382; x++) { + bool a = ink(actual.pixel(x + dx, y + dy)); + bool b = ink(expected.pixel(x, y)); + overlap += a && b; + count += a + b; + } + if (count) best = std::max(best, 2.0 * overlap / count); + } + return best; +} + +bool autosave_off_highlight(const ImageViewRGB32& screen, double y) { + auto red_count = [&](const ImageFloatBox& box){ + ImageViewRGB32 image = extract_box_reference(screen, box); + size_t red = 0; + for (size_t py = 0; py < image.height(); py++) + for (size_t px = 0; px < image.width(); px++) { + uint32_t p = image.pixel(px, py); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + red += r > g + 30 && r > b + 30; + } + return red; + }; + size_t off = red_count({0.63, y - 0.022, 0.055, 0.044}); + size_t on = red_count({0.34, y - 0.022, 0.055, 0.044}); + // Require a clear Off highlight and reject any competing On highlight. + return off >= 10 && on * 10 < off; +} + +void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& context) { + context.wait_for_all_requests(); + MenuDetector menu; + VideoOverlaySet detector_overlays(stream.overlay()); + menu.make_overlays(detector_overlays); + detector_overlays.add(COLOR_CYAN, {0.04, 0.065, 0.93, 0.70}, ""); + detector_overlays.add(COLOR_BLUE, {0.01, 0.005, 0.20, 0.025}, ""); + detector_overlays.add(COLOR_RED, {0.08, 0.08, 0.82, 0.67}, ""); + auto initial = stream.video().snapshot(); + if (!initial || !menu.detect(initial)) { + pbf_press_button(context, BUTTON_X, 80ms, 1000ms); + context.wait_for_all_requests(); + } + MenuWatcher menu_open; + if (wait_until(stream, context, 5s, {{menu_open}}) != 0) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Main menu not detected.", stream); + } + stream.log("[Tutorial settings] Main menu detected; settling for 500ms before Options navigation."); + pbf_wait(context, 500ms); + context.wait_for_all_requests(); + TutorialOptionsDetector options_screen; + TutorialOptionPositionDetector position; + // Only two icons are available at this initial checkpoint. + // Verify the main menu before navigating to the neighboring icon. + bool options_open = false; + for (size_t attempt = 0; attempt < 2; attempt++) { + pbf_press_dpad(context, attempt == 0 ? DPAD_RIGHT : DPAD_LEFT, 80ms, 350ms); + pbf_press_button(context, BUTTON_A, 80ms, 0ms); + if (wait_for_options_screen(stream, context, options_screen, 5s)) { + options_open = true; + break; + } + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + context.wait_for_all_requests(); + MenuWatcher back_to_menu; + if (wait_until(stream, context, 5s, {{back_to_menu}}) != 0) break; + } + if (!options_open) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Options screen not detected.", stream); + } + stream.log("[Tutorial settings] Options screen detected."); + pbf_wait(context, 500ms); + context.wait_for_all_requests(); + struct Setting { const char* label; const char* value; }; + const Setting settings[] = { + {"textspeed", "fast"}, {"changewindow", "type1"}, + {"battleeffects", "off"}, {"battlestyle", "set"}, + {"sendtoboxes", "automatic"}, {"givenicknames", "dontgive"}, + {"gyroscope", "use"}, {"autosave", "off"}, + {"shownicknames", "show"}, {"backgroundmusic", "8"}, + {"soundeffects", "8"}, + }; + for (size_t row = 0; row < sizeof(settings) / sizeof(settings[0]); row++) { + // Detect the selected row afresh after every navigation/correction. + double y = -1; + const std::string base = RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/" + settings[row].label; + const ImageRGB32 label_reference(base + "_label.png"); + const ImageRGB32 value_reference(base + "_value.png"); + bool slider_reset = false; + bool correct = false; + for (size_t attempt = 0; attempt < 30; attempt++) { + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + if (frame && options_screen.detect(frame)) y = position.selected_row(frame); + else y = -1; + const bool value_visible = y >= 0 && position.selected_value(frame, y); + const double label_score = y >= 0 ? option_image_score(frame, + {0.08, y - 0.025, 0.24, 0.05}, label_reference, false) : 0; + stream.log("[Tutorial settings] Expected row=" + std::string(settings[row].label) + + " row_y=" + std::to_string(y) + " highlighted_value=" + std::to_string(value_visible) + + " label_match=" + std::to_string(label_score)); + if (y < 0 || !value_visible || label_score < 0.80) { + // A menu can be detected while its row/text is still + // fading in. Re-read without directional inputs first. + if (attempt < 4) { + pbf_wait(context, 200ms); + context.wait_for_all_requests(); + continue; + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Unable to confirm expected row " + std::string(settings[row].label), stream); + } + // These are regular video-overlay boxes, so the existing + // overlay display control can show/hide every active ROI. + OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, + {0.07, y - 0.04, 0.84, 0.08}, ""); + OverlayBoxScope label_overlay(stream.overlay(), COLOR_YELLOW, + {0.08, y - 0.025, 0.24, 0.05}, ""); + OverlayBoxScope value_overlay(stream.overlay(), COLOR_GREEN, + {0.33, y - 0.025, 0.54, 0.05}, ""); + VideoOverlaySet autosave_rois(stream.overlay()); + if (row == 7) { + autosave_rois.add(COLOR_GREEN, {0.63, y - 0.022, 0.055, 0.044}); + autosave_rois.add(COLOR_RED, {0.34, y - 0.022, 0.055, 0.044}); + } + double score; + if (row >= 9) { + // Compare the numeric value rather than the long slider: + // neighboring slider levels otherwise look too similar. + ImageViewRGB32 number = value_reference.sub_image( + value_reference.width() * 82 / 100, 0, + value_reference.width() - value_reference.width() * 82 / 100, + value_reference.height()); + ImageRGB32 number_reference = number.scale_to(96, 32); + score = option_image_score(frame, + {0.7728, y - 0.025, 0.0972, 0.05}, number_reference, true); + } else { + score = option_image_score(frame, + {0.33, y - 0.025, 0.54, 0.05}, value_reference, true); + } + stream.log("[Tutorial settings] " + std::string(settings[row].label) + + " target match=" + std::to_string(score)); + // The reference recording uses inactive pale-red text; + // the focused live row has a thicker dark-red glyph mask. + // Verified Fast scores 0.734 while Slow/Normal score 0. + if (score >= 0.65 && (row != 7 || autosave_off_highlight(frame, y))) { + if (row == 7) { + for (size_t confirmation = 0; confirmation < 3; confirmation++) { + pbf_wait(context, 200ms); + context.wait_for_all_requests(); + auto check = stream.video().snapshot(); + double check_y = check ? position.selected_row(check) : -1; + if (!check || !options_screen.detect(check) || check_y < 0 || + option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, label_reference, false) < 0.80 || + !autosave_off_highlight(check, check_y)) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Autosave Off could not be confirmed. Stopping before movement.", stream); + } + } + } + stream.log("[Tutorial settings] Verified " + std::string(settings[row].label) + "=" + settings[row].value); + correct = true; + break; + } + stream.log("[Tutorial settings] Correcting " + std::string(settings[row].label)); + if (row >= 9 && !slider_reset) { + // Sliders clamp at zero. Establish a known endpoint, + // then increment; verify the target image afterward. + for (size_t step = 0; step < 12; step++) + pbf_press_dpad(context, DPAD_LEFT, 80ms, 100ms); + slider_reset = true; + } else { + pbf_press_dpad(context, DPAD_RIGHT, 80ms, 350ms); + } + context.wait_for_all_requests(); + } + if (!correct) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Could not apply " + std::string(settings[row].label), stream); + } + if (row + 1 < sizeof(settings) / sizeof(settings[0])) + pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + } + // Confirm settings before exiting: A A A A B B. + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_press_button(context, BUTTON_A, 80ms, 1200ms); + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + context.wait_for_all_requests(); + OverworldWatcher overworld; + if (wait_until(stream, context, 10s, {{overworld}}) != 0) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Did not return to the overworld after saving options.", stream); + } + // Reopen the saved Options screen and independently verify Autosave. + // No route movement is permitted until this persisted-value check passes. + pbf_press_button(context, BUTTON_X, 80ms, 1000ms); + context.wait_for_all_requests(); + MenuWatcher verify_menu; + if (wait_until(stream, context, 5s, {{verify_menu}}) != 0) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Autosave verification: Main menu not detected.", stream); + } + bool reopened = false; + for (size_t attempt = 0; attempt < 2; attempt++) { + pbf_press_button(context, BUTTON_A, 80ms, 0ms); + if (wait_for_options_screen(stream, context, options_screen, 5s)) { reopened = true; break; } + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + context.wait_for_all_requests(); + MenuWatcher return_menu; + if (wait_until(stream, context, 5s, {{return_menu}}) != 0) break; + pbf_press_dpad(context, DPAD_RIGHT, 80ms, 350ms); + } + if (!reopened) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Autosave verification: Could not reopen saved Options.", stream); + } + // Options starts on Text Speed; Autosave is seven rows below it. + for (size_t step = 0; step < 7; step++) + pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + context.wait_for_all_requests(); + const ImageRGB32 autosave_label(RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/autosave_label.png"); + for (size_t confirmation = 0; confirmation < 3; confirmation++) { + auto check = stream.video().snapshot(); + double check_y = check ? position.selected_row(check) : -1; + if (!check || !options_screen.detect(check) || check_y < 0 || + option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, autosave_label, false) < 0.80 || + !autosave_off_highlight(check, check_y)) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Saved Autosave setting is not verified Off. Stopping before movement.", stream); + } + pbf_wait(context, 200ms); + context.wait_for_all_requests(); + } + stream.log("[Tutorial settings] Autosave = Off: confirmed after reopening saved Options (3 frames)."); + // Confirm settings before exiting: A A A A B B. + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_press_button(context, BUTTON_A, 80ms, 1200ms); + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + context.wait_for_all_requests(); + OverworldWatcher verified_exit; + if (wait_until(stream, context, 10s, {{verified_exit}}) != 0) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Autosave verification: Could not return to the overworld.", stream); + } + // Report inferred values explicitly: these are visual matches, + // allowing the user to compare our conclusions with the screen. + stream.log("[Tutorial settings] Completed. Values believed applied (visual matches before saving):"); + stream.log("[Tutorial settings] Text Speed = Fast; Change Window = Type 1; Battle Effects = Off; Battle Style = Set."); + stream.log("[Tutorial settings] Send to Boxes = Automatic; Give Nicknames = Don't give; Gyroscope = Use."); + stream.log("[Tutorial settings] Autosave = Off; Show Nicknames = Show; Background Music = 8; Sound Effects = 8."); +} + +} + +void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context) { + // Settings-only recovery: initial attempt plus three retries. + // Never permit route movement unless the complete check succeeds. + constexpr size_t MAX_RETRIES = 3; + for (size_t attempt = 0; attempt <= MAX_RETRIES; attempt++) { + stream.log("[Tutorial settings] Attempt " + std::to_string(attempt + 1) + "/4."); + try { + verify_tutorial_settings_once(stream, context); + return; + } catch (OperationFailedException& error) { + stream.log("[Tutorial settings] Attempt failed: " + error.message(), COLOR_ORANGE); + if (attempt == MAX_RETRIES) { + stream.log("[Tutorial settings] All three retries exhausted; stopping before movement.", COLOR_RED); + throw; + } + // Back out of Options, save prompts, bag, or menu without + // moving the player or resetting/loading the game. + stream.log("[Tutorial settings] Returning to overworld before retry."); + context.wait_for_all_requests(); + TutorialOptionsDetector options_screen; + const auto failed_frame = stream.video().snapshot(); + if (failed_frame && options_screen.detect(failed_frame)) { + // Finish the known Options save/exit sequence before + // observing state; never cancel it on an Options frame + // misclassified as overworld by the generic detector. + for (size_t press = 0; press < 4; press++) + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + } + pbf_mash_button(context, BUTTON_B, 3000ms); + context.wait_for_all_requests(); + OverworldDetector overworld; + MenuDetector menu; + BlackScreenDetector black; + bool recovered = false; + size_t confirmations = 0; + for (size_t check = 0; check < 15; check++) { + const auto frame = stream.video().snapshot(); + const bool clear = frame && !options_screen.detect(frame) && + !menu.detect(frame) && !black.detect(frame) && bag_pocket_position(frame) == 0 && overworld.detect(frame); + if (clear) { + if (++confirmations >= 3) { recovered = true; break; } + pbf_wait(context, 200ms); + } else { + confirmations = 0; + pbf_mash_button(context, BUTTON_B, 500ms); + } + context.wait_for_all_requests(); + } + stream.log(recovered ? "[Tutorial settings] Recovery confirmed outside Options/menu; retrying." + : "[Tutorial settings] Recovery not confirmed; next attempt will recheck the screen."); + pbf_wait(context, 500ms); + context.wait_for_all_requests(); + } + } +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h new file mode 100644 index 0000000000..2fa2d27641 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h @@ -0,0 +1,31 @@ +/* BDSP AutoStory - Settings + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Settings_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Settings_H + +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// From the overworld: opens Options and sets what the story needs (fast +// text, Send to Boxes automatic, no nickname prompt, Autosave off), saves +// them and returns to the overworld. Tries four times, backing out to the +// overworld between tries, and fails with a screenshot before any movement. +void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context); + + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp new file mode 100644 index 0000000000..b0333a7963 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp @@ -0,0 +1,444 @@ +/* BDSP AutoStory - Wild battles + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/OCR/OCR_RawTesseractOCR.h" +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_EndBattleDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_SelectionArrow.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_ReceivePokemonDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_RouteWalk.h" +#include "PokemonBDSP_AutoStory_Battle.h" +#include "PokemonBDSP_AutoStory_WildBattle.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +namespace{ + +// Two grass rows at 240 ms each, with room to spare: the wall stops us. +constexpr Milliseconds CATCH_GRASS_WALK = 900ms; + +void encounter_one_wild_pokemon(VideoStream& stream, ProControllerContext& context) { + context.wait_for_all_requests(); + pbf_wait(context, 1000ms); + + bool encounter_found = false; + const auto deadline = std::chrono::steady_clock::now() + 180s; + + // Walk up and down the catch grass until a battle starts. Its three rows + // sit between tree walls, so each walk ends at a wall, on the same tile + // every time, after two new grass tiles. Short sideways steps drift and + // can rock across one tile edge without entering new grass. + while (!encounter_found && std::chrono::steady_clock::now() < deadline) { + BlackScreenWatcher battle_start; + BattleMenuWatcher battle_menu(BattleType::STANDARD, COLOR_YELLOW); + BattleDialogWatcher battle_dialog(COLOR_YELLOW); + + route_turn(stream, context, Pokemon::Direction::NORTH); + pbf_move_left_joystick(context, {0, 1}, CATCH_GRASS_WALK, 200ms); + context.wait_for_all_requests(); + + int ret = wait_until( + stream, + context, + 1500ms, + { + {battle_start}, + {battle_menu}, + {battle_dialog}, + } + ); + + if (ret >= 0) { + encounter_found = true; + break; + } + + route_turn(stream, context, Pokemon::Direction::SOUTH); + pbf_move_left_joystick(context, {0, -1}, CATCH_GRASS_WALK, 200ms); + context.wait_for_all_requests(); + + ret = wait_until( + stream, + context, + 1500ms, + { + {battle_start}, + {battle_menu}, + {battle_dialog}, + } + ); + + if (ret >= 0) { + encounter_found = true; + break; + } + } + + if (!encounter_found) OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Wild encounter: walking up and down the grass found no battle within three minutes.", stream); + stream.log("[AutoStory] Wild battle detected!", COLOR_GREEN); + + // Wait until the actual battle menu appears. + BattleMenuWatcher confirm_menu(BattleType::STANDARD); + + int menu_ret = wait_until( + stream, + context, + 15s, + { + {confirm_menu}, + } + ); + + if (menu_ret < 0) { + stream.log( + "[AutoStory] Failed to detect battle menu.", + COLOR_RED + ); + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "[AutoStory] Failed to detect battle menu.", stream); + } + + stream.log("[AutoStory] Battle menu detected.", COLOR_GREEN); + + context.wait_for_all_requests(); + pbf_wait(context, 500ms); + +} + +bool mine_confirm_escape_via_menu(VideoStream& stream, ProControllerContext& context, BattleType battle_type){ + MenuWatcher game_menu; + OverworldWatcher overworld; + BattleMenuWatcher battle_menu(battle_type, COLOR_YELLOW); + for (size_t attempt = 1; attempt <= 3; attempt++){ + // Do not accept the absence-based overworld detector before menu proof. + const int state = wait_until(stream, context, 300ms, {{game_menu}, {battle_menu}}); + if (state == 1){ + stream.log("[Mine escape] Battle menu still present; retrying Run."); + return false; + } + int detected = state; + if (state != 0){ + stream.log("[Mine escape] Main menu fallback attempt " + std::to_string(attempt) + "/3."); + detected = run_until(stream, + context, + [](ProControllerContext& controller){ + pbf_press_button(controller, BUTTON_X, 160ms, 840ms); + pbf_wait(controller, 3s); + controller.wait_for_all_requests(); + }, + {game_menu, battle_menu}); + if (detected == 1){ + return false; + } + } + if (detected != 0){ + continue; + } + stream.log("[Mine escape] Main game menu confirmed; battle has ended.", COLOR_GREEN); + // Close the confirmed menu before returning control to navigation. + MenuDetector menu_visible; + for (size_t close = 0; close < 3; close++){ + pbf_press_button(context, BUTTON_B, 80ms, 500ms); + context.wait_for_all_requests(); + if (wait_until(stream, context, 1s, {{overworld}}) == 0){ + return true; + } + BlackScreenWatcher visible(COLOR_YELLOW, + {0.1, 0.1, 0.8, 0.8}, + 100, + 10, + BlackScreenWatcher::FinderType::GONE, + 500ms); + if (wait_until(stream, context, 2s, {{visible}}) == 0){ + const VideoSnapshot frame = stream.video().snapshot(); + if (frame && !menu_visible.detect(frame)){ + stream.log("[Mine escape] Confirmed game menu closed; returning to navigation.", COLOR_GREEN); + return true; + } + } + } + stream.log("[Mine escape] Main menu detected but could not confirm it closed.", COLOR_ORANGE); + return false; + } + return false; +} + +bool mine_select_run(VideoStream& stream, ProControllerContext& context, BattleType battle_type){ + BattleMenuDetector cursor(battle_type, COLOR_YELLOW); + size_t matching_frames = 0; + for (size_t check = 0; check < 16; check++){ + context.wait_for_all_requests(); + const VideoSnapshot frame = stream.video().snapshot(); + if (!frame){ + return false; + } + const BattleMenuOption selected = battle_menu_selected_option(cursor, frame); + stream.log( + "[Mine escape] Battle cursor row=" + std::to_string(static_cast(selected) + 1) + "; target Run=4."); + if (selected == BattleMenuOption::RUN){ + if (++matching_frames >= 3){ + stream.log("[Mine escape] Run cursor confirmed.", COLOR_GREEN); + return true; + } + pbf_wait(context, 100ms); + }else{ + matching_frames = 0; + if (selected == BattleMenuOption::UNKNOWN){ + // Do not send a direction or A without a known cursor. + pbf_wait(context, 150ms); + }else{ + const DpadPosition direction = selected == BattleMenuOption::BATTLE ? DPAD_UP : DPAD_DOWN; + pbf_press_dpad(context, direction, 80ms, 250ms); + } + } + } + stream.log("[Mine escape] Could not confirm Run selection; A was not pressed.", COLOR_ORANGE); + return false; +} + +} + +void fight_starly(VideoStream& stream, ProControllerContext& context) { + stream.log("[AutoStory] Starting Starly battle...", COLOR_BLUE); + + // Declare OUTSIDE the loop so accumulated detector state + // (trigger counts, black-screen transitions) is preserved across frames. + // STARTER type: player has no Pokeballs, so ball-count areas won't + // be white. STARTER skips that check and only verifies status bars + // and the Fight button color. + BattleMenuWatcher battle_menu(BattleType::STARTER, COLOR_YELLOW); + EndBattleWatcher end_battle; + + while (true) { + context.wait_for_all_requests(); + + // Phase 1: wait for either battle menu or battle end. + int ret = run_until( + stream, context, + [](ProControllerContext& ctx) { + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, +{ + {battle_menu}, + {end_battle}, +} +); + + switch (ret) { + case 0: + { + stream.log("[AutoStory] Battle menu detected. Selecting move...", COLOR_BLUE); + + // Select Fight, then move slot 1. + pbf_press_button(context, BUTTON_A, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 3000ms); + // Loop back - mash B through animations until next menu or end. + break; + } + case 1: + { + stream.log("[AutoStory] Starly battle complete.", COLOR_GREEN); + + // Mash through XP / level-up / dialog screens back to overworld. + OverworldWatcher overworld; + int cleanup_ret = run_until( + stream, context, + [](ProControllerContext& ctx) { + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, + { {overworld} } + ); + if (cleanup_ret != 0) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Starly battle: Overworld cleanup timed out.", stream); + } + return; + } + default: + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Starly battle timed out.", stream); + } + } +} + +void battle_1_pokemon(VideoStream& stream, ProControllerContext& context) { + stream.log("[Training] Finding one wild Pokemon using the catch encounter wiggle.", COLOR_BLUE); + encounter_one_wild_pokemon(stream, context); + // The best move on the menu, whatever the starter. + fight_wild_battle(stream, context, "Training"); + context.wait_for_all_requests(); + stream.log("[Training] One wild battle completed.", COLOR_GREEN); +} + +void catch_1_pokemon( + VideoStream& stream, + ProControllerContext& context +) { + stream.log("[AutoStory] Starting catch_1_pokemon helper...", COLOR_BLUE); + + encounter_one_wild_pokemon(stream, context); + + // Catch the pokemon. + stream.log("[AutoStory] Throwing Poke Ball (manual bag open)...", COLOR_BLUE); + ReceivePokemonDetector received_pokemon; + // Preserve exit-transition state from the ball outcome through + // receipt/experience cleanup; do not recreate it between phases. + EndBattleWatcher battle_ended; + bool caught = false; + bool exit_seen = false; + for (int throw_attempt = 0; throw_attempt < 10; throw_attempt++) { + // Open bag from battle menu (X), confirm first slot (ZL). + pbf_press_button(context, BUTTON_X, 160ms, 840ms); + context.wait_for_all_requests(); + pbf_press_button(context, BUTTON_ZL, 160ms, 1000ms); + context.wait_for_all_requests(); + + // Wait to see if the battle menu comes back (ball failed) + // or if the battle ends (caught or fainted). + BattleMenuWatcher menu_back(BattleType::STANDARD, COLOR_YELLOW); + int throw_ret = run_until( + stream, context, + [](ProControllerContext& ctx) { + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, + {{menu_back}, {received_pokemon}, {battle_ended}} + ); + + if (throw_ret == 1 || (throw_ret == 2 && received_pokemon.received())) { + caught = true; + exit_seen = throw_ret == 2; + stream.log("[AutoStory] Pokemon receipt verified.", COLOR_GREEN); + break; + } + if (throw_ret < 0 || throw_ret == 2) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Catch failed: No verified Pokemon receipt before battle ended or timed out.", stream); + } + stream.log( + "[AutoStory] Ball failed, retrying (attempt " + std::to_string(throw_attempt + 2) + ")...", + COLOR_ORANGE + ); + } + + context.wait_for_all_requests(); + + if (!caught) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Catch failed: Exhausted ten ball attempts.", stream); + } + // Receipt alone is not an overworld return: clear experience, + // automatic move learning, dex registration and nickname screens + // until the same completed-black-transition watcher used in battles fires. + if (!exit_seen) { + stream.log("[Catch cleanup] Receipt confirmed; waiting for battle exit transition."); + const int cleanup_ret = run_until(stream, context, + [](ProControllerContext& ctx) { pbf_mash_button(ctx, BUTTON_B, 120000ms); }, + {{battle_ended}}); + if (cleanup_ret != 0) { + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Catch cleanup: Battle exit transition did not finish.", stream); + } + } + context.wait_for_all_requests(); + stream.log("[Catch cleanup] Battle exit transition confirmed; checking stable overworld."); + confirm_battle_overworld(stream, context, "Catch cleanup"); + stream.log("[Catch cleanup] Verified receipt, completed battle exit, and four seconds of clear overworld.", COLOR_GREEN); +} + +void escape_wild_battle(VideoStream& stream, + ProControllerContext& context, + BattleType battle_type, + std::chrono::seconds timeout){ + EndBattleWatcher battle_end; + bool exit_transition = false; + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline){ + pbf_press_button(context, BUTTON_B, 40ms, 200ms); + context.wait_for_all_requests(); + BattleMenuWatcher menu_ready(battle_type, COLOR_YELLOW); + const int ready = wait_until(stream, context, 3s, {{battle_end}, {menu_ready}}); + if (ready == 0){ + exit_transition = true; + break; + } + if (ready != 1){ + continue; + } + if (!mine_select_run(stream, context, battle_type)){ + continue; + } + stream.log("[Mine escape] Run input sent; watching for black-screen exit."); + const int result = run_until(stream, + context, + [](ProControllerContext& controller){ + pbf_press_button(controller, BUTTON_A, 60ms, 0ms); + pbf_wait(controller, 4s); + controller.wait_for_all_requests(); + }, + {battle_end}); + if (result == 0){ + exit_transition = true; + break; + } + } + if (!exit_transition && wait_until(stream, context, 10s, {{battle_end}}) == 0){ + exit_transition = true; + } + if (exit_transition){ + // EndBattleWatcher requires black followed by a visible screen. + stream.log("[Mine escape] Black-screen exit transition confirmed.", COLOR_GREEN); + pbf_wait(context, 800ms); + context.wait_for_all_requests(); + return; + } + // Only probe X after normal escape attempts and the final transition wait. + // Absence of battle UI by itself is not evidence of an overworld return. + stream.log("[Mine escape] Exit transition missed; trying late main-menu fallback."); + if (mine_confirm_escape_via_menu(stream, context, battle_type)){ + return; + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "escape_wild_battle: Exit transition and late main-menu fallback could not confirm escape.", + stream); +} + +void run_from_wild_battle(VideoStream& stream, ProControllerContext& context){ + escape_wild_battle(stream, context, BattleType::STANDARD); +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h new file mode 100644 index 0000000000..5377a3c1b4 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h @@ -0,0 +1,56 @@ +/* BDSP AutoStory - Wild battles + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * The story's wild battles: the Starly the starter is chosen against, one + * found in the grass to fight or to catch, and running from one that + * interrupts a walk. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_WildBattle_H +#define PokemonAutomation_PokemonBDSP_AutoStory_WildBattle_H + +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The Starly battle at Lake Verity, from its first text to the overworld. +void fight_starly(VideoStream& stream, ProControllerContext& context); + +// Finds one wild Pokemon in the grass by walking up and down a column +// between two tree walls, and fights it with the best move. Does not heal. +void battle_1_pokemon(VideoStream& stream, ProControllerContext& context); + +// Finds one wild Pokemon the same way and catches it with the first ball +// in the Bag, up to ten throws; ends on the overworld. +void catch_1_pokemon(VideoStream& stream, ProControllerContext& context); + +// Runs from the wild battle on screen: Run on the battle menu until the +// exit's black screen shows. Fails with a screenshot after `timeout`. +void escape_wild_battle( + VideoStream& stream, + ProControllerContext& context, + BattleType battle_type = BattleType::STANDARD, + std::chrono::seconds timeout = std::chrono::seconds(30) +); + +// escape_wild_battle() for a standard wild battle, as the walker takes it +// (WildBattleEscape): how the story's walks leave the battles that +// interrupt them. +void run_from_wild_battle(VideoStream& stream, ProControllerContext& context); + + + +} +} +} +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 9c879c1e8b..0edb4d359d 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1350,6 +1350,8 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.h + Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp + Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h Source/NintendoSwitch/NintendoSwitch_ConsoleHandle.cpp @@ -1572,6 +1574,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.h Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h + Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp + Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h Source/PokemonBDSP/Inference/PokemonBDSP_PokeballSpriteMatcher.cpp Source/PokemonBDSP/Inference/PokemonBDSP_PokeballSpriteMatcher.h Source/PokemonBDSP/Inference/PokemonBDSP_ReceivePokemonDetector.cpp @@ -1624,6 +1628,26 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h Source/PokemonBDSP/PokemonBDSP_Tests.cpp Source/PokemonBDSP/PokemonBDSP_Tests.h + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h Source/PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_Maps.cpp Source/PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_Maps.h Source/PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_PlanCosts.cpp From cf389f36bc0048e742efa0187e5d2767bfaef925 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 14:02:35 -0800 Subject: [PATCH 02/23] BDSP AutoStory actions: drop the unused Bag icon box and the moved banners Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp | 4 ---- .../Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp | 5 ----- 2 files changed, 9 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp index 70a24e5350..60864695c0 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp @@ -30,10 +30,6 @@ using namespace std::chrono_literals; namespace{ -ImageFloatBox bag_list_icon_box(size_t row){ - return ImageFloatBox(0.449, 0.212 + (row - 1) * 0.075, 0.033, 0.050); -} - ImageFloatBox bag_list_name_box(size_t row){ // Previous height included the faint lower selection border in rows 2/6. // return ImageFloatBox(0.492, 0.213 + (row - 1) * 0.075, 0.335, 0.052); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp index 8635622af2..df473e30ca 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp @@ -441,9 +441,7 @@ void fight_using_move_list( switch (ret) { - // ------------------------------------------------- // Battle menu detected. - // ------------------------------------------------- case 0: battle_menu_seen = true; // Early battles take a few turns; this many means the chosen @@ -483,9 +481,7 @@ void fight_using_move_list( break; - // ------------------------------------------------- // End battle detected. - // ------------------------------------------------- case 1: { stream.log( "[AutoStory] " + label + " battle complete.", @@ -596,7 +592,6 @@ void fight_wild_battle(VideoStream& stream, ProControllerContext& context, const } - } } } From 18d71e03ee7ba6cbd19333cfebd5b4ae52d8bd13 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 14:09:13 -0800 Subject: [PATCH 03/23] BDSP AutoStory actions: credit #1717 in the battle menu and X menu readers Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp index a54eadcc5e..407bd7818e 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp @@ -2,6 +2,8 @@ * * From: https://github.com/PokemonAutomation/ * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * */ #include From 842a043d0991bd0c65ec345fc3e338fa0dff31ab Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 14:10:25 -0800 Subject: [PATCH 04/23] BDSP AutoStory actions: CRLF line endings, and credit #1717 in the X menu reader Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../NintendoSwitch_SystemKeyboardDetector.cpp | 312 +++--- .../NintendoSwitch_SystemKeyboardDetector.h | 134 +-- .../PokemonBDSP_NameListDetector.cpp | 192 ++-- .../Inference/PokemonBDSP_NameListDetector.h | 82 +- ...kemonBDSP_AutoStory_MenuCursorDetector.cpp | 586 ++++++------ ...PokemonBDSP_AutoStory_MenuCursorDetector.h | 294 +++--- .../PokemonBDSP_AutoStory_Center.cpp | 890 +++++++++--------- .../AutoStory/PokemonBDSP_AutoStory_Center.h | 236 ++--- .../PokemonBDSP_AutoStory_NewGame.cpp | 572 +++++------ .../AutoStory/PokemonBDSP_AutoStory_NewGame.h | 142 +-- 10 files changed, 1722 insertions(+), 1718 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp index 765baba05f..b8d75872dd 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -1,156 +1,156 @@ -/* System Keyboard Detector - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include -#include -#include -#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" -#include "CommonFramework/ImageTools/ImageStats.h" -#include "CommonFramework/VideoPipeline/VideoFeed.h" -#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" -#include "CommonTools/Images/SolidColorTest.h" -#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" -#include "NintendoSwitch_SystemKeyboardDetector.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ - -using namespace std::chrono_literals; - - -// Key grid of the QWERTY layout and where each key sits: rows "1".."z", columns "1".."-". -static const char* const KEY_ROWS[] = {"1234567890-", "qwertyuiop/", "asdfghjkl:'", "zxcvbnm,.?!"}; -static constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770}; -static constexpr double KEY_COL_X0 = 0.077; -static constexpr double KEY_COL_STEP = 0.075; - - -SystemKeyboardDetector::SystemKeyboardDetector(Color color) - : m_color(color) - , m_key_panel(0.15, 0.555, 0.65, 0.020) - , m_bottom_bar(0.10, 0.935, 0.35, 0.030) - , m_ok_button(0.875, 0.760, 0.045, 0.020) -{} -void SystemKeyboardDetector::make_overlays(VideoOverlaySet& items) const{ - items.add(m_color, m_key_panel); - items.add(m_color, m_bottom_bar); - items.add(m_color, m_ok_button); -} -bool SystemKeyboardDetector::detect(const ImageViewRGB32& screen){ - if (!is_white(extract_box_reference(screen, m_key_panel)) || - !is_white(extract_box_reference(screen, m_bottom_bar)) - ){ - return false; - } - // OK is blue, or flat grey while the field is empty (disabled). - const ImageStats ok = image_stats(extract_box_reference(screen, m_ok_button)); - return is_solid(ok, {0.172, 0.276, 0.552}) || is_grey(ok, 600, 690); -} - - -bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col){ - // The cursor is a cyan border around one key. Neighbouring keys overlap that border, - // so take the centroid of all cyan pixels and map it to the nearest key. - const ImageFloatBox area(0.02, 0.44, 0.84, 0.38); - ImageViewRGB32 keys = extract_box_reference(screen, area); - double count = 0, sum_x = 0, sum_y = 0; - for (size_t y = 0; y < keys.height(); y++){ - for (size_t x = 0; x < keys.width(); x++){ - const uint32_t pixel = keys.pixel(x, y); - const int red = (pixel >> 16) & 0xff; - const int green = (pixel >> 8) & 0xff; - const int blue = pixel & 0xff; - if (red < 150 && green > 200 && blue > 190){ - count++; - sum_x += (double)x; - sum_y += (double)y; - } - } - } - // The border is about 0.15% of the screen; a few stray pixels are not a cursor. - if (count < 0.0005 * screen.width() * screen.height()){ - return false; - } - const double cx = area.x + sum_x / count / keys.width() * area.width; - const double cy = area.y + sum_y / count / keys.height() * area.height; - col = (int)std::lround((cx - KEY_COL_X0) / KEY_COL_STEP); - row = 0; - for (int r = 1; r < 4; r++){ - if (std::abs(KEY_ROW_Y[r] - cy) < std::abs(KEY_ROW_Y[row] - cy)){ - row = r; - } - } - return 0 <= col && col < 11; -} - -bool system_keyboard_field_has_text(const ImageViewRGB32& screen){ - // Left part of the field, after the caret's position on an empty field. - const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.11, 0.235, 0.19, 0.030))); - return stats.stddev.sum() > 30; -} - - -void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text){ - for (char ch : text){ - const char key = (char)std::tolower((unsigned char)ch); - int target_row = -1; - int target_col = -1; - for (int r = 0; r < 4 && target_row < 0; r++){ - const char* p = std::strchr(KEY_ROWS[r], key); - if (p != nullptr && key != '\0'){ - target_row = r; - target_col = (int)(p - KEY_ROWS[r]); - } - } - if (target_row < 0){ - OperationFailedExceptionWithScreenshot::fire( - ErrorReportMode::SEND_ERROR_REPORT, - std::string("System keyboard: no key for '") + ch + "'.", - stream - ); - } - - // A button pressed while the keyboard slides in is ignored, so read the cursor - // again after each move until it is on the target. - bool arrived = false; - for (size_t attempt = 0; attempt < 10 && !arrived; attempt++){ - context.wait_for_all_requests(); - int row = -1; - int col = -1; - VideoSnapshot screen = stream.video().snapshot(); - if (!screen || !find_system_keyboard_cursor(screen, row, col)){ - pbf_wait(context, 300ms); - continue; - } - if (row == target_row && col == target_col){ - arrived = true; - break; - } - for (; row < target_row; row++) pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms); - for (; row > target_row; row--) pbf_press_dpad(context, DPAD_UP, 100ms, 200ms); - for (; col < target_col; col++) pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms); - for (; col > target_col; col--) pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms); - pbf_wait(context, 300ms); - } - if (!arrived){ - OperationFailedExceptionWithScreenshot::fire( - ErrorReportMode::SEND_ERROR_REPORT, - std::string("System keyboard: the cursor did not reach '") + key + "'.", - stream - ); - } - if (std::isupper((unsigned char)ch)){ - pbf_press_button(context, BUTTON_LCLICK, 100ms, 250ms); - } - pbf_press_button(context, BUTTON_A, 80ms, 300ms); - } - context.wait_for_all_requests(); -} - - - -} -} +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch_SystemKeyboardDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + +using namespace std::chrono_literals; + + +// Key grid of the QWERTY layout and where each key sits: rows "1".."z", columns "1".."-". +static const char* const KEY_ROWS[] = {"1234567890-", "qwertyuiop/", "asdfghjkl:'", "zxcvbnm,.?!"}; +static constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770}; +static constexpr double KEY_COL_X0 = 0.077; +static constexpr double KEY_COL_STEP = 0.075; + + +SystemKeyboardDetector::SystemKeyboardDetector(Color color) + : m_color(color) + , m_key_panel(0.15, 0.555, 0.65, 0.020) + , m_bottom_bar(0.10, 0.935, 0.35, 0.030) + , m_ok_button(0.875, 0.760, 0.045, 0.020) +{} +void SystemKeyboardDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_key_panel); + items.add(m_color, m_bottom_bar); + items.add(m_color, m_ok_button); +} +bool SystemKeyboardDetector::detect(const ImageViewRGB32& screen){ + if (!is_white(extract_box_reference(screen, m_key_panel)) || + !is_white(extract_box_reference(screen, m_bottom_bar)) + ){ + return false; + } + // OK is blue, or flat grey while the field is empty (disabled). + const ImageStats ok = image_stats(extract_box_reference(screen, m_ok_button)); + return is_solid(ok, {0.172, 0.276, 0.552}) || is_grey(ok, 600, 690); +} + + +bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col){ + // The cursor is a cyan border around one key. Neighbouring keys overlap that border, + // so take the centroid of all cyan pixels and map it to the nearest key. + const ImageFloatBox area(0.02, 0.44, 0.84, 0.38); + ImageViewRGB32 keys = extract_box_reference(screen, area); + double count = 0, sum_x = 0, sum_y = 0; + for (size_t y = 0; y < keys.height(); y++){ + for (size_t x = 0; x < keys.width(); x++){ + const uint32_t pixel = keys.pixel(x, y); + const int red = (pixel >> 16) & 0xff; + const int green = (pixel >> 8) & 0xff; + const int blue = pixel & 0xff; + if (red < 150 && green > 200 && blue > 190){ + count++; + sum_x += (double)x; + sum_y += (double)y; + } + } + } + // The border is about 0.15% of the screen; a few stray pixels are not a cursor. + if (count < 0.0005 * screen.width() * screen.height()){ + return false; + } + const double cx = area.x + sum_x / count / keys.width() * area.width; + const double cy = area.y + sum_y / count / keys.height() * area.height; + col = (int)std::lround((cx - KEY_COL_X0) / KEY_COL_STEP); + row = 0; + for (int r = 1; r < 4; r++){ + if (std::abs(KEY_ROW_Y[r] - cy) < std::abs(KEY_ROW_Y[row] - cy)){ + row = r; + } + } + return 0 <= col && col < 11; +} + +bool system_keyboard_field_has_text(const ImageViewRGB32& screen){ + // Left part of the field, after the caret's position on an empty field. + const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.11, 0.235, 0.19, 0.030))); + return stats.stddev.sum() > 30; +} + + +void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text){ + for (char ch : text){ + const char key = (char)std::tolower((unsigned char)ch); + int target_row = -1; + int target_col = -1; + for (int r = 0; r < 4 && target_row < 0; r++){ + const char* p = std::strchr(KEY_ROWS[r], key); + if (p != nullptr && key != '\0'){ + target_row = r; + target_col = (int)(p - KEY_ROWS[r]); + } + } + if (target_row < 0){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::string("System keyboard: no key for '") + ch + "'.", + stream + ); + } + + // A button pressed while the keyboard slides in is ignored, so read the cursor + // again after each move until it is on the target. + bool arrived = false; + for (size_t attempt = 0; attempt < 10 && !arrived; attempt++){ + context.wait_for_all_requests(); + int row = -1; + int col = -1; + VideoSnapshot screen = stream.video().snapshot(); + if (!screen || !find_system_keyboard_cursor(screen, row, col)){ + pbf_wait(context, 300ms); + continue; + } + if (row == target_row && col == target_col){ + arrived = true; + break; + } + for (; row < target_row; row++) pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms); + for (; row > target_row; row--) pbf_press_dpad(context, DPAD_UP, 100ms, 200ms); + for (; col < target_col; col++) pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms); + for (; col > target_col; col--) pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms); + pbf_wait(context, 300ms); + } + if (!arrived){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::string("System keyboard: the cursor did not reach '") + key + "'.", + stream + ); + } + if (std::isupper((unsigned char)ch)){ + pbf_press_button(context, BUTTON_LCLICK, 100ms, 250ms); + } + pbf_press_button(context, BUTTON_A, 80ms, 300ms); + } + context.wait_for_all_requests(); +} + + + +} +} diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h index aa23e7ec81..c29b21931e 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h @@ -1,67 +1,67 @@ -/* System Keyboard Detector - * - * From: https://github.com/PokemonAutomation/ - * - * The Switch's own software keyboard, which games open for names (e.g. the - * player's name in BDSP): detect it, find the key under its cursor, and type - * a string by moving the cursor and checking where it landed. - * - * Measured on 1920x1080 captures of the QWERTY layout. - * - */ - -#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H -#define PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H - -#include -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/Tools/VideoStream.h" -#include "CommonTools/VisualDetector.h" -#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ - - -// White key panel and bottom bar, and the blue OK button. -class SystemKeyboardDetector : public StaticScreenDetector{ -public: - SystemKeyboardDetector(Color color = COLOR_CYAN); - - virtual void make_overlays(VideoOverlaySet& items) const override; - virtual bool detect(const ImageViewRGB32& screen) override; - -private: - Color m_color; - ImageFloatBox m_key_panel; - ImageFloatBox m_bottom_bar; - ImageFloatBox m_ok_button; -}; -class SystemKeyboardWatcher : public DetectorToFinder{ -public: - SystemKeyboardWatcher(Color color = COLOR_CYAN) - : DetectorToFinder("SystemKeyboardWatcher", std::chrono::milliseconds(250), color) - {} -}; - - -// The key under the cursor, as row (0: digits .. 3: "zxcv") and column (0..10). -// The keyboard remembers the cursor between uses, so read it, don't assume it. -// Returns false if no cursor border is visible. -bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col); - -// Whether the text field holds any text. Pressing B on an empty field closes the -// keyboard, so delete only while this is true. -bool system_keyboard_field_has_text(const ImageViewRGB32& screen); - -// Types `text` (letters, digits and the symbols on the QWERTY grid). For each character, -// reads the cursor, moves it with the D-pad and checks it arrived before pressing A; an -// uppercase letter is typed with Shift (left stick click). Throws, with a screenshot, if -// the cursor can't be read or doesn't arrive. -void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text); - - - -} -} -#endif +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The Switch's own software keyboard, which games open for names (e.g. the + * player's name in BDSP): detect it, find the key under its cursor, and type + * a string by moving the cursor and checking where it landed. + * + * Measured on 1920x1080 captures of the QWERTY layout. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H +#define PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H + +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonTools/VisualDetector.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// White key panel and bottom bar, and the blue OK button. +class SystemKeyboardDetector : public StaticScreenDetector{ +public: + SystemKeyboardDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_key_panel; + ImageFloatBox m_bottom_bar; + ImageFloatBox m_ok_button; +}; +class SystemKeyboardWatcher : public DetectorToFinder{ +public: + SystemKeyboardWatcher(Color color = COLOR_CYAN) + : DetectorToFinder("SystemKeyboardWatcher", std::chrono::milliseconds(250), color) + {} +}; + + +// The key under the cursor, as row (0: digits .. 3: "zxcv") and column (0..10). +// The keyboard remembers the cursor between uses, so read it, don't assume it. +// Returns false if no cursor border is visible. +bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col); + +// Whether the text field holds any text. Pressing B on an empty field closes the +// keyboard, so delete only while this is true. +bool system_keyboard_field_has_text(const ImageViewRGB32& screen); + +// Types `text` (letters, digits and the symbols on the QWERTY grid). For each character, +// reads the cursor, moves it with the D-pad and checks it arrived before pressing A; an +// uppercase letter is typed with Shift (left stick click). Throws, with a screenshot, if +// the cursor can't be read or doesn't arrive. +void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text); + + + +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp index 456e914374..f9bf85a2a6 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp @@ -1,96 +1,96 @@ -/* Name List Detector - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include -#include -#include "Kernels/Waterfill/Kernels_Waterfill.h" -#include "CommonFramework/GlobalAutoPaths.h" -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonTools/Images/BinaryImage_FilterRgb32.h" -#include "CommonTools/ImageMatch/ExactImageMatcher.h" -#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" -#include "CommonTools/Images/SolidColorTest.h" -#include "PokemonBDSP_NameListDetector.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -namespace{ -// Row centers, as a fraction of the screen height (1080p: 307, 370, 433, ...). -const double FIRST_ROW_Y = 0.284; -const double ROW_STEP_Y = 0.0587; -const int ROWS = 5; - -// The menu's selection arrows in `image`, as PokemonBDSP_SelectionArrow.cpp -// finds them: that file keeps its finder out of its header, so the same -// waterfill and template match are repeated here rather than changing it. -const ImageMatch::ExactImageMatcher& name_list_arrow(){ - static ImageMatch::ExactImageMatcher matcher(RESOURCE_PATH() + "PokemonBDSP/SelectionArrow.png"); - return matcher; -} -std::vector find_name_list_arrows(const ImageViewRGB32& image){ - using namespace Kernels; - using namespace Kernels::Waterfill; - PackedBinaryMatrix matrix = compress_rgb32_to_binary_max(image, 200, 200, 200); - std::vector objects = find_objects_inplace(matrix, 200); - std::vector ret; - for (const WaterfillObject& object : objects){ - if (object.width() > object.height() || object.height() > 3 * object.width()){ - continue; - } - if (name_list_arrow().rmsd(extract_box_reference(image, object)) <= 90){ - ret.emplace_back(object.min_x, object.min_y, object.max_x, object.max_y); - } - } - return ret; -} -} - - -NameListDetector::NameListDetector(Color color) - : m_color(color) - , m_arrows(0.137, 0.245, 0.030, 0.310) - , m_panel(0.140, 0.245, 0.140, 0.300) -{} -void NameListDetector::make_overlays(VideoOverlaySet& items) const{ - items.add(m_color, m_arrows); - items.add(m_color, m_panel); -} -bool NameListDetector::detect(const ImageViewRGB32& screen){ - return selected_row(screen) >= 0; -} -int NameListDetector::selected_row(const ImageViewRGB32& screen) const{ - // The list is a white panel with a little dark text. A thin margin strip is not - // enough: the border's antialiasing reaches into it on some frames. - const ImageViewRGB32 panel = extract_box_reference(screen, m_panel); - size_t white = 0; - for (size_t y = 0; y < panel.height(); y += 2){ - for (size_t x = 0; x < panel.width(); x += 2){ - const uint32_t p = panel.pixel(x, y); - white += ((p >> 16) & 0xff) > 220 && ((p >> 8) & 0xff) > 220 && (p & 0xff) > 220; - } - } - const size_t samples = ((panel.width() + 1) / 2) * ((panel.height() + 1) / 2); - if (samples == 0 || white < samples * 6 / 10){ - return -1; - } - ImageViewRGB32 area = extract_box_reference(screen, m_arrows); - std::vector arrows = find_name_list_arrows(area); - if (arrows.size() != 1){ - return -1; - } - const double center_y = m_arrows.y + - m_arrows.height * (arrows[0].min_y + arrows[0].max_y) / 2.0 / (double)area.height(); - const int row = (int)std::lround((center_y - FIRST_ROW_Y) / ROW_STEP_Y); - return row >= 0 && row < ROWS ? row : -1; -} - - -} -} -} +/* Name List Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Kernels/Waterfill/Kernels_Waterfill.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/Images/BinaryImage_FilterRgb32.h" +#include "CommonTools/ImageMatch/ExactImageMatcher.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "PokemonBDSP_NameListDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ +// Row centers, as a fraction of the screen height (1080p: 307, 370, 433, ...). +const double FIRST_ROW_Y = 0.284; +const double ROW_STEP_Y = 0.0587; +const int ROWS = 5; + +// The menu's selection arrows in `image`, as PokemonBDSP_SelectionArrow.cpp +// finds them: that file keeps its finder out of its header, so the same +// waterfill and template match are repeated here rather than changing it. +const ImageMatch::ExactImageMatcher& name_list_arrow(){ + static ImageMatch::ExactImageMatcher matcher(RESOURCE_PATH() + "PokemonBDSP/SelectionArrow.png"); + return matcher; +} +std::vector find_name_list_arrows(const ImageViewRGB32& image){ + using namespace Kernels; + using namespace Kernels::Waterfill; + PackedBinaryMatrix matrix = compress_rgb32_to_binary_max(image, 200, 200, 200); + std::vector objects = find_objects_inplace(matrix, 200); + std::vector ret; + for (const WaterfillObject& object : objects){ + if (object.width() > object.height() || object.height() > 3 * object.width()){ + continue; + } + if (name_list_arrow().rmsd(extract_box_reference(image, object)) <= 90){ + ret.emplace_back(object.min_x, object.min_y, object.max_x, object.max_y); + } + } + return ret; +} +} + + +NameListDetector::NameListDetector(Color color) + : m_color(color) + , m_arrows(0.137, 0.245, 0.030, 0.310) + , m_panel(0.140, 0.245, 0.140, 0.300) +{} +void NameListDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_arrows); + items.add(m_color, m_panel); +} +bool NameListDetector::detect(const ImageViewRGB32& screen){ + return selected_row(screen) >= 0; +} +int NameListDetector::selected_row(const ImageViewRGB32& screen) const{ + // The list is a white panel with a little dark text. A thin margin strip is not + // enough: the border's antialiasing reaches into it on some frames. + const ImageViewRGB32 panel = extract_box_reference(screen, m_panel); + size_t white = 0; + for (size_t y = 0; y < panel.height(); y += 2){ + for (size_t x = 0; x < panel.width(); x += 2){ + const uint32_t p = panel.pixel(x, y); + white += ((p >> 16) & 0xff) > 220 && ((p >> 8) & 0xff) > 220 && (p & 0xff) > 220; + } + } + const size_t samples = ((panel.width() + 1) / 2) * ((panel.height() + 1) / 2); + if (samples == 0 || white < samples * 6 / 10){ + return -1; + } + ImageViewRGB32 area = extract_box_reference(screen, m_arrows); + std::vector arrows = find_name_list_arrows(area); + if (arrows.size() != 1){ + return -1; + } + const double center_y = m_arrows.y + + m_arrows.height * (arrows[0].min_y + arrows[0].max_y) / 2.0 / (double)area.height(); + const int row = (int)std::lround((center_y - FIRST_ROW_Y) / ROW_STEP_Y); + return row >= 0 && row < ROWS ? row : -1; +} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h index 59fa1efbeb..f247f1da74 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h @@ -1,41 +1,41 @@ -/* Name List Detector - * - * From: https://github.com/PokemonAutomation/ - * - * The list of preset names shown for the rival: "I'll tell you...", then four names. - * - */ - -#ifndef PokemonAutomation_PokemonBDSP_NameListDetector_H -#define PokemonAutomation_PokemonBDSP_NameListDetector_H - -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonTools/VisualDetector.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -class NameListDetector : public StaticScreenDetector{ -public: - NameListDetector(Color color = COLOR_MAGENTA); - - virtual void make_overlays(VideoOverlaySet& items) const override; - virtual bool detect(const ImageViewRGB32& screen) override; - - // The row the arrow is on: 0 is "I'll tell you...", 1 the first preset name. - // -1 if the list is not showing. - int selected_row(const ImageViewRGB32& screen) const; - -private: - Color m_color; - ImageFloatBox m_arrows; - ImageFloatBox m_panel; -}; - - -} -} -} -#endif +/* Name List Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The list of preset names shown for the rival: "I'll tell you...", then four names. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_NameListDetector_H +#define PokemonAutomation_PokemonBDSP_NameListDetector_H + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +class NameListDetector : public StaticScreenDetector{ +public: + NameListDetector(Color color = COLOR_MAGENTA); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + + // The row the arrow is on: 0 is "I'll tell you...", 1 the first preset name. + // -1 if the list is not showing. + int selected_row(const ImageViewRGB32& screen) const; + +private: + Color m_color; + ImageFloatBox m_arrows; + ImageFloatBox m_panel; +}; + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp index 9cf2086703..7ceacae47c 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp @@ -1,292 +1,294 @@ -/* BDSP AutoStory - Menu Cursor Detector - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include -#include - -#include "CommonFramework/VideoPipeline/VideoFeed.h" - -#include "CommonTools/Images/ImageFilter.h" - -#include "PokemonBDSP_AutoStory_MenuCursorDetector.h" - -namespace PokemonAutomation { -namespace NintendoSwitch { -namespace PokemonBDSP { - - -std::string menu_cursor_to_string(MenuCursorPosition pos){ - switch (pos){ - case MenuCursorPosition::POKEDEX: - return "POKEDEX"; - case MenuCursorPosition::POKEMON: - return "POKEMON"; - case MenuCursorPosition::BAG: - return "BAG"; - case MenuCursorPosition::CARD: - return "CARD"; - case MenuCursorPosition::OPTIONS: - return "OPTIONS"; - case MenuCursorPosition::MAP: - return "MAP"; - case MenuCursorPosition::CAPSULES: - return "CAPSULES"; - case MenuCursorPosition::MYSTERYGIFT: - return "MYSTERYGIFT"; - default: - return "UNKNOWN"; - } -} - - -MenuCursorDetector::MenuCursorDetector(Color color) : - m_color(color) - // ------------------------------------------------------------------------- - // Top row — left to right - // ------------------------------------------------------------------------- - // TODO: replace each ImageFloatBox(0,0,0,0) with the real normalized - // (x, y, width, height) for a small region that gains dark pixels when - // that icon is hovered. Use the test program overlays to tune coordinates. - // ------------------------------------------------------------------------- - , m_pokedex_box{ - ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), - }, - m_pokemon_box{ - ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), - }, - m_bag_box{ - ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), - }, - m_card_box{ - ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), - } // ------------------------------------------------------------------------- - // Bottom row — left to right - // 5-icon layout: OPTIONS is the only bottom icon (far-left position). - // 8-icon layout: MAP, CAPSULES, OPTIONS, MYSTERYGIFT from left to right. - // OPTIONS sits at a different x-position in each layout — use a separate - // MenuCursorDetector instance if you need to detect OPTIONS in both. - // ------------------------------------------------------------------------- - , m_options_box{ - ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), - }, - m_map_box{ - ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), - }, - m_capsules_box{ - ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), - }, - m_mysterygift_box{ - ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), - } // ------------------------------------------------------------------------- - // Approach 2 search boxes — one per icon, same area as the dark-pixel box. - // The template PNG is slid across this area at its native size. - // ------------------------------------------------------------------------- - , m_icon_match_box{ - ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), // 0 POKEDEX - ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), // 1 POKEMON - ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), // 2 BAG - ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), // 3 CARD - ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), // 4 OPTIONS (8-icon layout) - ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), // 5 MAP - ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), // 6 CAPSULES - ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), // 7 MYSTERYGIFT - } { -} - - -void MenuCursorDetector::make_overlays(VideoOverlaySet& items) const { - for (const ImageFloatBox* icon_boxes : { - m_pokedex_box, - m_pokemon_box, - m_bag_box, - m_card_box, - m_options_box, - m_map_box, - m_capsules_box, - m_mysterygift_box, - }){ - for (int i = 0; i < BOXES_PER_ICON; i++){ - if (icon_boxes[i].width > 0.0 && icon_boxes[i].height > 0.0){ - items.add(m_color, icon_boxes[i]); - } - } - } - for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ - if (m_icon_match_box[i].width > 0.0 && m_icon_match_box[i].height > 0.0){ - items.add(m_color, m_icon_match_box[i]); - } - } -} - - -namespace { - -// Count how many of `n` boxes contain >= MIN_DARK_PIXEL_COUNT pixels with -// R, G, B all <= BLACK_CHANNEL_MAX. Boxes with zero width are skipped. -int count_dark_boxes(const ImageViewRGB32& screen, const ImageFloatBox* boxes, int n){ - const uint32_t ch = uint32_t(MenuCursorDetector::BLACK_CHANNEL_MAX); - const uint32_t maxs = 0xff000000u | (ch << 16) | (ch << 8) | ch; - - int count = 0; - for (int i = 0; i < n; i++){ - if (boxes[i].width == 0.0){ - continue; - } - size_t dark_pixels = 0; - filter_rgb32_range(dark_pixels, - extract_box_reference(screen, boxes[i]), - 0xff000000u, - maxs, - Color(0xffffffff), - false); - if (dark_pixels >= MenuCursorDetector::MIN_DARK_PIXEL_COUNT){ - count++; - } - } - return count; -} - -// Pixel-wise RMSD between two same-size images over RGB channels only. -double compute_rmsd(const ImageViewRGB32& a, const ImageViewRGB32& b){ - size_t n = a.width() * a.height(); - if (n == 0){ - return 0.0; - } - double sum_sq = 0.0; - for (size_t y = 0; y < a.height(); y++){ - for (size_t x = 0; x < a.width(); x++){ - uint32_t pa = a.pixel(x, y); - uint32_t pb = b.pixel(x, y); - int dr = int((pa >> 16) & 0xff) - int((pb >> 16) & 0xff); - int dg = int((pa >> 8) & 0xff) - int((pb >> 8) & 0xff); - int db = int(pa & 0xff) - int(pb & 0xff); - sum_sq += double(dr * dr + dg * dg + db * db); - } - } - return std::sqrt(sum_sq / (3.0 * n)); -} - -// Slide `tmpl` (at its native pixel size) across the region of `screen` -// defined by `box`. Returns the minimum RMSD found over all window positions. -double template_match_rmsd(const ImageViewRGB32& screen, const ImageFloatBox& box, const ImageViewRGB32& tmpl){ - ImageViewRGB32 region = extract_box_reference(screen, box); - size_t rw = tmpl.width(), rh = tmpl.height(); - size_t lw = region.width(), lh = region.height(); - if (rw == 0 || rh == 0 || rw > lw || rh > lh){ - return 0.0; - } - - double min_rmsd = std::numeric_limits::max(); - for (size_t dy = 0; dy <= lh - rh; dy++){ - for (size_t dx = 0; dx <= lw - rw; dx++){ - double r = compute_rmsd(region.sub_image(dx, dy, rw, rh), tmpl); - if (r < min_rmsd){ - min_rmsd = r; - } - } - } - return min_rmsd; -} - -} // anonymous namespace - - -MenuCursorPosition MenuCursorDetector::detect_dark_pixels(const ImageViewRGB32& screen) const { - struct IconEntry { - MenuCursorPosition position; - const ImageFloatBox* boxes; - }; - const IconEntry icons[] = { - {MenuCursorPosition::POKEDEX, m_pokedex_box}, - {MenuCursorPosition::POKEMON, m_pokemon_box}, - {MenuCursorPosition::BAG, m_bag_box}, - {MenuCursorPosition::CARD, m_card_box}, - {MenuCursorPosition::OPTIONS, m_options_box}, - {MenuCursorPosition::MAP, m_map_box}, - {MenuCursorPosition::CAPSULES, m_capsules_box}, - {MenuCursorPosition::MYSTERYGIFT, m_mysterygift_box}, - }; - - MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; - int best_count = 0; - - for (const auto& entry : icons){ - int count = count_dark_boxes(screen, entry.boxes, BOXES_PER_ICON); - if (count > best_count){ - best_count = count; - best_pos = entry.position; - } - } - - return (best_count >= REQUIRED_VOTES) ? best_pos : MenuCursorPosition::UNKNOWN; -} - - -MenuCursorPosition MenuCursorDetector::detect_image_match(const ImageViewRGB32& screen) const { - MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; - double best_rmsd = MATCH_RMSD_THRESHOLD; // must exceed this - - for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ - if (!m_icon_references[i].has_value()){ - continue; - } - if (m_icon_match_box[i].width == 0.0){ - continue; - } - double r = template_match_rmsd(screen, m_icon_match_box[i], *m_icon_references[i]); - if (r > best_rmsd){ - best_rmsd = r; - best_pos = static_cast(i); - } - } - return best_pos; -} - - -MenuCursorPosition MenuCursorDetector::detect(const ImageViewRGB32& screen) const { - MenuCursorPosition p1 = detect_dark_pixels(screen); - MenuCursorPosition p2 = detect_image_match(screen); - - if (p1 == p2){ - return p1; // both agree (or both UNKNOWN) - } - if (p1 == MenuCursorPosition::UNKNOWN){ - return p2; // only approach 2 fired - } - if (p2 == MenuCursorPosition::UNKNOWN){ - return p1; // only approach 1 fired - } - return p1; // conflict: prefer approach 1 -} - - -void MenuCursorDetector::set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference){ - m_icon_references[static_cast(icon)].emplace(std::move(reference)); -} - - -double MenuCursorDetector::icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const { - int idx = static_cast(icon); - if (!m_icon_references[idx].has_value() || m_icon_match_box[idx].width == 0.0){ - return -1.0; - } - return template_match_rmsd(screen, m_icon_match_box[idx], *m_icon_references[idx]); -} - - -MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& /*context*/){ - VideoSnapshot screen = stream.video().snapshot(); - if (!screen){ - return MenuCursorPosition::UNKNOWN; - } - MenuCursorDetector detector; - return detector.detect(screen); -} - - -} -} -} +/* BDSP AutoStory - Menu Cursor Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include + +#include "CommonFramework/VideoPipeline/VideoFeed.h" + +#include "CommonTools/Images/ImageFilter.h" + +#include "PokemonBDSP_AutoStory_MenuCursorDetector.h" + +namespace PokemonAutomation { +namespace NintendoSwitch { +namespace PokemonBDSP { + + +std::string menu_cursor_to_string(MenuCursorPosition pos){ + switch (pos){ + case MenuCursorPosition::POKEDEX: + return "POKEDEX"; + case MenuCursorPosition::POKEMON: + return "POKEMON"; + case MenuCursorPosition::BAG: + return "BAG"; + case MenuCursorPosition::CARD: + return "CARD"; + case MenuCursorPosition::OPTIONS: + return "OPTIONS"; + case MenuCursorPosition::MAP: + return "MAP"; + case MenuCursorPosition::CAPSULES: + return "CAPSULES"; + case MenuCursorPosition::MYSTERYGIFT: + return "MYSTERYGIFT"; + default: + return "UNKNOWN"; + } +} + + +MenuCursorDetector::MenuCursorDetector(Color color) : + m_color(color) + // ------------------------------------------------------------------------- + // Top row — left to right + // ------------------------------------------------------------------------- + // TODO: replace each ImageFloatBox(0,0,0,0) with the real normalized + // (x, y, width, height) for a small region that gains dark pixels when + // that icon is hovered. Use the test program overlays to tune coordinates. + // ------------------------------------------------------------------------- + , m_pokedex_box{ + ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), + }, + m_pokemon_box{ + ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), + }, + m_bag_box{ + ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), + }, + m_card_box{ + ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), + } // ------------------------------------------------------------------------- + // Bottom row — left to right + // 5-icon layout: OPTIONS is the only bottom icon (far-left position). + // 8-icon layout: MAP, CAPSULES, OPTIONS, MYSTERYGIFT from left to right. + // OPTIONS sits at a different x-position in each layout — use a separate + // MenuCursorDetector instance if you need to detect OPTIONS in both. + // ------------------------------------------------------------------------- + , m_options_box{ + ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), + }, + m_map_box{ + ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), + }, + m_capsules_box{ + ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), + }, + m_mysterygift_box{ + ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), + } // ------------------------------------------------------------------------- + // Approach 2 search boxes — one per icon, same area as the dark-pixel box. + // The template PNG is slid across this area at its native size. + // ------------------------------------------------------------------------- + , m_icon_match_box{ + ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), // 0 POKEDEX + ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), // 1 POKEMON + ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), // 2 BAG + ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), // 3 CARD + ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), // 4 OPTIONS (8-icon layout) + ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), // 5 MAP + ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), // 6 CAPSULES + ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), // 7 MYSTERYGIFT + } { +} + + +void MenuCursorDetector::make_overlays(VideoOverlaySet& items) const { + for (const ImageFloatBox* icon_boxes : { + m_pokedex_box, + m_pokemon_box, + m_bag_box, + m_card_box, + m_options_box, + m_map_box, + m_capsules_box, + m_mysterygift_box, + }){ + for (int i = 0; i < BOXES_PER_ICON; i++){ + if (icon_boxes[i].width > 0.0 && icon_boxes[i].height > 0.0){ + items.add(m_color, icon_boxes[i]); + } + } + } + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + if (m_icon_match_box[i].width > 0.0 && m_icon_match_box[i].height > 0.0){ + items.add(m_color, m_icon_match_box[i]); + } + } +} + + +namespace { + +// Count how many of `n` boxes contain >= MIN_DARK_PIXEL_COUNT pixels with +// R, G, B all <= BLACK_CHANNEL_MAX. Boxes with zero width are skipped. +int count_dark_boxes(const ImageViewRGB32& screen, const ImageFloatBox* boxes, int n){ + const uint32_t ch = uint32_t(MenuCursorDetector::BLACK_CHANNEL_MAX); + const uint32_t maxs = 0xff000000u | (ch << 16) | (ch << 8) | ch; + + int count = 0; + for (int i = 0; i < n; i++){ + if (boxes[i].width == 0.0){ + continue; + } + size_t dark_pixels = 0; + filter_rgb32_range(dark_pixels, + extract_box_reference(screen, boxes[i]), + 0xff000000u, + maxs, + Color(0xffffffff), + false); + if (dark_pixels >= MenuCursorDetector::MIN_DARK_PIXEL_COUNT){ + count++; + } + } + return count; +} + +// Pixel-wise RMSD between two same-size images over RGB channels only. +double compute_rmsd(const ImageViewRGB32& a, const ImageViewRGB32& b){ + size_t n = a.width() * a.height(); + if (n == 0){ + return 0.0; + } + double sum_sq = 0.0; + for (size_t y = 0; y < a.height(); y++){ + for (size_t x = 0; x < a.width(); x++){ + uint32_t pa = a.pixel(x, y); + uint32_t pb = b.pixel(x, y); + int dr = int((pa >> 16) & 0xff) - int((pb >> 16) & 0xff); + int dg = int((pa >> 8) & 0xff) - int((pb >> 8) & 0xff); + int db = int(pa & 0xff) - int(pb & 0xff); + sum_sq += double(dr * dr + dg * dg + db * db); + } + } + return std::sqrt(sum_sq / (3.0 * n)); +} + +// Slide `tmpl` (at its native pixel size) across the region of `screen` +// defined by `box`. Returns the minimum RMSD found over all window positions. +double template_match_rmsd(const ImageViewRGB32& screen, const ImageFloatBox& box, const ImageViewRGB32& tmpl){ + ImageViewRGB32 region = extract_box_reference(screen, box); + size_t rw = tmpl.width(), rh = tmpl.height(); + size_t lw = region.width(), lh = region.height(); + if (rw == 0 || rh == 0 || rw > lw || rh > lh){ + return 0.0; + } + + double min_rmsd = std::numeric_limits::max(); + for (size_t dy = 0; dy <= lh - rh; dy++){ + for (size_t dx = 0; dx <= lw - rw; dx++){ + double r = compute_rmsd(region.sub_image(dx, dy, rw, rh), tmpl); + if (r < min_rmsd){ + min_rmsd = r; + } + } + } + return min_rmsd; +} + +} // anonymous namespace + + +MenuCursorPosition MenuCursorDetector::detect_dark_pixels(const ImageViewRGB32& screen) const { + struct IconEntry { + MenuCursorPosition position; + const ImageFloatBox* boxes; + }; + const IconEntry icons[] = { + {MenuCursorPosition::POKEDEX, m_pokedex_box}, + {MenuCursorPosition::POKEMON, m_pokemon_box}, + {MenuCursorPosition::BAG, m_bag_box}, + {MenuCursorPosition::CARD, m_card_box}, + {MenuCursorPosition::OPTIONS, m_options_box}, + {MenuCursorPosition::MAP, m_map_box}, + {MenuCursorPosition::CAPSULES, m_capsules_box}, + {MenuCursorPosition::MYSTERYGIFT, m_mysterygift_box}, + }; + + MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; + int best_count = 0; + + for (const auto& entry : icons){ + int count = count_dark_boxes(screen, entry.boxes, BOXES_PER_ICON); + if (count > best_count){ + best_count = count; + best_pos = entry.position; + } + } + + return (best_count >= REQUIRED_VOTES) ? best_pos : MenuCursorPosition::UNKNOWN; +} + + +MenuCursorPosition MenuCursorDetector::detect_image_match(const ImageViewRGB32& screen) const { + MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; + double best_rmsd = MATCH_RMSD_THRESHOLD; // must exceed this + + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + if (!m_icon_references[i].has_value()){ + continue; + } + if (m_icon_match_box[i].width == 0.0){ + continue; + } + double r = template_match_rmsd(screen, m_icon_match_box[i], *m_icon_references[i]); + if (r > best_rmsd){ + best_rmsd = r; + best_pos = static_cast(i); + } + } + return best_pos; +} + + +MenuCursorPosition MenuCursorDetector::detect(const ImageViewRGB32& screen) const { + MenuCursorPosition p1 = detect_dark_pixels(screen); + MenuCursorPosition p2 = detect_image_match(screen); + + if (p1 == p2){ + return p1; // both agree (or both UNKNOWN) + } + if (p1 == MenuCursorPosition::UNKNOWN){ + return p2; // only approach 2 fired + } + if (p2 == MenuCursorPosition::UNKNOWN){ + return p1; // only approach 1 fired + } + return p1; // conflict: prefer approach 1 +} + + +void MenuCursorDetector::set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference){ + m_icon_references[static_cast(icon)].emplace(std::move(reference)); +} + + +double MenuCursorDetector::icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const { + int idx = static_cast(icon); + if (!m_icon_references[idx].has_value() || m_icon_match_box[idx].width == 0.0){ + return -1.0; + } + return template_match_rmsd(screen, m_icon_match_box[idx], *m_icon_references[idx]); +} + + +MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& /*context*/){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + return MenuCursorPosition::UNKNOWN; + } + MenuCursorDetector detector; + return detector.detect(screen); +} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h index e7090deafb..48512bdabf 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h @@ -1,146 +1,148 @@ -/* BDSP AutoStory - Menu Cursor Detector - * - * From: https://github.com/PokemonAutomation/ - * - * Detects which icon the cursor is hovering over in the X-menu. - * - * Two independent detection approaches are provided for testing: - * - * Approach 1 — dark pixel count (detect_dark_pixels): - * Each icon has BOXES_PER_ICON small boxes placed over a region that gains - * dark pixels when hovered. A box fires if >= MIN_DARK_PIXEL_COUNT pixels - * in it have R, G, B all <= BLACK_CHANNEL_MAX. An icon is selected when its - * fire count reaches REQUIRED_VOTES. - * Tune BLACK_CHANNEL_MAX upward if real dark pixels are being missed. - * Tune it downward (or raise MIN_DARK_PIXEL_COUNT) to reduce false positives. - * - * Approach 2 — template matching (detect_image_match): - * Each icon has one larger search box and a hovered-state reference PNG loaded - * at runtime via set_icon_reference(). The reference PNG is slid (at its native - * pixel size, without resizing) across the search box and the minimum RMSD over - * all window positions is computed. The icon with the lowest min-RMSD that is - * also <= MATCH_RMSD_THRESHOLD is considered selected. - * Tune MATCH_RMSD_THRESHOLD downward to require a closer match. - * - * detect() is the production method and currently delegates to approach 1. - * It will become a combined vote once both approaches are validated. - * - * Menu layouts: - * 5-icon (early game): POKEDEX POKEMON BAG CARD / OPTIONS - * 8-icon (late game): POKEDEX POKEMON BAG CARD / MAP CAPSULES OPTIONS MYSTERYGIFT - * - * Note: OPTIONS sits at a different screen position in each layout. - * Use separate MenuCursorDetector instances with different box coordinates - * if you need to detect OPTIONS reliably in both layouts. - */ - -#ifndef PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H -#define PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H - -#include -#include -#include - -#include "Common/Cpp/Color.h" - -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTypes/ImageRGB32.h" -#include "CommonFramework/ImageTypes/ImageViewRGB32.h" -#include "CommonFramework/Tools/VideoStream.h" -#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" - -#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" - -namespace PokemonAutomation { -namespace NintendoSwitch { -namespace PokemonBDSP { - - -enum class MenuCursorPosition { - UNKNOWN = -1, - POKEDEX = 0, - POKEMON = 1, - BAG = 2, - CARD = 3, - OPTIONS = 4, - MAP = 5, - CAPSULES = 6, - MYSTERYGIFT = 7, -}; - -static constexpr int MENU_CURSOR_NUM_ICONS = 8; - -std::string menu_cursor_to_string(MenuCursorPosition pos); - - -class MenuCursorDetector { -public: - // ── Approach 1: dark pixel detection ───────────────────────────────────── - static constexpr int BOXES_PER_ICON = 3; - static constexpr int REQUIRED_VOTES = 1; // out of BOXES_PER_ICON - static constexpr uint8_t BLACK_CHANNEL_MAX = 50; // R,G,B <= this → dark pixel - static constexpr size_t MIN_DARK_PIXEL_COUNT = 1; // dark pixels per box to fire - - // ── Approach 2: template match ─────────────────────────────────────────── - // Reference PNGs are the UN-hovered state of each icon. When hovered, the - // icon gains dark pixels and diverges from the reference, so RMSD increases. - // detect_image_match() selects the icon with the HIGHEST RMSD >= threshold. - // Tune upward to reduce false positives; downward if the correct icon is missed. - static constexpr double MATCH_RMSD_THRESHOLD = 53.0; - - MenuCursorDetector(Color color = COLOR_RED); - - void make_overlays(VideoOverlaySet& items) const; - - // Approach 1: votes across dark-pixel boxes, returns icon with highest - // vote count >= REQUIRED_VOTES, or UNKNOWN. - MenuCursorPosition detect_dark_pixels(const ImageViewRGB32& screen) const; - - // Approach 2: returns the icon with the highest RMSD against its reference, - // provided that RMSD >= MATCH_RMSD_THRESHOLD (high RMSD = hovered). - // Returns UNKNOWN if no references are loaded or nothing exceeds threshold. - MenuCursorPosition detect_image_match(const ImageViewRGB32& screen) const; - - // Production method — currently delegates to detect_dark_pixels(). - MenuCursorPosition detect(const ImageViewRGB32& screen) const; - - // Load a hovered-state reference image for one icon (enables approach 2 for - // that icon). The match box for that icon must already be set in the - // constructor (m_icon_match_box); this method just attaches the reference. - void set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference); - - // Returns the raw RMSD of `icon` against its reference on the given frame. - // Returns -1.0 if no reference is loaded for that icon. - double icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const; - -private: - Color m_color; - - // ── Approach 1 boxes ───────────────────────────────────────────────────── - // 3 boxes per icon targeting regions that gain dark pixels when hovered. - // All initialized to (0,0,0,0) — replace with real normalized coordinates. - ImageFloatBox m_pokedex_box[BOXES_PER_ICON]; - ImageFloatBox m_pokemon_box[BOXES_PER_ICON]; - ImageFloatBox m_bag_box[BOXES_PER_ICON]; - ImageFloatBox m_card_box[BOXES_PER_ICON]; - ImageFloatBox m_options_box[BOXES_PER_ICON]; - ImageFloatBox m_map_box[BOXES_PER_ICON]; - ImageFloatBox m_capsules_box[BOXES_PER_ICON]; - ImageFloatBox m_mysterygift_box[BOXES_PER_ICON]; - - // ── Approach 2 search boxes + references ──────────────────────────────── - // One search box per icon (the area to slide the template across). - // References are populated via set_icon_reference() at runtime. - ImageFloatBox m_icon_match_box[MENU_CURSOR_NUM_ICONS]; - std::array, MENU_CURSOR_NUM_ICONS> m_icon_references; -}; - - -// Captures the current frame from `stream` and runs detect() on it. -MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& context); - - -} -} -} -#endif +/* BDSP AutoStory - Menu Cursor Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Detects which icon the cursor is hovering over in the X-menu. + * + * Two independent detection approaches are provided for testing: + * + * Approach 1 — dark pixel count (detect_dark_pixels): + * Each icon has BOXES_PER_ICON small boxes placed over a region that gains + * dark pixels when hovered. A box fires if >= MIN_DARK_PIXEL_COUNT pixels + * in it have R, G, B all <= BLACK_CHANNEL_MAX. An icon is selected when its + * fire count reaches REQUIRED_VOTES. + * Tune BLACK_CHANNEL_MAX upward if real dark pixels are being missed. + * Tune it downward (or raise MIN_DARK_PIXEL_COUNT) to reduce false positives. + * + * Approach 2 — template matching (detect_image_match): + * Each icon has one larger search box and a hovered-state reference PNG loaded + * at runtime via set_icon_reference(). The reference PNG is slid (at its native + * pixel size, without resizing) across the search box and the minimum RMSD over + * all window positions is computed. The icon with the lowest min-RMSD that is + * also <= MATCH_RMSD_THRESHOLD is considered selected. + * Tune MATCH_RMSD_THRESHOLD downward to require a closer match. + * + * detect() is the production method and currently delegates to approach 1. + * It will become a combined vote once both approaches are validated. + * + * Menu layouts: + * 5-icon (early game): POKEDEX POKEMON BAG CARD / OPTIONS + * 8-icon (late game): POKEDEX POKEMON BAG CARD / MAP CAPSULES OPTIONS MYSTERYGIFT + * + * Note: OPTIONS sits at a different screen position in each layout. + * Use separate MenuCursorDetector instances with different box coordinates + * if you need to detect OPTIONS reliably in both layouts. + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H + +#include +#include +#include + +#include "Common/Cpp/Color.h" + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" + +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation { +namespace NintendoSwitch { +namespace PokemonBDSP { + + +enum class MenuCursorPosition { + UNKNOWN = -1, + POKEDEX = 0, + POKEMON = 1, + BAG = 2, + CARD = 3, + OPTIONS = 4, + MAP = 5, + CAPSULES = 6, + MYSTERYGIFT = 7, +}; + +static constexpr int MENU_CURSOR_NUM_ICONS = 8; + +std::string menu_cursor_to_string(MenuCursorPosition pos); + + +class MenuCursorDetector { +public: + // ── Approach 1: dark pixel detection ───────────────────────────────────── + static constexpr int BOXES_PER_ICON = 3; + static constexpr int REQUIRED_VOTES = 1; // out of BOXES_PER_ICON + static constexpr uint8_t BLACK_CHANNEL_MAX = 50; // R,G,B <= this → dark pixel + static constexpr size_t MIN_DARK_PIXEL_COUNT = 1; // dark pixels per box to fire + + // ── Approach 2: template match ─────────────────────────────────────────── + // Reference PNGs are the UN-hovered state of each icon. When hovered, the + // icon gains dark pixels and diverges from the reference, so RMSD increases. + // detect_image_match() selects the icon with the HIGHEST RMSD >= threshold. + // Tune upward to reduce false positives; downward if the correct icon is missed. + static constexpr double MATCH_RMSD_THRESHOLD = 53.0; + + MenuCursorDetector(Color color = COLOR_RED); + + void make_overlays(VideoOverlaySet& items) const; + + // Approach 1: votes across dark-pixel boxes, returns icon with highest + // vote count >= REQUIRED_VOTES, or UNKNOWN. + MenuCursorPosition detect_dark_pixels(const ImageViewRGB32& screen) const; + + // Approach 2: returns the icon with the highest RMSD against its reference, + // provided that RMSD >= MATCH_RMSD_THRESHOLD (high RMSD = hovered). + // Returns UNKNOWN if no references are loaded or nothing exceeds threshold. + MenuCursorPosition detect_image_match(const ImageViewRGB32& screen) const; + + // Production method — currently delegates to detect_dark_pixels(). + MenuCursorPosition detect(const ImageViewRGB32& screen) const; + + // Load a hovered-state reference image for one icon (enables approach 2 for + // that icon). The match box for that icon must already be set in the + // constructor (m_icon_match_box); this method just attaches the reference. + void set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference); + + // Returns the raw RMSD of `icon` against its reference on the given frame. + // Returns -1.0 if no reference is loaded for that icon. + double icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const; + +private: + Color m_color; + + // ── Approach 1 boxes ───────────────────────────────────────────────────── + // 3 boxes per icon targeting regions that gain dark pixels when hovered. + // All initialized to (0,0,0,0) — replace with real normalized coordinates. + ImageFloatBox m_pokedex_box[BOXES_PER_ICON]; + ImageFloatBox m_pokemon_box[BOXES_PER_ICON]; + ImageFloatBox m_bag_box[BOXES_PER_ICON]; + ImageFloatBox m_card_box[BOXES_PER_ICON]; + ImageFloatBox m_options_box[BOXES_PER_ICON]; + ImageFloatBox m_map_box[BOXES_PER_ICON]; + ImageFloatBox m_capsules_box[BOXES_PER_ICON]; + ImageFloatBox m_mysterygift_box[BOXES_PER_ICON]; + + // ── Approach 2 search boxes + references ──────────────────────────────── + // One search box per icon (the area to slide the template across). + // References are populated via set_icon_reference() at runtime. + ImageFloatBox m_icon_match_box[MENU_CURSOR_NUM_ICONS]; + std::array, MENU_CURSOR_NUM_ICONS> m_icon_references; +}; + + +// Captures the current frame from `stream` and runs detect() on it. +MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& context); + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp index 35ed89c090..1e98e411d4 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp @@ -1,445 +1,445 @@ -/* BDSP AutoStory - Pokemon Center - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include -#include -#include -#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTools/ImageStats.h" -#include "CommonFramework/ImageTypes/ImageRGB32.h" -#include "CommonFramework/VideoPipeline/VideoFeed.h" -#include "CommonTools/Async/InferenceRoutines.h" -#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" -#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" -#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_RouteWalk.h" -#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h" -#include "PokemonBDSP_AutoStory_WildBattle.h" -#include "PokemonBDSP_AutoStory_Center.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - -using namespace std::chrono_literals; -using namespace Pokemon; - - -namespace{ - -// How many tiles the 1080p view spans across: a tile is about 125 px. -constexpr double SCREEN_WIDTH_TILES = 1920.0 / 125; -// The camera keeps the player in the middle of the screen across, and the -// centre of his tile a little above the middle down (r121a and r122a: his -// feet at y = 590 of 1080). -constexpr double PLAYER_X = 0.5; -constexpr double PLAYER_Y = 0.505; -// How much of the screen's height one row of tiles takes near the player: -// the doorway straight ahead of him (r121a) is centred this far above his -// tile's centre. Rows further north look shorter, so this undercounts them. -constexpr double SCREEN_ROW_HEIGHT = 0.12; -// Doorways are looked for down to here, 0 to 1: a door the player can walk -// into is north of him. -constexpr double DOORWAY_SEARCH_BOTTOM = 0.67; -// A doorway found within this many rows of straight ahead is at the front -// row; further, the walk to the door ended short. -constexpr double DOORWAY_FRONT_ROWS = 1.5; - -// The doorway is looked for on a view this size: 4 px of 1080p per pixel. -constexpr size_t SMALL_WIDTH = 480; -constexpr size_t SMALL_HEIGHT = 270; -// A doorway's lit columns run at least this many small rows (the Mart's 12, -// the Center's 15); the buildings' lit roof bands 3 to 5. -constexpr size_t DOORWAY_COLUMN_ROWS = 8; -// A doorway has this many lit small pixels (the Mart's 260 to 350); a roof -// band's run 60 to 75. -constexpr double DOORWAY_MIN_MASS = 80; - -// Whether a pixel is the pale yellow-white of a lit doorway: the Mart's is -// (235, 243, 205), the Center's (255, 255, 185). The buildings' white roof -// bands are bluer, their orange bands and the light they throw on the -// ground outside are darker. -bool lit_doorway_pixel(uint32_t pixel){ - const int r = (pixel >> 16) & 0xff; - const int g = (pixel >> 8) & 0xff; - const int b = pixel & 0xff; - return r >= 215 && g >= 215 && b >= 150 && b <= g - 15 && r >= g - 25; -} - -// Each column of the small view: how many lit doorway pixels it has in the -// search band, and the sum of their rows. -struct LitColumns{ - std::vector count; - std::vector row_sum; -}; -LitColumns lit_columns(const ImageViewRGB32& screen){ - const ImageRGB32 small = screen.scale_to(SMALL_WIDTH, SMALL_HEIGHT); - LitColumns columns{std::vector(small.width(), 0), std::vector(small.width(), 0)}; - const size_t bottom = (size_t)(DOORWAY_SEARCH_BOTTOM * (double)small.height()); - for (size_t y = 0; y < bottom; y++){ - for (size_t x = 0; x < small.width(); x++){ - if (lit_doorway_pixel(small.pixel(x, y))){ - columns.count[x]++; - columns.row_sum[x] += (double)y; - } - } - } - return columns; -} - -// A run of lit columns: its lit pixels, and the sums of their columns and rows. -struct LitRun{ - double mass = 0; - double column_sum = 0; - double row_sum = 0; -}; -// The run of columns with at least DOORWAY_COLUMN_ROWS lit that starts at -// `x`, allowing gaps of up to 3 columns (the door's frame). Moves `x` past it. -LitRun lit_run(const LitColumns& columns, size_t& x){ - LitRun run; - size_t gap = 0; - for (; x < columns.count.size() && gap <= 3; x++){ - if (columns.count[x] < DOORWAY_COLUMN_ROWS){ - gap++; - continue; - } - run.mass += (double)columns.count[x]; - run.column_sum += (double)columns.count[x] * (double)x; - run.row_sum += columns.row_sum[x]; - gap = 0; - } - return run; -} - -// Where a building's lit doorway is across and down the screen, 0 to 1: the -// biggest run of lit columns within 0.35 of the middle (a second building's -// door may show further off). Nothing if no doorway shows. -std::optional> lit_doorway_position(const ImageViewRGB32& screen){ - const LitColumns columns = lit_columns(screen); - const double width = (double)columns.count.size(); - std::optional> best; - double best_mass = 0; - for (size_t x = 0; x < columns.count.size(); ){ - if (columns.count[x] < DOORWAY_COLUMN_ROWS){ - x++; - continue; - } - const LitRun run = lit_run(columns, x); - const double center = run.column_sum / run.mass / width; - if (run.mass < DOORWAY_MIN_MASS || std::abs(center - PLAYER_X) >= 0.35 || run.mass <= best_mass){ - continue; - } - best = {center, run.row_sum / run.mass / (double)SMALL_HEIGHT}; - best_mass = run.mass; - } - return best; -} - -// Whether the whole view is a different scene from "before". -bool scene_changed_since(VideoStream& stream, ProControllerContext& context, const VideoSnapshot& before){ - context.wait_for_all_requests(); - VideoSnapshot now = stream.video().snapshot(); - return before && now && walk_screen_change(*before.frame, *now.frame) > WALK_SCENE_CHANGE; -} - -// Moves across by "tiles", east if positive, west if negative. -void step_across(VideoStream& stream, ProControllerContext& context, double tiles){ - const Direction side = tiles > 0 ? Direction::EAST : Direction::WEST; - route_turn(stream, context, side); - const Milliseconds hold = std::max(Milliseconds(40), - Milliseconds((int64_t)(std::abs(tiles) * ROUTE_MS_PER_TILE_ACROSS.count()))); - pbf_move_left_joystick(context, {tiles > 0 ? 1.0 : -1.0, 0}, hold, 300ms); -} - -// Whether the screen now shows a Pokemon Center's inside. -bool center_inside_now(VideoStream& stream, ProControllerContext& context){ - context.wait_for_all_requests(); - VideoSnapshot now = stream.video().snapshot(); - return now && pokemon_center_inside(*now.frame); -} - -// Walks north into the door straight ahead, `doorway` away. Returns whether -// we are inside. -bool walk_into_door( - VideoStream& stream, ProControllerContext& context, - const VideoSnapshot& outside, const LitDoorway& doorway, const std::string& label -){ - // Time for the rows to the door (none when it is straight ahead), the - // door opening, and a start delay. - const double rows = std::max(0.0, doorway.rows_up - 1); - const Milliseconds limit = 1500ms + Milliseconds((int64_t)(rows * ROUTE_MS_PER_TILE_UP_DOWN.count())); - // The Center's fade is too light to read as black (r126 runs 2 and 3: a - // screen change of 49 at 400 ms), so its inside, once it shows, stops - // the walk too. - const WalkStop stop = walk_until(stream, context, run_from_wild_battle, 0, 1, limit, label + ": into the door", - [](const ImageViewRGB32& screen){ return pokemon_center_inside(screen); }).stop; - if (stop == WalkStop::TRANSITION || stop == WalkStop::ARRIVED){ - pbf_wait(context, 2000ms); - return true; - } - // The door slides open before it takes us in, and its fade is not always - // black enough to read as a transition, and may finish after the walk; a - // whole new scene says we are in. Not after a walk of several rows, which - // scrolls the view: the next look decides. - pbf_wait(context, 2000ms); - if (center_inside_now(stream, context)){ - stream.log("[Route] " + label + ": a Pokemon Center's inside shows; inside."); - return true; - } - if (doorway.rows_up > DOORWAY_FRONT_ROWS || !scene_changed_since(stream, context, outside)){ - return false; - } - stream.log("[Route] " + label + ": the scene changed; inside."); - return true; -} - -// Where the doorway is on screen, in the log. -void log_doorway(VideoStream& stream, const std::string& label, const LitDoorway& doorway){ - stream.log("[Route] " + label + ": doorway " + std::to_string(doorway.tiles_across) + - " tiles across, " + std::to_string(doorway.rows_up) + " rows up (at " + - std::to_string(doorway.x) + ", " + std::to_string(doorway.y) + " of the screen)."); -} - -} - - -bool pokemon_center_inside(const ImageViewRGB32& screen){ - // The orange back wall along the top, in two of three places: a plant - // or a bookcase can stand in one (Jubilife's top left), and at night - // it is darker (r 160..200 in Jubilife, 254 in Sandgem by day). - size_t wall = 0; - for (double x : {0.02, 0.25, 0.65}){ - const ImageStats box = image_stats(extract_box_reference(screen, ImageFloatBox(x, 0.03, 0.08, 0.06))); - const double r = box.average.r, g = box.average.g, b = box.average.b; - wall += r >= 140 && r > g + 40 && g > b + 40; - } - if (wall < 2){ - return false; - } - // Black below the floor, either side of the step out of the door. - for (const ImageFloatBox& box : {ImageFloatBox(0.02, 0.95, 0.30, 0.04), ImageFloatBox(0.65, 0.95, 0.33, 0.04)}){ - if (image_stats(extract_box_reference(screen, box)).average.sum() > 90){ - return false; - } - } - // And the floor lit between them: not a fade (349 in Jubilife at night). - return image_stats(extract_box_reference(screen, ImageFloatBox(0.30, 0.55, 0.40, 0.15))).average.sum() > 250; -} - -void fail_if_blacked_out(VideoStream& stream, ProControllerContext& context, const std::string& label){ - for (size_t look = 0; look < 10; look++){ - context.wait_for_all_requests(); - VideoSnapshot screen = stream.video().snapshot(); - if (screen && indoors(*screen.frame)){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": the party fainted; the game took us home (a room shows after the battle).", stream); - } - // The blackout's text and fade are dark; the grass is not. - if (screen && image_stats(*screen.frame).average.sum() > 150){ - return; - } - pbf_wait(context, 1000ms); - } -} - -std::optional find_lit_lamp_x(const ImageViewRGB32& screen, const ImageFloatBox& box){ - const size_t x0 = (size_t)(box.x * screen.width()), x1 = (size_t)((box.x + box.width) * screen.width()); - const size_t y0 = (size_t)(box.y * screen.height()), y1 = (size_t)((box.y + box.height) * screen.height()); - double sum = 0; - size_t count = 0; - for (size_t y = y0; y < y1 && y < screen.height(); y++){ - for (size_t x = x0; x < x1 && x < screen.width(); x++){ - const Color pixel(screen.pixel(x, y)); - if (pixel.red() > 235 && pixel.green() > 220 && pixel.blue() > 150){ - sum += x; - count++; - } - } - } - // Jubilife's lamps at night: 50 to 100 such pixels at 480 px across. - if (count * 480 * 480 < 30 * screen.width() * screen.width()){ - return std::nullopt; - } - return sum / count / screen.width(); -} - -double tiles_across_at(double screen_x){ - return (screen_x - PLAYER_X) * SCREEN_WIDTH_TILES; -} - -std::optional find_lit_doorway(const ImageViewRGB32& screen){ - // A Center's lit floor has the doorway's colour. - if (pokemon_center_inside(screen)){ - return std::nullopt; - } - const std::optional> position = lit_doorway_position(screen); - if (!position){ - return std::nullopt; - } - return LitDoorway{ - position->first, - position->second, - (position->first - PLAYER_X) * SCREEN_WIDTH_TILES, - (PLAYER_Y - position->second) / SCREEN_ROW_HEIGHT, - }; -} - -std::vector tiles_short_of_door( - VideoStream& stream, ProControllerContext& context, - const StoryWorld& grid, const TilePos& door, const std::string& label -){ - context.wait_for_all_requests(); - VideoSnapshot screen = stream.video().snapshot(); - const std::optional doorway = screen ? find_lit_doorway(*screen.frame) : std::nullopt; - if (!doorway){ - stream.log("[Route] " + label + ": no lit doorway on screen; not walking again."); - return {}; - } - log_doorway(stream, label, *doorway); - if (doorway->rows_up <= DOORWAY_FRONT_ROWS){ - return {}; - } - return tiles_by_lit_doorway(*doorway, grid, door); -} - -std::vector tiles_by_lit_doorway(const LitDoorway& doorway, const StoryWorld& grid, const TilePos& door){ - const TilePos guess{door.x - (int)std::lround(doorway.tiles_across), door.y + (int)std::lround(doorway.rows_up)}; - std::vector tiles; - for (int y = guess.y - 1; y <= guess.y + 1; y++){ - for (int x = guess.x - 1; x <= guess.x + 1; x++){ - if (grid.standable(TilePos{x, y})){ - tiles.push_back(TilePos{x, y}); - } - } - } - return tiles; -} - -bool line_up_with_lit_door( - VideoStream& stream, ProControllerContext& context, - const std::string& label -){ - route_status(stream, label + ": lining up with the door."); - for (size_t tries = 0; tries < 6; tries++){ - context.wait_for_all_requests(); - VideoSnapshot screen = stream.video().snapshot(); - if (screen && pokemon_center_inside(*screen.frame)){ - stream.log("[Route] " + label + ": a Pokemon Center's inside shows; not in front of the door."); - return false; - } - const std::optional doorway = screen ? find_lit_doorway(*screen.frame) : std::nullopt; - if (!doorway){ - stream.log("[Route] " + label + ": no lit doorway on screen."); - return false; - } - log_doorway(stream, label, *doorway); - // Steps overshoot: within a quarter tile, r128 measured 0.29, -0.28, - // 0.48, -0.67 and 0.48 tiles in turn and never settled. The door - // still takes us in from 0.4 (enter_lit_door()). - if (std::abs(doorway->tiles_across) <= 0.4){ - return true; - } - step_across(stream, context, doorway->tiles_across); - // Let a door that took us in show its inside before looking again. - pbf_wait(context, 1000ms); - } - stream.log("[Route] " + label + ": not lined up after 6 steps."); - return false; -} - -void enter_lit_door( - VideoStream& stream, ProControllerContext& context, - const std::string& label -){ - // Movement is not locked to tiles, and the door takes us in only when we - // are lined up with it. Its doorway is lit: find it on screen, where the - // camera keeps the player in the middle, move across by its offset, and - // walk in once it is straight ahead, looking again after every move. - route_status(stream, label + ": lining up with the door."); - VideoSnapshot last_outside; - for (size_t tries = 0; tries < 12; tries++){ - context.wait_for_all_requests(); - VideoSnapshot outside = stream.video().snapshot(); - if (!outside){ - continue; - } - // Inside a Center, its lit floor would read as another doorway. - if (pokemon_center_inside(*outside.frame)){ - stream.log("[Route] " + label + ": a Pokemon Center's inside shows; inside."); - return; - } - const std::optional doorway = find_lit_doorway(*outside.frame); - if (!doorway){ - // The door may have taken us in after all, more slowly than we watched. - pbf_wait(context, 1500ms); - if (!scene_changed_since(stream, context, last_outside)){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": no lit doorway on screen.", stream); - } - stream.log("[Route] " + label + ": the scene changed; inside."); - pbf_wait(context, 1500ms); - return; - } - last_outside = outside; - log_doorway(stream, label, *doorway); - // We stand on tiles, so within half a tile is the door's column; - // a step moves about 0.7 of a reading, and at 0.25 r144a stepped - // between 0.3 and -0.4 twelve times. Try the door there, and step - // toward it if it does not take us. - if (std::abs(doorway->tiles_across) > 0.5){ - step_across(stream, context, doorway->tiles_across); - continue; - } - if (walk_into_door(stream, context, outside, *doorway, label)){ - pbf_wait(context, 1500ms); - return; - } - step_across(stream, context, doorway->tiles_across); - } - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": not through the door after 12 tries.", stream); -} - -void heal_at_pokemon_center( - VideoStream& stream, ProControllerContext& context, - const std::string& label -){ - enter_lit_door(stream, context, label); - - // The nurse's counter is straight ahead of the door. - route_status(stream, label + ": talking to the nurse."); - if (walk_to_talk(stream, context, run_from_wild_battle, 0, 1, 6000ms, label + ": to the nurse").stop != WalkStop::DIALOG){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": nurse not reached.", stream); - } - // A through her lines; the cursor starts on "Yes". Stop once no dialogue - // has shown for a while, so A does not start the talk over. - size_t quiet = 0; - for (size_t presses = 0; presses < 30 && quiet < 2; presses++){ - ShortDialogWatcher dialog(COLOR_YELLOW); - if (wait_until(stream, context, 1500ms, {{dialog}}) == 0){ - quiet = 0; - pbf_press_button(context, BUTTON_A, 80ms, 700ms); - }else{ - quiet++; - } - } - context.wait_for_all_requests(); - - route_status(stream, label + ": healed; walking out."); - leave_door_south(stream, context, run_from_wild_battle, label + ": out the door"); -} - - - -void move_across(VideoStream& stream, ProControllerContext& context, double tiles){ - step_across(stream, context, tiles); -} - - -} -} -} +/* BDSP AutoStory - Pokemon Center + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_RouteWalk.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h" +#include "PokemonBDSP_AutoStory_WildBattle.h" +#include "PokemonBDSP_AutoStory_Center.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; +using namespace Pokemon; + + +namespace{ + +// How many tiles the 1080p view spans across: a tile is about 125 px. +constexpr double SCREEN_WIDTH_TILES = 1920.0 / 125; +// The camera keeps the player in the middle of the screen across, and the +// centre of his tile a little above the middle down (r121a and r122a: his +// feet at y = 590 of 1080). +constexpr double PLAYER_X = 0.5; +constexpr double PLAYER_Y = 0.505; +// How much of the screen's height one row of tiles takes near the player: +// the doorway straight ahead of him (r121a) is centred this far above his +// tile's centre. Rows further north look shorter, so this undercounts them. +constexpr double SCREEN_ROW_HEIGHT = 0.12; +// Doorways are looked for down to here, 0 to 1: a door the player can walk +// into is north of him. +constexpr double DOORWAY_SEARCH_BOTTOM = 0.67; +// A doorway found within this many rows of straight ahead is at the front +// row; further, the walk to the door ended short. +constexpr double DOORWAY_FRONT_ROWS = 1.5; + +// The doorway is looked for on a view this size: 4 px of 1080p per pixel. +constexpr size_t SMALL_WIDTH = 480; +constexpr size_t SMALL_HEIGHT = 270; +// A doorway's lit columns run at least this many small rows (the Mart's 12, +// the Center's 15); the buildings' lit roof bands 3 to 5. +constexpr size_t DOORWAY_COLUMN_ROWS = 8; +// A doorway has this many lit small pixels (the Mart's 260 to 350); a roof +// band's run 60 to 75. +constexpr double DOORWAY_MIN_MASS = 80; + +// Whether a pixel is the pale yellow-white of a lit doorway: the Mart's is +// (235, 243, 205), the Center's (255, 255, 185). The buildings' white roof +// bands are bluer, their orange bands and the light they throw on the +// ground outside are darker. +bool lit_doorway_pixel(uint32_t pixel){ + const int r = (pixel >> 16) & 0xff; + const int g = (pixel >> 8) & 0xff; + const int b = pixel & 0xff; + return r >= 215 && g >= 215 && b >= 150 && b <= g - 15 && r >= g - 25; +} + +// Each column of the small view: how many lit doorway pixels it has in the +// search band, and the sum of their rows. +struct LitColumns{ + std::vector count; + std::vector row_sum; +}; +LitColumns lit_columns(const ImageViewRGB32& screen){ + const ImageRGB32 small = screen.scale_to(SMALL_WIDTH, SMALL_HEIGHT); + LitColumns columns{std::vector(small.width(), 0), std::vector(small.width(), 0)}; + const size_t bottom = (size_t)(DOORWAY_SEARCH_BOTTOM * (double)small.height()); + for (size_t y = 0; y < bottom; y++){ + for (size_t x = 0; x < small.width(); x++){ + if (lit_doorway_pixel(small.pixel(x, y))){ + columns.count[x]++; + columns.row_sum[x] += (double)y; + } + } + } + return columns; +} + +// A run of lit columns: its lit pixels, and the sums of their columns and rows. +struct LitRun{ + double mass = 0; + double column_sum = 0; + double row_sum = 0; +}; +// The run of columns with at least DOORWAY_COLUMN_ROWS lit that starts at +// `x`, allowing gaps of up to 3 columns (the door's frame). Moves `x` past it. +LitRun lit_run(const LitColumns& columns, size_t& x){ + LitRun run; + size_t gap = 0; + for (; x < columns.count.size() && gap <= 3; x++){ + if (columns.count[x] < DOORWAY_COLUMN_ROWS){ + gap++; + continue; + } + run.mass += (double)columns.count[x]; + run.column_sum += (double)columns.count[x] * (double)x; + run.row_sum += columns.row_sum[x]; + gap = 0; + } + return run; +} + +// Where a building's lit doorway is across and down the screen, 0 to 1: the +// biggest run of lit columns within 0.35 of the middle (a second building's +// door may show further off). Nothing if no doorway shows. +std::optional> lit_doorway_position(const ImageViewRGB32& screen){ + const LitColumns columns = lit_columns(screen); + const double width = (double)columns.count.size(); + std::optional> best; + double best_mass = 0; + for (size_t x = 0; x < columns.count.size(); ){ + if (columns.count[x] < DOORWAY_COLUMN_ROWS){ + x++; + continue; + } + const LitRun run = lit_run(columns, x); + const double center = run.column_sum / run.mass / width; + if (run.mass < DOORWAY_MIN_MASS || std::abs(center - PLAYER_X) >= 0.35 || run.mass <= best_mass){ + continue; + } + best = {center, run.row_sum / run.mass / (double)SMALL_HEIGHT}; + best_mass = run.mass; + } + return best; +} + +// Whether the whole view is a different scene from "before". +bool scene_changed_since(VideoStream& stream, ProControllerContext& context, const VideoSnapshot& before){ + context.wait_for_all_requests(); + VideoSnapshot now = stream.video().snapshot(); + return before && now && walk_screen_change(*before.frame, *now.frame) > WALK_SCENE_CHANGE; +} + +// Moves across by "tiles", east if positive, west if negative. +void step_across(VideoStream& stream, ProControllerContext& context, double tiles){ + const Direction side = tiles > 0 ? Direction::EAST : Direction::WEST; + route_turn(stream, context, side); + const Milliseconds hold = std::max(Milliseconds(40), + Milliseconds((int64_t)(std::abs(tiles) * ROUTE_MS_PER_TILE_ACROSS.count()))); + pbf_move_left_joystick(context, {tiles > 0 ? 1.0 : -1.0, 0}, hold, 300ms); +} + +// Whether the screen now shows a Pokemon Center's inside. +bool center_inside_now(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + VideoSnapshot now = stream.video().snapshot(); + return now && pokemon_center_inside(*now.frame); +} + +// Walks north into the door straight ahead, `doorway` away. Returns whether +// we are inside. +bool walk_into_door( + VideoStream& stream, ProControllerContext& context, + const VideoSnapshot& outside, const LitDoorway& doorway, const std::string& label +){ + // Time for the rows to the door (none when it is straight ahead), the + // door opening, and a start delay. + const double rows = std::max(0.0, doorway.rows_up - 1); + const Milliseconds limit = 1500ms + Milliseconds((int64_t)(rows * ROUTE_MS_PER_TILE_UP_DOWN.count())); + // The Center's fade is too light to read as black (r126 runs 2 and 3: a + // screen change of 49 at 400 ms), so its inside, once it shows, stops + // the walk too. + const WalkStop stop = walk_until(stream, context, run_from_wild_battle, 0, 1, limit, label + ": into the door", + [](const ImageViewRGB32& screen){ return pokemon_center_inside(screen); }).stop; + if (stop == WalkStop::TRANSITION || stop == WalkStop::ARRIVED){ + pbf_wait(context, 2000ms); + return true; + } + // The door slides open before it takes us in, and its fade is not always + // black enough to read as a transition, and may finish after the walk; a + // whole new scene says we are in. Not after a walk of several rows, which + // scrolls the view: the next look decides. + pbf_wait(context, 2000ms); + if (center_inside_now(stream, context)){ + stream.log("[Route] " + label + ": a Pokemon Center's inside shows; inside."); + return true; + } + if (doorway.rows_up > DOORWAY_FRONT_ROWS || !scene_changed_since(stream, context, outside)){ + return false; + } + stream.log("[Route] " + label + ": the scene changed; inside."); + return true; +} + +// Where the doorway is on screen, in the log. +void log_doorway(VideoStream& stream, const std::string& label, const LitDoorway& doorway){ + stream.log("[Route] " + label + ": doorway " + std::to_string(doorway.tiles_across) + + " tiles across, " + std::to_string(doorway.rows_up) + " rows up (at " + + std::to_string(doorway.x) + ", " + std::to_string(doorway.y) + " of the screen)."); +} + +} + + +bool pokemon_center_inside(const ImageViewRGB32& screen){ + // The orange back wall along the top, in two of three places: a plant + // or a bookcase can stand in one (Jubilife's top left), and at night + // it is darker (r 160..200 in Jubilife, 254 in Sandgem by day). + size_t wall = 0; + for (double x : {0.02, 0.25, 0.65}){ + const ImageStats box = image_stats(extract_box_reference(screen, ImageFloatBox(x, 0.03, 0.08, 0.06))); + const double r = box.average.r, g = box.average.g, b = box.average.b; + wall += r >= 140 && r > g + 40 && g > b + 40; + } + if (wall < 2){ + return false; + } + // Black below the floor, either side of the step out of the door. + for (const ImageFloatBox& box : {ImageFloatBox(0.02, 0.95, 0.30, 0.04), ImageFloatBox(0.65, 0.95, 0.33, 0.04)}){ + if (image_stats(extract_box_reference(screen, box)).average.sum() > 90){ + return false; + } + } + // And the floor lit between them: not a fade (349 in Jubilife at night). + return image_stats(extract_box_reference(screen, ImageFloatBox(0.30, 0.55, 0.40, 0.15))).average.sum() > 250; +} + +void fail_if_blacked_out(VideoStream& stream, ProControllerContext& context, const std::string& label){ + for (size_t look = 0; look < 10; look++){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + if (screen && indoors(*screen.frame)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": the party fainted; the game took us home (a room shows after the battle).", stream); + } + // The blackout's text and fade are dark; the grass is not. + if (screen && image_stats(*screen.frame).average.sum() > 150){ + return; + } + pbf_wait(context, 1000ms); + } +} + +std::optional find_lit_lamp_x(const ImageViewRGB32& screen, const ImageFloatBox& box){ + const size_t x0 = (size_t)(box.x * screen.width()), x1 = (size_t)((box.x + box.width) * screen.width()); + const size_t y0 = (size_t)(box.y * screen.height()), y1 = (size_t)((box.y + box.height) * screen.height()); + double sum = 0; + size_t count = 0; + for (size_t y = y0; y < y1 && y < screen.height(); y++){ + for (size_t x = x0; x < x1 && x < screen.width(); x++){ + const Color pixel(screen.pixel(x, y)); + if (pixel.red() > 235 && pixel.green() > 220 && pixel.blue() > 150){ + sum += x; + count++; + } + } + } + // Jubilife's lamps at night: 50 to 100 such pixels at 480 px across. + if (count * 480 * 480 < 30 * screen.width() * screen.width()){ + return std::nullopt; + } + return sum / count / screen.width(); +} + +double tiles_across_at(double screen_x){ + return (screen_x - PLAYER_X) * SCREEN_WIDTH_TILES; +} + +std::optional find_lit_doorway(const ImageViewRGB32& screen){ + // A Center's lit floor has the doorway's colour. + if (pokemon_center_inside(screen)){ + return std::nullopt; + } + const std::optional> position = lit_doorway_position(screen); + if (!position){ + return std::nullopt; + } + return LitDoorway{ + position->first, + position->second, + (position->first - PLAYER_X) * SCREEN_WIDTH_TILES, + (PLAYER_Y - position->second) / SCREEN_ROW_HEIGHT, + }; +} + +std::vector tiles_short_of_door( + VideoStream& stream, ProControllerContext& context, + const StoryWorld& grid, const TilePos& door, const std::string& label +){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + const std::optional doorway = screen ? find_lit_doorway(*screen.frame) : std::nullopt; + if (!doorway){ + stream.log("[Route] " + label + ": no lit doorway on screen; not walking again."); + return {}; + } + log_doorway(stream, label, *doorway); + if (doorway->rows_up <= DOORWAY_FRONT_ROWS){ + return {}; + } + return tiles_by_lit_doorway(*doorway, grid, door); +} + +std::vector tiles_by_lit_doorway(const LitDoorway& doorway, const StoryWorld& grid, const TilePos& door){ + const TilePos guess{door.x - (int)std::lround(doorway.tiles_across), door.y + (int)std::lround(doorway.rows_up)}; + std::vector tiles; + for (int y = guess.y - 1; y <= guess.y + 1; y++){ + for (int x = guess.x - 1; x <= guess.x + 1; x++){ + if (grid.standable(TilePos{x, y})){ + tiles.push_back(TilePos{x, y}); + } + } + } + return tiles; +} + +bool line_up_with_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + route_status(stream, label + ": lining up with the door."); + for (size_t tries = 0; tries < 6; tries++){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + if (screen && pokemon_center_inside(*screen.frame)){ + stream.log("[Route] " + label + ": a Pokemon Center's inside shows; not in front of the door."); + return false; + } + const std::optional doorway = screen ? find_lit_doorway(*screen.frame) : std::nullopt; + if (!doorway){ + stream.log("[Route] " + label + ": no lit doorway on screen."); + return false; + } + log_doorway(stream, label, *doorway); + // Steps overshoot: within a quarter tile, r128 measured 0.29, -0.28, + // 0.48, -0.67 and 0.48 tiles in turn and never settled. The door + // still takes us in from 0.4 (enter_lit_door()). + if (std::abs(doorway->tiles_across) <= 0.4){ + return true; + } + step_across(stream, context, doorway->tiles_across); + // Let a door that took us in show its inside before looking again. + pbf_wait(context, 1000ms); + } + stream.log("[Route] " + label + ": not lined up after 6 steps."); + return false; +} + +void enter_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + // Movement is not locked to tiles, and the door takes us in only when we + // are lined up with it. Its doorway is lit: find it on screen, where the + // camera keeps the player in the middle, move across by its offset, and + // walk in once it is straight ahead, looking again after every move. + route_status(stream, label + ": lining up with the door."); + VideoSnapshot last_outside; + for (size_t tries = 0; tries < 12; tries++){ + context.wait_for_all_requests(); + VideoSnapshot outside = stream.video().snapshot(); + if (!outside){ + continue; + } + // Inside a Center, its lit floor would read as another doorway. + if (pokemon_center_inside(*outside.frame)){ + stream.log("[Route] " + label + ": a Pokemon Center's inside shows; inside."); + return; + } + const std::optional doorway = find_lit_doorway(*outside.frame); + if (!doorway){ + // The door may have taken us in after all, more slowly than we watched. + pbf_wait(context, 1500ms); + if (!scene_changed_since(stream, context, last_outside)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": no lit doorway on screen.", stream); + } + stream.log("[Route] " + label + ": the scene changed; inside."); + pbf_wait(context, 1500ms); + return; + } + last_outside = outside; + log_doorway(stream, label, *doorway); + // We stand on tiles, so within half a tile is the door's column; + // a step moves about 0.7 of a reading, and at 0.25 r144a stepped + // between 0.3 and -0.4 twelve times. Try the door there, and step + // toward it if it does not take us. + if (std::abs(doorway->tiles_across) > 0.5){ + step_across(stream, context, doorway->tiles_across); + continue; + } + if (walk_into_door(stream, context, outside, *doorway, label)){ + pbf_wait(context, 1500ms); + return; + } + step_across(stream, context, doorway->tiles_across); + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": not through the door after 12 tries.", stream); +} + +void heal_at_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + enter_lit_door(stream, context, label); + + // The nurse's counter is straight ahead of the door. + route_status(stream, label + ": talking to the nurse."); + if (walk_to_talk(stream, context, run_from_wild_battle, 0, 1, 6000ms, label + ": to the nurse").stop != WalkStop::DIALOG){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": nurse not reached.", stream); + } + // A through her lines; the cursor starts on "Yes". Stop once no dialogue + // has shown for a while, so A does not start the talk over. + size_t quiet = 0; + for (size_t presses = 0; presses < 30 && quiet < 2; presses++){ + ShortDialogWatcher dialog(COLOR_YELLOW); + if (wait_until(stream, context, 1500ms, {{dialog}}) == 0){ + quiet = 0; + pbf_press_button(context, BUTTON_A, 80ms, 700ms); + }else{ + quiet++; + } + } + context.wait_for_all_requests(); + + route_status(stream, label + ": healed; walking out."); + leave_door_south(stream, context, run_from_wild_battle, label + ": out the door"); +} + + + +void move_across(VideoStream& stream, ProControllerContext& context, double tiles){ + step_across(stream, context, tiles); +} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h index f5e3577de5..ec94028fbe 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h @@ -1,118 +1,118 @@ -/* BDSP AutoStory - Pokemon Center - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Center_H -#define PokemonAutomation_PokemonBDSP_AutoStory_Center_H - -#include -#include -#include -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTypes/ImageViewRGB32.h" -#include "CommonFramework/Tools/VideoStream.h" -#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" -#include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryWorld.h" -#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Indoors.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -// A building's lit doorway seen on the overworld, relative to the player, -// whom the camera keeps in the middle of the screen. -struct LitDoorway{ - double x; // centre across the screen, 0 to 1 - double y; // centre down the screen, 0 to 1 - double tiles_across; // east of the player if positive, west if negative - double rows_up; // rows north of the player's tile, roughly: 1 straight ahead -}; - -// Whether `screen` is the inside of a Pokemon Center seen from its door: -// the orange back wall at the top left, and the black below the entrance -// mat along the bottom edge. Its lit floor reads as a doorway -// (find_lit_doorway()), so this is what says we are in (r126 runs 2, 3). -bool pokemon_center_inside(const ImageViewRGB32& screen); - -// Finds a lit doorway (a Pokemon Center's or a Poke Mart's) in the upper -// two thirds of `screen`, the pale yellow-white of its open door. Nothing if -// none shows there, or inside a Pokemon Center, whose floor is that colour. -std::optional find_lit_doorway(const ImageViewRGB32& screen); - -// After a wild battle out of doors: fails with a screenshot if the screen -// turns into a room within 10 s, which means the party fainted and the -// game took us home (r258 lost its third training battle in Route 202's -// grass, woke in Twinleaf, and walked the Sandgem plan there). Returns -// once the outside shows, not black and not a room. -void fail_if_blacked_out(VideoStream& stream, ProControllerContext& context, const std::string& label); - -// A lit street lamp's head inside `box` (0 to 1 of the screen): the mean -// x of its white core, or nothing if fewer than a lamp's worth of white -// pixels show there (by day the lamps are not lit). -std::optional find_lit_lamp_x(const ImageViewRGB32& screen, const ImageFloatBox& box); - -// How many tiles a thing seen at screen x is across from the player -// (positive: east). -double tiles_across_at(double screen_x); - -// Moves across by `tiles`, east if positive, west if negative, facing that -// way first. -void move_across(VideoStream& stream, ProControllerContext& context, double tiles); - -// The tiles a player who sees `doorway` (the door on world tile `door`) may -// stand on: the one its offset says, give or take one each way, since the -// rows are a rough count. Only standable tiles of `grid`. -std::vector tiles_by_lit_doorway( - const LitDoorway& doorway, const StoryWorld& grid, const Pokemon::TilePos& door -); - -// For a player who walked to the row below the door on world tile `door`: -// if its lit doorway shows further up the screen than straight ahead, the -// walk ended short (someone in the way stops an open leg, and the walk -// cannot tell), and the tiles he may stand on instead, read off the -// doorway's offset, are returned to walk from again. Empty when he is at -// the front row, or no doorway shows. -std::vector tiles_short_of_door( - VideoStream& stream, ProControllerContext& context, - const StoryWorld& grid, const Pokemon::TilePos& door, const std::string& label -); - -// For a player on the row below a building's door, its lit doorway on -// screen: steps across by the doorway's offset, looking again after each -// step, until it is straight ahead. Does not go in, and does not turn north -// to face it: a turn there walked us in (r127c). Returns false if no -// doorway shows, a Center's inside shows, or after 6 steps. -bool line_up_with_lit_door( - VideoStream& stream, ProControllerContext& context, - const std::string& label -); - -// Goes in through a building's door (a Pokemon Center's or a Poke Mart's). -// Starts with its lit doorway on screen, within a few tiles: lines up with -// the doorway by sight and walks in, looking again after every move. Whole- -// tile steps sideways miss it, since movement is not locked to tiles, and -// its fade may not read as black: a new scene also counts as inside. Fails -// with a screenshot when no doorway shows, or after 12 moves. -void enter_lit_door( - VideoStream& stream, ProControllerContext& context, - const std::string& label -); - -// Heals the party at a Pokemon Center. Starts in front of the Center with its -// lit doorway on screen, within a few tiles of it: lines up with the doorway -// by sight, walks in, talks to the nurse, accepts, and walks back out. Ends -// on the tile below the door. -void heal_at_pokemon_center( - VideoStream& stream, ProControllerContext& context, - const std::string& label -); - - - -} -} -} -#endif +/* BDSP AutoStory - Pokemon Center + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Center_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Center_H + +#include +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryWorld.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Indoors.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// A building's lit doorway seen on the overworld, relative to the player, +// whom the camera keeps in the middle of the screen. +struct LitDoorway{ + double x; // centre across the screen, 0 to 1 + double y; // centre down the screen, 0 to 1 + double tiles_across; // east of the player if positive, west if negative + double rows_up; // rows north of the player's tile, roughly: 1 straight ahead +}; + +// Whether `screen` is the inside of a Pokemon Center seen from its door: +// the orange back wall at the top left, and the black below the entrance +// mat along the bottom edge. Its lit floor reads as a doorway +// (find_lit_doorway()), so this is what says we are in (r126 runs 2, 3). +bool pokemon_center_inside(const ImageViewRGB32& screen); + +// Finds a lit doorway (a Pokemon Center's or a Poke Mart's) in the upper +// two thirds of `screen`, the pale yellow-white of its open door. Nothing if +// none shows there, or inside a Pokemon Center, whose floor is that colour. +std::optional find_lit_doorway(const ImageViewRGB32& screen); + +// After a wild battle out of doors: fails with a screenshot if the screen +// turns into a room within 10 s, which means the party fainted and the +// game took us home (r258 lost its third training battle in Route 202's +// grass, woke in Twinleaf, and walked the Sandgem plan there). Returns +// once the outside shows, not black and not a room. +void fail_if_blacked_out(VideoStream& stream, ProControllerContext& context, const std::string& label); + +// A lit street lamp's head inside `box` (0 to 1 of the screen): the mean +// x of its white core, or nothing if fewer than a lamp's worth of white +// pixels show there (by day the lamps are not lit). +std::optional find_lit_lamp_x(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// How many tiles a thing seen at screen x is across from the player +// (positive: east). +double tiles_across_at(double screen_x); + +// Moves across by `tiles`, east if positive, west if negative, facing that +// way first. +void move_across(VideoStream& stream, ProControllerContext& context, double tiles); + +// The tiles a player who sees `doorway` (the door on world tile `door`) may +// stand on: the one its offset says, give or take one each way, since the +// rows are a rough count. Only standable tiles of `grid`. +std::vector tiles_by_lit_doorway( + const LitDoorway& doorway, const StoryWorld& grid, const Pokemon::TilePos& door +); + +// For a player who walked to the row below the door on world tile `door`: +// if its lit doorway shows further up the screen than straight ahead, the +// walk ended short (someone in the way stops an open leg, and the walk +// cannot tell), and the tiles he may stand on instead, read off the +// doorway's offset, are returned to walk from again. Empty when he is at +// the front row, or no doorway shows. +std::vector tiles_short_of_door( + VideoStream& stream, ProControllerContext& context, + const StoryWorld& grid, const Pokemon::TilePos& door, const std::string& label +); + +// For a player on the row below a building's door, its lit doorway on +// screen: steps across by the doorway's offset, looking again after each +// step, until it is straight ahead. Does not go in, and does not turn north +// to face it: a turn there walked us in (r127c). Returns false if no +// doorway shows, a Center's inside shows, or after 6 steps. +bool line_up_with_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + +// Goes in through a building's door (a Pokemon Center's or a Poke Mart's). +// Starts with its lit doorway on screen, within a few tiles: lines up with +// the doorway by sight and walks in, looking again after every move. Whole- +// tile steps sideways miss it, since movement is not locked to tiles, and +// its fade may not read as black: a new scene also counts as inside. Fails +// with a screenshot when no doorway shows, or after 12 moves. +void enter_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + +// Heals the party at a Pokemon Center. Starts in front of the Center with its +// lit doorway on screen, within a few tiles of it: lines up with the doorway +// by sight, walks in, talks to the nurse, accepts, and walks back out. Ends +// on the tile below the door. +void heal_at_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index ac311dcba9..c081981796 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -1,286 +1,286 @@ -/* BDSP AutoStory - New Game - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include "Common/Cpp/Filesystem/Filesystem.h" -#include "CommonFramework/ImageTools/ImageDiff.h" -#include "CommonFramework/ImageTypes/ImageRGB32.h" -#include "CommonFramework/VideoPipeline/VideoFeed.h" -#include "CommonTools/Async/InferenceRoutines.h" -#include "CommonTools/ScreenMachine/ScreenMachine.h" -#include "CommonTools/ScreenMachine/StepTrace.h" -#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" -#include "NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h" -#include "NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.h" -#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" -#include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h" -#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" -#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" -#include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" -#include "PokemonBDSP_AutoStory_NewGame.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - -using namespace std::chrono_literals; - - -std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ - // The Switch's own screens come first: BDSP's dialogue detector also fires on them. - if (console != nullptr && StartGameUserSelectDetector(*console).detect(screen)){ - return "user-select"; - } - if (console != nullptr && HomeMenuDetector(*console, COLOR_RED).detect(screen)){ - return "home"; - } - if (SystemKeyboardDetector().detect(screen)){ - return "keyboard"; - } - switch (NameListDetector().selected_row(screen)){ - case -1: - break; - case 0: - return "rival-list-tell"; - default: - return "rival-list-preset"; - } - if (MenuDetector().detect(screen)){ - return "menu"; - } - // Includes the appearance grid, whose prompt sits in a dialogue box. - if (ShortDialogDetector().detect(screen)){ - return "dialog"; - } - // Language, cutscenes, fades, the bedroom with no menu open. - return "other"; -} - -IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ - if (screen == "user-select") return IntroStep::SELECT_USER; - if (screen == "home") return IntroStep::FAIL_NOT_IN_GAME; - if (screen == "keyboard"){ - return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; - } - if (screen == "rival-list-tell") return IntroStep::MOVE_TO_PRESET; - if (screen == "rival-list-preset") return IntroStep::PICK_PRESET; - if (screen == "menu") return IntroStep::CLOSE_MENU_AND_FINISH; - if (screen == "dialog") return IntroStep::ADVANCE; - // The bedroom comes only after the rival is named. Before that, every unknown - // screen (language, the appearance preview, cutscenes) takes A. - return progress.rival_named ? IntroStep::PROBE_FOR_MENU : IntroStep::ADVANCE; -} -bool advance_after_probe(bool screen_changed){ - return !screen_changed; -} - -size_t run_intro_screen_tests(Logger& logger, const std::string& folder){ - size_t passed = 0; - size_t failed = 0; - for (const Filesystem::Path& dir : Filesystem::list_directory(Filesystem::Path(folder))){ - if (!Filesystem::is_directory(dir)){ - continue; - } - const std::string expected = dir.filename().string(); - for (const Filesystem::Path& file : Filesystem::list_directory(dir)){ - if (file.extension().string() != ".png"){ - continue; - } - const ImageRGB32 image(file.string()); - // 1080p and 720p. At 960x540 the keyboard's and the name list's checks - // do not hold (blurred key text, an arrow below the object size limit). - for (size_t width : {size_t(1920), size_t(1280)}){ - const std::string actual = classify_intro_screen(nullptr, image.scale_to(width, width * 9 / 16)); - if (actual == expected){ - passed++; - continue; - } - failed++; - logger.log("[Intro tests] FAIL " + expected + "/" + file.filename().string() + - " at " + std::to_string(width) + "px: read as " + actual, COLOR_RED); - } - } - } - logger.log("[Intro tests] " + std::to_string(passed) + " passed, " + std::to_string(failed) + " failed.", - failed == 0 ? COLOR_BLUE : COLOR_RED); - return failed; -} - - -namespace{ - -// Waits until the keyboard is gone (or, with `gone` false, has appeared). -bool wait_for_keyboard(VideoStream& stream, ProControllerContext& context, bool gone, Milliseconds timeout){ - context.wait_for_all_requests(); - const WallClock deadline = current_time() + timeout; - while (current_time() < deadline){ - VideoSnapshot screen = stream.video().snapshot(); - if (screen && SystemKeyboardDetector().detect(screen) != gone){ - return true; - } - context.wait_for(100ms); - } - return false; -} - -void enter_player_name(VideoStream& stream, ProControllerContext& context, StepTrace& trace, const std::string& name){ - // Let the keyboard finish sliding in: inputs during that are dropped. - pbf_wait(context, 800ms); - context.wait_for_all_requests(); - // B deletes a character, and closes the keyboard on an empty field, so delete only - // while there is text (e.g. after a retry). - for (size_t c = 0; c < 14; c++){ - VideoSnapshot screen = stream.video().snapshot(); - if (!screen || !system_keyboard_field_has_text(screen)){ - break; - } - trace.action("Name field holds text; deleting a character."); - pbf_press_button(context, BUTTON_B, 60ms, 250ms); - context.wait_for_all_requests(); - } - trace.action("Typing the player's name: " + name); - type_on_system_keyboard(stream, context, name); - pbf_press_button(context, BUTTON_PLUS, 80ms, 0ms); - if (!wait_for_keyboard(stream, context, true, 5s)){ - trace.fail("The keyboard stayed open after PLUS (OK)."); - } - trace.action("Name confirmed."); -} - -} - - - -void play_new_game_intro( - SingleSwitchProgramEnvironment& env, - ProControllerContext& context, - const std::string& player_name -){ - VideoStream& stream = env.console; - StepTrace trace(stream, "Intro"); - - // Only close and reopen the game: the states below take it from the user select - // on. reset_game_from_home() waits for a loaded save's black screen to end, and a - // new game's first screen is a dialog on black, so it would sit there until its - // timeout. - trace.action("Restarting the game."); - go_home(env.console, context); - from_home_close_and_reopen_game(env.console, context, true); - - IntroProgress progress; - size_t x_without_menu = 0; - auto press = [&](Button button){ - pbf_press_button(context, button, 80ms, 0ms); - }; - - // Carries out what decide_intro_step() chose for `screen`. - auto act = [&](const char* screen){ - switch (decide_intro_step(screen, progress)){ - case IntroStep::SELECT_USER: - // The cursor starts on the user that launched the game. - press(BUTTON_A); - pbf_wait(context, 1000ms); - return; - case IntroStep::FAIL_NOT_IN_GAME: - trace.fail("The Switch HOME menu is showing: the game is not running in front."); - case IntroStep::ENTER_PLAYER_NAME: - enter_player_name(stream, context, trace, player_name); - progress.player_named = true; - return; - case IntroStep::CANCEL_RIVAL_KEYBOARD: - // Opened from "I'll tell you..."; inputs during the slide-in are dropped. - pbf_wait(context, 800ms); - press(BUTTON_X); - return; - case IntroStep::MOVE_TO_PRESET: - pbf_press_dpad(context, DPAD_DOWN, 100ms, 300ms); - return; - case IntroStep::PICK_PRESET: - press(BUTTON_A); - pbf_wait(context, 500ms); - progress.rival_named = true; - return; - case IntroStep::CLOSE_MENU_AND_FINISH:{ - trace.action("Main menu is open: the player is in control in the bedroom. Closing it."); - press(BUTTON_B); - pbf_wait(context, 1000ms); - context.wait_for_all_requests(); - VideoSnapshot screen_now = stream.video().snapshot(); - if (!screen_now || MenuDetector().detect(screen_now)){ - trace.fail("The main menu did not close."); - } - return; - } - case IntroStep::ADVANCE: - pbf_press_button(context, BUTTON_A, 60ms, 400ms); - return; - case IntroStep::PROBE_FOR_MENU:{ - context.wait_for_all_requests(); - VideoSnapshot before = stream.video().snapshot(); - press(BUTTON_X); - context.wait_for_all_requests(); - MenuWatcher menu; - if (wait_until(stream, context, 3s, {{menu}}) == 0){ - return; // The next read sees the menu and finishes. - } - VideoSnapshot after = stream.video().snapshot(); - const bool changed = !before || !after || ImageMatch::pixel_RMSD(before, after) > 10; - if (!advance_after_probe(changed)){ - trace.action("X changed the screen but no known menu opened; not pressing A."); - if (++x_without_menu >= 10){ - trace.fail("X keeps changing the screen without a known menu opening."); - } - return; - } - x_without_menu = 0; - pbf_press_button(context, BUTTON_A, 60ms, 400ms); - return; - } - } - }; - - auto screen_is = [&](const ImageViewRGB32& screen, const char* name){ - return classify_intro_screen(&env.console, screen) == name; - }; - // The machine's states: the screens classify_intro_screen() tells apart, with the - // keyboard split by whose name it asks for. Each state's step is act(). - const std::vector states{ - {"user-select", - [&](const ImageViewRGB32& s){ return screen_is(s, "user-select"); }, - [&]{ act("user-select"); }, {}, 3}, - {"home", - [&](const ImageViewRGB32& s){ return screen_is(s, "home"); }, - [&]{ act("home"); }, {}, 1}, - {"player-keyboard", - [&](const ImageViewRGB32& s){ return !progress.player_named && screen_is(s, "keyboard"); }, - [&]{ act("keyboard"); }, {"dialog", "other"}, 1}, - {"rival-keyboard", - [&](const ImageViewRGB32& s){ return progress.player_named && screen_is(s, "keyboard"); }, - [&]{ act("keyboard"); }, {"rival-list-tell", "dialog", "other"}, 3}, - {"rival-list-tell", - [&](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, - [&]{ act("rival-list-tell"); }, {"rival-list-preset", "other"}, 6}, - {"rival-list-preset", - [&](const ImageViewRGB32& s){ return screen_is(s, "rival-list-preset"); }, - [&]{ act("rival-list-preset"); }, {"dialog", "other"}, 3}, - {"menu", - [&](const ImageViewRGB32& s){ return screen_is(s, "menu"); }, - [&]{ act("menu"); }, {}, 2, true}, - {"dialog", - [&](const ImageViewRGB32& s){ return screen_is(s, "dialog"); }, - [&]{ act("dialog"); }, {}, 400}, - {"other", - [](const ImageViewRGB32&){ return true; }, - [&]{ act("other"); }, {}, 300, false, 60}, - }; - run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); - trace.action("Intro done."); -} - - - -} -} -} +/* BDSP AutoStory - New Game + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/ScreenMachine/ScreenMachine.h" +#include "CommonTools/ScreenMachine/StepTrace.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" +#include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" +#include "PokemonBDSP_AutoStory_NewGame.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ + // The Switch's own screens come first: BDSP's dialogue detector also fires on them. + if (console != nullptr && StartGameUserSelectDetector(*console).detect(screen)){ + return "user-select"; + } + if (console != nullptr && HomeMenuDetector(*console, COLOR_RED).detect(screen)){ + return "home"; + } + if (SystemKeyboardDetector().detect(screen)){ + return "keyboard"; + } + switch (NameListDetector().selected_row(screen)){ + case -1: + break; + case 0: + return "rival-list-tell"; + default: + return "rival-list-preset"; + } + if (MenuDetector().detect(screen)){ + return "menu"; + } + // Includes the appearance grid, whose prompt sits in a dialogue box. + if (ShortDialogDetector().detect(screen)){ + return "dialog"; + } + // Language, cutscenes, fades, the bedroom with no menu open. + return "other"; +} + +IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ + if (screen == "user-select") return IntroStep::SELECT_USER; + if (screen == "home") return IntroStep::FAIL_NOT_IN_GAME; + if (screen == "keyboard"){ + return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; + } + if (screen == "rival-list-tell") return IntroStep::MOVE_TO_PRESET; + if (screen == "rival-list-preset") return IntroStep::PICK_PRESET; + if (screen == "menu") return IntroStep::CLOSE_MENU_AND_FINISH; + if (screen == "dialog") return IntroStep::ADVANCE; + // The bedroom comes only after the rival is named. Before that, every unknown + // screen (language, the appearance preview, cutscenes) takes A. + return progress.rival_named ? IntroStep::PROBE_FOR_MENU : IntroStep::ADVANCE; +} +bool advance_after_probe(bool screen_changed){ + return !screen_changed; +} + +size_t run_intro_screen_tests(Logger& logger, const std::string& folder){ + size_t passed = 0; + size_t failed = 0; + for (const Filesystem::Path& dir : Filesystem::list_directory(Filesystem::Path(folder))){ + if (!Filesystem::is_directory(dir)){ + continue; + } + const std::string expected = dir.filename().string(); + for (const Filesystem::Path& file : Filesystem::list_directory(dir)){ + if (file.extension().string() != ".png"){ + continue; + } + const ImageRGB32 image(file.string()); + // 1080p and 720p. At 960x540 the keyboard's and the name list's checks + // do not hold (blurred key text, an arrow below the object size limit). + for (size_t width : {size_t(1920), size_t(1280)}){ + const std::string actual = classify_intro_screen(nullptr, image.scale_to(width, width * 9 / 16)); + if (actual == expected){ + passed++; + continue; + } + failed++; + logger.log("[Intro tests] FAIL " + expected + "/" + file.filename().string() + + " at " + std::to_string(width) + "px: read as " + actual, COLOR_RED); + } + } + } + logger.log("[Intro tests] " + std::to_string(passed) + " passed, " + std::to_string(failed) + " failed.", + failed == 0 ? COLOR_BLUE : COLOR_RED); + return failed; +} + + +namespace{ + +// Waits until the keyboard is gone (or, with `gone` false, has appeared). +bool wait_for_keyboard(VideoStream& stream, ProControllerContext& context, bool gone, Milliseconds timeout){ + context.wait_for_all_requests(); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + VideoSnapshot screen = stream.video().snapshot(); + if (screen && SystemKeyboardDetector().detect(screen) != gone){ + return true; + } + context.wait_for(100ms); + } + return false; +} + +void enter_player_name(VideoStream& stream, ProControllerContext& context, StepTrace& trace, const std::string& name){ + // Let the keyboard finish sliding in: inputs during that are dropped. + pbf_wait(context, 800ms); + context.wait_for_all_requests(); + // B deletes a character, and closes the keyboard on an empty field, so delete only + // while there is text (e.g. after a retry). + for (size_t c = 0; c < 14; c++){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen || !system_keyboard_field_has_text(screen)){ + break; + } + trace.action("Name field holds text; deleting a character."); + pbf_press_button(context, BUTTON_B, 60ms, 250ms); + context.wait_for_all_requests(); + } + trace.action("Typing the player's name: " + name); + type_on_system_keyboard(stream, context, name); + pbf_press_button(context, BUTTON_PLUS, 80ms, 0ms); + if (!wait_for_keyboard(stream, context, true, 5s)){ + trace.fail("The keyboard stayed open after PLUS (OK)."); + } + trace.action("Name confirmed."); +} + +} + + + +void play_new_game_intro( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& player_name +){ + VideoStream& stream = env.console; + StepTrace trace(stream, "Intro"); + + // Only close and reopen the game: the states below take it from the user select + // on. reset_game_from_home() waits for a loaded save's black screen to end, and a + // new game's first screen is a dialog on black, so it would sit there until its + // timeout. + trace.action("Restarting the game."); + go_home(env.console, context); + from_home_close_and_reopen_game(env.console, context, true); + + IntroProgress progress; + size_t x_without_menu = 0; + auto press = [&](Button button){ + pbf_press_button(context, button, 80ms, 0ms); + }; + + // Carries out what decide_intro_step() chose for `screen`. + auto act = [&](const char* screen){ + switch (decide_intro_step(screen, progress)){ + case IntroStep::SELECT_USER: + // The cursor starts on the user that launched the game. + press(BUTTON_A); + pbf_wait(context, 1000ms); + return; + case IntroStep::FAIL_NOT_IN_GAME: + trace.fail("The Switch HOME menu is showing: the game is not running in front."); + case IntroStep::ENTER_PLAYER_NAME: + enter_player_name(stream, context, trace, player_name); + progress.player_named = true; + return; + case IntroStep::CANCEL_RIVAL_KEYBOARD: + // Opened from "I'll tell you..."; inputs during the slide-in are dropped. + pbf_wait(context, 800ms); + press(BUTTON_X); + return; + case IntroStep::MOVE_TO_PRESET: + pbf_press_dpad(context, DPAD_DOWN, 100ms, 300ms); + return; + case IntroStep::PICK_PRESET: + press(BUTTON_A); + pbf_wait(context, 500ms); + progress.rival_named = true; + return; + case IntroStep::CLOSE_MENU_AND_FINISH:{ + trace.action("Main menu is open: the player is in control in the bedroom. Closing it."); + press(BUTTON_B); + pbf_wait(context, 1000ms); + context.wait_for_all_requests(); + VideoSnapshot screen_now = stream.video().snapshot(); + if (!screen_now || MenuDetector().detect(screen_now)){ + trace.fail("The main menu did not close."); + } + return; + } + case IntroStep::ADVANCE: + pbf_press_button(context, BUTTON_A, 60ms, 400ms); + return; + case IntroStep::PROBE_FOR_MENU:{ + context.wait_for_all_requests(); + VideoSnapshot before = stream.video().snapshot(); + press(BUTTON_X); + context.wait_for_all_requests(); + MenuWatcher menu; + if (wait_until(stream, context, 3s, {{menu}}) == 0){ + return; // The next read sees the menu and finishes. + } + VideoSnapshot after = stream.video().snapshot(); + const bool changed = !before || !after || ImageMatch::pixel_RMSD(before, after) > 10; + if (!advance_after_probe(changed)){ + trace.action("X changed the screen but no known menu opened; not pressing A."); + if (++x_without_menu >= 10){ + trace.fail("X keeps changing the screen without a known menu opening."); + } + return; + } + x_without_menu = 0; + pbf_press_button(context, BUTTON_A, 60ms, 400ms); + return; + } + } + }; + + auto screen_is = [&](const ImageViewRGB32& screen, const char* name){ + return classify_intro_screen(&env.console, screen) == name; + }; + // The machine's states: the screens classify_intro_screen() tells apart, with the + // keyboard split by whose name it asks for. Each state's step is act(). + const std::vector states{ + {"user-select", + [&](const ImageViewRGB32& s){ return screen_is(s, "user-select"); }, + [&]{ act("user-select"); }, {}, 3}, + {"home", + [&](const ImageViewRGB32& s){ return screen_is(s, "home"); }, + [&]{ act("home"); }, {}, 1}, + {"player-keyboard", + [&](const ImageViewRGB32& s){ return !progress.player_named && screen_is(s, "keyboard"); }, + [&]{ act("keyboard"); }, {"dialog", "other"}, 1}, + {"rival-keyboard", + [&](const ImageViewRGB32& s){ return progress.player_named && screen_is(s, "keyboard"); }, + [&]{ act("keyboard"); }, {"rival-list-tell", "dialog", "other"}, 3}, + {"rival-list-tell", + [&](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, + [&]{ act("rival-list-tell"); }, {"rival-list-preset", "other"}, 6}, + {"rival-list-preset", + [&](const ImageViewRGB32& s){ return screen_is(s, "rival-list-preset"); }, + [&]{ act("rival-list-preset"); }, {"dialog", "other"}, 3}, + {"menu", + [&](const ImageViewRGB32& s){ return screen_is(s, "menu"); }, + [&]{ act("menu"); }, {}, 2, true}, + {"dialog", + [&](const ImageViewRGB32& s){ return screen_is(s, "dialog"); }, + [&]{ act("dialog"); }, {}, 400}, + {"other", + [](const ImageViewRGB32&){ return true; }, + [&]{ act("other"); }, {}, 300, false, 60}, + }; + run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); + trace.action("Intro done."); +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h index 7ab9e4fb5c..eb7d4f01ca 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h @@ -1,71 +1,71 @@ -/* BDSP AutoStory - New Game - * - * From: https://github.com/PokemonAutomation/ - * - * Plays a new game's intro (language, Professor Rowan, appearance, names) up to - * the bedroom with the player in control, which is where the Tutorial starts. - * - */ - -#ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H -#define PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H - -#include -#include "CommonFramework/ImageTypes/ImageViewRGB32.h" -#include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -// Which intro screen `screen` shows, as the intro's steps tell them apart: -// "user-select", "home", "keyboard", "rival-list-tell" (the rival's name list on -// "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or -// "other". The Switch's own screens need `console`; without it they read as -// whatever else they look like. -std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen); - -// What the intro does on a screen. Decided from the screen and how far the intro has -// got only, so it is tested without a console. -enum class IntroStep{ - SELECT_USER, // A on the Switch's user select - FAIL_NOT_IN_GAME, // HOME is showing - ENTER_PLAYER_NAME, // type the name on the first keyboard - CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list - MOVE_TO_PRESET, // Down: off "I'll tell you..." - PICK_PRESET, // A on a preset name - CLOSE_MENU_AND_FINISH, // B: the menu opened in the bedroom, the intro is over - ADVANCE, // A: dialogue, language, appearance, cutscenes - PROBE_FOR_MENU, // X, to see whether the player is in control -}; -struct IntroProgress{ - bool player_named = false; - bool rival_named = false; -}; -IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); - -// After PROBE_FOR_MENU found no menu: whether A may follow. Only when X changed -// nothing, since A on something X opened would pick from it (the Bag). -bool advance_after_probe(bool screen_changed); - -// Checks classify_intro_screen() on //*.png at 1920 and -// 1280 px wide. Logs each mismatch; returns the number of them. -size_t run_intro_screen_tests(Logger& logger, const std::string& folder); - -// From the title screen or any intro screen of a game with no save. Restarts the -// game first, so it always starts from the title. Names the player `player_name` -// and picks the rival's default name (Barry) from the list. Each step reads the -// screen and waits for the expected next one; an unexpected screen fails with a -// screenshot and the recent screens. -void play_new_game_intro( - SingleSwitchProgramEnvironment& env, - ProControllerContext& context, - const std::string& player_name -); - - -} -} -} -#endif +/* BDSP AutoStory - New Game + * + * From: https://github.com/PokemonAutomation/ + * + * Plays a new game's intro (language, Professor Rowan, appearance, names) up to + * the bedroom with the player in control, which is where the Tutorial starts. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H +#define PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Which intro screen `screen` shows, as the intro's steps tell them apart: +// "user-select", "home", "keyboard", "rival-list-tell" (the rival's name list on +// "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or +// "other". The Switch's own screens need `console`; without it they read as +// whatever else they look like. +std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen); + +// What the intro does on a screen. Decided from the screen and how far the intro has +// got only, so it is tested without a console. +enum class IntroStep{ + SELECT_USER, // A on the Switch's user select + FAIL_NOT_IN_GAME, // HOME is showing + ENTER_PLAYER_NAME, // type the name on the first keyboard + CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list + MOVE_TO_PRESET, // Down: off "I'll tell you..." + PICK_PRESET, // A on a preset name + CLOSE_MENU_AND_FINISH, // B: the menu opened in the bedroom, the intro is over + ADVANCE, // A: dialogue, language, appearance, cutscenes + PROBE_FOR_MENU, // X, to see whether the player is in control +}; +struct IntroProgress{ + bool player_named = false; + bool rival_named = false; +}; +IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); + +// After PROBE_FOR_MENU found no menu: whether A may follow. Only when X changed +// nothing, since A on something X opened would pick from it (the Bag). +bool advance_after_probe(bool screen_changed); + +// Checks classify_intro_screen() on //*.png at 1920 and +// 1280 px wide. Logs each mismatch; returns the number of them. +size_t run_intro_screen_tests(Logger& logger, const std::string& folder); + +// From the title screen or any intro screen of a game with no save. Restarts the +// game first, so it always starts from the title. Names the player `player_name` +// and picks the rival's default name (Barry) from the list. Each step reads the +// screen and waits for the expected next one; an unexpected screen fails with a +// screenshot and the recent screens. +void play_new_game_intro( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& player_name +); + + +} +} +} +#endif From a9b2ba996e69cba36105f74d6a6848f2e52e4bcc Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 14:46:30 -0800 Subject: [PATCH 05/23] BDSP AutoStory actions: code health, without changing what they do The repo's control-flow formatting throughout; long functions split into helpers that run the same statements in the same order; the move reading moved to BattleMoves. Battle drops the parameters and branches no caller used (a confirmed default target, move lists, no post-battle cleanup): every battle takes the best move each turn, and fight_story_battle() takes BattleFormat instead of a bool. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../NintendoSwitch_SystemKeyboardDetector.cpp | 16 +- ...PokemonBDSP_AutoStory_BattleMenuReader.cpp | 218 +++--- ...kemonBDSP_AutoStory_MenuCursorDetector.cpp | 22 +- ...PokemonBDSP_AutoStory_MenuCursorDetector.h | 6 +- .../PokemonBDSP_AutoStory_Actions.cpp | 336 ++++----- .../AutoStory/PokemonBDSP_AutoStory_Bag.cpp | 260 ++++--- .../PokemonBDSP_AutoStory_Battle.cpp | 682 +++++++----------- .../AutoStory/PokemonBDSP_AutoStory_Battle.h | 10 +- .../PokemonBDSP_AutoStory_BattleMoves.cpp | 276 +++++++ .../PokemonBDSP_AutoStory_BattleMoves.h | 37 + .../PokemonBDSP_AutoStory_Center.cpp | 25 +- .../PokemonBDSP_AutoStory_Settings.cpp | 521 +++++++------ .../PokemonBDSP_AutoStory_WildBattle.cpp | 352 +++++---- SerialPrograms/cmake/SourceFiles.cmake | 2 + 14 files changed, 1533 insertions(+), 1230 deletions(-) create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp index b8d75872dd..a7ec920307 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -129,10 +129,18 @@ void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, arrived = true; break; } - for (; row < target_row; row++) pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms); - for (; row > target_row; row--) pbf_press_dpad(context, DPAD_UP, 100ms, 200ms); - for (; col < target_col; col++) pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms); - for (; col > target_col; col--) pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms); + for (; row < target_row; row++){ + pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms); + } + for (; row > target_row; row--){ + pbf_press_dpad(context, DPAD_UP, 100ms, 200ms); + } + for (; col < target_col; col++){ + pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms); + } + for (; col > target_col; col--){ + pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms); + } pbf_wait(context, 300ms); } if (!arrived){ diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp index 407bd7818e..7cb7d4f296 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp @@ -18,135 +18,165 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ -BattleMenuOption battle_menu_selected_option(BattleMenuDetector& menu, const ImageViewRGB32& screen){ - if (!screen || !menu.detect(screen)){ - return BattleMenuOption::UNKNOWN; - } - int selected = -1; - for (int row = 0; row < 4; row++){ - const double offset = row * 0.1075; - bool brackets = false; - // The corner brackets wiggle; all four must agree on one small offset. - const double offsets[] = {0.0, -0.003, 0.003, -0.006, 0.006}; - for (double dy : offsets){ - for (double dx : offsets){ - const ImageFloatBox corners[] = { - {0.793 + dx, 0.562 + offset + dy, 0.018, 0.023}, - {0.976 + dx, 0.562 + offset + dy, 0.018, 0.023}, - {0.793 + dx, 0.652 + offset + dy, 0.018, 0.023}, - {0.976 + dx, 0.652 + offset + dy, 0.018, 0.023}, - }; - bool candidate = true; - for (const ImageFloatBox& box : corners){ - const ImageViewRGB32 patch = extract_box_reference(screen, box); - size_t white = 0; - const size_t total = patch.width() * patch.height(); - for (size_t y = 0; y < patch.height(); y++){ - for (size_t x = 0; x < patch.width(); x++){ - const uint32_t pixel = patch.pixel(x, y); - const unsigned r = (pixel >> 16) & 255; - const unsigned g = (pixel >> 8) & 255; - const unsigned b = pixel & 255; - const unsigned low = std::min(r, std::min(g, b)); - const unsigned high = std::max(r, std::max(g, b)); - white += low > 210 && high - low < 35; - } - } - if (total == 0 || static_cast(white) / total < 0.15){ - candidate = false; - break; - } - } - if (candidate){ - brackets = true; - break; - } - } - if (brackets){ - break; - } +namespace{ + +// Whether at least 15% of the box's pixels are white (min channel over +// `low`, spread under `spread`): a cursor bracket's corner. +bool white_corner(const ImageViewRGB32& screen, const ImageFloatBox& box, unsigned low_min, unsigned spread_max){ + const ImageViewRGB32 patch = extract_box_reference(screen, box); + size_t white = 0; + const size_t total = patch.width() * patch.height(); + for (size_t y = 0; y < patch.height(); y++){ + for (size_t x = 0; x < patch.width(); x++){ + const uint32_t pixel = patch.pixel(x, y); + const unsigned r = (pixel >> 16) & 255; + const unsigned g = (pixel >> 8) & 255; + const unsigned b = pixel & 255; + const unsigned low = std::min(r, std::min(g, b)); + const unsigned high = std::max(r, std::max(g, b)); + white += low > low_min && high - low < spread_max; } - if (brackets){ - if (selected >= 0){ - return BattleMenuOption::UNKNOWN; + } + return total != 0 && static_cast(white) / total >= 0.15; +} + +// Whether the command menu's row `row` has the cursor's corner brackets. +// They wiggle; all four must agree on one small offset. +bool row_has_brackets(const ImageViewRGB32& screen, int row){ + const double offset = row * 0.1075; + const double offsets[] = {0.0, -0.003, 0.003, -0.006, 0.006}; + for (double dy : offsets){ + for (double dx : offsets){ + const ImageFloatBox corners[] = { + {0.793 + dx, 0.562 + offset + dy, 0.018, 0.023}, + {0.976 + dx, 0.562 + offset + dy, 0.018, 0.023}, + {0.793 + dx, 0.652 + offset + dy, 0.018, 0.023}, + {0.976 + dx, 0.652 + offset + dy, 0.018, 0.023}, + }; + if (std::all_of(std::begin(corners), std::end(corners), [&](const ImageFloatBox& box){ + return white_corner(screen, box, 210, 35); + })){ + return true; } - selected = row; } } - return static_cast(selected); + return false; } -BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen){ - BattleMoveMenuState result; - if (!screen) return result; - // Every populated move has a white PP panel, irrespective of move type or - // selected row. Find connected panels rather than assume a 4-row layout. - const ImageFloatBox search{0.90, 0.48, 0.075, 0.505}; - const ImageViewRGB32 patch = extract_box_reference(screen, search); +// The white PP panels in `patch` (the search box), top to bottom: nothing +// for no panel, more than four, or geometry that is not a move list. +std::vector find_pp_panels(const ImageViewRGB32& screen, const ImageViewRGB32& patch){ cv::Mat white((int)patch.height(), (int)patch.width(), CV_8UC1); - for (size_t y = 0; y < patch.height(); y++) for (size_t x = 0; x < patch.width(); x++) { - uint32_t p = patch.pixel(x, y); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - white.at((int)y, (int)x) = std::min(r, std::min(g, b)) > 225 && - std::max(r, std::max(g, b)) - std::min(r, std::min(g, b)) < 30 ? 255 : 0; + for (size_t y = 0; y < patch.height(); y++){ + for (size_t x = 0; x < patch.width(); x++){ + uint32_t p = patch.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + white.at((int)y, (int)x) = std::min(r, std::min(g, b)) > 225 && + std::max(r, std::max(g, b)) - std::min(r, std::min(g, b)) < 30 ? 255 : 0; + } } cv::Mat labels, stats, centers; int components = cv::connectedComponentsWithStats(white, labels, stats, centers); std::vector panels; - for (int component = 1; component < components; component++) { + for (int component = 1; component < components; component++){ cv::Rect box(stats.at(component, cv::CC_STAT_LEFT), stats.at(component, cv::CC_STAT_TOP), stats.at(component, cv::CC_STAT_WIDTH), stats.at(component, cv::CC_STAT_HEIGHT)); + const double area = (double)stats.at(component, cv::CC_STAT_AREA); if (box.width < patch.width() * 0.60 || box.height < screen.height() * 0.055 || - box.height > screen.height() * 0.13) continue; - double area = (double)stats.at(component, cv::CC_STAT_AREA); - if (area / box.area() < 0.55) continue; + box.height > screen.height() * 0.13 || area / box.area() < 0.55){ + continue; + } panels.push_back(box); } - if (panels.empty() || panels.size() > 4) return result; + if (panels.empty() || panels.size() > 4){ + return {}; + } std::sort(panels.begin(), panels.end(), [](const cv::Rect& a, const cv::Rect& b){ return a.y < b.y; }); // Reject inconsistent/overlapping geometry rather than count noise as moves. - for (size_t i = 1; i < panels.size(); i++) { + for (size_t i = 1; i < panels.size(); i++){ if (panels[i].y <= panels[i - 1].y + panels[i - 1].height || - std::abs(panels[i].height - panels[0].height) > screen.height() * 0.025) return result; + std::abs(panels[i].height - panels[0].height) > screen.height() * 0.025){ + return {}; + } } - result.move_count = panels.size(); + return panels; +} + +// Whether the move button from `top` to `bottom` (of the screen) has the +// cursor's four corners, at a small offset. +bool move_button_selected(const ImageViewRGB32& screen, double top, double bottom){ auto is_corner = [&](double x, double y){ - if (x < 0 || y < 0 || x + 0.018 > 1 || y + 0.023 > 1) return false; - ImageViewRGB32 corner = extract_box_reference(screen, ImageFloatBox{x, y, 0.018, 0.023}); - size_t white_count = 0, total = corner.width() * corner.height(); - for (size_t py = 0; py < corner.height(); py++) for (size_t px = 0; px < corner.width(); px++) { - uint32_t p = corner.pixel(px, py); - int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; - white_count += std::min(r, std::min(g, b)) > 210 && - std::max(r, std::max(g, b)) - std::min(r, std::min(g, b)) < 35; + if (x < 0 || y < 0 || x + 0.018 > 1 || y + 0.023 > 1){ + return false; } - return total && (double)white_count / total >= 0.15; + return white_corner(screen, ImageFloatBox{x, y, 0.018, 0.023}, 210, 35); }; - for (size_t slot = 0; slot < panels.size(); slot++) { + for (double dy : {0.0, -0.003, 0.003, -0.006, 0.006, -0.010, 0.010}){ + for (double dx : {0.0, -0.003, 0.003}){ + if (is_corner(0.663 + dx, top - 0.010 + dy) && + is_corner(0.976 + dx, top - 0.010 + dy) && + is_corner(0.663 + dx, bottom - 0.014 + dy) && + is_corner(0.976 + dx, bottom - 0.014 + dy)){ + return true; + } + } + } + return false; +} + +} + + +BattleMenuOption battle_menu_selected_option(BattleMenuDetector& menu, const ImageViewRGB32& screen){ + if (!screen || !menu.detect(screen)){ + return BattleMenuOption::UNKNOWN; + } + int selected = -1; + for (int row = 0; row < 4; row++){ + if (!row_has_brackets(screen, row)){ + continue; + } + if (selected >= 0){ + return BattleMenuOption::UNKNOWN; + } + selected = row; + } + return static_cast(selected); +} + +BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen){ + BattleMoveMenuState result; + if (!screen){ + return result; + } + // Every populated move has a white PP panel, irrespective of move type or + // selected row. Find connected panels rather than assume a 4-row layout. + const ImageFloatBox search{0.90, 0.48, 0.075, 0.505}; + const std::vector panels = find_pp_panels(screen, extract_box_reference(screen, search)); + if (panels.empty()){ + return result; + } + result.move_count = panels.size(); + for (size_t slot = 0; slot < panels.size(); slot++){ double top = search.y + (double)panels[slot].y / screen.height(); double bottom = top + (double)panels[slot].height / screen.height(); result.buttons[slot] = ImageFloatBox{0.665, std::max(0.0, top - 0.01), 0.333, std::min(1.0, bottom + 0.01) - std::max(0.0, top - 0.01)}; - bool selected = false; - for (double dy : {0.0, -0.003, 0.003, -0.006, 0.006, -0.010, 0.010}) { - for (double dx : {0.0, -0.003, 0.003}) { - if (is_corner(0.663 + dx, top - 0.010 + dy) && - is_corner(0.976 + dx, top - 0.010 + dy) && - is_corner(0.663 + dx, bottom - 0.014 + dy) && - is_corner(0.976 + dx, bottom - 0.014 + dy)) { selected = true; break; } - } - if (selected) break; + if (!move_button_selected(screen, top, bottom)){ + continue; } - if (selected) { - if (result.selected_slot) { result.selected_slot = 0; return result; } - result.selected_slot = slot + 1; + if (result.selected_slot){ + result.selected_slot = 0; + return result; } + result.selected_slot = slot + 1; } return result; } + } } } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp index 7ceacae47c..47db816a50 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp @@ -15,9 +15,9 @@ #include "PokemonBDSP_AutoStory_MenuCursorDetector.h" -namespace PokemonAutomation { -namespace NintendoSwitch { -namespace PokemonBDSP { +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ std::string menu_cursor_to_string(MenuCursorPosition pos){ @@ -46,13 +46,7 @@ std::string menu_cursor_to_string(MenuCursorPosition pos){ MenuCursorDetector::MenuCursorDetector(Color color) : m_color(color) - // ------------------------------------------------------------------------- - // Top row — left to right - // ------------------------------------------------------------------------- - // TODO: replace each ImageFloatBox(0,0,0,0) with the real normalized - // (x, y, width, height) for a small region that gains dark pixels when - // that icon is hovered. Use the test program overlays to tune coordinates. - // ------------------------------------------------------------------------- + // Top row, left to right. , m_pokedex_box{ ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), }, @@ -64,13 +58,12 @@ MenuCursorDetector::MenuCursorDetector(Color color) : }, m_card_box{ ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), - } // ------------------------------------------------------------------------- + } // Bottom row — left to right // 5-icon layout: OPTIONS is the only bottom icon (far-left position). // 8-icon layout: MAP, CAPSULES, OPTIONS, MYSTERYGIFT from left to right. // OPTIONS sits at a different x-position in each layout — use a separate // MenuCursorDetector instance if you need to detect OPTIONS in both. - // ------------------------------------------------------------------------- , m_options_box{ ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), }, @@ -82,10 +75,9 @@ MenuCursorDetector::MenuCursorDetector(Color color) : }, m_mysterygift_box{ ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), - } // ------------------------------------------------------------------------- + } // Approach 2 search boxes — one per icon, same area as the dark-pixel box. // The template PNG is slid across this area at its native size. - // ------------------------------------------------------------------------- , m_icon_match_box{ ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), // 0 POKEDEX ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), // 1 POKEMON @@ -124,7 +116,7 @@ void MenuCursorDetector::make_overlays(VideoOverlaySet& items) const { } -namespace { +namespace{ // Count how many of `n` boxes contain >= MIN_DARK_PIXEL_COUNT pixels with // R, G, B all <= BLACK_CHANNEL_MAX. Boxes with zero width are skipped. diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h index 48512bdabf..ca855d8e79 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h @@ -53,9 +53,9 @@ #include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" -namespace PokemonAutomation { -namespace NintendoSwitch { -namespace PokemonBDSP { +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ enum class MenuCursorPosition { diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp index 2e8826bbef..870a989c4d 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp @@ -92,6 +92,164 @@ IconGrid icon_to_grid(MenuCursorPosition pos, int num_icons){ } } +enum class SaveOutcome{ SAVED, WRONG_PLACE, FAILED }; +struct SaveAttempt{ + SaveOutcome outcome = SaveOutcome::FAILED; + std::string wrong_place; +}; + +// One feedback-checked save: the save prompt, its card checked against +// `expected_location` if given, then the save and back to the overworld. +SaveAttempt save_once( + SingleSwitchProgramEnvironment& env, ProControllerContext& context, + const std::string& expected_location, size_t attempt +){ + try{ + env.console.log("Checkpoint save attempt: " + std::to_string(attempt + 1)); + + // Only mash B on retries - no need to on first attempt + // if already cleanly in the overworld. + if (attempt > 0){ + pbf_mash_button(context, BUTTON_B, 1000ms); + } + + // The feedback-based save of save_game(VideoStream&, ...), split at the + // save prompt to check where it stands. + const std::string location = open_save_prompt(env.console, context); + if (!expected_location.empty() && !location_banner_matches(location, expected_location)){ + SaveAttempt wrong{SaveOutcome::WRONG_PLACE, + "About to save in \"" + location + "\", expected " + expected_location + "; not saving."}; + env.console.log(wrong.wrong_place, COLOR_RED); + leave_save_prompt(env.console, context); + return wrong; + } + pbf_press_button(context, BUTTON_ZL, 80ms, 5000ms); + + // Return to overworld cleanly. + pbf_mash_button(context, BUTTON_B, 1000ms); + + // Drain R/confirm/B before detection; an earlier overworld frame + // must not report success while saving commands are still queued. + context.wait_for_all_requests(); + + // Verify overworld. Generous timeout: save animation + fade can be slow. + OverworldWatcher overworld; + if (wait_until(env.console, context, std::chrono::seconds(15), {overworld}) < 0){ + throw OperationFailedExceptionWithScreenshot(ErrorReportMode::SEND_ERROR_REPORT, + "Failed to detect overworld after save.", + env.console); + } + + env.console.log("Checkpoint save verified."); + return {SaveOutcome::SAVED, ""}; + }catch (OperationFailedException& e){ + env.console.log("Checkpoint save failed: " + e.message(), COLOR_RED); + context.wait_for(1000ms); + return {SaveOutcome::FAILED, ""}; + } +} + +// The X menu's icons as hovered, where packaged: the cursor detector's +// template match. Without them the dark-pixel detector still works. +void load_menu_icon_references(MenuCursorDetector& detector){ + static const char* ICON_NAMES[MENU_CURSOR_NUM_ICONS] = { + "POKEDEX", + "POKEMON", + "BAG", + "CARD", + "OPTIONS", + "MAP", + "CAPSULES", + "MYSTERYGIFT", + }; + const std::string ref_dir = RESOURCE_PATH() + "PokemonBDSP/Autostory"; + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + const std::string path = ref_dir + "/" + ICON_NAMES[i] + ".png"; + if (!Filesystem::is_regular_file(path)){ + continue; + } + ImageRGB32 img(path); + if (img){ + detector.set_icon_reference(static_cast(i), std::move(img)); + } + } +} + +// The cursor on the open menu, from up to ten frames; UNKNOWN if none reads. +MenuCursorPosition read_menu_cursor(VideoStream& stream, ProControllerContext& context, const MenuCursorDetector& detector){ + MenuCursorPosition cur = MenuCursorPosition::UNKNOWN; + for (int frame = 0; frame < 10 && cur == MenuCursorPosition::UNKNOWN; frame++){ + context.wait_for(50ms); + const VideoSnapshot snapshot = stream.video().snapshot(); + if (snapshot){ + cur = detector.detect(snapshot); + } + } + return cur; +} + +// Opens the menu unless it is already open: a late-opening menu may show +// after a timeout, and X would close it. +void open_menu_unless_open(VideoStream& stream, ProControllerContext& context){ + MenuWatcher already_open; + if (wait_until(stream, context, 300ms, {{already_open}}) < 0){ + overworld_to_menu(stream, context); + }else{ + stream.log("[Open menu] Menu already visible; skipping X."); + } +} + +// Opens the menu and reads its cursor, in three attempts. Once the menu +// is confirmed open, retries only look: no input that could close it. +MenuCursorPosition open_menu_and_find_cursor(VideoStream& stream, ProControllerContext& context, const MenuCursorDetector& detector){ + MenuCursorPosition cur = MenuCursorPosition::UNKNOWN; + bool menu_confirmed_open = false; + for (size_t attempt = 1; attempt <= 3 && cur == MenuCursorPosition::UNKNOWN; attempt++){ + stream.log("[Open menu] Open and detect attempt " + std::to_string(attempt) + "/3."); + try{ + if (!menu_confirmed_open){ + open_menu_unless_open(stream, context); + menu_confirmed_open = true; + }else{ + stream.log("[Open menu] Retrying cursor detection in the open menu."); + } + pbf_wait(context, 1000ms); + context.wait_for_all_requests(); + cur = read_menu_cursor(stream, context, detector); + stream.log("[Open menu] Menu cursor " + std::string(cur == MenuCursorPosition::UNKNOWN ? "not detected" : "detected") + + " on attempt " + std::to_string(attempt) + ".", cur == MenuCursorPosition::UNKNOWN ? COLOR_ORANGE : COLOR_GREEN); + }catch (OperationFailedException& error){ + stream.log("[Open menu] Attempt " + std::to_string(attempt) + " failed: " + error.message(), COLOR_ORANGE); + } + } + if (cur == MenuCursorPosition::UNKNOWN){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Menu or cursor detection failed after 3 opening attempts.", + stream); + } + return cur; +} + +// From the icon `cur` to `target`: a row up or down, then across. +void move_menu_cursor(VideoStream& stream, ProControllerContext& context, MenuCursorPosition cur, MenuCursorPosition target, int num_icons){ + const IconGrid cur_g = icon_to_grid(cur, num_icons); + const IconGrid tgt_g = icon_to_grid(target, num_icons); + if (tgt_g.row == -1){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Unknown target menu icon.", + stream); + } + if (cur_g.row != tgt_g.row){ + pbf_press_dpad(context, cur_g.row < tgt_g.row ? DPAD_DOWN : DPAD_UP, 80ms, 300ms); + context.wait_for_all_requests(); + } + const int col_diff = tgt_g.col - cur_g.col; + for (int i = 0; i < std::abs(col_diff); i++){ + pbf_press_dpad(context, col_diff > 0 ? DPAD_RIGHT : DPAD_LEFT, 80ms, 300ms); + context.wait_for_all_requests(); + } +} + } std::string read_location_from_save_screen(VideoStream& stream, ProControllerContext& context){ @@ -104,58 +262,18 @@ bool save_game( SingleSwitchProgramEnvironment& env, ProControllerContext& context, const std::string& expected_location -) { +){ static constexpr size_t MAX_RETRIES = 3; - std::string wrong_place; - for (size_t attempt = 0; attempt < MAX_RETRIES && wrong_place.empty(); attempt++){ - try { - env.console.log("Checkpoint save attempt: " + std::to_string(attempt + 1)); - - // Only mash B on retries - no need to on first attempt - // if already cleanly in the overworld. - if (attempt > 0){ - pbf_mash_button(context, BUTTON_B, 1000ms); - } - - // The feedback-based save of save_game(VideoStream&, ...), split at the - // save prompt to check where it stands. - const std::string location = open_save_prompt(env.console, context); - if (!expected_location.empty() && !location_banner_matches(location, expected_location)){ - wrong_place = "About to save in \"" + location + "\", expected " + expected_location + "; not saving."; - env.console.log(wrong_place, COLOR_RED); - leave_save_prompt(env.console, context); - break; - } - pbf_press_button(context, BUTTON_ZL, 80ms, 5000ms); - - // Return to overworld cleanly. - pbf_mash_button(context, BUTTON_B, 1000ms); - - // Drain R/confirm/B before detection; an earlier overworld frame - // must not report success while saving commands are still queued. - context.wait_for_all_requests(); - - // Verify overworld. Generous timeout: save animation + fade can be slow. - OverworldWatcher overworld; - int ret = wait_until(env.console, context, std::chrono::seconds(15), {overworld}); - if (ret < 0){ - throw OperationFailedExceptionWithScreenshot(ErrorReportMode::SEND_ERROR_REPORT, - "Failed to detect overworld after save.", - env.console); - } - - env.console.log("Checkpoint save verified."); + for (size_t attempt = 0; attempt < MAX_RETRIES; attempt++){ + const SaveAttempt result = save_once(env, context, expected_location, attempt); + if (result.outcome == SaveOutcome::SAVED){ return true; - - } catch (OperationFailedException& e){ - env.console.log("Checkpoint save failed: " + e.message(), COLOR_RED); - context.wait_for(1000ms); } - } - // No blind fallback for the wrong place: it would save there. - if (!wrong_place.empty()){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, wrong_place, env.console); + // No blind fallback for the wrong place: it would save there. + if (result.outcome == SaveOutcome::WRONG_PLACE){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, result.wrong_place, env.console); + } } // Blind fallback: reuse the non-feedback save_game(context) from GameNavigation @@ -209,127 +327,11 @@ void wait_for_dialogue(VideoStream& stream, } void open_menu(VideoStream& stream, ProControllerContext& context, MenuCursorPosition target, int num_icons){ - using namespace std::chrono_literals; - - // Previous single opening attempt: - // context.wait_for_all_requests(); - // overworld_to_menu(stream, context); - // pbf_wait(context, 1000ms); - // context.wait_for_all_requests(); context.wait_for_all_requests(); - MenuCursorDetector detector; - { - static const char* ICON_NAMES[MENU_CURSOR_NUM_ICONS] = { - "POKEDEX", - "POKEMON", - "BAG", - "CARD", - "OPTIONS", - "MAP", - "CAPSULES", - "MYSTERYGIFT", - }; - // Previous optional template directory: PokemonBDSP/AutoStory. - std::string ref_dir = RESOURCE_PATH() + "PokemonBDSP/Autostory"; - for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ - std::string path = ref_dir + "/" + ICON_NAMES[i] + ".png"; - // These templates are optional. The dark-pixel cursor detector - // remains available when a template is not packaged. - if (!Filesystem::is_regular_file(path)){ - continue; - } - // Previously constructed the image before checking its availability. - ImageRGB32 img(path); - if (img){ - detector.set_icon_reference(static_cast(i), std::move(img)); - } - } - } - // Previous single detection attempt retained for comparison. - // MenuCursorPosition cur = MenuCursorPosition::UNKNOWN; - // for (int i = 0; i < 10 && cur == MenuCursorPosition::UNKNOWN; i++){ - // context.wait_for(50ms); - // VideoSnapshot snap = stream.video().snapshot(); - // if (snap) cur = detector.detect(snap); - // } - // if (cur == MenuCursorPosition::UNKNOWN){ - // OperationFailedExceptionWithScreenshot::fire( - // ErrorReportMode::SEND_ERROR_REPORT, - // "Menu cursor not detected after opening menu.", - // stream - // ); - // } - - MenuCursorPosition cur = MenuCursorPosition::UNKNOWN; - bool menu_confirmed_open = false; - for (size_t attempt = 1; attempt <= 3; attempt++){ - stream.log("[Open menu] Open and detect attempt " + std::to_string(attempt) + "/3."); - try { - // A late-opening menu may already be visible after a timeout. - // Check before pressing X again, since X would close that menu. - if (!menu_confirmed_open){ - MenuWatcher already_open; - if (wait_until(stream, context, 300ms, {{already_open}}) < 0){ - overworld_to_menu(stream, context); - }else{ - stream.log("[Open menu] Menu already visible; skipping X."); - } - menu_confirmed_open = true; - }else{ - // Cursor detection retries send no input that could close the - // confirmed menu. Keep observing it instead of toggling X. - stream.log("[Open menu] Retrying cursor detection in the open menu."); - } - pbf_wait(context, 1000ms); - context.wait_for_all_requests(); - for (int frame = 0; frame < 10 && cur == MenuCursorPosition::UNKNOWN; frame++){ - context.wait_for(50ms); - const VideoSnapshot snapshot = stream.video().snapshot(); - if (snapshot){ - cur = detector.detect(snapshot); - } - } - if (cur != MenuCursorPosition::UNKNOWN){ - stream.log("[Open menu] Menu and cursor detected on attempt " + std::to_string(attempt) + ".", - COLOR_GREEN); - break; - } - stream.log("[Open menu] Menu cursor not detected on attempt " + std::to_string(attempt) + ".", - COLOR_ORANGE); - } catch (OperationFailedException& error){ - stream.log("[Open menu] Attempt " + std::to_string(attempt) + " failed: " + error.message(), COLOR_ORANGE); - } - } - if (cur == MenuCursorPosition::UNKNOWN){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Menu or cursor detection failed after 3 opening attempts.", - stream); - } - - IconGrid cur_g = icon_to_grid(cur, num_icons); - IconGrid tgt_g = icon_to_grid(target, num_icons); - if (tgt_g.row == -1){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Unknown target menu icon.", - stream); - } - - if (cur_g.row != tgt_g.row){ - DpadPosition vert = (cur_g.row < tgt_g.row) ? DPAD_DOWN : DPAD_UP; - pbf_press_dpad(context, vert, 80ms, 300ms); - context.wait_for_all_requests(); - } - - int col_diff = tgt_g.col - cur_g.col; - if (col_diff != 0){ - DpadPosition horiz = (col_diff > 0) ? DPAD_RIGHT : DPAD_LEFT; - for (int i = 0; i < std::abs(col_diff); i++){ - pbf_press_dpad(context, horiz, 80ms, 300ms); - context.wait_for_all_requests(); - } - } - + load_menu_icon_references(detector); + const MenuCursorPosition cur = open_menu_and_find_cursor(stream, context, detector); + move_menu_cursor(stream, context, cur, target, num_icons); pbf_press_button(context, BUTTON_A, 80ms, 200ms); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp index 60864695c0..7de60c3a36 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp @@ -126,6 +126,133 @@ double confirmed_potion_hp(VideoStream& stream, ProControllerContext& context){ stream); } +// A walk down a pocket's list, reading the cursor's row. +struct BagScan{ + DpadPosition direction = DPAD_DOWN; + std::string previous; + size_t same = 0; + size_t confirmed = 0; + size_t unknown_cursor = 0; +}; +enum class BagLook{ GO_ON, FOUND, NOT_FOUND }; + +// The name on the cursor's row `cursor`, as lower-case letters. +std::string read_bag_row(const ImageViewRGB32& frame, int cursor){ + const std::string text = OCR::tesseract_ocr_read(Language::English, + extract_box_reference(frame, bag_list_name_box(cursor))); + std::string normalized; + for (unsigned char ch : text){ + if (std::isalpha(ch)){ + normalized.push_back((char)std::tolower(ch)); + } + } + return normalized; +} + +// One look at the list: the row under the cursor is `name` (twice in a +// row), or the list ran out both ways, or the cursor moves on. +BagLook bag_scan_step(VideoStream& stream, ProControllerContext& context, BagScan& scan, const std::string& name){ + context.wait_for_all_requests(); + const VideoSnapshot frame = stream.video().snapshot(); + if (!frame || bag_list_is_empty(*frame.frame)){ + stream.log("[Bag search] " + name + ": pocket is empty.", COLOR_ORANGE); + return BagLook::NOT_FOUND; + } + const int cursor = bag_cursor_position(*frame.frame); + if (cursor == 0){ + if (++scan.unknown_cursor >= 3){ + stream.log("[Bag search] " + name + ": cursor not found.", COLOR_ORANGE); + return BagLook::NOT_FOUND; + } + pbf_wait(context, 200ms); + return BagLook::GO_ON; + } + scan.unknown_cursor = 0; + const std::string normalized = read_bag_row(*frame.frame, cursor); + stream.log("[Bag search] Row " + std::to_string(cursor) + " reads \"" + normalized + "\"."); + if (normalized == name){ + if (++scan.confirmed >= 2){ + stream.log("[Bag search] " + name + " confirmed in row " + std::to_string(cursor) + ".", COLOR_GREEN); + return BagLook::FOUND; + } + pbf_wait(context, 150ms); + return BagLook::GO_ON; + } + scan.confirmed = 0; + scan.same = normalized == scan.previous ? scan.same + 1 : 0; + scan.previous = normalized; + if (scan.same >= 2){ + if (scan.direction == DPAD_UP){ + stream.log("[Bag search] " + name + " not found.", COLOR_ORANGE); + return BagLook::NOT_FOUND; + } + scan.direction = DPAD_UP; + scan.same = 0; + } + pbf_press_dpad(context, scan.direction, 80ms, 300ms); + return BagLook::GO_ON; +} + +// B out of the Bag to the overworld; fails with `failure` if it does not show. +void bag_back_to_overworld(VideoStream& stream, ProControllerContext& context, const std::string& failure){ + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + OverworldWatcher overworld; + if (wait_until(stream, context, std::chrono::seconds(10), {{overworld}}) != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, failure, stream); + } +} + +enum class Dose{ HEALED, GIVEN }; + +// One Potion on the lead, from the overworld, its HP read before and after +// in the Bag. HEALED when no dose is needed (HP full, or above +// `heal_below` on the first look, or no Potion when only checking) or the +// dose filled it; GIVEN when HP rose but is not full yet. +Dose potion_dose(VideoStream& stream, ProControllerContext& context, size_t dose, double heal_below){ + stream.log("[Potion] Finding Bag/Medicine/Potion; dose " + std::to_string(dose + 1) + "/3."); + if (!find_bag_item_by_name(stream, context, "potion", 1, 5)){ + if (dose == 0 && heal_below < 1.0){ + // Only a check: without Potions there is nothing to do here. + stream.log("[Potion] No Potion in the Bag; skipping the HP check.", COLOR_ORANGE); + return Dose::HEALED; + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Could not verify Potion in the Medicine pocket.", + stream); + } + const double before = confirmed_potion_hp(stream, context); + stream.log("[Potion] Slot 1 HP bar before use=" + std::to_string(before * 100) + "%."); + if (before >= 0.98){ + return Dose::HEALED; + } + if (dose == 0 && before >= heal_below){ + stream.log("[Potion] Slot 1 HP is at least " + std::to_string(heal_below * 100) + "%; no Potion needed."); + return Dose::HEALED; + } + if (before <= 0.005){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Slot 1 appears fainted; a Potion cannot revive it.", + stream); + } + pbf_press_button(context, BUTTON_A, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + bag_back_to_overworld(stream, context, "Potion: Did not return to the overworld after use."); + // Reopen Bag to verify the result, including when the last Potion was used. + open_menu(stream, context, MenuCursorPosition::BAG, 5); + pbf_wait(context, 750ms); + context.wait_for_all_requests(); + const double after = confirmed_potion_hp(stream, context); + stream.log("[Potion] Slot 1 HP bar after use=" + std::to_string(after * 100) + "%."); + if (after <= before + 0.005){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Slot 1 HP did not increase; refusing to continue movement.", + stream); + } + return after >= 0.98 ? Dose::HEALED : Dose::GIVEN; +} + } int bag_pocket_position(const ImageViewRGB32& frame){ @@ -240,7 +367,9 @@ bool find_bag_item_in_open_bag( int target_pocket ){ context.wait_for_all_requests(); - if (!bag_navigate_to_pocket(stream, context, target_pocket)) return false; + if (!bag_navigate_to_pocket(stream, context, target_pocket)){ + return false; + } // No reader for the game's language: the cursor's row, the pocket's // first, if the pocket has one. Early in the story the pocket holds only @@ -255,140 +384,39 @@ bool find_bag_item_in_open_bag( // Walk down the list reading the cursor row. The list does not wrap, so // turn around once the cursor stops moving. - DpadPosition direction = DPAD_DOWN; - std::string previous; - size_t same = 0; - size_t confirmed = 0; - size_t unknown_cursor = 0; - for (size_t scan = 0; scan < 128; scan++){ - context.wait_for_all_requests(); - const VideoSnapshot frame = stream.video().snapshot(); - if (!frame || bag_list_is_empty(*frame.frame)){ - stream.log("[Bag search] " + name + ": pocket is empty.", COLOR_ORANGE); - return false; - } - const int cursor = bag_cursor_position(*frame.frame); - if (cursor == 0){ - if (++unknown_cursor >= 3){ - stream.log("[Bag search] " + name + ": cursor not found.", COLOR_ORANGE); - return false; - } - pbf_wait(context, 200ms); - continue; + BagScan scan; + for (size_t look = 0; look < 128; look++){ + const BagLook result = bag_scan_step(stream, context, scan, name); + if (result != BagLook::GO_ON){ + return result == BagLook::FOUND; } - unknown_cursor = 0; - const std::string text = OCR::tesseract_ocr_read(Language::English, - extract_box_reference(*frame.frame, bag_list_name_box(cursor))); - std::string normalized; - for (unsigned char ch : text){ - if (std::isalpha(ch)) normalized.push_back((char)std::tolower(ch)); - } - stream.log("[Bag search] Row " + std::to_string(cursor) + " reads \"" + normalized + "\"."); - if (normalized == name){ - if (++confirmed >= 2){ - stream.log("[Bag search] " + name + " confirmed in row " + std::to_string(cursor) + ".", COLOR_GREEN); - return true; - } - pbf_wait(context, 150ms); - continue; - } - confirmed = 0; - same = normalized == previous ? same + 1 : 0; - previous = normalized; - if (same >= 2){ - if (direction == DPAD_UP) break; - direction = DPAD_UP; - same = 0; - } - pbf_press_dpad(context, direction, 80ms, 300ms); } stream.log("[Bag search] " + name + " not found.", COLOR_ORANGE); return false; } -bool bag_has_item(VideoStream& stream, ProControllerContext& context, const std::string& name, int target_pocket) { +bool bag_has_item(VideoStream& stream, ProControllerContext& context, const std::string& name, int target_pocket){ const bool found = find_bag_item_by_name(stream, context, name, target_pocket, 5); - pbf_mash_button(context, BUTTON_B, 2000ms); - context.wait_for_all_requests(); - OverworldWatcher overworld; - if (wait_until(stream, context, std::chrono::seconds(10), {{overworld}}) != 0) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Bag: Did not return to the overworld after looking for " + name + ".", stream); - } + bag_back_to_overworld(stream, context, "Bag: Did not return to the overworld after looking for " + name + "."); return found; } -void use_potion_first_pokemon(VideoStream& stream, ProControllerContext& context, double heal_below) { +void use_potion_first_pokemon(VideoStream& stream, ProControllerContext& context, double heal_below){ OverlayBoxScope hp_overlay(stream.overlay(), COLOR_GREEN, {0.137, 0.160, 0.202, 0.016}, ""); bool healed = false; - for (size_t dose = 0; dose < 3; dose++){ - stream.log("[Potion] Finding Bag/Medicine/Potion; dose " + std::to_string(dose + 1) + "/3."); - if (!find_bag_item_by_name(stream, context, "potion", 1, 5)) { - if (dose == 0 && heal_below < 1.0) { - // Only a check: without Potions there is nothing to do here. - stream.log("[Potion] No Potion in the Bag; skipping the HP check.", COLOR_ORANGE); - healed = true; - break; - } - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Potion: Could not verify Potion in the Medicine pocket.", - stream); - } - const double before = confirmed_potion_hp(stream, context); - stream.log("[Potion] Slot 1 HP bar before use=" + std::to_string(before * 100) + "%."); - if (before >= 0.98) { healed = true; break; } - if (dose == 0 && before >= heal_below) { - stream.log("[Potion] Slot 1 HP is at least " + std::to_string(heal_below * 100) + "%; no Potion needed."); - healed = true; - break; - } - if (before <= 0.005) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Potion: Slot 1 appears fainted; a Potion cannot revive it.", - stream); - } - pbf_press_button(context, BUTTON_A, 80ms, 500ms); - pbf_press_button(context, BUTTON_A, 80ms, 500ms); - pbf_press_button(context, BUTTON_A, 80ms, 1000ms); - pbf_mash_button(context, BUTTON_B, 2000ms); - context.wait_for_all_requests(); - OverworldWatcher overworld; - if (wait_until(stream, context, std::chrono::seconds(10), {{overworld}}) != 0){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Potion: Did not return to the overworld after use.", - stream); - } - // Reopen Bag to verify the result, including when the last Potion was used. - open_menu(stream, context, MenuCursorPosition::BAG, 5); - pbf_wait(context, 750ms); - context.wait_for_all_requests(); - const double after = confirmed_potion_hp(stream, context); - stream.log("[Potion] Slot 1 HP bar after use=" + std::to_string(after * 100) + "%."); - if (after <= before + 0.005){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Potion: Slot 1 HP did not increase; refusing to continue movement.", - stream); - } - if (after >= 0.98){ - healed = true; - break; + for (size_t dose = 0; dose < 3 && !healed; dose++){ + healed = potion_dose(stream, context, dose, heal_below) == Dose::HEALED; + if (!healed){ + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); } - pbf_mash_button(context, BUTTON_B, 2000ms); - context.wait_for_all_requests(); } if (!healed){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Potion: Full HP not confirmed after three uses.", stream); } - pbf_mash_button(context, BUTTON_B, 2000ms); - context.wait_for_all_requests(); - OverworldWatcher overworld; - if (wait_until(stream, context, std::chrono::seconds(10), {{overworld}}) != 0){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Potion: Final overworld return not confirmed.", - stream); - } + bag_back_to_overworld(stream, context, "Potion: Final overworld return not confirmed."); stream.log("[Potion] Slot 1 HP bar full and overworld confirmed."); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp index df473e30ca..6b9a49ea24 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp @@ -29,6 +29,7 @@ #include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" #include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_Bag.h" +#include "PokemonBDSP_AutoStory_BattleMoves.h" #include "PokemonBDSP_AutoStory_Battle.h" namespace PokemonAutomation{ @@ -49,276 +50,118 @@ BattleDialogWatcher::BattleDialogWatcher(Color color) namespace{ -// Early moves the starters and their foes have, as type, power and -// most PP. A status move has power 0. -struct KnownMove { const char* name; const char* type; int power; int pp; }; -const KnownMove KNOWN_MOVES[] = { - {"pound", "normal", 40, 35}, {"tackle", "normal", 40, 35}, {"scratch", "normal", 40, 35}, - {"quickattack", "normal", 40, 30}, {"headbutt", "normal", 70, 15}, - {"watergun", "water", 40, 25}, {"bubble", "water", 40, 30}, {"bubblebeam", "water", 65, 20}, - {"ember", "fire", 40, 25}, {"flamewheel", "fire", 60, 25}, {"machpunch", "fighting", 40, 30}, - {"absorb", "grass", 20, 25}, {"razorleaf", "grass", 55, 25}, {"bite", "dark", 60, 25}, - {"peck", "flying", 35, 35}, {"wingattack", "flying", 60, 35}, {"metalclaw", "steel", 50, 35}, - {"growl", "normal", 0, 40}, {"leer", "normal", 0, 30}, {"tailwhip", "normal", 0, 30}, - {"withdraw", "water", 0, 40}, {"taunt", "dark", 0, 20}, -}; - -// A move button's PP left, "8/25" at its right end; -1 if it cannot be -// read. The text is dark, or yellow to red as PP runs low, so text is -// whatever differs from the button's own color. `max_pp` helps when -// the OCR reads the slash as a digit. -int read_move_pp(const ImageViewRGB32& screen, const ImageFloatBox& button, int max_pp) { - const ImageViewRGB32 box = extract_box_reference(screen, ImageFloatBox{ - button.x + button.width * 0.75, button.y + button.height * 0.10, - button.width * 0.21, button.height * 0.80}); - if (box.width() == 0 || box.height() == 0) return -1; - std::vector pixels; - for (size_t y = 0; y < box.height(); y++) for (size_t x = 0; x < box.width(); x++) pixels.push_back(box.pixel(x, y)); - auto brightness = [](uint32_t p) { return ((p >> 16) & 255) + ((p >> 8) & 255) + (p & 255); }; - std::vector sorted = pixels; - std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end(), - [&](uint32_t a, uint32_t b) { return brightness(a) < brightness(b); }); - const uint32_t background = sorted[sorted.size() / 2]; - const size_t pad = 8; - ImageRGB32 text(box.width() + 2 * pad, box.height() + 2 * pad); - text.fill(0xffffffff); - for (size_t y = 0; y < box.height(); y++) { - for (size_t x = 0; x < box.width(); x++) { - const uint32_t p = box.pixel(x, y); - int distance = 0; - for (int shift : {16, 8, 0}) distance += std::abs((int)((p >> shift) & 255) - (int)((background >> shift) & 255)); - if (distance > 120) text.pixel(x + pad, y + pad) = 0xff000000; - } - } - const std::string read = OCR::tesseract_ocr_read(Language::English, text); - const size_t slash = read.find('/'); - std::string digits; - for (size_t i = 0; i < read.size() && i < slash; i++) if (std::isdigit((unsigned char)read[i])) digits.push_back(read[i]); - if (slash == std::string::npos && max_pp > 0) { - // "29135" for 29/35: drop the most PP and the misread slash. - const std::string max = std::to_string(max_pp); - if (digits.size() > max.size() + 1 && digits.compare(digits.size() - max.size(), max.size(), max) == 0) { - digits.resize(digits.size() - max.size() - 1); - } else { - return -1; - } - } - if (digits.empty() || digits.size() > 2) return -1; - return std::stoi(digits); -} - -// The best move on the move menu: the most powerful one we know, and -// on a tie one that is not Normal (an early starter's own type). -// Reads the names with OCR; 1 if none is known. -size_t pick_best_move(VideoStream& stream, const ImageViewRGB32& screen, - const BattleMoveMenuState& moves, const std::string& label) { - // Keep the menu as it looked, to check the reads against. - { - const std::string folder = SCREENSHOTS_PATH() + "AutoStory/moves/"; - Filesystem::create_directories(Filesystem::Path(folder)); - screen.save(folder + now_to_filestring() + "-" + label + ".png"); - } - size_t best = 1; - int best_score = -1; - bool any_attack = false; - size_t last_not_status = 1; - for (size_t i = 0; i < moves.move_count; i++) { - // The name is dark text on a light button (grey, or light blue under the - // cursor). Take the box right of the type icon and left of the PP, tall - // enough for a status move's name, which sits lower: it has no "Effective" - // line under it. - const ImageFloatBox& button = moves.buttons[i]; - const ImageFloatBox name_box{button.x + button.width * 0.17, button.y + button.height * 0.08, - button.width * 0.53, button.height * 0.80}; - const ImageViewRGB32 box = extract_box_reference(screen, name_box); - // Text is darker than halfway between the background and the darkest pixel; - // a fixed cut thins the strokes on the blue button. - std::vector gray(box.width() * box.height()); - for (size_t y = 0; y < box.height(); y++) { - for (size_t x = 0; x < box.width(); x++) { - const uint32_t pixel = box.pixel(x, y); - gray[y * box.width() + x] = (int)(((pixel >> 16) & 0xff) + ((pixel >> 8) & 0xff) + (pixel & 0xff)) / 3; - } - } - std::vector sorted = gray; - std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end()); - const int background = sorted[sorted.size() / 2]; - const int darkest = *std::min_element(gray.begin(), gray.end()); - const int threshold = (background + darkest) / 2; - // The name is the tallest band of text rows; "Effective" is smaller. - size_t band_top = 0, band_bottom = 0; - for (size_t y = 0; y < box.height();) { - auto has_text = [&](size_t row) { - for (size_t x = 0; x < box.width(); x++) if (gray[row * box.width() + x] < threshold) return true; - return false; - }; - if (!has_text(y)) { y++; continue; } - size_t end = y; - while (end < box.height() && has_text(end)) end++; - if (end - y > band_bottom - band_top) { band_top = y; band_bottom = end; } - y = end; - } - // Black text on white, with a margin, for the OCR. - const size_t pad = 6; - const size_t top = band_top >= 4 ? band_top - 4 : 0; - const size_t bottom = std::min(box.height(), band_bottom + 4); - ImageRGB32 text(box.width() + 2 * pad, bottom - top + 2 * pad); - text.fill(0xffffffff); - for (size_t y = top; y < bottom; y++) { - for (size_t x = 0; x < box.width(); x++) { - if (gray[y * box.width() + x] < threshold) text.pixel(x + pad, y - top + pad) = 0xff000000; - } - } - std::string name; - // Move names only where there is a reader for the game's - // language; an unread move counts as a plain attack. - for (unsigned char ch : story_reads_text() ? OCR::tesseract_ocr_read(g_story_language, text) : std::string()) { - if (std::isalpha(ch)) name.push_back((char)std::tolower(ch)); - } - // A slot read well once keeps its name when a later read fails. - static std::array last_read; - const KnownMove* known = nullptr; - for (int pass = 0; pass < 2 && !known; pass++) { - if (pass == 1) name = last_read[i]; - for (const KnownMove& move : KNOWN_MOVES) { - // The longest name that appears wins: "bubblebeam" over "bubble". - if (name.find(move.name) != std::string::npos && - (!known || std::string(move.name).size() > std::string(known->name).size())) { - known = &move; - } - } - } - if (known) last_read[i] = known->name; - // A move with no PP left cannot be chosen; the game says so and - // keeps the menu up. - const int pp = read_move_pp(screen, button, known ? known->pp : 0); - stream.log("[" + label + "] Move " + std::to_string(i + 1) + " PP left: " + (pp < 0 ? std::string("unread") : std::to_string(pp)) + "."); - if (pp == 0) { - continue; - } - // An unreadable move counts as a plain attack, ahead of known - // status moves. - const int score = known ? known->power * 2 + (std::string(known->type) != "normal" ? 1 : 0) : 40 * 2 - 1; - stream.log("[" + label + "] Move " + std::to_string(i + 1) + " reads \"" + name + "\"" + - (known ? std::string(" = ") + known->name + " (" + known->type + ", " + std::to_string(known->power) + ")" : " (unknown)") + "."); - if (score > best_score) { best_score = score; best = i + 1; } - if (known && known->power > 0) any_attack = true; - if (!known || known->power > 0) last_not_status = i + 1; - } - // No attack read: the newest move is the starter's own-type attack - // (Water Gun, Ember, Absorb), unless it is a status move. - if (!any_attack) best = last_not_status; - stream.log("[" + label + "] Best move: " + std::to_string(best) + "."); - return best; -} - -void select_battle_move_slot(VideoStream& stream, ProControllerContext& context, - BattleType battle_type, size_t target, const std::string& label, - bool confirm_default_target = false, size_t current_move = 1) { +// On the battle menu: the cursor onto Battle (the command list's first +// row), confirmed on three frames, then A. +void choose_battle_command(VideoStream& stream, ProControllerContext& context, BattleType battle_type, const std::string& label){ BattleMenuDetector commands(battle_type); VideoOverlaySet overlays(stream.overlay()); commands.make_overlays(overlays); overlays.add(COLOR_GREEN, {0.90, 0.48, 0.075, 0.505}); - // Reenter Fight from an already-open list without assuming - // that this resets the remembered move cursor. - if (confirm_default_target) { - context.wait_for_all_requests(); - const auto frame=stream.video().snapshot(); - if (frame && detect_battle_move_menu(frame).move_count) { - pbf_press_button(context,BUTTON_B,80ms,500ms); - } - } - bool fight = false; size_t confirmed = 0; - for (size_t check = 0; check < 30; check++) { + for (size_t check = 0; check < 30 && confirmed < 3; check++){ context.wait_for_all_requests(); auto frame = stream.video().snapshot(); BattleMenuOption selected = frame ? battle_menu_selected_option(commands, frame) : BattleMenuOption::UNKNOWN; stream.log("[" + label + "] Command cursor=" + std::to_string((int)selected + 1) + "; target Battle=1."); - if (selected == BattleMenuOption::BATTLE) { - if (++confirmed >= 3) { fight = true; break; } - pbf_wait(context, 100ms); - } else { + if (selected == BattleMenuOption::BATTLE){ + if (++confirmed < 3){ + pbf_wait(context, 100ms); + } + }else if (selected == BattleMenuOption::UNKNOWN){ confirmed = 0; - if (selected == BattleMenuOption::UNKNOWN) pbf_wait(context, 150ms); - else pbf_press_dpad(context, DPAD_UP, 80ms, 250ms); + pbf_wait(context, 150ms); + }else{ + confirmed = 0; + pbf_press_dpad(context, DPAD_UP, 80ms, 250ms); } } - if (!fight) OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": Could not confirm Battle command; A not pressed.", stream); - pbf_press_button(context,BUTTON_A,80ms,confirm_default_target ? 1000ms : 500ms); - if (confirm_default_target) { - // Reuse the farming move/target confirmation sequence, but - // navigate relative to this battle's retained move cursor. - // Reopening Fight does not reset it to move 1. - size_t available=0,stable=0; - for (size_t check=0;check<20 && stable<3;check++) { - context.wait_for_all_requests(); - const auto frame=stream.video().snapshot(); - const auto moves=frame ? detect_battle_move_menu(frame) : BattleMoveMenuState{}; - if (moves.move_count && moves.move_count==available) stable++; - else {available=moves.move_count;stable=moves.move_count ? 1:0;} - if (stable<3) pbf_wait(context,100ms); - } - if (stable<3 || available "+std::to_string(target)+"."); - for (size_t slot=current_move;slottarget;slot--) pbf_press_dpad(context,DPAD_UP,80ms,400ms); - pbf_press_button(context,BUTTON_A,80ms,1000ms); - pbf_press_button(context,BUTTON_A,80ms,3000ms); - return; + if (confirmed < 3){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Could not confirm Battle command; A not pressed.", stream); } - confirmed = 0; - size_t previous_move_count = 0, stable_move_count = 0; - for (size_t check = 0; check < 30; check++) { - context.wait_for_all_requests(); - auto frame = stream.video().snapshot(); - const BattleMoveMenuState moves = frame ? detect_battle_move_menu(frame) : BattleMoveMenuState{}; - const int slot = (int)moves.selected_slot; - VideoOverlaySet move_overlays(stream.overlay()); - for (size_t i = 0; i < moves.move_count; i++) - move_overlays.add(moves.selected_slot == i + 1 ? COLOR_GREEN : COLOR_YELLOW, moves.buttons[i]); - stream.log("[" + label + "] Available moves=" + std::to_string(moves.move_count) + - "; selected slot=" + std::to_string(slot) + "/4; target=" + std::to_string(target) + "."); - if (moves.move_count == previous_move_count && moves.move_count > 0) stable_move_count++; - else { previous_move_count = moves.move_count; stable_move_count = 1; confirmed = 0; } - // Target 0: pick the best move once the menu has settled. - if (target == 0 && stable_move_count >= 3 && frame) { - target = pick_best_move(stream, *frame.frame, moves, label); + pbf_press_button(context, BUTTON_A, 80ms, 500ms); +} + +// Watching the move menu settle: how many frames in a row showed the same +// number of moves, and how many had the cursor on the target. +struct MoveMenuWatch{ + size_t previous_move_count = 0; + size_t stable_move_count = 0; + size_t confirmed = 0; + // The move to use, 1-based; 0 until the menu has settled and the best is picked. + size_t target = 0; +}; + +// One look at the move menu. Returns whether the target is confirmed +// under the cursor; otherwise moves the cursor toward it or waits. +bool move_menu_step(VideoStream& stream, ProControllerContext& context, MoveMenuWatch& watch, const std::string& label){ + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + const BattleMoveMenuState moves = frame ? detect_battle_move_menu(frame) : BattleMoveMenuState{}; + const int slot = (int)moves.selected_slot; + VideoOverlaySet move_overlays(stream.overlay()); + for (size_t i = 0; i < moves.move_count; i++){ + move_overlays.add(moves.selected_slot == i + 1 ? COLOR_GREEN : COLOR_YELLOW, moves.buttons[i]); + } + stream.log("[" + label + "] Available moves=" + std::to_string(moves.move_count) + + "; selected slot=" + std::to_string(slot) + "/4; target=" + std::to_string(watch.target) + "."); + if (moves.move_count == watch.previous_move_count && moves.move_count > 0){ + watch.stable_move_count++; + }else{ + watch.previous_move_count = moves.move_count; + watch.stable_move_count = 1; + watch.confirmed = 0; + } + // Pick the best move once the menu has settled. + if (watch.target == 0 && watch.stable_move_count >= 3 && frame){ + watch.target = pick_best_move(stream, *frame.frame, moves, label); + } + if (moves.move_count > 0 && moves.move_count < watch.target && watch.stable_move_count >= 3){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Move slot " + std::to_string(watch.target) + " is absent; no move confirmed.", stream); + } + if (slot == (int)watch.target && watch.stable_move_count >= 3){ + if (++watch.confirmed >= 3){ + return true; } - if (moves.move_count > 0 && moves.move_count < target && stable_move_count >= 3) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": Move slot " + std::to_string(target) + " is absent; no move confirmed.", stream); + pbf_wait(context, 100ms); + }else{ + watch.confirmed = 0; + if (slot == 0 || slot == (int)watch.target){ + pbf_wait(context, 150ms); + }else{ + pbf_press_dpad(context, slot < (int)watch.target ? DPAD_DOWN : DPAD_UP, 80ms, 250ms); } - if (slot == (int)target && stable_move_count >= 3) { - if (++confirmed >= 3) { - stream.log("[" + label + "] Move " + std::to_string(target) + " confirmed; using it."); - pbf_press_button(context, BUTTON_A, 80ms, 3000ms); - return; - } - pbf_wait(context, 100ms); - } else { - confirmed = 0; - if (slot == 0 || slot == (int)target) pbf_wait(context, 150ms); - else pbf_press_dpad(context, slot < (int)target ? DPAD_DOWN : DPAD_UP, 80ms, 250ms); + } + context.wait_for_all_requests(); + return false; +} + +// On the battle menu: Battle, then the best move on the move menu once +// it has settled, confirmed under the cursor, then A. +void use_best_move(VideoStream& stream, ProControllerContext& context, BattleType battle_type, const std::string& label){ + choose_battle_command(stream, context, battle_type, label); + MoveMenuWatch watch; + for (size_t check = 0; check < 30; check++){ + if (move_menu_step(stream, context, watch, label)){ + stream.log("[" + label + "] Move " + std::to_string(watch.target) + " confirmed; using it."); + pbf_press_button(context, BUTTON_A, 80ms, 3000ms); + return; } - context.wait_for_all_requests(); } OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": Could not confirm move " + std::to_string(target) + "; A not pressed.", stream); + label + ": Could not confirm move " + std::to_string(watch.target) + "; A not pressed.", stream); } // Our HP bar in the battle's bottom-left panel, as a fraction; -1 if // no bar is there. Measured on 1920x1080 frames: the bar runs from // x 88 to 466 on rows 961 to 965. -double battle_player_hp(const ImageViewRGB32& frame) { +double battle_player_hp(const ImageViewRGB32& frame){ const ImageViewRGB32 bar = extract_box_reference(frame, ImageFloatBox{0.046, 0.890, 0.197, 0.004}); size_t filled = 0; - for (size_t x = 0; x < bar.width(); x++) { + for (size_t x = 0; x < bar.width(); x++){ size_t color = 0; - for (size_t y = 0; y < bar.height(); y++) { + for (size_t y = 0; y < bar.height(); y++){ const uint32_t p = bar.pixel(x, y); const int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; const bool green = g > r + 30 && g > b + 20; @@ -331,39 +174,60 @@ double battle_player_hp(const ImageViewRGB32& frame) { return bar.width() && filled ? (double)filled / bar.width() : -1; } -enum class BattleHeal { NOT_NEEDED, USED, NONE_LEFT }; - -// At the battle menu: if our HP is under `heal_below`, use a Potion -// from the Bag on the lead, which takes the turn. Otherwise the -// battle menu is left up. -BattleHeal battle_heal_if_low(VideoStream& stream, ProControllerContext& context, - BattleType battle_type, double heal_below, const std::string& label) { +// Our HP at the battle menu, from up to five looks; -1 if unread. +double read_battle_hp(VideoStream& stream, ProControllerContext& context){ context.wait_for_all_requests(); double hp = -1; - for (size_t check = 0; check < 5 && hp < 0; check++) { + for (size_t check = 0; check < 5 && hp < 0; check++){ const auto frame = stream.video().snapshot(); - if (frame) hp = battle_player_hp(*frame.frame); - if (hp < 0) pbf_wait(context, 150ms); + if (frame){ + hp = battle_player_hp(*frame.frame); + } + if (hp < 0){ + pbf_wait(context, 150ms); + } context.wait_for_all_requests(); } - stream.log("[" + label + "] Our HP bar reads " + std::to_string((int)(hp * 100)) + "%."); - if (hp < 0 || hp >= heal_below) return BattleHeal::NOT_NEEDED; + return hp; +} + +// On the battle menu: the cursor onto Bag. Whether it got there. +bool cursor_to_bag(VideoStream& stream, ProControllerContext& context, BattleType battle_type){ BattleMenuDetector commands(battle_type); - bool on_bag = false; - for (size_t check = 0; check < 30 && !on_bag; check++) { + for (size_t check = 0; check < 30; check++){ context.wait_for_all_requests(); const auto frame = stream.video().snapshot(); const BattleMenuOption selected = frame ? battle_menu_selected_option(commands, frame) : BattleMenuOption::UNKNOWN; - if (selected == BattleMenuOption::BAG) on_bag = true; - else if (selected == BattleMenuOption::UNKNOWN) pbf_wait(context, 150ms); - else pbf_press_dpad(context, (int)selected < (int)BattleMenuOption::BAG ? DPAD_DOWN : DPAD_UP, 80ms, 300ms); + if (selected == BattleMenuOption::BAG){ + return true; + } + if (selected == BattleMenuOption::UNKNOWN){ + pbf_wait(context, 150ms); + }else{ + pbf_press_dpad(context, (int)selected < (int)BattleMenuOption::BAG ? DPAD_DOWN : DPAD_UP, 80ms, 300ms); + } + } + return false; +} + +enum class BattleHeal{ NOT_NEEDED, USED, NONE_LEFT }; + +// At the battle menu: if our HP is under `heal_below`, use a Potion +// from the Bag on the lead, which takes the turn. Otherwise the +// battle menu is left up. +BattleHeal battle_heal_if_low(VideoStream& stream, ProControllerContext& context, + BattleType battle_type, double heal_below, const std::string& label){ + const double hp = read_battle_hp(stream, context); + stream.log("[" + label + "] Our HP bar reads " + std::to_string((int)(hp * 100)) + "%."); + if (hp < 0 || hp >= heal_below){ + return BattleHeal::NOT_NEEDED; } - if (!on_bag) { + if (!cursor_to_bag(stream, context, battle_type)){ stream.log("[" + label + "] Could not reach the Bag command; fighting on.", COLOR_ORANGE); return BattleHeal::NONE_LEFT; } pbf_press_button(context, BUTTON_A, 80ms, 2000ms); - if (!find_bag_item_in_open_bag(stream, context, "potion", 1)) { + if (!find_bag_item_in_open_bag(stream, context, "potion", 1)){ stream.log("[" + label + "] No Potion left; fighting on.", COLOR_ORANGE); pbf_press_button(context, BUTTON_B, 80ms, 1500ms); pbf_press_button(context, BUTTON_B, 80ms, 1500ms); @@ -379,175 +243,141 @@ BattleHeal battle_heal_if_low(VideoStream& stream, ProControllerContext& context return BattleHeal::USED; } -void fight_using_move_list( - VideoStream& stream, ProControllerContext& context, - BattleType battle_type, const std::vector& move_slots, - const std::string& label, bool confirm_default_target, bool post_battle_dialogue_cleanup, - bool doubles = false, bool switch_on_faint = false -) { - if (move_slots.empty() || std::any_of(move_slots.begin(),move_slots.end(), - [](size_t slot) { return slot>4; })) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label+": Supply a nonempty move list containing only positions 0 (best) through 4.",stream); - } - size_t move_index=0; - // The first Fight menu starts at move 1. Later turns retain - // this Pokemon's previous selection; a new battle resets it. - size_t current_move=1; - stream.log( - "[AutoStory] Starting " + label + " battle...", - COLOR_BLUE - ); - - // Scripted battles wait on the menu detector below, so no - // extra fixed delay is needed after a caller confirms it. - // Preserve the existing early-game transition delay. - if (!confirm_default_target) pbf_wait(context,5000ms); +// How a battle is fought: every battle here takes the best move each turn. +struct BestMoveBattle{ + BattleType battle_type = BattleType::TRAINER; + BattleFormat format = BattleFormat::SINGLES; + // After a faint the game shows the party: send out the next Pokemon. + // Without, a quiet two minutes is a timeout. + bool switch_on_faint = false; +}; +// What a battle has done so far, across turns. +struct BattleProgress{ bool battle_menu_seen = false; bool out_of_potions = false; size_t potions_used = 0; size_t turns = 0; size_t faint_switches = 0; - const bool watch_learn_move = confirm_default_target || doubles || switch_on_faint; - EndBattleWatcher end_battle; - - while (true) { - - context.wait_for_all_requests(); - - BattleMenuWatcher battle_menu(battle_type); - // Reuse the farming handlers' move-learning prompt recovery. - SelectionArrowFinder learn_move(stream.overlay(),{0.50,0.62,0.40,0.18},COLOR_YELLOW); - - int ret = run_until( - stream, - context, - [&](ProControllerContext& context) { - // Clear narration with B while watching for commands. - // A here can open Fight before selection begins. - // With faint switching, a quiet 30 s means the party - // screen after a faint, which B cannot leave. - pbf_mash_button(context,BUTTON_B,switch_on_faint ? 30000ms : 120000ms); - }, - { - {battle_menu}, - battle_menu_seen - ? PeriodicInferenceCallback{end_battle} - : PeriodicInferenceCallback{}, - watch_learn_move ? PeriodicInferenceCallback{learn_move} : PeriodicInferenceCallback{}, - } - ); - - switch (ret) { - - // Battle menu detected. - case 0: - battle_menu_seen = true; - // Early battles take a few turns; this many means the chosen - // move is not being used (no PP, a misread menu). - if (++turns > 25) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": still at the battle menu after 25 turns.", stream); - } - - stream.log( - "[AutoStory] Battle menu detected.", - COLOR_BLUE - ); - - // A trainer battle we may lose: a Potion when HP is low. - if (battle_type == BattleType::TRAINER && !out_of_potions && potions_used < 4) { - const BattleHeal heal = battle_heal_if_low(stream, context, battle_type, 0.35, label); - if (heal == BattleHeal::USED) { - potions_used++; - break; - } - out_of_potions = heal == BattleHeal::NONE_LEFT; - } - stream.log("["+label+"] Move list position "+std::to_string(move_index+1)+ - ": use move "+std::to_string(move_slots[move_index])+"."); - select_battle_move_slot(stream, context, battle_type, move_slots[move_index], label, - confirm_default_target,current_move); - current_move=move_slots[move_index]; - // A double battle asks for the target next; B would - // back out of it. The default target is an opponent. - if (doubles) { - pbf_wait(context,1200ms); - pbf_press_button(context,BUTTON_A,80ms,1500ms); - } - // Advance once per move selection; retain the final entry. - if (move_index+1(stream, context, - [](ProControllerContext& ctx) { - pbf_mash_button(ctx, BUTTON_B, 120000ms); - }, {{overworld}}); - if (cleanup_ret != 0) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": Post-battle cleanup timed out.", stream); - } - context.wait_for_all_requests(); - confirm_battle_overworld(stream, context, label + " cleanup"); +// One turn at the battle menu: a Potion when a trainer's battle has left +// our HP low, else the best move (and the default target in doubles). +void take_turn( + VideoStream& stream, ProControllerContext& context, + const BestMoveBattle& battle, BattleProgress& progress, const std::string& label +){ + progress.battle_menu_seen = true; + // Early battles take a few turns; this many means the chosen + // move is not being used (no PP, a misread menu). + if (++progress.turns > 25){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": still at the battle menu after 25 turns.", stream); + } + stream.log("[AutoStory] Battle menu detected.", COLOR_BLUE); + // A trainer battle we may lose: a Potion when HP is low. + if (battle.battle_type == BattleType::TRAINER && !progress.out_of_potions && progress.potions_used < 4){ + const BattleHeal heal = battle_heal_if_low(stream, context, battle.battle_type, 0.35, label); + if (heal == BattleHeal::USED){ + progress.potions_used++; return; } + progress.out_of_potions = heal == BattleHeal::NONE_LEFT; + } + stream.log("[" + label + "] Move list position 1: use move 0."); + use_best_move(stream, context, battle.battle_type, label); + // A double battle asks for the target next; B would + // back out of it. The default target is an opponent. + if (battle.format == BattleFormat::DOUBLES){ + pbf_wait(context, 1200ms); + pbf_press_button(context, BUTTON_A, 80ms, 1500ms); + } +} - case 2: - stream.log("["+label+"] Declining move learning; preserving scripted move positions."); - pbf_move_right_joystick(context,{0,-1},160ms,840ms); - pbf_press_button(context,BUTTON_ZL,160ms,840ms); - break; +// The battle has ended: its text to the overworld, which must stay +// dialogue-free for four seconds. +void finish_battle(VideoStream& stream, ProControllerContext& context, const std::string& label){ + stream.log("[AutoStory] " + label + " battle complete.", COLOR_GREEN); + OverworldWatcher overworld; + const int cleanup_ret = run_until(stream, context, + [](ProControllerContext& ctx){ + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, {{overworld}}); + if (cleanup_ret != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Post-battle cleanup timed out.", stream); + } + context.wait_for_all_requests(); + confirm_battle_overworld(stream, context, label + " cleanup"); +} - default: - // After a faint the game shows the party: Down to the - // next Pokemon, A, then A on "Send out" (by hand, - // 2026-10-29 against Roark). - if (switch_on_faint && faint_switches < 5) { - faint_switches++; - stream.log("[" + label + "] No battle menu for 30 s: a faint; sending out the next Pokemon.", COLOR_ORANGE); - pbf_press_dpad(context, DPAD_DOWN, 80ms, 500ms); - pbf_press_button(context, BUTTON_A, 80ms, 1200ms); - pbf_press_button(context, BUTTON_A, 80ms, 2000ms); - break; - } - stream.log( - "[AutoStory] " + label + " battle timeout.", - COLOR_RED - ); +// B through the battle's text until its menu, its end, or a move-learning +// prompt shows: 0, 1 or 2; -1 after the wait. +int wait_for_battle_screen( + VideoStream& stream, ProControllerContext& context, const BestMoveBattle& battle, + const BattleProgress& progress, EndBattleWatcher& end_battle +){ + context.wait_for_all_requests(); + BattleMenuWatcher battle_menu(battle.battle_type); + // Reuse the farming handlers' move-learning prompt recovery. + SelectionArrowFinder learn_move(stream.overlay(), {0.50, 0.62, 0.40, 0.18}, COLOR_YELLOW); + const bool watch_learn_move = battle.format == BattleFormat::DOUBLES || battle.switch_on_faint; + return run_until( + stream, + context, + [&](ProControllerContext& context){ + // Clear narration with B while watching for commands. + // A here can open Fight before selection begins. + // With faint switching, a quiet 30 s means the party + // screen after a faint, which B cannot leave. + pbf_mash_button(context, BUTTON_B, battle.switch_on_faint ? 30000ms : 120000ms); + }, + { + {battle_menu}, + progress.battle_menu_seen ? PeriodicInferenceCallback{end_battle} : PeriodicInferenceCallback{}, + watch_learn_move ? PeriodicInferenceCallback{learn_move} : PeriodicInferenceCallback{}, + } + ); +} +// Fights the battle on screen to its end and the overworld after it. +void fight_with_best_move(VideoStream& stream, ProControllerContext& context, const BestMoveBattle& battle, const std::string& label){ + stream.log("[AutoStory] Starting " + label + " battle...", COLOR_BLUE); + // Preserve the existing early-game transition delay. + pbf_wait(context, 5000ms); + BattleProgress progress; + EndBattleWatcher end_battle; + while (true){ + const int ret = wait_for_battle_screen(stream, context, battle, progress, end_battle); + if (ret == 0){ + take_turn(stream, context, battle, progress, label); + }else if (ret == 1){ + finish_battle(stream, context, label); + return; + }else if (ret == 2){ + stream.log("[" + label + "] Declining move learning; preserving scripted move positions."); + pbf_move_right_joystick(context, {0, -1}, 160ms, 840ms); + pbf_press_button(context, BUTTON_ZL, 160ms, 840ms); + }else if (battle.switch_on_faint && progress.faint_switches < 5){ + // After a faint the game shows the party: Down to the next + // Pokemon, A, then A on "Send out" (by hand, 2026-10-29 against Roark). + progress.faint_switches++; + stream.log("[" + label + "] No battle menu for 30 s: a faint; sending out the next Pokemon.", COLOR_ORANGE); + pbf_press_dpad(context, DPAD_DOWN, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 1200ms); + pbf_press_button(context, BUTTON_A, 80ms, 2000ms); + }else{ + stream.log("[AutoStory] " + label + " battle timeout.", COLOR_RED); OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "[AutoStory] " + label + " battle timeout.", stream); } } } -// Preserve the existing early-game callers and their B text cleanup. -void fight_using_move_slot(VideoStream& stream, ProControllerContext& context, - BattleType battle_type, size_t move_slot, const std::string& label) { - fight_using_move_list(stream,context,battle_type,{move_slot},label,false,true); -} - } void confirm_battle_overworld(VideoStream& stream, ProControllerContext& context, - const std::string& label) { + const std::string& label){ OverworldDetector overworld; ShortDialogDetector dialogue(COLOR_YELLOW); BlackScreenDetector black; @@ -558,37 +388,41 @@ void confirm_battle_overworld(VideoStream& stream, ProControllerContext& context const auto deadline = std::chrono::steady_clock::now() + 180s; auto quiet_since = std::chrono::steady_clock::now(); bool finished = false; - while (std::chrono::steady_clock::now() < deadline) { + while (std::chrono::steady_clock::now() < deadline){ const auto frame = stream.video().snapshot(); - if (!frame) OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": No video frame.", stream); + if (!frame){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": No video frame.", stream); + } const auto now = std::chrono::steady_clock::now(); - if (dialogue.detect(frame) || black.detect(frame) || !overworld.detect(frame)) { + if (dialogue.detect(frame) || black.detect(frame) || !overworld.detect(frame)){ quiet_since = now; - } else if (now - quiet_since >= 4s) { + }else if (now - quiet_since >= 4s){ finished = true; break; } pbf_mash_button(context, BUTTON_B, 100ms); context.wait_for_all_requests(); } - if (!finished) OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": Dialogue-free overworld not confirmed for four seconds.", stream); + if (!finished){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Dialogue-free overworld not confirmed for four seconds.", stream); + } context.wait_for_all_requests(); } -void fight_story_battle(VideoStream& stream, ProControllerContext& context, const std::string& label, bool doubles) { +void fight_story_battle(VideoStream& stream, ProControllerContext& context, const std::string& label, BattleFormat format){ // The best move each turn; the next Pokemon after a faint; no new moves. - fight_using_move_list(stream, context, BattleType::TRAINER, {0}, label, false, true, doubles, true); + fight_with_best_move(stream, context, {.battle_type = BattleType::TRAINER, .format = format, .switch_on_faint = true}, label); } -void fight_trainer(VideoStream& stream, ProControllerContext& context, const std::string& label) { - // Move 0: the best move on the menu each turn, whatever the starter. - fight_using_move_slot(stream, context, BattleType::TRAINER, 0, label); +void fight_trainer(VideoStream& stream, ProControllerContext& context, const std::string& label){ + // The best move on the menu each turn, whatever the starter. + fight_with_best_move(stream, context, {.battle_type = BattleType::TRAINER, .format = BattleFormat::SINGLES, .switch_on_faint = false}, label); } void fight_wild_battle(VideoStream& stream, ProControllerContext& context, const std::string& label){ - fight_using_move_slot(stream, context, BattleType::STANDARD, 0, label); + fight_with_best_move(stream, context, {.battle_type = BattleType::STANDARD, .format = BattleFormat::SINGLES, .switch_on_faint = false}, label); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h index efc7e47697..c038cbadbe 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h @@ -29,6 +29,12 @@ class BattleDialogWatcher : public DetectorToFinder{ BattleDialogWatcher(Color color = COLOR_RED); }; +// One opponent, or two (a tag or twin battle, which asks for a target). +enum class BattleFormat{ + SINGLES, + DOUBLES, +}; + // Mashes B until the overworld has shown with no dialogue for four seconds. // Fails with a screenshot after three minutes. void confirm_battle_overworld(VideoStream& stream, ProControllerContext& context, const std::string& label); @@ -36,9 +42,9 @@ void confirm_battle_overworld(VideoStream& stream, ProControllerContext& context // A story battle that may go badly: the best move each turn, Potions // under 35 %, the next Pokemon when one faints, and "Keep old moves" // on level-up prompts. `doubles` confirms the default target each -// turn (tag and twin battles). Returns once the battle has ended, +// turn in DOUBLES. Returns once the battle has ended, // won or lost; after a loss we stand in the last Pokemon Center. -void fight_story_battle(VideoStream& stream, ProControllerContext& context, const std::string& label, bool doubles = false); +void fight_story_battle(VideoStream& stream, ProControllerContext& context, const std::string& label, BattleFormat format); // Any trainer battle, whatever the starter: the best move on the menu each // turn until the battle ends, then the text after it. diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp new file mode 100644 index 0000000000..868a407580 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp @@ -0,0 +1,276 @@ +/* BDSP AutoStory - Battle moves + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/OCR/OCR_RawTesseractOCR.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_BattleMoves.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// Early moves the starters and their foes have, as type, power and +// most PP. A status move has power 0. +struct KnownMove{ + const char* name; + const char* type; + int power; + int pp; +}; +const KnownMove KNOWN_MOVES[] = { + {"pound", "normal", 40, 35}, {"tackle", "normal", 40, 35}, {"scratch", "normal", 40, 35}, + {"quickattack", "normal", 40, 30}, {"headbutt", "normal", 70, 15}, + {"watergun", "water", 40, 25}, {"bubble", "water", 40, 30}, {"bubblebeam", "water", 65, 20}, + {"ember", "fire", 40, 25}, {"flamewheel", "fire", 60, 25}, {"machpunch", "fighting", 40, 30}, + {"absorb", "grass", 20, 25}, {"razorleaf", "grass", 55, 25}, {"bite", "dark", 60, 25}, + {"peck", "flying", 35, 35}, {"wingattack", "flying", 60, 35}, {"metalclaw", "steel", 50, 35}, + {"growl", "normal", 0, 40}, {"leer", "normal", 0, 30}, {"tailwhip", "normal", 0, 30}, + {"withdraw", "water", 0, 40}, {"taunt", "dark", 0, 20}, +}; + +// The box's text, whatever differs from its median colour by more than +// 120, as black on white with an 8-pixel margin. +ImageRGB32 pp_text_image(const ImageViewRGB32& box){ + std::vector pixels; + for (size_t y = 0; y < box.height(); y++){ + for (size_t x = 0; x < box.width(); x++){ + pixels.push_back(box.pixel(x, y)); + } + } + auto brightness = [](uint32_t p){ return ((p >> 16) & 255) + ((p >> 8) & 255) + (p & 255); }; + std::nth_element(pixels.begin(), pixels.begin() + pixels.size() / 2, pixels.end(), + [&](uint32_t a, uint32_t b){ return brightness(a) < brightness(b); }); + const uint32_t background = pixels[pixels.size() / 2]; + const size_t pad = 8; + ImageRGB32 text(box.width() + 2 * pad, box.height() + 2 * pad); + text.fill(0xffffffff); + for (size_t y = 0; y < box.height(); y++){ + for (size_t x = 0; x < box.width(); x++){ + const uint32_t p = box.pixel(x, y); + int distance = 0; + for (int shift : {16, 8, 0}){ + distance += std::abs((int)((p >> shift) & 255) - (int)((background >> shift) & 255)); + } + if (distance > 120){ + text.pixel(x + pad, y + pad) = 0xff000000; + } + } + } + return text; +} + +// A move button's PP left, "8/25" at its right end; -1 if it cannot be +// read. The text is dark, or yellow to red as PP runs low, so text is +// whatever differs from the button's own color. `max_pp` helps when +// the OCR reads the slash as a digit. +int read_move_pp(const ImageViewRGB32& screen, const ImageFloatBox& button, int max_pp){ + const ImageViewRGB32 box = extract_box_reference(screen, ImageFloatBox{ + button.x + button.width * 0.75, button.y + button.height * 0.10, + button.width * 0.21, button.height * 0.80}); + if (box.width() == 0 || box.height() == 0){ + return -1; + } + const std::string read = OCR::tesseract_ocr_read(Language::English, pp_text_image(box)); + const size_t slash = read.find('/'); + std::string digits; + for (size_t i = 0; i < read.size() && i < slash; i++){ + if (std::isdigit((unsigned char)read[i])){ + digits.push_back(read[i]); + } + } + if (slash == std::string::npos && max_pp > 0){ + // "29135" for 29/35: drop the most PP and the misread slash. + const std::string max = std::to_string(max_pp); + if (digits.size() <= max.size() + 1 || digits.compare(digits.size() - max.size(), max.size(), max) != 0){ + return -1; + } + digits.resize(digits.size() - max.size() - 1); + } + if (digits.empty() || digits.size() > 2){ + return -1; + } + return std::stoi(digits); +} + +// A box's gray levels, row by row, and the cut between text and background: +// halfway between the background and the darkest pixel (a fixed cut thins +// the strokes on the blue button). +struct GrayBox{ + std::vector gray; + size_t width = 0; + size_t height = 0; + int threshold = 0; + bool text_in_row(size_t row) const{ + for (size_t x = 0; x < width; x++){ + if (gray[row * width + x] < threshold){ + return true; + } + } + return false; + } +}; +GrayBox gray_box(const ImageViewRGB32& box){ + GrayBox ret{std::vector(box.width() * box.height()), box.width(), box.height(), 0}; + for (size_t y = 0; y < box.height(); y++){ + for (size_t x = 0; x < box.width(); x++){ + const uint32_t pixel = box.pixel(x, y); + ret.gray[y * box.width() + x] = (int)(((pixel >> 16) & 0xff) + ((pixel >> 8) & 0xff) + (pixel & 0xff)) / 3; + } + } + std::vector sorted = ret.gray; + std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end()); + const int background = sorted[sorted.size() / 2]; + const int darkest = *std::min_element(ret.gray.begin(), ret.gray.end()); + ret.threshold = (background + darkest) / 2; + return ret; +} + +// The tallest band of text rows, [top, bottom): the name; "Effective" is smaller. +std::pair tallest_text_band(const GrayBox& box){ + size_t band_top = 0, band_bottom = 0; + for (size_t y = 0; y < box.height;){ + if (!box.text_in_row(y)){ + y++; + continue; + } + size_t end = y; + while (end < box.height && box.text_in_row(end)){ + end++; + } + if (end - y > band_bottom - band_top){ + band_top = y; + band_bottom = end; + } + y = end; + } + return {band_top, band_bottom}; +} + +// The name is dark text on a light button (grey, or light blue under the +// cursor). Take the box right of the type icon and left of the PP, tall +// enough for a status move's name, which sits lower: it has no "Effective" +// line under it. Read as lower-case letters; nothing where there is no +// reader for the game's language (an unread move counts as a plain attack). +std::string read_move_name(const ImageViewRGB32& screen, const ImageFloatBox& button){ + const ImageFloatBox name_box{button.x + button.width * 0.17, button.y + button.height * 0.08, + button.width * 0.53, button.height * 0.80}; + const GrayBox box = gray_box(extract_box_reference(screen, name_box)); + const auto [band_top, band_bottom] = tallest_text_band(box); + // Black text on white, with a margin, for the OCR. + const size_t pad = 6; + const size_t top = band_top >= 4 ? band_top - 4 : 0; + const size_t bottom = std::min(box.height, band_bottom + 4); + ImageRGB32 text(box.width + 2 * pad, bottom - top + 2 * pad); + text.fill(0xffffffff); + for (size_t y = top; y < bottom; y++){ + for (size_t x = 0; x < box.width; x++){ + if (box.gray[y * box.width + x] < box.threshold){ + text.pixel(x + pad, y - top + pad) = 0xff000000; + } + } + } + std::string name; + for (unsigned char ch : story_reads_text() ? OCR::tesseract_ocr_read(g_story_language, text) : std::string()){ + if (std::isalpha(ch)){ + name.push_back((char)std::tolower(ch)); + } + } + return name; +} + +// The longest known move whose name appears in `name`: "bubblebeam" over "bubble". +const KnownMove* known_move(const std::string& name){ + const KnownMove* known = nullptr; + for (const KnownMove& move : KNOWN_MOVES){ + if (name.find(move.name) != std::string::npos && + (!known || std::string(move.name).size() > std::string(known->name).size())){ + known = &move; + } + } + return known; +} + +// The move in slot `i`: its read name, and what it is if known. A slot +// read well once keeps its name when a later read fails. +std::pair identify_move(const ImageViewRGB32& screen, const ImageFloatBox& button, size_t i){ + static std::array last_read; + std::string name = read_move_name(screen, button); + const KnownMove* known = known_move(name); + if (!known){ + name = last_read[i]; + known = known_move(name); + } + if (known){ + last_read[i] = known->name; + } + return {name, known}; +} + +} + + +size_t pick_best_move(VideoStream& stream, const ImageViewRGB32& screen, + const BattleMoveMenuState& moves, const std::string& label){ + // Keep the menu as it looked, to check the reads against. + const std::string folder = SCREENSHOTS_PATH() + "AutoStory/moves/"; + Filesystem::create_directories(Filesystem::Path(folder)); + screen.save(folder + now_to_filestring() + "-" + label + ".png"); + size_t best = 1; + int best_score = -1; + bool any_attack = false; + size_t last_not_status = 1; + for (size_t i = 0; i < moves.move_count; i++){ + const auto [name, known] = identify_move(screen, moves.buttons[i], i); + // A move with no PP left cannot be chosen; the game says so and + // keeps the menu up. + const int pp = read_move_pp(screen, moves.buttons[i], known ? known->pp : 0); + stream.log("[" + label + "] Move " + std::to_string(i + 1) + " PP left: " + (pp < 0 ? std::string("unread") : std::to_string(pp)) + "."); + if (pp == 0){ + continue; + } + // An unreadable move counts as a plain attack, ahead of known + // status moves. + const int score = known ? known->power * 2 + (std::string(known->type) != "normal" ? 1 : 0) : 40 * 2 - 1; + stream.log("[" + label + "] Move " + std::to_string(i + 1) + " reads \"" + name + "\"" + + (known ? std::string(" = ") + known->name + " (" + known->type + ", " + std::to_string(known->power) + ")" : " (unknown)") + "."); + if (score > best_score){ + best_score = score; + best = i + 1; + } + any_attack = any_attack || (known && known->power > 0); + if (!known || known->power > 0){ + last_not_status = i + 1; + } + } + // No attack read: the newest move is the starter's own-type attack + // (Water Gun, Ember, Absorb), unless it is a status move. + if (!any_attack){ + best = last_not_status; + } + stream.log("[" + label + "] Best move: " + std::to_string(best) + "."); + return best; +} + + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h new file mode 100644 index 0000000000..e5618b329f --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h @@ -0,0 +1,37 @@ +/* BDSP AutoStory - Battle moves + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Which move to use: the move menu's names and PP read with OCR, scored by + * the power of the early moves the starters and their foes have. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_BattleMoves_H +#define PokemonAutomation_PokemonBDSP_AutoStory_BattleMoves_H + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The best move on the move menu `moves` of `screen`, 1-based: the most +// powerful one we know, and on a tie one that is not Normal (an early +// starter's own type). A move with no PP left is passed over; with no +// attack read, the newest move that is not a status move. Keeps a +// screenshot of the menu, labelled `label`. +size_t pick_best_move(VideoStream& stream, const ImageViewRGB32& screen, const BattleMoveMenuState& moves, const std::string& label); + + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp index 1e98e411d4..ce23b9eadf 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp @@ -350,6 +350,22 @@ bool line_up_with_lit_door( return false; } +namespace{ + +// No lit doorway shows: the door may have taken us in after all, more +// slowly than we watched. Fails unless the scene changed since `last_outside`. +void confirm_inside_without_doorway(VideoStream& stream, ProControllerContext& context, const VideoSnapshot& last_outside, const std::string& label){ + pbf_wait(context, 1500ms); + if (!scene_changed_since(stream, context, last_outside)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": no lit doorway on screen.", stream); + } + stream.log("[Route] " + label + ": the scene changed; inside."); + pbf_wait(context, 1500ms); +} + +} + void enter_lit_door( VideoStream& stream, ProControllerContext& context, const std::string& label @@ -373,14 +389,7 @@ void enter_lit_door( } const std::optional doorway = find_lit_doorway(*outside.frame); if (!doorway){ - // The door may have taken us in after all, more slowly than we watched. - pbf_wait(context, 1500ms); - if (!scene_changed_since(stream, context, last_outside)){ - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - label + ": no lit doorway on screen.", stream); - } - stream.log("[Route] " + label + ": the scene changed; inside."); - pbf_wait(context, 1500ms); + confirm_inside_without_doorway(stream, context, last_outside, label); return; } last_outside = outside; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp index ac9a427ccd..c9f5b50e24 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp @@ -37,9 +37,11 @@ class TutorialOptionsDetector { ImageViewRGB32 panel = extract_box_reference(screen, ImageFloatBox{0.04, 0.065, 0.93, 0.70}); size_t white = 0; for (size_t y = 0; y < panel.height(); y += 4) - for (size_t x = 0; x < panel.width(); x += 4) { + for (size_t x = 0; x < panel.width(); x += 4){ uint32_t p = panel.pixel(x, y); - if (((p >> 16) & 255) > 225 && ((p >> 8) & 255) > 225 && (p & 255) > 225) white++; + if (((p >> 16) & 255) > 225 && ((p >> 8) & 255) > 225 && (p & 255) > 225){ + white++; + } } double samples = (double)(((panel.width() + 3) / 4) * ((panel.height() + 3) / 4)); // Distinguish Options from the icon menu and bag using its @@ -47,7 +49,7 @@ class TutorialOptionsDetector { ImageViewRGB32 heading = extract_box_reference(screen, ImageFloatBox{0.01, 0.005, 0.20, 0.025}); size_t blue = 0, total = 0; for (size_t y = 0; y < heading.height(); y++) - for (size_t x = 0; x < heading.width(); x++) { + for (size_t x = 0; x < heading.width(); x++){ uint32_t p = heading.pixel(x, y); int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; blue += b > r + 20 && g > r + 10; @@ -59,12 +61,14 @@ class TutorialOptionsDetector { // Options fades in after A on its icon: poll instead of judging one frame. bool wait_for_options_screen(VideoStream& stream, ProControllerContext& context, - const TutorialOptionsDetector& options_screen, Milliseconds timeout) { + const TutorialOptionsDetector& options_screen, Milliseconds timeout){ context.wait_for_all_requests(); const WallClock deadline = current_time() + timeout; - while (current_time() < deadline) { + while (current_time() < deadline){ auto frame = stream.video().snapshot(); - if (frame && options_screen.detect(frame)) return true; + if (frame && options_screen.detect(frame)){ + return true; + } context.wait_for(100ms); } return false; @@ -76,20 +80,24 @@ class TutorialOptionPositionDetector { // selected row. Scrolling changes their position automatically. double selected_row(const ImageViewRGB32& screen) const { int first = -1, last = -1; - for (size_t y = (size_t)(screen.height() * 0.08); y < screen.height() * 0.75; y++) { + for (size_t y = (size_t)(screen.height() * 0.08); y < screen.height() * 0.75; y++){ size_t red = 0, total = 0; - for (size_t x = (size_t)(screen.width() * 0.08); x < screen.width() * 0.90; x += 2) { + for (size_t x = (size_t)(screen.width() * 0.08); x < screen.width() * 0.90; x += 2){ uint32_t p = screen.pixel(x, y); int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; red += r > 150 && r > g + 40 && r > b + 40; total++; } - if (total && red > total * 0.70) { - if (first < 0) first = (int)y; + if (total && red > total * 0.70){ + if (first < 0){ + first = (int)y; + } last = (int)y; } } - if (first < 0 || last - first < screen.height() * 0.04 || last - first > screen.height() * 0.12) return -1; + if (first < 0 || last - first < screen.height() * 0.04 || last - first > screen.height() * 0.12){ + return -1; + } return (first + last) / (2.0 * screen.height()); } // Locate selected red value text separately from the row border. @@ -97,7 +105,7 @@ class TutorialOptionPositionDetector { ImageViewRGB32 value = extract_box_reference(screen, ImageFloatBox{0.33, y - 0.025, 0.54, 0.05}); size_t red = 0; for (size_t py = 0; py < value.height(); py++) - for (size_t px = 0; px < value.width(); px++) { + for (size_t px = 0; px < value.width(); px++){ uint32_t p = value.pixel(px, py); int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; red += r > g + 20 && r > b + 20; @@ -110,7 +118,7 @@ class TutorialOptionPositionDetector { // Compare binary visual shapes with small alignment tolerance. // No text recognition is needed for fixed English option labels. double option_image_score(const ImageViewRGB32& screen, - const ImageFloatBox& box, const ImageRGB32& reference, bool red_only) { + const ImageFloatBox& box, const ImageRGB32& reference, bool red_only){ ImageRGB32 actual = extract_box_reference(screen, box).scale_to(384, 32); ImageRGB32 expected = reference.scale_to(384, 32); auto ink = [red_only](uint32_t p){ @@ -118,25 +126,31 @@ double option_image_score(const ImageViewRGB32& screen, return red_only ? r > g + 20 && r > b + 20 : r + g + b < 500; }; double best = 0; - for (int dy = -2; dy <= 2; dy++) for (int dx = -2; dx <= 2; dx++) { - size_t overlap = 0, count = 0; - for (int y = 2; y < 30; y++) for (int x = 2; x < 382; x++) { - bool a = ink(actual.pixel(x + dx, y + dy)); - bool b = ink(expected.pixel(x, y)); - overlap += a && b; - count += a + b; + for (int dy = -2; dy <= 2; dy++){ + for (int dx = -2; dx <= 2; dx++){ + size_t overlap = 0, count = 0; + for (int y = 2; y < 30; y++){ + for (int x = 2; x < 382; x++){ + bool a = ink(actual.pixel(x + dx, y + dy)); + bool b = ink(expected.pixel(x, y)); + overlap += a && b; + count += a + b; + } + } + if (count){ + best = std::max(best, 2.0 * overlap / count); + } } - if (count) best = std::max(best, 2.0 * overlap / count); } return best; } -bool autosave_off_highlight(const ImageViewRGB32& screen, double y) { +bool autosave_off_highlight(const ImageViewRGB32& screen, double y){ auto red_count = [&](const ImageFloatBox& box){ ImageViewRGB32 image = extract_box_reference(screen, box); size_t red = 0; for (size_t py = 0; py < image.height(); py++) - for (size_t px = 0; px < image.width(); px++) { + for (size_t px = 0; px < image.width(); px++){ uint32_t p = image.pixel(px, py); int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; red += r > g + 30 && r > b + 30; @@ -149,7 +163,37 @@ bool autosave_off_highlight(const ImageViewRGB32& screen, double y) { return off >= 10 && on * 10 < off; } -void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& context) { +// The options the story needs, as each reference image names them: the +// row's label and the value to select. +struct Setting{ + const char* label; + const char* value; +}; +const Setting SETTINGS[] = { + {"textspeed", "fast"}, {"changewindow", "type1"}, + {"battleeffects", "off"}, {"battlestyle", "set"}, + {"sendtoboxes", "automatic"}, {"givenicknames", "dontgive"}, + {"gyroscope", "use"}, {"autosave", "off"}, + {"shownicknames", "show"}, {"backgroundmusic", "8"}, + {"soundeffects", "8"}, +}; +constexpr size_t SETTING_COUNT = sizeof(SETTINGS) / sizeof(SETTINGS[0]); +constexpr size_t AUTOSAVE_ROW = 7; +constexpr size_t FIRST_SLIDER_ROW = 9; + +// The Options screen and its selected row, as one reader. +struct OptionsScreen{ + TutorialOptionsDetector options_screen; + TutorialOptionPositionDetector position; +}; +// A row's label and target value, as the reference images show them. +struct OptionReferences{ + ImageRGB32 label; + ImageRGB32 value; +}; + +// From the overworld (or the main menu): the main menu, settled. +void open_main_menu(VideoStream& stream, ProControllerContext& context){ context.wait_for_all_requests(); MenuDetector menu; VideoOverlaySet detector_overlays(stream.overlay()); @@ -158,153 +202,186 @@ void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& co detector_overlays.add(COLOR_BLUE, {0.01, 0.005, 0.20, 0.025}, ""); detector_overlays.add(COLOR_RED, {0.08, 0.08, 0.82, 0.67}, ""); auto initial = stream.video().snapshot(); - if (!initial || !menu.detect(initial)) { + if (!initial || !menu.detect(initial)){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); } MenuWatcher menu_open; - if (wait_until(stream, context, 5s, {{menu_open}}) != 0) { + if (wait_until(stream, context, 5s, {{menu_open}}) != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Settings checkpoint: Main menu not detected.", stream); } stream.log("[Tutorial settings] Main menu detected; settling for 500ms before Options navigation."); pbf_wait(context, 500ms); context.wait_for_all_requests(); - TutorialOptionsDetector options_screen; - TutorialOptionPositionDetector position; - // Only two icons are available at this initial checkpoint. - // Verify the main menu before navigating to the neighboring icon. +} + +// From the main menu: Options. Only two icons are available at this +// initial checkpoint; verify the main menu before trying the other. +void open_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ bool options_open = false; - for (size_t attempt = 0; attempt < 2; attempt++) { + for (size_t attempt = 0; attempt < 2; attempt++){ pbf_press_dpad(context, attempt == 0 ? DPAD_RIGHT : DPAD_LEFT, 80ms, 350ms); pbf_press_button(context, BUTTON_A, 80ms, 0ms); - if (wait_for_options_screen(stream, context, options_screen, 5s)) { + if (wait_for_options_screen(stream, context, options_screen, 5s)){ options_open = true; break; } pbf_press_button(context, BUTTON_B, 80ms, 1000ms); context.wait_for_all_requests(); MenuWatcher back_to_menu; - if (wait_until(stream, context, 5s, {{back_to_menu}}) != 0) break; + if (wait_until(stream, context, 5s, {{back_to_menu}}) != 0){ + break; + } } - if (!options_open) { + if (!options_open){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Settings checkpoint: Options screen not detected.", stream); } stream.log("[Tutorial settings] Options screen detected."); pbf_wait(context, 500ms); context.wait_for_all_requests(); - struct Setting { const char* label; const char* value; }; - const Setting settings[] = { - {"textspeed", "fast"}, {"changewindow", "type1"}, - {"battleeffects", "off"}, {"battlestyle", "set"}, - {"sendtoboxes", "automatic"}, {"givenicknames", "dontgive"}, - {"gyroscope", "use"}, {"autosave", "off"}, - {"shownicknames", "show"}, {"backgroundmusic", "8"}, - {"soundeffects", "8"}, - }; - for (size_t row = 0; row < sizeof(settings) / sizeof(settings[0]); row++) { +} + +// How well the selected row's value matches the reference. +double option_value_score(const ImageViewRGB32& frame, double y, size_t row, const ImageRGB32& value_reference){ + if (row >= FIRST_SLIDER_ROW){ + // Compare the numeric value rather than the long slider: + // neighboring slider levels otherwise look too similar. + ImageViewRGB32 number = value_reference.sub_image( + value_reference.width() * 82 / 100, 0, + value_reference.width() - value_reference.width() * 82 / 100, + value_reference.height()); + ImageRGB32 number_reference = number.scale_to(96, 32); + return option_image_score(frame, + {0.7728, y - 0.025, 0.0972, 0.05}, number_reference, true); + } + return option_image_score(frame, + {0.33, y - 0.025, 0.54, 0.05}, value_reference, true); +} + +// Whether Options shows Autosave selected and set to Off: a frame's look. +bool autosave_off_on_screen( + const ImageViewRGB32& check, const TutorialOptionsDetector& options_screen, + const TutorialOptionPositionDetector& position, const ImageRGB32& autosave_label +){ + const double check_y = check ? position.selected_row(check) : -1; + return check && options_screen.detect(check) && check_y >= 0 && + option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, autosave_label, false) >= 0.80 && + autosave_off_highlight(check, check_y); +} + +// Autosave Off, on three more frames 200 ms apart. +void confirm_autosave_off( + VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen, + const TutorialOptionPositionDetector& position, const ImageRGB32& label_reference +){ + for (size_t confirmation = 0; confirmation < 3; confirmation++){ + pbf_wait(context, 200ms); + context.wait_for_all_requests(); + auto check = stream.video().snapshot(); + if (!autosave_off_on_screen(check, options_screen, position, label_reference)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Autosave Off could not be confirmed. Stopping before movement.", stream); + } + } +} + +// One press toward the target value. Sliders clamp at zero: establish a +// known endpoint, then increment; the target image is verified afterward. +void correct_option(ProControllerContext& context, size_t row, bool& slider_reset){ + if (row >= FIRST_SLIDER_ROW && !slider_reset){ + for (size_t step = 0; step < 12; step++){ + pbf_press_dpad(context, DPAD_LEFT, 80ms, 100ms); + } + slider_reset = true; + }else{ + pbf_press_dpad(context, DPAD_RIGHT, 80ms, 350ms); + } + context.wait_for_all_requests(); +} + +// The selected row on screen, if it is the expected one with its value +// highlighted; -1 otherwise. +double expected_row_y( + VideoStream& stream, const ImageViewRGB32& frame, const TutorialOptionsDetector& options_screen, + const TutorialOptionPositionDetector& position, size_t row, const ImageRGB32& label_reference +){ + const double y = frame && options_screen.detect(frame) ? position.selected_row(frame) : -1; + const bool value_visible = y >= 0 && position.selected_value(frame, y); + const double label_score = y >= 0 ? option_image_score(frame, + {0.08, y - 0.025, 0.24, 0.05}, label_reference, false) : 0; + stream.log("[Tutorial settings] Expected row=" + std::string(SETTINGS[row].label) + + " row_y=" + std::to_string(y) + " highlighted_value=" + std::to_string(value_visible) + + " label_match=" + std::to_string(label_score)); + return y >= 0 && value_visible && label_score >= 0.80 ? y : -1; +} + +// Whether the selected row (at `y`) shows the target value: checked on +// the frame, with the row's boxes on the overlay. +bool option_verified( + VideoStream& stream, ProControllerContext& context, const ImageViewRGB32& frame, double y, size_t row, + const OptionsScreen& screen, const OptionReferences& references +){ + const ImageRGB32& value_reference = references.value; + // These are regular video-overlay boxes, so the existing overlay + // display control can show/hide every active ROI. + OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, {0.07, y - 0.04, 0.84, 0.08}, ""); + OverlayBoxScope label_overlay(stream.overlay(), COLOR_YELLOW, {0.08, y - 0.025, 0.24, 0.05}, ""); + OverlayBoxScope value_overlay(stream.overlay(), COLOR_GREEN, {0.33, y - 0.025, 0.54, 0.05}, ""); + VideoOverlaySet autosave_rois(stream.overlay()); + if (row == AUTOSAVE_ROW){ + autosave_rois.add(COLOR_GREEN, {0.63, y - 0.022, 0.055, 0.044}); + autosave_rois.add(COLOR_RED, {0.34, y - 0.022, 0.055, 0.044}); + } + const double score = option_value_score(frame, y, row, value_reference); + stream.log("[Tutorial settings] " + std::string(SETTINGS[row].label) + " target match=" + std::to_string(score)); + // The reference recording uses inactive pale-red text; the focused live + // row has a thicker dark-red glyph mask. Verified Fast scores 0.734 + // while Slow/Normal score 0. + if (score < 0.65 || (row == AUTOSAVE_ROW && !autosave_off_highlight(frame, y))){ + return false; + } + if (row == AUTOSAVE_ROW){ + confirm_autosave_off(stream, context, screen.options_screen, screen.position, references.label); + } + return true; +} + +// Sets the option on `row`, the row selected now, to its target value. +void set_option(VideoStream& stream, ProControllerContext& context, size_t row, const OptionsScreen& screen){ + const std::string base = RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/" + SETTINGS[row].label; + const OptionReferences references{ImageRGB32(base + "_label.png"), ImageRGB32(base + "_value.png")}; + bool slider_reset = false; + for (size_t attempt = 0; attempt < 30; attempt++){ // Detect the selected row afresh after every navigation/correction. - double y = -1; - const std::string base = RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/" + settings[row].label; - const ImageRGB32 label_reference(base + "_label.png"); - const ImageRGB32 value_reference(base + "_value.png"); - bool slider_reset = false; - bool correct = false; - for (size_t attempt = 0; attempt < 30; attempt++) { - context.wait_for_all_requests(); - auto frame = stream.video().snapshot(); - if (frame && options_screen.detect(frame)) y = position.selected_row(frame); - else y = -1; - const bool value_visible = y >= 0 && position.selected_value(frame, y); - const double label_score = y >= 0 ? option_image_score(frame, - {0.08, y - 0.025, 0.24, 0.05}, label_reference, false) : 0; - stream.log("[Tutorial settings] Expected row=" + std::string(settings[row].label) + - " row_y=" + std::to_string(y) + " highlighted_value=" + std::to_string(value_visible) + - " label_match=" + std::to_string(label_score)); - if (y < 0 || !value_visible || label_score < 0.80) { - // A menu can be detected while its row/text is still - // fading in. Re-read without directional inputs first. - if (attempt < 4) { - pbf_wait(context, 200ms); - context.wait_for_all_requests(); - continue; - } - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Settings checkpoint: Unable to confirm expected row " + std::string(settings[row].label), stream); - } - // These are regular video-overlay boxes, so the existing - // overlay display control can show/hide every active ROI. - OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, - {0.07, y - 0.04, 0.84, 0.08}, ""); - OverlayBoxScope label_overlay(stream.overlay(), COLOR_YELLOW, - {0.08, y - 0.025, 0.24, 0.05}, ""); - OverlayBoxScope value_overlay(stream.overlay(), COLOR_GREEN, - {0.33, y - 0.025, 0.54, 0.05}, ""); - VideoOverlaySet autosave_rois(stream.overlay()); - if (row == 7) { - autosave_rois.add(COLOR_GREEN, {0.63, y - 0.022, 0.055, 0.044}); - autosave_rois.add(COLOR_RED, {0.34, y - 0.022, 0.055, 0.044}); - } - double score; - if (row >= 9) { - // Compare the numeric value rather than the long slider: - // neighboring slider levels otherwise look too similar. - ImageViewRGB32 number = value_reference.sub_image( - value_reference.width() * 82 / 100, 0, - value_reference.width() - value_reference.width() * 82 / 100, - value_reference.height()); - ImageRGB32 number_reference = number.scale_to(96, 32); - score = option_image_score(frame, - {0.7728, y - 0.025, 0.0972, 0.05}, number_reference, true); - } else { - score = option_image_score(frame, - {0.33, y - 0.025, 0.54, 0.05}, value_reference, true); - } - stream.log("[Tutorial settings] " + std::string(settings[row].label) + - " target match=" + std::to_string(score)); - // The reference recording uses inactive pale-red text; - // the focused live row has a thicker dark-red glyph mask. - // Verified Fast scores 0.734 while Slow/Normal score 0. - if (score >= 0.65 && (row != 7 || autosave_off_highlight(frame, y))) { - if (row == 7) { - for (size_t confirmation = 0; confirmation < 3; confirmation++) { - pbf_wait(context, 200ms); - context.wait_for_all_requests(); - auto check = stream.video().snapshot(); - double check_y = check ? position.selected_row(check) : -1; - if (!check || !options_screen.detect(check) || check_y < 0 || - option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, label_reference, false) < 0.80 || - !autosave_off_highlight(check, check_y)) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Autosave Off could not be confirmed. Stopping before movement.", stream); - } - } - } - stream.log("[Tutorial settings] Verified " + std::string(settings[row].label) + "=" + settings[row].value); - correct = true; - break; - } - stream.log("[Tutorial settings] Correcting " + std::string(settings[row].label)); - if (row >= 9 && !slider_reset) { - // Sliders clamp at zero. Establish a known endpoint, - // then increment; verify the target image afterward. - for (size_t step = 0; step < 12; step++) - pbf_press_dpad(context, DPAD_LEFT, 80ms, 100ms); - slider_reset = true; - } else { - pbf_press_dpad(context, DPAD_RIGHT, 80ms, 350ms); + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + const double y = expected_row_y(stream, frame, screen.options_screen, screen.position, row, references.label); + if (y < 0){ + // A menu can be detected while its row/text is still fading + // in. Re-read without directional inputs first. + if (attempt < 4){ + pbf_wait(context, 200ms); + context.wait_for_all_requests(); + continue; } - context.wait_for_all_requests(); - } - if (!correct) { OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Settings checkpoint: Could not apply " + std::string(settings[row].label), stream); + "Settings checkpoint: Unable to confirm expected row " + std::string(SETTINGS[row].label), stream); } - if (row + 1 < sizeof(settings) / sizeof(settings[0])) - pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + if (option_verified(stream, context, frame, y, row, screen, references)){ + stream.log("[Tutorial settings] Verified " + std::string(SETTINGS[row].label) + "=" + SETTINGS[row].value); + return; + } + stream.log("[Tutorial settings] Correcting " + std::string(SETTINGS[row].label)); + correct_option(context, row, slider_reset); } - // Confirm settings before exiting: A A A A B B. + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Could not apply " + std::string(SETTINGS[row].label), stream); +} + +// Confirms the settings and leaves Options (A A A A B B), back to the overworld. +void save_options(VideoStream& stream, ProControllerContext& context, const std::string& failure){ pbf_press_button(context, BUTTON_A, 80ms, 1000ms); pbf_press_button(context, BUTTON_A, 80ms, 1000ms); pbf_press_button(context, BUTTON_A, 80ms, 1000ms); @@ -313,44 +390,55 @@ void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& co pbf_press_button(context, BUTTON_B, 80ms, 1000ms); context.wait_for_all_requests(); OverworldWatcher overworld; - if (wait_until(stream, context, 10s, {{overworld}}) != 0) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Settings checkpoint: Did not return to the overworld after saving options.", stream); + if (wait_until(stream, context, 10s, {{overworld}}) != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, failure, stream); } - // Reopen the saved Options screen and independently verify Autosave. - // No route movement is permitted until this persisted-value check passes. +} + +// From the overworld: the saved Options screen again. +void reopen_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); MenuWatcher verify_menu; - if (wait_until(stream, context, 5s, {{verify_menu}}) != 0) { + if (wait_until(stream, context, 5s, {{verify_menu}}) != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Autosave verification: Main menu not detected.", stream); } bool reopened = false; - for (size_t attempt = 0; attempt < 2; attempt++) { + for (size_t attempt = 0; attempt < 2; attempt++){ pbf_press_button(context, BUTTON_A, 80ms, 0ms); - if (wait_for_options_screen(stream, context, options_screen, 5s)) { reopened = true; break; } + if (wait_for_options_screen(stream, context, options_screen, 5s)){ + reopened = true; + break; + } pbf_press_button(context, BUTTON_B, 80ms, 1000ms); context.wait_for_all_requests(); MenuWatcher return_menu; - if (wait_until(stream, context, 5s, {{return_menu}}) != 0) break; + if (wait_until(stream, context, 5s, {{return_menu}}) != 0){ + break; + } pbf_press_dpad(context, DPAD_RIGHT, 80ms, 350ms); } - if (!reopened) { + if (!reopened){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Autosave verification: Could not reopen saved Options.", stream); } - // Options starts on Text Speed; Autosave is seven rows below it. - for (size_t step = 0; step < 7; step++) +} + +// In the saved Options: Autosave is Off on three frames. Options starts +// on Text Speed; Autosave is seven rows below it. +void check_saved_autosave( + VideoStream& stream, ProControllerContext& context, + const TutorialOptionsDetector& options_screen, const TutorialOptionPositionDetector& position +){ + for (size_t step = 0; step < AUTOSAVE_ROW; step++){ pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + } context.wait_for_all_requests(); const ImageRGB32 autosave_label(RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/autosave_label.png"); - for (size_t confirmation = 0; confirmation < 3; confirmation++) { + for (size_t confirmation = 0; confirmation < 3; confirmation++){ auto check = stream.video().snapshot(); - double check_y = check ? position.selected_row(check) : -1; - if (!check || !options_screen.detect(check) || check_y < 0 || - option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, autosave_label, false) < 0.80 || - !autosave_off_highlight(check, check_y)) { + if (!autosave_off_on_screen(check, options_screen, position, autosave_label)){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Saved Autosave setting is not verified Off. Stopping before movement.", stream); } @@ -358,19 +446,24 @@ void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& co context.wait_for_all_requests(); } stream.log("[Tutorial settings] Autosave = Off: confirmed after reopening saved Options (3 frames)."); - // Confirm settings before exiting: A A A A B B. - pbf_press_button(context, BUTTON_A, 80ms, 1000ms); - pbf_press_button(context, BUTTON_A, 80ms, 1000ms); - pbf_press_button(context, BUTTON_A, 80ms, 1000ms); - pbf_press_button(context, BUTTON_A, 80ms, 1200ms); - pbf_press_button(context, BUTTON_B, 80ms, 1000ms); - pbf_press_button(context, BUTTON_B, 80ms, 1000ms); - context.wait_for_all_requests(); - OverworldWatcher verified_exit; - if (wait_until(stream, context, 10s, {{verified_exit}}) != 0) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Autosave verification: Could not return to the overworld.", stream); +} + +void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& context){ + open_main_menu(stream, context); + OptionsScreen screen; + open_options(stream, context, screen.options_screen); + for (size_t row = 0; row < SETTING_COUNT; row++){ + set_option(stream, context, row, screen); + if (row + 1 < SETTING_COUNT){ + pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + } } + save_options(stream, context, "Settings checkpoint: Did not return to the overworld after saving options."); + // Reopen the saved Options screen and independently verify Autosave. + // No route movement is permitted until this persisted-value check passes. + reopen_options(stream, context, screen.options_screen); + check_saved_autosave(stream, context, screen.options_screen, screen.position); + save_options(stream, context, "Autosave verification: Could not return to the overworld."); // Report inferred values explicitly: these are visual matches, // allowing the user to compare our conclusions with the screen. stream.log("[Tutorial settings] Completed. Values believed applied (visual matches before saving):"); @@ -379,60 +472,66 @@ void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& co stream.log("[Tutorial settings] Autosave = Off; Show Nicknames = Show; Background Music = 8; Sound Effects = 8."); } +// After a failed attempt: back out of Options, save prompts, the Bag or +// the menu without moving the player or resetting/loading the game. +void back_to_overworld(VideoStream& stream, ProControllerContext& context){ + stream.log("[Tutorial settings] Returning to overworld before retry."); + context.wait_for_all_requests(); + TutorialOptionsDetector options_screen; + const auto failed_frame = stream.video().snapshot(); + if (failed_frame && options_screen.detect(failed_frame)){ + // Finish the known Options save/exit sequence before observing + // state; never cancel it on an Options frame misclassified as + // overworld by the generic detector. + for (size_t press = 0; press < 4; press++){ + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + } + } + pbf_mash_button(context, BUTTON_B, 3000ms); + context.wait_for_all_requests(); + OverworldDetector overworld; + MenuDetector menu; + BlackScreenDetector black; + bool recovered = false; + size_t confirmations = 0; + for (size_t check = 0; check < 15 && !recovered; check++){ + const auto frame = stream.video().snapshot(); + const bool clear = frame && !options_screen.detect(frame) && + !menu.detect(frame) && !black.detect(frame) && bag_pocket_position(frame) == 0 && overworld.detect(frame); + if (clear && ++confirmations >= 3){ + recovered = true; + }else if (clear){ + pbf_wait(context, 200ms); + }else{ + confirmations = 0; + pbf_mash_button(context, BUTTON_B, 500ms); + } + context.wait_for_all_requests(); + } + stream.log(recovered ? "[Tutorial settings] Recovery confirmed outside Options/menu; retrying." + : "[Tutorial settings] Recovery not confirmed; next attempt will recheck the screen."); + pbf_wait(context, 500ms); + context.wait_for_all_requests(); } -void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context) { +} + +void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context){ // Settings-only recovery: initial attempt plus three retries. // Never permit route movement unless the complete check succeeds. constexpr size_t MAX_RETRIES = 3; - for (size_t attempt = 0; attempt <= MAX_RETRIES; attempt++) { + for (size_t attempt = 0; attempt <= MAX_RETRIES; attempt++){ stream.log("[Tutorial settings] Attempt " + std::to_string(attempt + 1) + "/4."); - try { + try{ verify_tutorial_settings_once(stream, context); return; - } catch (OperationFailedException& error) { + }catch (OperationFailedException& error){ stream.log("[Tutorial settings] Attempt failed: " + error.message(), COLOR_ORANGE); - if (attempt == MAX_RETRIES) { + if (attempt == MAX_RETRIES){ stream.log("[Tutorial settings] All three retries exhausted; stopping before movement.", COLOR_RED); throw; } - // Back out of Options, save prompts, bag, or menu without - // moving the player or resetting/loading the game. - stream.log("[Tutorial settings] Returning to overworld before retry."); - context.wait_for_all_requests(); - TutorialOptionsDetector options_screen; - const auto failed_frame = stream.video().snapshot(); - if (failed_frame && options_screen.detect(failed_frame)) { - // Finish the known Options save/exit sequence before - // observing state; never cancel it on an Options frame - // misclassified as overworld by the generic detector. - for (size_t press = 0; press < 4; press++) - pbf_press_button(context, BUTTON_A, 80ms, 1000ms); - } - pbf_mash_button(context, BUTTON_B, 3000ms); - context.wait_for_all_requests(); - OverworldDetector overworld; - MenuDetector menu; - BlackScreenDetector black; - bool recovered = false; - size_t confirmations = 0; - for (size_t check = 0; check < 15; check++) { - const auto frame = stream.video().snapshot(); - const bool clear = frame && !options_screen.detect(frame) && - !menu.detect(frame) && !black.detect(frame) && bag_pocket_position(frame) == 0 && overworld.detect(frame); - if (clear) { - if (++confirmations >= 3) { recovered = true; break; } - pbf_wait(context, 200ms); - } else { - confirmations = 0; - pbf_mash_button(context, BUTTON_B, 500ms); - } - context.wait_for_all_requests(); - } - stream.log(recovered ? "[Tutorial settings] Recovery confirmed outside Options/menu; retrying." - : "[Tutorial settings] Recovery not confirmed; next attempt will recheck the screen."); - pbf_wait(context, 500ms); - context.wait_for_all_requests(); + back_to_overworld(stream, context); } } } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp index b0333a7963..9a6f01695f 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp @@ -47,7 +47,34 @@ namespace{ // Two grass rows at 240 ms each, with room to spare: the wall stops us. constexpr Milliseconds CATCH_GRASS_WALK = 900ms; -void encounter_one_wild_pokemon(VideoStream& stream, ProControllerContext& context) { +// Whether a wild battle has started: its black screen, menu or text. +bool wild_battle_started(VideoStream& stream, ProControllerContext& context){ + BlackScreenWatcher battle_start; + BattleMenuWatcher battle_menu(BattleType::STANDARD, COLOR_YELLOW); + BattleDialogWatcher battle_dialog(COLOR_YELLOW); + const int ret = wait_until( + stream, + context, + 1500ms, + { + {battle_start}, + {battle_menu}, + {battle_dialog}, + } + ); + return ret >= 0; +} + +// One walk across the catch grass `direction`, to the wall; whether a +// battle started. +bool walk_catch_grass(VideoStream& stream, ProControllerContext& context, Pokemon::Direction direction){ + route_turn(stream, context, direction); + pbf_move_left_joystick(context, {0, direction == Pokemon::Direction::NORTH ? 1.0 : -1.0}, CATCH_GRASS_WALK, 200ms); + context.wait_for_all_requests(); + return wild_battle_started(stream, context); +} + +void encounter_one_wild_pokemon(VideoStream& stream, ProControllerContext& context){ context.wait_for_all_requests(); pbf_wait(context, 1000ms); @@ -58,73 +85,21 @@ void encounter_one_wild_pokemon(VideoStream& stream, ProControllerContext& conte // sit between tree walls, so each walk ends at a wall, on the same tile // every time, after two new grass tiles. Short sideways steps drift and // can rock across one tile edge without entering new grass. - while (!encounter_found && std::chrono::steady_clock::now() < deadline) { - BlackScreenWatcher battle_start; - BattleMenuWatcher battle_menu(BattleType::STANDARD, COLOR_YELLOW); - BattleDialogWatcher battle_dialog(COLOR_YELLOW); - - route_turn(stream, context, Pokemon::Direction::NORTH); - pbf_move_left_joystick(context, {0, 1}, CATCH_GRASS_WALK, 200ms); - context.wait_for_all_requests(); - - int ret = wait_until( - stream, - context, - 1500ms, - { - {battle_start}, - {battle_menu}, - {battle_dialog}, - } - ); - - if (ret >= 0) { - encounter_found = true; - break; - } - - route_turn(stream, context, Pokemon::Direction::SOUTH); - pbf_move_left_joystick(context, {0, -1}, CATCH_GRASS_WALK, 200ms); - context.wait_for_all_requests(); - - ret = wait_until( - stream, - context, - 1500ms, - { - {battle_start}, - {battle_menu}, - {battle_dialog}, - } - ); - - if (ret >= 0) { - encounter_found = true; - break; - } + while (!encounter_found && std::chrono::steady_clock::now() < deadline){ + encounter_found = walk_catch_grass(stream, context, Pokemon::Direction::NORTH) || + walk_catch_grass(stream, context, Pokemon::Direction::SOUTH); } - if (!encounter_found) OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Wild encounter: walking up and down the grass found no battle within three minutes.", stream); + if (!encounter_found){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Wild encounter: walking up and down the grass found no battle within three minutes.", stream); + } stream.log("[AutoStory] Wild battle detected!", COLOR_GREEN); // Wait until the actual battle menu appears. BattleMenuWatcher confirm_menu(BattleType::STANDARD); - - int menu_ret = wait_until( - stream, - context, - 15s, - { - {confirm_menu}, - } - ); - - if (menu_ret < 0) { - stream.log( - "[AutoStory] Failed to detect battle menu.", - COLOR_RED - ); + if (wait_until(stream, context, 15s, {{confirm_menu}}) < 0){ + stream.log("[AutoStory] Failed to detect battle menu.", COLOR_RED); OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "[AutoStory] Failed to detect battle menu.", stream); } @@ -133,12 +108,40 @@ void encounter_one_wild_pokemon(VideoStream& stream, ProControllerContext& conte context.wait_for_all_requests(); pbf_wait(context, 500ms); +} +// With the game's main menu open after a battle: B until it closes, the +// overworld or a screen without it. Whether it was seen closed. +bool close_menu_after_battle(VideoStream& stream, ProControllerContext& context){ + OverworldWatcher overworld; + MenuDetector menu_visible; + for (size_t close = 0; close < 3; close++){ + pbf_press_button(context, BUTTON_B, 80ms, 500ms); + context.wait_for_all_requests(); + if (wait_until(stream, context, 1s, {{overworld}}) == 0){ + return true; + } + BlackScreenWatcher visible(COLOR_YELLOW, + {0.1, 0.1, 0.8, 0.8}, + 100, + 10, + BlackScreenWatcher::FinderType::GONE, + 500ms); + if (wait_until(stream, context, 2s, {{visible}}) != 0){ + continue; + } + const VideoSnapshot frame = stream.video().snapshot(); + if (frame && !menu_visible.detect(frame)){ + stream.log("[Mine escape] Confirmed game menu closed; returning to navigation.", COLOR_GREEN); + return true; + } + } + stream.log("[Mine escape] Main menu detected but could not confirm it closed.", COLOR_ORANGE); + return false; } bool mine_confirm_escape_via_menu(VideoStream& stream, ProControllerContext& context, BattleType battle_type){ MenuWatcher game_menu; - OverworldWatcher overworld; BattleMenuWatcher battle_menu(battle_type, COLOR_YELLOW); for (size_t attempt = 1; attempt <= 3; attempt++){ // Do not accept the absence-based overworld detector before menu proof. @@ -167,29 +170,7 @@ bool mine_confirm_escape_via_menu(VideoStream& stream, ProControllerContext& con } stream.log("[Mine escape] Main game menu confirmed; battle has ended.", COLOR_GREEN); // Close the confirmed menu before returning control to navigation. - MenuDetector menu_visible; - for (size_t close = 0; close < 3; close++){ - pbf_press_button(context, BUTTON_B, 80ms, 500ms); - context.wait_for_all_requests(); - if (wait_until(stream, context, 1s, {{overworld}}) == 0){ - return true; - } - BlackScreenWatcher visible(COLOR_YELLOW, - {0.1, 0.1, 0.8, 0.8}, - 100, - 10, - BlackScreenWatcher::FinderType::GONE, - 500ms); - if (wait_until(stream, context, 2s, {{visible}}) == 0){ - const VideoSnapshot frame = stream.video().snapshot(); - if (frame && !menu_visible.detect(frame)){ - stream.log("[Mine escape] Confirmed game menu closed; returning to navigation.", COLOR_GREEN); - return true; - } - } - } - stream.log("[Mine escape] Main menu detected but could not confirm it closed.", COLOR_ORANGE); - return false; + return close_menu_after_battle(stream, context); } return false; } @@ -227,9 +208,59 @@ bool mine_select_run(VideoStream& stream, ProControllerContext& context, BattleT return false; } +// Run on the battle menu, until the battle's exit shows or `timeout` +// runs out. Whether the exit was seen. +bool run_until_escaped( + VideoStream& stream, ProControllerContext& context, BattleType battle_type, + std::chrono::seconds timeout, EndBattleWatcher& battle_end +){ + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline){ + pbf_press_button(context, BUTTON_B, 40ms, 200ms); + context.wait_for_all_requests(); + BattleMenuWatcher menu_ready(battle_type, COLOR_YELLOW); + const int ready = wait_until(stream, context, 3s, {{battle_end}, {menu_ready}}); + if (ready == 0){ + return true; + } + if (ready != 1 || !mine_select_run(stream, context, battle_type)){ + continue; + } + stream.log("[Mine escape] Run input sent; watching for black-screen exit."); + const int result = run_until(stream, + context, + [](ProControllerContext& controller){ + pbf_press_button(controller, BUTTON_A, 60ms, 0ms); + pbf_wait(controller, 4s); + controller.wait_for_all_requests(); + }, + {battle_end}); + if (result == 0){ + return true; + } + } + return false; +} + +// B until the overworld shows; fails with `failure` after two minutes. +void mash_to_overworld(VideoStream& stream, ProControllerContext& context, const std::string& failure){ + // Mash through XP / level-up / dialog screens back to overworld. + OverworldWatcher overworld; + const int cleanup_ret = run_until( + stream, context, + [](ProControllerContext& ctx){ + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, + {{overworld}} + ); + if (cleanup_ret != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, failure, stream); + } +} + } -void fight_starly(VideoStream& stream, ProControllerContext& context) { +void fight_starly(VideoStream& stream, ProControllerContext& context){ stream.log("[AutoStory] Starting Starly battle...", COLOR_BLUE); // Declare OUTSIDE the loop so accumulated detector state @@ -240,59 +271,35 @@ void fight_starly(VideoStream& stream, ProControllerContext& context) { BattleMenuWatcher battle_menu(BattleType::STARTER, COLOR_YELLOW); EndBattleWatcher end_battle; - while (true) { + while (true){ context.wait_for_all_requests(); // Phase 1: wait for either battle menu or battle end. - int ret = run_until( + const int ret = run_until( stream, context, - [](ProControllerContext& ctx) { + [](ProControllerContext& ctx){ pbf_mash_button(ctx, BUTTON_B, 120000ms); }, -{ - {battle_menu}, - {end_battle}, -} -); - - switch (ret) { - case 0: - { - stream.log("[AutoStory] Battle menu detected. Selecting move...", COLOR_BLUE); - - // Select Fight, then move slot 1. - pbf_press_button(context, BUTTON_A, 80ms, 500ms); - pbf_press_button(context, BUTTON_A, 80ms, 3000ms); - // Loop back - mash B through animations until next menu or end. - break; - } - case 1: - { + {{battle_menu}, {end_battle}} + ); + if (ret == 1){ stream.log("[AutoStory] Starly battle complete.", COLOR_GREEN); - - // Mash through XP / level-up / dialog screens back to overworld. - OverworldWatcher overworld; - int cleanup_ret = run_until( - stream, context, - [](ProControllerContext& ctx) { - pbf_mash_button(ctx, BUTTON_B, 120000ms); - }, - { {overworld} } - ); - if (cleanup_ret != 0) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Starly battle: Overworld cleanup timed out.", stream); - } + mash_to_overworld(stream, context, "Starly battle: Overworld cleanup timed out."); return; } - default: + if (ret != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Starly battle timed out.", stream); } + stream.log("[AutoStory] Battle menu detected. Selecting move...", COLOR_BLUE); + // Select Fight, then move slot 1; loop back and mash B through the + // animations until the next menu or the end. + pbf_press_button(context, BUTTON_A, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 3000ms); } } -void battle_1_pokemon(VideoStream& stream, ProControllerContext& context) { +void battle_1_pokemon(VideoStream& stream, ProControllerContext& context){ stream.log("[Training] Finding one wild Pokemon using the catch encounter wiggle.", COLOR_BLUE); encounter_one_wild_pokemon(stream, context); // The best move on the menu, whatever the starter. @@ -301,23 +308,13 @@ void battle_1_pokemon(VideoStream& stream, ProControllerContext& context) { stream.log("[Training] One wild battle completed.", COLOR_GREEN); } -void catch_1_pokemon( - VideoStream& stream, - ProControllerContext& context -) { - stream.log("[AutoStory] Starting catch_1_pokemon helper...", COLOR_BLUE); - - encounter_one_wild_pokemon(stream, context); +namespace{ - // Catch the pokemon. - stream.log("[AutoStory] Throwing Poke Ball (manual bag open)...", COLOR_BLUE); +// Throws the first ball in the Bag at the wild Pokemon, up to ten times, +// until its receipt shows. Returns whether the battle's exit was seen. +bool throw_until_caught(VideoStream& stream, ProControllerContext& context, EndBattleWatcher& battle_ended){ ReceivePokemonDetector received_pokemon; - // Preserve exit-transition state from the ball outcome through - // receipt/experience cleanup; do not recreate it between phases. - EndBattleWatcher battle_ended; - bool caught = false; - bool exit_seen = false; - for (int throw_attempt = 0; throw_attempt < 10; throw_attempt++) { + for (int throw_attempt = 0; throw_attempt < 10; throw_attempt++){ // Open bag from battle menu (X), confirm first slot (ZL). pbf_press_button(context, BUTTON_X, 160ms, 840ms); context.wait_for_all_requests(); @@ -327,21 +324,20 @@ void catch_1_pokemon( // Wait to see if the battle menu comes back (ball failed) // or if the battle ends (caught or fainted). BattleMenuWatcher menu_back(BattleType::STANDARD, COLOR_YELLOW); - int throw_ret = run_until( + const int throw_ret = run_until( stream, context, - [](ProControllerContext& ctx) { + [](ProControllerContext& ctx){ pbf_mash_button(ctx, BUTTON_B, 120000ms); }, {{menu_back}, {received_pokemon}, {battle_ended}} ); - if (throw_ret == 1 || (throw_ret == 2 && received_pokemon.received())) { - caught = true; - exit_seen = throw_ret == 2; + if (throw_ret == 1 || (throw_ret == 2 && received_pokemon.received())){ stream.log("[AutoStory] Pokemon receipt verified.", COLOR_GREEN); - break; + context.wait_for_all_requests(); + return throw_ret == 2; } - if (throw_ret < 0 || throw_ret == 2) { + if (throw_ret < 0 || throw_ret == 2){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Catch failed: No verified Pokemon receipt before battle ended or timed out.", stream); } @@ -350,22 +346,36 @@ void catch_1_pokemon( COLOR_ORANGE ); } - context.wait_for_all_requests(); + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Catch failed: Exhausted ten ball attempts.", stream); +} - if (!caught) { - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Catch failed: Exhausted ten ball attempts.", stream); - } +} + +void catch_1_pokemon( + VideoStream& stream, + ProControllerContext& context +){ + stream.log("[AutoStory] Starting catch_1_pokemon helper...", COLOR_BLUE); + + encounter_one_wild_pokemon(stream, context); + + // Catch the pokemon. + stream.log("[AutoStory] Throwing Poke Ball (manual bag open)...", COLOR_BLUE); + // Preserve exit-transition state from the ball outcome through + // receipt/experience cleanup; do not recreate it between phases. + EndBattleWatcher battle_ended; + const bool exit_seen = throw_until_caught(stream, context, battle_ended); // Receipt alone is not an overworld return: clear experience, // automatic move learning, dex registration and nickname screens // until the same completed-black-transition watcher used in battles fires. - if (!exit_seen) { + if (!exit_seen){ stream.log("[Catch cleanup] Receipt confirmed; waiting for battle exit transition."); const int cleanup_ret = run_until(stream, context, - [](ProControllerContext& ctx) { pbf_mash_button(ctx, BUTTON_B, 120000ms); }, + [](ProControllerContext& ctx){ pbf_mash_button(ctx, BUTTON_B, 120000ms); }, {{battle_ended}}); - if (cleanup_ret != 0) { + if (cleanup_ret != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Catch cleanup: Battle exit transition did not finish.", stream); } @@ -381,37 +391,7 @@ void escape_wild_battle(VideoStream& stream, BattleType battle_type, std::chrono::seconds timeout){ EndBattleWatcher battle_end; - bool exit_transition = false; - const auto deadline = std::chrono::steady_clock::now() + timeout; - while (std::chrono::steady_clock::now() < deadline){ - pbf_press_button(context, BUTTON_B, 40ms, 200ms); - context.wait_for_all_requests(); - BattleMenuWatcher menu_ready(battle_type, COLOR_YELLOW); - const int ready = wait_until(stream, context, 3s, {{battle_end}, {menu_ready}}); - if (ready == 0){ - exit_transition = true; - break; - } - if (ready != 1){ - continue; - } - if (!mine_select_run(stream, context, battle_type)){ - continue; - } - stream.log("[Mine escape] Run input sent; watching for black-screen exit."); - const int result = run_until(stream, - context, - [](ProControllerContext& controller){ - pbf_press_button(controller, BUTTON_A, 60ms, 0ms); - pbf_wait(controller, 4s); - controller.wait_for_all_requests(); - }, - {battle_end}); - if (result == 0){ - exit_transition = true; - break; - } - } + bool exit_transition = run_until_escaped(stream, context, battle_type, timeout, battle_end); if (!exit_transition && wait_until(stream, context, 10s, {{battle_end}}) == 0){ exit_transition = true; } diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 0edb4d359d..8bf4c90bb9 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1636,6 +1636,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp + Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp From 9ff952bcac13079d717904387272ca71536d4f00 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 14:46:30 -0800 Subject: [PATCH 06/23] BDSP AutoStory new game: refuse to start from HOME, check the game is BDSP, read arrowless dialogue r1 (2026-11-01) started with no game running: closing it failed, HOME was pressed again, its cursor moved to the first tile, and FireRed was launched and played for 30 s. The intro now refuses to start with HOME in front, and before a screen only BDSP shows (its dialogue box, the rival's name list, its menu) presses A on unknown screens at most 20 times, then fails as another game. r2 stopped on Rowan's last line: a dialogue box without the "more" arrow, which ShortDialogDetector needs, read as "other", and the background's sparkles read as X changing the screen. The intro now reads such a box as dialogue, and the probe compares only the box's area. Tests: the intro's decisions with the new gate; the intro screens from CommandLineTests (PokemonBDSP/AutoStory/IntroScreens), skipped without them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 75 +++- .../PokemonBDSP_AutoStory_NewGame.cpp | 327 ++++++++++++------ .../AutoStory/PokemonBDSP_AutoStory_NewGame.h | 21 +- 3 files changed, 303 insertions(+), 120 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp index 6abbf23dde..8f5bed9469 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -41,9 +41,9 @@ class Test_IntroDecisions : public UnitTest{ IntroProgress progress; IntroStep expected; }; - const IntroProgress start{false, false}; - const IntroProgress named{true, false}; - const IntroProgress done{true, true}; + const IntroProgress start{.player_named = false, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0}; + const IntroProgress named{.player_named = true, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0}; + const IntroProgress done{.player_named = true, .rival_named = true, .game_confirmed = true, .unconfirmed_presses = 0}; const Case cases[] = { {"user-select", start, IntroStep::SELECT_USER}, {"home", done, IntroStep::FAIL_NOT_IN_GAME}, @@ -80,6 +80,73 @@ class Test_IntroDecisions : public UnitTest{ }; +// The intro gates its A presses on a BDSP screen: past the limit it is +// another game (r1, 2026-11-01, launched FireRed). +class Test_IntroGate : public UnitTest{ +public: + Test_IntroGate() + : UnitTest("PokemonBDSP::AutoStory - new-game intro: BDSP or stop") + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + struct Case{ + const char* screen; + IntroProgress progress; + IntroStep expected; + }; + const IntroProgress unconfirmed{.player_named = false, .rival_named = false, .game_confirmed = false, .unconfirmed_presses = 3}; + const IntroProgress exhausted{.player_named = false, .rival_named = false, .game_confirmed = false, + .unconfirmed_presses = INTRO_PRESSES_BEFORE_BDSP}; + const IntroProgress confirmed{.player_named = false, .rival_named = false, .game_confirmed = true, + .unconfirmed_presses = INTRO_PRESSES_BEFORE_BDSP}; + const Case cases[] = { + {"other", unconfirmed, IntroStep::ADVANCE}, + {"other", exhausted, IntroStep::FAIL_NOT_BDSP}, + {"other", confirmed, IntroStep::ADVANCE}, + }; + for (const Case& c : cases){ + if (decide_intro_step(c.screen, c.progress) != c.expected){ + return std::string("Screen \"") + c.screen + "\" after " + std::to_string(c.progress.unconfirmed_presses) + + " unconfirmed presses: unexpected step."; + } + } + for (const char* screen : {"dialog", "rival-list-tell", "rival-list-preset", "menu"}){ + if (!intro_screen_confirms_bdsp(screen)){ + return std::string("Expected \"") + screen + "\" to confirm BDSP."; + } + } + for (const char* screen : {"user-select", "home", "keyboard", "other"}){ + if (intro_screen_confirms_bdsp(screen)){ + return std::string("Expected \"") + screen + "\" not to confirm BDSP."; + } + } + return true; + } +}; + +// The intro's screens, classified: /PokemonBDSP/AutoStory/IntroScreens/ +// /*.png (in CommandLineTests), at 1080p and 720p. Among +// them r2's "dialog/020-other.png": Rowan's last line, a dialogue box with +// no "more" arrow. +class Test_IntroScreens : public UnitTest{ +public: + Test_IntroScreens() + : UnitTest("PokemonBDSP::AutoStory - new-game intro screens") + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + const std::string folder = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/IntroScreens/"; + if (!Filesystem::is_directory(Filesystem::Path(folder))){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshots not found: " + folder); + } + const size_t failed = run_intro_screen_tests(logger, folder); + if (failed != 0){ + return std::to_string(failed) + " intro screens misread; see the log."; + } + return true; + } +}; + // The Mart's doorway on the failure screenshots of r121a and r122a, kept in // the unit test resources (the CommandLineTests checkout): where it is // relative to the player, who is in the middle of the screen. @@ -258,6 +325,8 @@ class Test_SchoolLamp : public UnitTest{ void add_tests_AutoStory_Actions(UnitTestDatabase& database){ database.add(); + database.add(); + database.add(); database.add(); database.add(); database.add(); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index c081981796..b77515b129 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -4,8 +4,12 @@ * */ +#include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonFramework/ImageTools/ImageDiff.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonTools/Images/SolidColorTest.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "CommonTools/Async/InferenceRoutines.h" @@ -28,6 +32,32 @@ namespace PokemonBDSP{ using namespace std::chrono_literals; +namespace{ + +// BDSP's dialogue box. ShortDialogDetector also wants something busy at the +// box's right end, which the "more text" arrow gives; a speaker's last line +// has no arrow and read as "other" (r2, 2026-11-01: Rowan's "Now, go on, and +// leap into the world of Pokemon!"). Without the arrow: white at both ends +// and along the bottom of the text area, and the box's edge on the left. +bool intro_dialog_showing(const ImageViewRGB32& screen){ + if (ShortDialogDetector().detect(screen)){ + return true; + } + const ImageStats left_white = image_stats(extract_box_reference(screen, ImageFloatBox(0.21, 0.835, 0.008, 0.12))); + const ImageStats right_white = image_stats(extract_box_reference(screen, ImageFloatBox(0.785, 0.835, 0.008, 0.12))); + const ImageStats bottom = image_stats(extract_box_reference(screen, ImageFloatBox(0.50, 0.91, 0.29, 0.05))); + const ImageStats left_edge = image_stats(extract_box_reference(screen, ImageFloatBox(0.18, 0.835, 0.02, 0.12))); + return is_white(left_white) && is_white(right_white) && is_white(bottom) && left_edge.stddev.sum() >= 30; +} + +// The part of the screen the probe compares before and after X: the +// dialogue box's area. The intro's background sparkles everywhere above +// it, which read as X having changed the screen (r2). +const ImageFloatBox PROBE_AREA(0.15, 0.75, 0.70, 0.25); + +} + + std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ // The Switch's own screens come first: BDSP's dialogue detector also fires on them. if (console != nullptr && StartGameUserSelectDetector(*console).detect(screen)){ @@ -51,7 +81,7 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& return "menu"; } // Includes the appearance grid, whose prompt sits in a dialogue box. - if (ShortDialogDetector().detect(screen)){ + if (intro_dialog_showing(screen)){ return "dialog"; } // Language, cutscenes, fades, the bedroom with no menu open. @@ -59,19 +89,42 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& } IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ - if (screen == "user-select") return IntroStep::SELECT_USER; - if (screen == "home") return IntroStep::FAIL_NOT_IN_GAME; + if (screen == "user-select"){ + return IntroStep::SELECT_USER; + } + if (screen == "home"){ + return IntroStep::FAIL_NOT_IN_GAME; + } if (screen == "keyboard"){ return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; } - if (screen == "rival-list-tell") return IntroStep::MOVE_TO_PRESET; - if (screen == "rival-list-preset") return IntroStep::PICK_PRESET; - if (screen == "menu") return IntroStep::CLOSE_MENU_AND_FINISH; - if (screen == "dialog") return IntroStep::ADVANCE; + if (screen == "rival-list-tell"){ + return IntroStep::MOVE_TO_PRESET; + } + if (screen == "rival-list-preset"){ + return IntroStep::PICK_PRESET; + } + if (screen == "menu"){ + return IntroStep::CLOSE_MENU_AND_FINISH; + } + if (screen == "dialog"){ + return IntroStep::ADVANCE; + } // The bedroom comes only after the rival is named. Before that, every unknown - // screen (language, the appearance preview, cutscenes) takes A. - return progress.rival_named ? IntroStep::PROBE_FOR_MENU : IntroStep::ADVANCE; + // screen (language, the appearance preview, cutscenes) takes A, but only so + // many before a screen only BDSP shows: past those it is another game. + if (progress.rival_named){ + return IntroStep::PROBE_FOR_MENU; + } + if (!progress.game_confirmed && progress.unconfirmed_presses >= INTRO_PRESSES_BEFORE_BDSP){ + return IntroStep::FAIL_NOT_BDSP; + } + return IntroStep::ADVANCE; } +bool intro_screen_confirms_bdsp(const std::string& screen){ + return screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; +} + bool advance_after_probe(bool screen_changed){ return !screen_changed; } @@ -149,6 +202,141 @@ void enter_player_name(VideoStream& stream, ProControllerContext& context, StepT trace.action("Name confirmed."); } +// A run of the intro: what its steps act on and how far it has got. +struct IntroRun{ + SingleSwitchProgramEnvironment& env; + ProControllerContext& context; + StepTrace& trace; + const std::string& player_name; + IntroProgress progress; + size_t x_without_menu = 0; +}; + +void press(IntroRun& run, Button button){ + pbf_press_button(run.context, button, 80ms, 0ms); +} + +void close_menu_and_finish(IntroRun& run){ + run.trace.action("Main menu is open: the player is in control in the bedroom. Closing it."); + press(run, BUTTON_B); + pbf_wait(run.context, 1000ms); + run.context.wait_for_all_requests(); + VideoSnapshot screen_now = run.env.console.video().snapshot(); + if (!screen_now || MenuDetector().detect(screen_now)){ + run.trace.fail("The main menu did not close."); + } +} + +void probe_for_menu(IntroRun& run){ + VideoStream& stream = run.env.console; + run.context.wait_for_all_requests(); + VideoSnapshot before = stream.video().snapshot(); + press(run, BUTTON_X); + run.context.wait_for_all_requests(); + MenuWatcher menu; + if (wait_until(stream, run.context, 3s, {{menu}}) == 0){ + return; // The next read sees the menu and finishes. + } + VideoSnapshot after = stream.video().snapshot(); + const bool changed = !before || !after || ImageMatch::pixel_RMSD( + extract_box_reference(*before.frame, PROBE_AREA), extract_box_reference(*after.frame, PROBE_AREA) + ) > 10; + if (!advance_after_probe(changed)){ + run.trace.action("X changed the screen but no known menu opened; not pressing A."); + if (++run.x_without_menu >= 10){ + run.trace.fail("X keeps changing the screen without a known menu opening."); + } + return; + } + run.x_without_menu = 0; + pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); +} + +// Carries out what decide_intro_step() chose for `screen`. +void act(IntroRun& run, const char* screen){ + if (intro_screen_confirms_bdsp(screen)){ + run.progress.game_confirmed = true; + } + switch (decide_intro_step(screen, run.progress)){ + case IntroStep::SELECT_USER: + // The cursor starts on the user that launched the game. + press(run, BUTTON_A); + pbf_wait(run.context, 1000ms); + return; + case IntroStep::FAIL_NOT_IN_GAME: + run.trace.fail("The Switch HOME menu is showing: the game is not running in front."); + case IntroStep::FAIL_NOT_BDSP: + run.trace.fail("The game in front is not BDSP: no BDSP screen after " + + std::to_string(run.progress.unconfirmed_presses) + " presses on unknown screens."); + case IntroStep::ENTER_PLAYER_NAME: + enter_player_name(run.env.console, run.context, run.trace, run.player_name); + run.progress.player_named = true; + return; + case IntroStep::CANCEL_RIVAL_KEYBOARD: + // Opened from "I'll tell you..."; inputs during the slide-in are dropped. + pbf_wait(run.context, 800ms); + press(run, BUTTON_X); + return; + case IntroStep::MOVE_TO_PRESET: + pbf_press_dpad(run.context, DPAD_DOWN, 100ms, 300ms); + return; + case IntroStep::PICK_PRESET: + press(run, BUTTON_A); + pbf_wait(run.context, 500ms); + run.progress.rival_named = true; + return; + case IntroStep::CLOSE_MENU_AND_FINISH: + close_menu_and_finish(run); + return; + case IntroStep::ADVANCE: + if (!run.progress.game_confirmed){ + run.progress.unconfirmed_presses++; + } + pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); + return; + case IntroStep::PROBE_FOR_MENU: + probe_for_menu(run); + return; + } +} + +// The machine's states: the screens classify_intro_screen() tells apart, with +// the keyboard split by whose name it asks for. Each state's step is act(). +std::vector intro_states(IntroRun& run){ + auto screen_is = [&run](const ImageViewRGB32& screen, const char* name){ + return classify_intro_screen(&run.env.console, screen) == name; + }; + return { + {"user-select", + [=](const ImageViewRGB32& s){ return screen_is(s, "user-select"); }, + [&run]{ act(run, "user-select"); }, {}, 3}, + {"home", + [=](const ImageViewRGB32& s){ return screen_is(s, "home"); }, + [&run]{ act(run, "home"); }, {}, 1}, + {"player-keyboard", + [=, &run](const ImageViewRGB32& s){ return !run.progress.player_named && screen_is(s, "keyboard"); }, + [&run]{ act(run, "keyboard"); }, {"dialog", "other"}, 1}, + {"rival-keyboard", + [=, &run](const ImageViewRGB32& s){ return run.progress.player_named && screen_is(s, "keyboard"); }, + [&run]{ act(run, "keyboard"); }, {"rival-list-tell", "dialog", "other"}, 3}, + {"rival-list-tell", + [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, + [&run]{ act(run, "rival-list-tell"); }, {"rival-list-preset", "other"}, 6}, + {"rival-list-preset", + [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-preset"); }, + [&run]{ act(run, "rival-list-preset"); }, {"dialog", "other"}, 3}, + {"menu", + [=](const ImageViewRGB32& s){ return screen_is(s, "menu"); }, + [&run]{ act(run, "menu"); }, {}, 2, true}, + {"dialog", + [=](const ImageViewRGB32& s){ return screen_is(s, "dialog"); }, + [&run]{ act(run, "dialog"); }, {}, 400}, + {"other", + [](const ImageViewRGB32&){ return true; }, + [&run]{ act(run, "other"); }, {}, 300, false, 60}, + }; +} + } @@ -161,6 +349,19 @@ void play_new_game_intro( VideoStream& stream = env.console; StepTrace trace(stream, "Intro"); + // The restart below reopens the game HOME's cursor is on, which is the + // game running in front only while it runs: with none running (r1, + // 2026-11-01), closing it fails, HOME is pressed again, the cursor goes + // to the first tile, and another game was launched. So BDSP must be + // running in front when the intro starts. + context.wait_for_all_requests(); + VideoSnapshot start = stream.video().snapshot(); + if (start && HomeMenuDetector(env.console, COLOR_RED).detect(start)){ + throw UserSetupError(env.console, + "Start a new game with BDSP running in front, not from the Switch's HOME menu: " + "the intro restarts the game in front."); + } + // Only close and reopen the game: the states below take it from the user select // on. reset_game_from_home() waits for a loaded save's black screen to end, and a // new game's first screen is a dialog on black, so it would sit there until its @@ -169,112 +370,8 @@ void play_new_game_intro( go_home(env.console, context); from_home_close_and_reopen_game(env.console, context, true); - IntroProgress progress; - size_t x_without_menu = 0; - auto press = [&](Button button){ - pbf_press_button(context, button, 80ms, 0ms); - }; - - // Carries out what decide_intro_step() chose for `screen`. - auto act = [&](const char* screen){ - switch (decide_intro_step(screen, progress)){ - case IntroStep::SELECT_USER: - // The cursor starts on the user that launched the game. - press(BUTTON_A); - pbf_wait(context, 1000ms); - return; - case IntroStep::FAIL_NOT_IN_GAME: - trace.fail("The Switch HOME menu is showing: the game is not running in front."); - case IntroStep::ENTER_PLAYER_NAME: - enter_player_name(stream, context, trace, player_name); - progress.player_named = true; - return; - case IntroStep::CANCEL_RIVAL_KEYBOARD: - // Opened from "I'll tell you..."; inputs during the slide-in are dropped. - pbf_wait(context, 800ms); - press(BUTTON_X); - return; - case IntroStep::MOVE_TO_PRESET: - pbf_press_dpad(context, DPAD_DOWN, 100ms, 300ms); - return; - case IntroStep::PICK_PRESET: - press(BUTTON_A); - pbf_wait(context, 500ms); - progress.rival_named = true; - return; - case IntroStep::CLOSE_MENU_AND_FINISH:{ - trace.action("Main menu is open: the player is in control in the bedroom. Closing it."); - press(BUTTON_B); - pbf_wait(context, 1000ms); - context.wait_for_all_requests(); - VideoSnapshot screen_now = stream.video().snapshot(); - if (!screen_now || MenuDetector().detect(screen_now)){ - trace.fail("The main menu did not close."); - } - return; - } - case IntroStep::ADVANCE: - pbf_press_button(context, BUTTON_A, 60ms, 400ms); - return; - case IntroStep::PROBE_FOR_MENU:{ - context.wait_for_all_requests(); - VideoSnapshot before = stream.video().snapshot(); - press(BUTTON_X); - context.wait_for_all_requests(); - MenuWatcher menu; - if (wait_until(stream, context, 3s, {{menu}}) == 0){ - return; // The next read sees the menu and finishes. - } - VideoSnapshot after = stream.video().snapshot(); - const bool changed = !before || !after || ImageMatch::pixel_RMSD(before, after) > 10; - if (!advance_after_probe(changed)){ - trace.action("X changed the screen but no known menu opened; not pressing A."); - if (++x_without_menu >= 10){ - trace.fail("X keeps changing the screen without a known menu opening."); - } - return; - } - x_without_menu = 0; - pbf_press_button(context, BUTTON_A, 60ms, 400ms); - return; - } - } - }; - - auto screen_is = [&](const ImageViewRGB32& screen, const char* name){ - return classify_intro_screen(&env.console, screen) == name; - }; - // The machine's states: the screens classify_intro_screen() tells apart, with the - // keyboard split by whose name it asks for. Each state's step is act(). - const std::vector states{ - {"user-select", - [&](const ImageViewRGB32& s){ return screen_is(s, "user-select"); }, - [&]{ act("user-select"); }, {}, 3}, - {"home", - [&](const ImageViewRGB32& s){ return screen_is(s, "home"); }, - [&]{ act("home"); }, {}, 1}, - {"player-keyboard", - [&](const ImageViewRGB32& s){ return !progress.player_named && screen_is(s, "keyboard"); }, - [&]{ act("keyboard"); }, {"dialog", "other"}, 1}, - {"rival-keyboard", - [&](const ImageViewRGB32& s){ return progress.player_named && screen_is(s, "keyboard"); }, - [&]{ act("keyboard"); }, {"rival-list-tell", "dialog", "other"}, 3}, - {"rival-list-tell", - [&](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, - [&]{ act("rival-list-tell"); }, {"rival-list-preset", "other"}, 6}, - {"rival-list-preset", - [&](const ImageViewRGB32& s){ return screen_is(s, "rival-list-preset"); }, - [&]{ act("rival-list-preset"); }, {"dialog", "other"}, 3}, - {"menu", - [&](const ImageViewRGB32& s){ return screen_is(s, "menu"); }, - [&]{ act("menu"); }, {}, 2, true}, - {"dialog", - [&](const ImageViewRGB32& s){ return screen_is(s, "dialog"); }, - [&]{ act("dialog"); }, {}, 400}, - {"other", - [](const ImageViewRGB32&){ return true; }, - [&]{ act("other"); }, {}, 300, false, 60}, - }; + IntroRun run{env, context, trace, player_name, {}, 0}; + const std::vector states = intro_states(run); run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); trace.action("Intro done."); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h index eb7d4f01ca..a48da130e9 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h @@ -10,6 +10,7 @@ #ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H #define PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H +#include #include #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" @@ -31,6 +32,7 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& enum class IntroStep{ SELECT_USER, // A on the Switch's user select FAIL_NOT_IN_GAME, // HOME is showing + FAIL_NOT_BDSP, // no BDSP screen after INTRO_PRESSES_BEFORE_BDSP presses ENTER_PLAYER_NAME, // type the name on the first keyboard CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list MOVE_TO_PRESET, // Down: off "I'll tell you..." @@ -42,7 +44,19 @@ enum class IntroStep{ struct IntroProgress{ bool player_named = false; bool rival_named = false; + // A screen only BDSP shows has been seen (intro_screen_confirms_bdsp()). + bool game_confirmed = false; + // A presses on unknown screens before that: the title, the language. + size_t unconfirmed_presses = 0; }; +// Unknown screens pressed through before a BDSP screen must show: the title +// and the language select take a few. Another game launched by mistake +// (r1, 2026-11-01: FireRed) fails here instead of being played. +inline constexpr size_t INTRO_PRESSES_BEFORE_BDSP = 20; +// Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: +// its dialogue box, the rival's name list, its main menu. The keyboard and +// the user select are the Switch's. +bool intro_screen_confirms_bdsp(const std::string& screen); IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); // After PROBE_FOR_MENU found no menu: whether A may follow. Only when X changed @@ -53,8 +67,11 @@ bool advance_after_probe(bool screen_changed); // 1280 px wide. Logs each mismatch; returns the number of them. size_t run_intro_screen_tests(Logger& logger, const std::string& folder); -// From the title screen or any intro screen of a game with no save. Restarts the -// game first, so it always starts from the title. Names the player `player_name` +// From the title screen or any intro screen of a game with no save, with BDSP +// running in front: from HOME it throws UserSetupError, since the restart would +// launch whatever tile HOME's cursor is on. Restarts the game first, so it +// always starts from the title, and fails if no BDSP screen shows within +// INTRO_PRESSES_BEFORE_BDSP presses. Names the player `player_name` // and picks the rival's default name (Barry) from the list. Each step reads the // screen and waits for the expected next one; an unexpected screen fails with a // screenshot and the recent screens. From 6158441f5e186437e92feb6ef2dd0d9a576b052d Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 14:47:12 -0800 Subject: [PATCH 07/23] System keyboard: the cursor's walk to a key as its own function, unchanged Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../NintendoSwitch_SystemKeyboardDetector.cpp | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp index a7ec920307..7403cfac90 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -93,6 +93,44 @@ bool system_keyboard_field_has_text(const ImageViewRGB32& screen){ } +namespace{ + +// Moves the keyboard's cursor onto (target_row, target_col). A button +// pressed while the keyboard slides in is ignored, so read the cursor again +// after each move until it is on the target. Whether it got there. +bool move_keyboard_cursor(VideoStream& stream, ProControllerContext& context, int target_row, int target_col){ + for (size_t attempt = 0; attempt < 10; attempt++){ + context.wait_for_all_requests(); + int row = -1; + int col = -1; + VideoSnapshot screen = stream.video().snapshot(); + if (!screen || !find_system_keyboard_cursor(screen, row, col)){ + pbf_wait(context, 300ms); + continue; + } + if (row == target_row && col == target_col){ + return true; + } + for (; row < target_row; row++){ + pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms); + } + for (; row > target_row; row--){ + pbf_press_dpad(context, DPAD_UP, 100ms, 200ms); + } + for (; col < target_col; col++){ + pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms); + } + for (; col > target_col; col--){ + pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms); + } + pbf_wait(context, 300ms); + } + return false; +} + +} + + void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text){ for (char ch : text){ const char key = (char)std::tolower((unsigned char)ch); @@ -112,38 +150,7 @@ void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, stream ); } - - // A button pressed while the keyboard slides in is ignored, so read the cursor - // again after each move until it is on the target. - bool arrived = false; - for (size_t attempt = 0; attempt < 10 && !arrived; attempt++){ - context.wait_for_all_requests(); - int row = -1; - int col = -1; - VideoSnapshot screen = stream.video().snapshot(); - if (!screen || !find_system_keyboard_cursor(screen, row, col)){ - pbf_wait(context, 300ms); - continue; - } - if (row == target_row && col == target_col){ - arrived = true; - break; - } - for (; row < target_row; row++){ - pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms); - } - for (; row > target_row; row--){ - pbf_press_dpad(context, DPAD_UP, 100ms, 200ms); - } - for (; col < target_col; col++){ - pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms); - } - for (; col > target_col; col--){ - pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms); - } - pbf_wait(context, 300ms); - } - if (!arrived){ + if (!move_keyboard_cursor(stream, context, target_row, target_col)){ OperationFailedExceptionWithScreenshot::fire( ErrorReportMode::SEND_ERROR_REPORT, std::string("System keyboard: the cursor did not reach '") + key + "'.", From 682fdadebfa73107527a0dd0c00005febc64cf9e Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 14:48:23 -0800 Subject: [PATCH 08/23] BDSP AutoStory new game: the arrowless box's right end below the first line, which a long line reaches Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index b77515b129..e10ca25723 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -37,14 +37,16 @@ namespace{ // BDSP's dialogue box. ShortDialogDetector also wants something busy at the // box's right end, which the "more text" arrow gives; a speaker's last line // has no arrow and read as "other" (r2, 2026-11-01: Rowan's "Now, go on, and -// leap into the world of Pokemon!"). Without the arrow: white at both ends -// and along the bottom of the text area, and the box's edge on the left. +// leap into the world of Pokemon!"). Without the arrow: white at the text +// area's left end, along its bottom and at its right end below the first +// line, and the box's edge on the left. bool intro_dialog_showing(const ImageViewRGB32& screen){ if (ShortDialogDetector().detect(screen)){ return true; } const ImageStats left_white = image_stats(extract_box_reference(screen, ImageFloatBox(0.21, 0.835, 0.008, 0.12))); - const ImageStats right_white = image_stats(extract_box_reference(screen, ImageFloatBox(0.785, 0.835, 0.008, 0.12))); + // Below the first line's end: a long line reaches it (r2's ran to 0.79). + const ImageStats right_white = image_stats(extract_box_reference(screen, ImageFloatBox(0.785, 0.915, 0.008, 0.04))); const ImageStats bottom = image_stats(extract_box_reference(screen, ImageFloatBox(0.50, 0.91, 0.29, 0.05))); const ImageStats left_edge = image_stats(extract_box_reference(screen, ImageFloatBox(0.18, 0.835, 0.02, 0.12))); return is_white(left_white) && is_white(right_white) && is_white(bottom) && left_edge.stddev.sum() >= 30; From 50aaf4d9774acb327c2c130bcb796219a7a713c8 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 14:52:37 -0800 Subject: [PATCH 09/23] BDSP AutoStory new game: B on unknown screens after the rival's name, then X; only the menu ends the intro Screens the classifier misses no longer stop the intro: once the rival is named, each unknown screen takes B (up to INTRO_B_BEFORE_X), then X checks for the menu, up to INTRO_X_WITHOUT_MENU times. Dialogue takes B too. The before/after comparison of the X probe is gone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 62 ++++++++++++---- .../PokemonBDSP_AutoStory_NewGame.cpp | 70 ++++++++++--------- .../AutoStory/PokemonBDSP_AutoStory_NewGame.h | 21 ++++-- 3 files changed, 105 insertions(+), 48 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp index 8f5bed9469..a3acbac653 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -11,6 +11,7 @@ #include #include +#include #include "Common/Cpp/Filesystem/Filesystem.h" #include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonFramework/GlobalAutoPaths.h" @@ -41,9 +42,9 @@ class Test_IntroDecisions : public UnitTest{ IntroProgress progress; IntroStep expected; }; - const IntroProgress start{.player_named = false, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0}; - const IntroProgress named{.player_named = true, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0}; - const IntroProgress done{.player_named = true, .rival_named = true, .game_confirmed = true, .unconfirmed_presses = 0}; + const IntroProgress start{.player_named = false, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0, .b_presses = 0}; + const IntroProgress named{.player_named = true, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0, .b_presses = 0}; + const IntroProgress done{.player_named = true, .rival_named = true, .game_confirmed = true, .unconfirmed_presses = 0, .b_presses = 0}; const Case cases[] = { {"user-select", start, IntroStep::SELECT_USER}, {"home", done, IntroStep::FAIL_NOT_IN_GAME}, @@ -54,11 +55,11 @@ class Test_IntroDecisions : public UnitTest{ {"menu", done, IntroStep::CLOSE_MENU_AND_FINISH}, // The appearance grid has its prompt in a dialogue box. {"dialog", start, IntroStep::ADVANCE}, - {"dialog", done, IntroStep::ADVANCE}, + {"dialog", done, IntroStep::ADVANCE_WITH_B}, // Language and the appearance preview: no X before the bedroom is possible. {"other", start, IntroStep::ADVANCE}, {"other", named, IntroStep::ADVANCE}, - {"other", done, IntroStep::PROBE_FOR_MENU}, + {"other", done, IntroStep::CLEAR_WITH_B}, }; for (const Case& c : cases){ const IntroStep actual = decide_intro_step(c.screen, c.progress); @@ -69,11 +70,47 @@ class Test_IntroDecisions : public UnitTest{ std::to_string((int)actual) + ", expected " + std::to_string((int)c.expected) + "."; } } - if (advance_after_probe(true)){ - return "A after X changed the screen would pick from whatever X opened."; + return true; + } +}; + + +// After the rival is named, unknown screens take INTRO_B_BEFORE_X B presses, +// then X; only the menu ends the intro (r2: Rowan's last line, misread). +class Test_IntroBeforeMenu : public UnitTest{ +public: + Test_IntroBeforeMenu() + : UnitTest("PokemonBDSP::AutoStory - new-game intro: B on unknown screens, then X for the menu") + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + IntroProgress progress{.player_named = true, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0, .b_presses = 0}; + record_intro_step(progress, "rival-list-preset", decide_intro_step("rival-list-preset", progress)); + if (!progress.rival_named){ + return "Picking the preset name must count the rival as named."; + } + std::vector expected; + for (size_t round = 0; round < 2; round++){ + expected.insert(expected.end(), INTRO_B_BEFORE_X, IntroStep::CLEAR_WITH_B); + expected.push_back(IntroStep::PROBE_FOR_MENU); + } + // A dialogue line in between takes B without counting. + expected.push_back(IntroStep::ADVANCE_WITH_B); + expected.push_back(IntroStep::CLEAR_WITH_B); + expected.push_back(IntroStep::CLOSE_MENU_AND_FINISH); + const size_t others = 2 * (INTRO_B_BEFORE_X + 1); + for (size_t c = 0; c < expected.size(); c++){ + const char* screen = c < others ? "other" : c == others ? "dialog" : c == others + 1 ? "other" : "menu"; + const IntroStep step = decide_intro_step(screen, progress); + record_intro_step(progress, screen, step); + if (step != expected[c]){ + return "Screen " + std::to_string(c) + " (\"" + screen + "\"): step " + + std::to_string((int)step) + ", expected " + std::to_string((int)expected[c]) + "."; + } } - if (!advance_after_probe(false)){ - return "A must advance a screen X left unchanged."; + if (decide_intro_step("other", IntroProgress{.player_named = true, .rival_named = true, .game_confirmed = true, + .unconfirmed_presses = 0, .b_presses = INTRO_B_BEFORE_X}) != IntroStep::PROBE_FOR_MENU){ + return "X must follow INTRO_B_BEFORE_X B presses."; } return true; } @@ -94,11 +131,11 @@ class Test_IntroGate : public UnitTest{ IntroProgress progress; IntroStep expected; }; - const IntroProgress unconfirmed{.player_named = false, .rival_named = false, .game_confirmed = false, .unconfirmed_presses = 3}; + const IntroProgress unconfirmed{.player_named = false, .rival_named = false, .game_confirmed = false, .unconfirmed_presses = 3, .b_presses = 0}; const IntroProgress exhausted{.player_named = false, .rival_named = false, .game_confirmed = false, - .unconfirmed_presses = INTRO_PRESSES_BEFORE_BDSP}; + .unconfirmed_presses = INTRO_PRESSES_BEFORE_BDSP, .b_presses = 0}; const IntroProgress confirmed{.player_named = false, .rival_named = false, .game_confirmed = true, - .unconfirmed_presses = INTRO_PRESSES_BEFORE_BDSP}; + .unconfirmed_presses = INTRO_PRESSES_BEFORE_BDSP, .b_presses = 0}; const Case cases[] = { {"other", unconfirmed, IntroStep::ADVANCE}, {"other", exhausted, IntroStep::FAIL_NOT_BDSP}, @@ -326,6 +363,7 @@ class Test_SchoolLamp : public UnitTest{ void add_tests_AutoStory_Actions(UnitTestDatabase& database){ database.add(); database.add(); + database.add(); database.add(); database.add(); database.add(); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index e10ca25723..fb135ad7d3 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -52,11 +52,6 @@ bool intro_dialog_showing(const ImageViewRGB32& screen){ return is_white(left_white) && is_white(right_white) && is_white(bottom) && left_edge.stddev.sum() >= 30; } -// The part of the screen the probe compares before and after X: the -// dialogue box's area. The intro's background sparkles everywhere above -// it, which read as X having changed the screen (r2). -const ImageFloatBox PROBE_AREA(0.15, 0.75, 0.70, 0.25); - } @@ -110,13 +105,13 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog return IntroStep::CLOSE_MENU_AND_FINISH; } if (screen == "dialog"){ - return IntroStep::ADVANCE; + return progress.rival_named ? IntroStep::ADVANCE_WITH_B : IntroStep::ADVANCE; } // The bedroom comes only after the rival is named. Before that, every unknown // screen (language, the appearance preview, cutscenes) takes A, but only so // many before a screen only BDSP shows: past those it is another game. if (progress.rival_named){ - return IntroStep::PROBE_FOR_MENU; + return progress.b_presses < INTRO_B_BEFORE_X ? IntroStep::CLEAR_WITH_B : IntroStep::PROBE_FOR_MENU; } if (!progress.game_confirmed && progress.unconfirmed_presses >= INTRO_PRESSES_BEFORE_BDSP){ return IntroStep::FAIL_NOT_BDSP; @@ -127,8 +122,31 @@ bool intro_screen_confirms_bdsp(const std::string& screen){ return screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; } -bool advance_after_probe(bool screen_changed){ - return !screen_changed; +void record_intro_step(IntroProgress& progress, const std::string& screen, IntroStep step){ + if (intro_screen_confirms_bdsp(screen)){ + progress.game_confirmed = true; + } + switch (step){ + case IntroStep::ENTER_PLAYER_NAME: + progress.player_named = true; + return; + case IntroStep::ADVANCE: + if (!progress.game_confirmed){ + progress.unconfirmed_presses++; + } + return; + case IntroStep::PICK_PRESET: + progress.rival_named = true; + return; + case IntroStep::CLEAR_WITH_B: + progress.b_presses++; + return; + case IntroStep::PROBE_FOR_MENU: + progress.b_presses = 0; + return; + default: + return; + } } size_t run_intro_screen_tests(Logger& logger, const std::string& folder){ @@ -229,37 +247,26 @@ void close_menu_and_finish(IntroRun& run){ } } +// X, then the menu or B presses again: whatever X opened instead, B closes. void probe_for_menu(IntroRun& run){ VideoStream& stream = run.env.console; - run.context.wait_for_all_requests(); - VideoSnapshot before = stream.video().snapshot(); press(run, BUTTON_X); run.context.wait_for_all_requests(); MenuWatcher menu; if (wait_until(stream, run.context, 3s, {{menu}}) == 0){ return; // The next read sees the menu and finishes. } - VideoSnapshot after = stream.video().snapshot(); - const bool changed = !before || !after || ImageMatch::pixel_RMSD( - extract_box_reference(*before.frame, PROBE_AREA), extract_box_reference(*after.frame, PROBE_AREA) - ) > 10; - if (!advance_after_probe(changed)){ - run.trace.action("X changed the screen but no known menu opened; not pressing A."); - if (++run.x_without_menu >= 10){ - run.trace.fail("X keeps changing the screen without a known menu opening."); - } - return; + run.trace.action("X opened no menu; B again before the next X."); + if (++run.x_without_menu >= INTRO_X_WITHOUT_MENU){ + run.trace.fail("No menu after " + std::to_string(run.x_without_menu) + " X presses: the player never got control."); } - run.x_without_menu = 0; - pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); } // Carries out what decide_intro_step() chose for `screen`. void act(IntroRun& run, const char* screen){ - if (intro_screen_confirms_bdsp(screen)){ - run.progress.game_confirmed = true; - } - switch (decide_intro_step(screen, run.progress)){ + const IntroStep step = decide_intro_step(screen, run.progress); + record_intro_step(run.progress, screen, step); + switch (step){ case IntroStep::SELECT_USER: // The cursor starts on the user that launched the game. press(run, BUTTON_A); @@ -272,7 +279,6 @@ void act(IntroRun& run, const char* screen){ std::to_string(run.progress.unconfirmed_presses) + " presses on unknown screens."); case IntroStep::ENTER_PLAYER_NAME: enter_player_name(run.env.console, run.context, run.trace, run.player_name); - run.progress.player_named = true; return; case IntroStep::CANCEL_RIVAL_KEYBOARD: // Opened from "I'll tell you..."; inputs during the slide-in are dropped. @@ -285,17 +291,17 @@ void act(IntroRun& run, const char* screen){ case IntroStep::PICK_PRESET: press(run, BUTTON_A); pbf_wait(run.context, 500ms); - run.progress.rival_named = true; return; case IntroStep::CLOSE_MENU_AND_FINISH: close_menu_and_finish(run); return; case IntroStep::ADVANCE: - if (!run.progress.game_confirmed){ - run.progress.unconfirmed_presses++; - } pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); return; + case IntroStep::ADVANCE_WITH_B: + case IntroStep::CLEAR_WITH_B: + pbf_press_button(run.context, BUTTON_B, 60ms, 400ms); + return; case IntroStep::PROBE_FOR_MENU: probe_for_menu(run); return; diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h index a48da130e9..cbd6366f53 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h @@ -39,6 +39,8 @@ enum class IntroStep{ PICK_PRESET, // A on a preset name CLOSE_MENU_AND_FINISH, // B: the menu opened in the bedroom, the intro is over ADVANCE, // A: dialogue, language, appearance, cutscenes + ADVANCE_WITH_B, // B: dialogue once the rival is named + CLEAR_WITH_B, // B: an unknown screen once the rival is named PROBE_FOR_MENU, // X, to see whether the player is in control }; struct IntroProgress{ @@ -48,6 +50,8 @@ struct IntroProgress{ bool game_confirmed = false; // A presses on unknown screens before that: the title, the language. size_t unconfirmed_presses = 0; + // B presses on unknown screens since the last X. + size_t b_presses = 0; }; // Unknown screens pressed through before a BDSP screen must show: the title // and the language select take a few. Another game launched by mistake @@ -57,11 +61,20 @@ inline constexpr size_t INTRO_PRESSES_BEFORE_BDSP = 20; // its dialogue box, the rival's name list, its main menu. The keyboard and // the user select are the Switch's. bool intro_screen_confirms_bdsp(const std::string& screen); -IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); -// After PROBE_FOR_MENU found no menu: whether A may follow. Only when X changed -// nothing, since A on something X opened would pick from it (the Bag). -bool advance_after_probe(bool screen_changed); +// Once the rival is named, the intro only has dialogue and cutscenes left +// before the bedroom, and screens the classifier misses (r2: a dialogue box +// without its arrow). Each unknown screen takes B, which advances dialogue, +// never starts a talk and does nothing in the bedroom; after this many, X +// checks for the menu, and only the menu ends the intro. +inline constexpr size_t INTRO_B_BEFORE_X = 5; +// X presses that opened no menu before the intro gives up: with +// INTRO_B_BEFORE_X B presses between them, about two minutes. +inline constexpr size_t INTRO_X_WITHOUT_MENU = 20; + +IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); +// Counts `step`, taken on `screen`, in `progress`. +void record_intro_step(IntroProgress& progress, const std::string& screen, IntroStep step); // Checks classify_intro_screen() on //*.png at 1920 and // 1280 px wide. Logs each mismatch; returns the number of them. From c60b30a848958d7eb9992fc386a96536335aa317 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 14:57:11 -0800 Subject: [PATCH 10/23] BDSP AutoStory actions: walk-v4's stick positions, StoryWorld free functions and story_language() Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp | 2 +- .../Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp index 868a407580..d96cc7da5d 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp @@ -188,7 +188,7 @@ std::string read_move_name(const ImageViewRGB32& screen, const ImageFloatBox& bu } } std::string name; - for (unsigned char ch : story_reads_text() ? OCR::tesseract_ocr_read(g_story_language, text) : std::string()){ + for (unsigned char ch : story_reads_text() ? OCR::tesseract_ocr_read(story_language(), text) : std::string()){ if (std::isalpha(ch)){ name.push_back((char)std::tolower(ch)); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp index ce23b9eadf..e4b20bd0e0 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp @@ -174,7 +174,7 @@ bool walk_into_door( // The Center's fade is too light to read as black (r126 runs 2 and 3: a // screen change of 49 at 400 ms), so its inside, once it shows, stops // the walk too. - const WalkStop stop = walk_until(stream, context, run_from_wild_battle, 0, 1, limit, label + ": into the door", + const WalkStop stop = walk_until(stream, context, run_from_wild_battle, {0, 1}, limit, label + ": into the door", [](const ImageViewRGB32& screen){ return pokemon_center_inside(screen); }).stop; if (stop == WalkStop::TRANSITION || stop == WalkStop::ARRIVED){ pbf_wait(context, 2000ms); @@ -310,7 +310,7 @@ std::vector tiles_by_lit_doorway(const LitDoorway& doorway, const Story std::vector tiles; for (int y = guess.y - 1; y <= guess.y + 1; y++){ for (int x = guess.x - 1; x <= guess.x + 1; x++){ - if (grid.standable(TilePos{x, y})){ + if (standable(grid, TilePos{x, y})){ tiles.push_back(TilePos{x, y}); } } @@ -420,7 +420,7 @@ void heal_at_pokemon_center( // The nurse's counter is straight ahead of the door. route_status(stream, label + ": talking to the nurse."); - if (walk_to_talk(stream, context, run_from_wild_battle, 0, 1, 6000ms, label + ": to the nurse").stop != WalkStop::DIALOG){ + if (walk_to_talk(stream, context, run_from_wild_battle, {0, 1}, 6000ms, label + ": to the nurse").stop != WalkStop::DIALOG){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, label + ": nurse not reached.", stream); } From 0050abfb57fbfe9c536a27ec6b230133bea0c96d Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 15:01:23 -0800 Subject: [PATCH 11/23] BDSP AutoStory new game: wait out the black boot, then a time limit for the wrong-game guard r3 spent the guard's 20 presses on BDSP's black boot (about 25 s after the restart) and stopped. Now the intro waits for a screen that is not black before it starts (up to INTRO_BOOT_LIMIT), black screens are their own class that takes no press, and the guard counts INTRO_TIME_BEFORE_BDSP from the first screen that is not black instead of presses. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 102 ++++++++++++------ .../PokemonBDSP_AutoStory_NewGame.cpp | 79 ++++++++++---- .../AutoStory/PokemonBDSP_AutoStory_NewGame.h | 41 ++++--- 3 files changed, 160 insertions(+), 62 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp index a3acbac653..6bb3ca56e6 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -30,6 +30,16 @@ namespace PokemonBDSP{ namespace{ +// How far the intro has got, with BDSP confirmed: 1, the player named; +// 2, the rival too. +IntroProgress intro_progress(size_t names_given){ + IntroProgress progress; + progress.player_named = names_given >= 1; + progress.rival_named = names_given >= 2; + progress.game_confirmed = true; + return progress; +} + class Test_IntroDecisions : public UnitTest{ public: Test_IntroDecisions() @@ -42,9 +52,9 @@ class Test_IntroDecisions : public UnitTest{ IntroProgress progress; IntroStep expected; }; - const IntroProgress start{.player_named = false, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0, .b_presses = 0}; - const IntroProgress named{.player_named = true, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0, .b_presses = 0}; - const IntroProgress done{.player_named = true, .rival_named = true, .game_confirmed = true, .unconfirmed_presses = 0, .b_presses = 0}; + const IntroProgress start = intro_progress(0); + const IntroProgress named = intro_progress(1); + const IntroProgress done = intro_progress(2); const Case cases[] = { {"user-select", start, IntroStep::SELECT_USER}, {"home", done, IntroStep::FAIL_NOT_IN_GAME}, @@ -56,10 +66,13 @@ class Test_IntroDecisions : public UnitTest{ // The appearance grid has its prompt in a dialogue box. {"dialog", start, IntroStep::ADVANCE}, {"dialog", done, IntroStep::ADVANCE_WITH_B}, - // Language and the appearance preview: no X before the bedroom is possible. + // Logos, the title, language and the appearance preview: no X + // before the bedroom is possible. {"other", start, IntroStep::ADVANCE}, {"other", named, IntroStep::ADVANCE}, {"other", done, IntroStep::CLEAR_WITH_B}, + {"black", start, IntroStep::WAIT}, + {"black", done, IntroStep::WAIT}, }; for (const Case& c : cases){ const IntroStep actual = decide_intro_step(c.screen, c.progress); @@ -84,8 +97,8 @@ class Test_IntroBeforeMenu : public UnitTest{ {} virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - IntroProgress progress{.player_named = true, .rival_named = false, .game_confirmed = true, .unconfirmed_presses = 0, .b_presses = 0}; - record_intro_step(progress, "rival-list-preset", decide_intro_step("rival-list-preset", progress)); + IntroProgress progress = intro_progress(1); + record_intro_step(progress, decide_intro_step("rival-list-preset", progress)); if (!progress.rival_named){ return "Picking the preset name must count the rival as named."; } @@ -102,14 +115,15 @@ class Test_IntroBeforeMenu : public UnitTest{ for (size_t c = 0; c < expected.size(); c++){ const char* screen = c < others ? "other" : c == others ? "dialog" : c == others + 1 ? "other" : "menu"; const IntroStep step = decide_intro_step(screen, progress); - record_intro_step(progress, screen, step); + record_intro_step(progress, step); if (step != expected[c]){ return "Screen " + std::to_string(c) + " (\"" + screen + "\"): step " + std::to_string((int)step) + ", expected " + std::to_string((int)expected[c]) + "."; } } - if (decide_intro_step("other", IntroProgress{.player_named = true, .rival_named = true, .game_confirmed = true, - .unconfirmed_presses = 0, .b_presses = INTRO_B_BEFORE_X}) != IntroStep::PROBE_FOR_MENU){ + IntroProgress pressed = intro_progress(2); + pressed.b_presses = INTRO_B_BEFORE_X; + if (decide_intro_step("other", pressed) != IntroStep::PROBE_FOR_MENU){ return "X must follow INTRO_B_BEFORE_X B presses."; } return true; @@ -117,8 +131,10 @@ class Test_IntroBeforeMenu : public UnitTest{ }; -// The intro gates its A presses on a BDSP screen: past the limit it is -// another game (r1, 2026-11-01, launched FireRed). +// The intro gates its A presses on a BDSP screen within a time from the +// first screen that is not black: past it, it is another game (r1, +// 2026-11-01, launched FireRed). The boot's black screens, about 25 s of +// them (r3), neither take presses nor count. class Test_IntroGate : public UnitTest{ public: Test_IntroGate() @@ -126,33 +142,57 @@ class Test_IntroGate : public UnitTest{ {} virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - struct Case{ - const char* screen; - IntroProgress progress; - IntroStep expected; - }; - const IntroProgress unconfirmed{.player_named = false, .rival_named = false, .game_confirmed = false, .unconfirmed_presses = 3, .b_presses = 0}; - const IntroProgress exhausted{.player_named = false, .rival_named = false, .game_confirmed = false, - .unconfirmed_presses = INTRO_PRESSES_BEFORE_BDSP, .b_presses = 0}; - const IntroProgress confirmed{.player_named = false, .rival_named = false, .game_confirmed = true, - .unconfirmed_presses = INTRO_PRESSES_BEFORE_BDSP, .b_presses = 0}; - const Case cases[] = { - {"other", unconfirmed, IntroStep::ADVANCE}, - {"other", exhausted, IntroStep::FAIL_NOT_BDSP}, - {"other", confirmed, IntroStep::ADVANCE}, - }; - for (const Case& c : cases){ - if (decide_intro_step(c.screen, c.progress) != c.expected){ - return std::string("Screen \"") + c.screen + "\" after " + std::to_string(c.progress.unconfirmed_presses) + - " unconfirmed presses: unexpected step."; + using std::chrono::milliseconds; + IntroProgress progress; + const std::string boot = check_boot(progress); + if (!boot.empty()){ + return boot; + } + const milliseconds shown = 2 * INTRO_TIME_BEFORE_BDSP + milliseconds(500); + note_intro_screen(progress, "other", shown); + if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ + return "The first logo must take A."; + } + note_intro_screen(progress, "black", shown + INTRO_TIME_BEFORE_BDSP / 2); + note_intro_screen(progress, "other", shown + INTRO_TIME_BEFORE_BDSP - milliseconds(1)); + if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ + return "Unknown screens take A until INTRO_TIME_BEFORE_BDSP has passed."; + } + IntroProgress other_game = progress; + note_intro_screen(other_game, "other", shown + INTRO_TIME_BEFORE_BDSP); + if (decide_intro_step("other", other_game) != IntroStep::FAIL_NOT_BDSP){ + return "Past INTRO_TIME_BEFORE_BDSP of unknown screens, it is another game."; + } + note_intro_screen(progress, "dialog", shown + INTRO_TIME_BEFORE_BDSP - milliseconds(1)); + note_intro_screen(progress, "other", shown + 10 * INTRO_TIME_BEFORE_BDSP); + if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ + return "Once BDSP's dialogue has shown, the clock stops."; + } + return check_confirming_screens(); + } + +private: + // Black from the restart until the first logo, well past the limit. + static std::string check_boot(IntroProgress& progress){ + using std::chrono::milliseconds; + for (milliseconds t(0); t <= 2 * INTRO_TIME_BEFORE_BDSP; t += milliseconds(500)){ + note_intro_screen(progress, "black", t); + if (decide_intro_step("black", progress) != IntroStep::WAIT){ + return "A black screen must take no press."; } } + if (progress.first_shown || progress.unconfirmed_for != milliseconds(0)){ + return "Black screens must not start the wrong-game clock."; + } + return ""; + } + static UnitTestResult check_confirming_screens(){ for (const char* screen : {"dialog", "rival-list-tell", "rival-list-preset", "menu"}){ if (!intro_screen_confirms_bdsp(screen)){ return std::string("Expected \"") + screen + "\" to confirm BDSP."; } } - for (const char* screen : {"user-select", "home", "keyboard", "other"}){ + for (const char* screen : {"user-select", "home", "keyboard", "black", "other"}){ if (intro_screen_confirms_bdsp(screen)){ return std::string("Expected \"") + screen + "\" not to confirm BDSP."; } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index fb135ad7d3..9f40fc46f3 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -15,6 +15,7 @@ #include "CommonTools/Async/InferenceRoutines.h" #include "CommonTools/ScreenMachine/ScreenMachine.h" #include "CommonTools/ScreenMachine/StepTrace.h" +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" #include "NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h" #include "NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.h" @@ -81,7 +82,11 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& if (intro_dialog_showing(screen)){ return "dialog"; } - // Language, cutscenes, fades, the bedroom with no menu open. + // The boot and fades. After the dialogue: Rowan's first lines show on black. + if (BlackScreenDetector().detect(screen)){ + return "black"; + } + // Logos, the title, language, cutscenes, the bedroom with no menu open. return "other"; } @@ -104,16 +109,20 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog if (screen == "menu"){ return IntroStep::CLOSE_MENU_AND_FINISH; } + if (screen == "black"){ + return IntroStep::WAIT; + } if (screen == "dialog"){ return progress.rival_named ? IntroStep::ADVANCE_WITH_B : IntroStep::ADVANCE; } // The bedroom comes only after the rival is named. Before that, every unknown - // screen (language, the appearance preview, cutscenes) takes A, but only so - // many before a screen only BDSP shows: past those it is another game. + // screen (logos, the title, language, the appearance preview, cutscenes) + // takes A, but only for so long before a screen only BDSP shows: past that + // it is another game. if (progress.rival_named){ return progress.b_presses < INTRO_B_BEFORE_X ? IntroStep::CLEAR_WITH_B : IntroStep::PROBE_FOR_MENU; } - if (!progress.game_confirmed && progress.unconfirmed_presses >= INTRO_PRESSES_BEFORE_BDSP){ + if (!progress.game_confirmed && progress.unconfirmed_for >= INTRO_TIME_BEFORE_BDSP){ return IntroStep::FAIL_NOT_BDSP; } return IntroStep::ADVANCE; @@ -122,19 +131,26 @@ bool intro_screen_confirms_bdsp(const std::string& screen){ return screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; } -void record_intro_step(IntroProgress& progress, const std::string& screen, IntroStep step){ +void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start){ if (intro_screen_confirms_bdsp(screen)){ progress.game_confirmed = true; } + if (screen == "black"){ + return; + } + if (!progress.first_shown){ + progress.first_shown = since_start; + } + if (!progress.game_confirmed){ + progress.unconfirmed_for = since_start - *progress.first_shown; + } +} + +void record_intro_step(IntroProgress& progress, IntroStep step){ switch (step){ case IntroStep::ENTER_PLAYER_NAME: progress.player_named = true; return; - case IntroStep::ADVANCE: - if (!progress.game_confirmed){ - progress.unconfirmed_presses++; - } - return; case IntroStep::PICK_PRESET: progress.rival_named = true; return; @@ -228,6 +244,7 @@ struct IntroRun{ ProControllerContext& context; StepTrace& trace; const std::string& player_name; + WallClock start; IntroProgress progress; size_t x_without_menu = 0; }; @@ -264,8 +281,10 @@ void probe_for_menu(IntroRun& run){ // Carries out what decide_intro_step() chose for `screen`. void act(IntroRun& run, const char* screen){ + note_intro_screen(run.progress, screen, + std::chrono::duration_cast(current_time() - run.start)); const IntroStep step = decide_intro_step(screen, run.progress); - record_intro_step(run.progress, screen, step); + record_intro_step(run.progress, step); switch (step){ case IntroStep::SELECT_USER: // The cursor starts on the user that launched the game. @@ -275,8 +294,8 @@ void act(IntroRun& run, const char* screen){ case IntroStep::FAIL_NOT_IN_GAME: run.trace.fail("The Switch HOME menu is showing: the game is not running in front."); case IntroStep::FAIL_NOT_BDSP: - run.trace.fail("The game in front is not BDSP: no BDSP screen after " + - std::to_string(run.progress.unconfirmed_presses) + " presses on unknown screens."); + run.trace.fail("The game in front is not BDSP: no BDSP screen in " + + std::to_string(run.progress.unconfirmed_for.count() / 1000) + " s of unknown screens."); case IntroStep::ENTER_PLAYER_NAME: enter_player_name(run.env.console, run.context, run.trace, run.player_name); return; @@ -305,7 +324,26 @@ void act(IntroRun& run, const char* screen){ case IntroStep::PROBE_FOR_MENU: probe_for_menu(run); return; + case IntroStep::WAIT: + pbf_wait(run.context, 500ms); + return; + } +} + +// After the restart BDSP boots on black for about 25 s (r3); presses then +// would count against the wrong-game guard, so none until it ends. +void wait_out_boot(VideoStream& stream, ProControllerContext& context, StepTrace& trace){ + context.wait_for_all_requests(); + const WallClock deadline = current_time() + INTRO_BOOT_LIMIT; + while (current_time() < deadline){ + VideoSnapshot screen = stream.video().snapshot(); + if (screen && !BlackScreenDetector().detect(screen)){ + trace.action("The boot's black screen ended."); + return; + } + context.wait_for(500ms); } + trace.fail("The screen stayed black for " + std::to_string(INTRO_BOOT_LIMIT.count() / 1000) + " s after the restart."); } // The machine's states: the screens classify_intro_screen() tells apart, with @@ -323,22 +361,26 @@ std::vector intro_states(IntroRun& run){ [&run]{ act(run, "home"); }, {}, 1}, {"player-keyboard", [=, &run](const ImageViewRGB32& s){ return !run.progress.player_named && screen_is(s, "keyboard"); }, - [&run]{ act(run, "keyboard"); }, {"dialog", "other"}, 1}, + [&run]{ act(run, "keyboard"); }, {"dialog", "black", "other"}, 1}, {"rival-keyboard", [=, &run](const ImageViewRGB32& s){ return run.progress.player_named && screen_is(s, "keyboard"); }, - [&run]{ act(run, "keyboard"); }, {"rival-list-tell", "dialog", "other"}, 3}, + [&run]{ act(run, "keyboard"); }, {"rival-list-tell", "dialog", "black", "other"}, 3}, {"rival-list-tell", [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, - [&run]{ act(run, "rival-list-tell"); }, {"rival-list-preset", "other"}, 6}, + [&run]{ act(run, "rival-list-tell"); }, {"rival-list-preset", "black", "other"}, 6}, {"rival-list-preset", [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-preset"); }, - [&run]{ act(run, "rival-list-preset"); }, {"dialog", "other"}, 3}, + [&run]{ act(run, "rival-list-preset"); }, {"dialog", "black", "other"}, 3}, {"menu", [=](const ImageViewRGB32& s){ return screen_is(s, "menu"); }, [&run]{ act(run, "menu"); }, {}, 2, true}, {"dialog", [=](const ImageViewRGB32& s){ return screen_is(s, "dialog"); }, [&run]{ act(run, "dialog"); }, {}, 400}, + // The boot is waited out before the machine starts; this is fades. + {"black", + [=](const ImageViewRGB32& s){ return screen_is(s, "black"); }, + [&run]{ act(run, "black"); }, {}, 300, false, 60}, {"other", [](const ImageViewRGB32&){ return true; }, [&run]{ act(run, "other"); }, {}, 300, false, 60}, @@ -378,7 +420,8 @@ void play_new_game_intro( go_home(env.console, context); from_home_close_and_reopen_game(env.console, context, true); - IntroRun run{env, context, trace, player_name, {}, 0}; + wait_out_boot(stream, context, trace); + IntroRun run{env, context, trace, player_name, current_time(), {}, 0}; const std::vector states = intro_states(run); run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); trace.action("Intro done."); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h index cbd6366f53..af832034f1 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h @@ -10,7 +10,9 @@ #ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H #define PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H +#include #include +#include #include #include "CommonFramework/ImageTypes/ImageViewRGB32.h" #include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" @@ -23,8 +25,8 @@ namespace PokemonBDSP{ // Which intro screen `screen` shows, as the intro's steps tell them apart: // "user-select", "home", "keyboard", "rival-list-tell" (the rival's name list on // "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or -// "other". The Switch's own screens need `console`; without it they read as -// whatever else they look like. +// "black" (the boot, fades) or "other". The Switch's own screens need +// `console`; without it they read as whatever else they look like. std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen); // What the intro does on a screen. Decided from the screen and how far the intro has @@ -32,7 +34,7 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& enum class IntroStep{ SELECT_USER, // A on the Switch's user select FAIL_NOT_IN_GAME, // HOME is showing - FAIL_NOT_BDSP, // no BDSP screen after INTRO_PRESSES_BEFORE_BDSP presses + FAIL_NOT_BDSP, // no BDSP screen within INTRO_TIME_BEFORE_BDSP ENTER_PLAYER_NAME, // type the name on the first keyboard CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list MOVE_TO_PRESET, // Down: off "I'll tell you..." @@ -42,21 +44,30 @@ enum class IntroStep{ ADVANCE_WITH_B, // B: dialogue once the rival is named CLEAR_WITH_B, // B: an unknown screen once the rival is named PROBE_FOR_MENU, // X, to see whether the player is in control + WAIT, // nothing: a black screen (the boot, a fade) }; struct IntroProgress{ bool player_named = false; bool rival_named = false; // A screen only BDSP shows has been seen (intro_screen_confirms_bdsp()). bool game_confirmed = false; - // A presses on unknown screens before that: the title, the language. - size_t unconfirmed_presses = 0; + // When the first screen that is not black showed, since the intro + // started; none yet while the game boots. + std::optional first_shown = std::nullopt; + // How long screens have shown without one only BDSP shows: the logos, + // the title, the language select. + std::chrono::milliseconds unconfirmed_for = std::chrono::milliseconds(0); // B presses on unknown screens since the last X. size_t b_presses = 0; }; -// Unknown screens pressed through before a BDSP screen must show: the title -// and the language select take a few. Another game launched by mistake -// (r1, 2026-11-01: FireRed) fails here instead of being played. -inline constexpr size_t INTRO_PRESSES_BEFORE_BDSP = 20; +// How long the intro presses A on unknown screens, from the first screen +// that is not black, before a BDSP screen must show: the logos, the title +// and the language select. Another game launched by mistake (r1, +// 2026-11-01: FireRed) fails here instead of being played. Black screens +// take no presses: BDSP boots on black for about 25 s (r3, 2026-11-01). +inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(60); +// How long the boot may stay black after the restart. +inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); // Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: // its dialogue box, the rival's name list, its main menu. The keyboard and // the user select are the Switch's. @@ -72,9 +83,12 @@ inline constexpr size_t INTRO_B_BEFORE_X = 5; // INTRO_B_BEFORE_X B presses between them, about two minutes. inline constexpr size_t INTRO_X_WITHOUT_MENU = 20; +// Notes `screen`, read `since_start` into the intro, in `progress`: the +// game is confirmed on a BDSP screen; black screens leave the clock alone. +void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start); IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); -// Counts `step`, taken on `screen`, in `progress`. -void record_intro_step(IntroProgress& progress, const std::string& screen, IntroStep step); +// Counts `step` in `progress`. +void record_intro_step(IntroProgress& progress, IntroStep step); // Checks classify_intro_screen() on //*.png at 1920 and // 1280 px wide. Logs each mismatch; returns the number of them. @@ -83,8 +97,9 @@ size_t run_intro_screen_tests(Logger& logger, const std::string& folder); // From the title screen or any intro screen of a game with no save, with BDSP // running in front: from HOME it throws UserSetupError, since the restart would // launch whatever tile HOME's cursor is on. Restarts the game first, so it -// always starts from the title, and fails if no BDSP screen shows within -// INTRO_PRESSES_BEFORE_BDSP presses. Names the player `player_name` +// always starts from the title, waits out the black boot (up to +// INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within +// INTRO_TIME_BEFORE_BDSP of screens. Names the player `player_name` // and picks the rival's default name (Barry) from the list. Each step reads the // screen and waits for the expected next one; an unexpected screen fails with a // screenshot and the recent screens. From 072a5f4110cbb4eeabf5b9f96a6d4d4122e345fe Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 15:08:56 -0800 Subject: [PATCH 12/23] BDSP AutoStory new game: A until the rival's picked name is confirmed r4 picked Barry, then B answered "Barry, is it?"'s Yes/No as No and the list came back until the state machine's visit limit. Picking a preset now leaves the name pending: dialog and unknown screens take A (Yes) until INTRO_SCREENS_AFTER_RIVAL_PICK of them pass without the list, and the list coming back is picked from again (up to 6 visits). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 79 +++++++++++++++---- .../PokemonBDSP_AutoStory_NewGame.cpp | 15 +++- .../AutoStory/PokemonBDSP_AutoStory_NewGame.h | 13 +++ 3 files changed, 90 insertions(+), 17 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp index 6bb3ca56e6..32c3efbc16 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -88,6 +88,56 @@ class Test_IntroDecisions : public UnitTest{ }; +// Runs `screens` through decide_intro_step() and record_intro_step() from +// `progress`; the steps taken. +std::vector intro_steps(IntroProgress& progress, const std::vector& screens){ + std::vector steps; + for (const std::string& screen : screens){ + steps.push_back(decide_intro_step(screen, progress)); + record_intro_step(progress, steps.back()); + } + return steps; +} + +// The rival's preset name is confirmed with A on "Barry, is it?" and its +// Yes/No; B there answers No and the list comes back (r4). Only +// INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick does B take over. +class Test_IntroRivalConfirm : public UnitTest{ +public: + Test_IntroRivalConfirm() + : UnitTest("PokemonBDSP::AutoStory - new-game intro: the rival's name confirmed with A") + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + IntroProgress progress = intro_progress(1); + // r4's frames 029 to 032: the pick, "Barry, is it?", Barry alone + // as the Yes/No comes up, the Yes/No. + const std::vector confirm = intro_steps(progress, {"rival-list-preset", "dialog", "other", "dialog"}); + const std::vector expected{IntroStep::PICK_PRESET, IntroStep::ADVANCE, IntroStep::ADVANCE, IntroStep::ADVANCE}; + if (confirm != expected){ + return "The pick and its confirmation must take A, never B."; + } + if (progress.rival_named){ + return "The name is not final until INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick."; + } + // The list back (No was answered): pick again, the count restarts. + if (intro_steps(progress, {"rival-list-tell", "rival-list-preset"}) != + std::vector{IntroStep::MOVE_TO_PRESET, IntroStep::PICK_PRESET}){ + return "The list coming back after a pick must be picked from again."; + } + for (size_t c = 0; c < INTRO_SCREENS_AFTER_RIVAL_PICK; c++){ + if (intro_steps(progress, {c % 2 ? "other" : "dialog"}) != std::vector{IntroStep::ADVANCE}){ + return "Screen " + std::to_string(c) + " after the pick must take A."; + } + } + if (!progress.rival_named){ + return "INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick, the name is final."; + } + return true; + } +}; + + // After the rival is named, unknown screens take INTRO_B_BEFORE_X B presses, // then X; only the menu ends the intro (r2: Rowan's last line, misread). class Test_IntroBeforeMenu : public UnitTest{ @@ -98,27 +148,25 @@ class Test_IntroBeforeMenu : public UnitTest{ virtual UnitTestResult run(Logger&, CancellableScope&) const override{ IntroProgress progress = intro_progress(1); - record_intro_step(progress, decide_intro_step("rival-list-preset", progress)); - if (!progress.rival_named){ - return "Picking the preset name must count the rival as named."; + std::vector screens{"rival-list-preset", "dialog", "other"}; + std::vector expected{IntroStep::PICK_PRESET, IntroStep::ADVANCE, IntroStep::ADVANCE}; + for (size_t c = 2; c < INTRO_SCREENS_AFTER_RIVAL_PICK; c++){ + screens.push_back("dialog"); + expected.push_back(IntroStep::ADVANCE); } - std::vector expected; for (size_t round = 0; round < 2; round++){ + screens.insert(screens.end(), INTRO_B_BEFORE_X + 1, "other"); expected.insert(expected.end(), INTRO_B_BEFORE_X, IntroStep::CLEAR_WITH_B); expected.push_back(IntroStep::PROBE_FOR_MENU); } // A dialogue line in between takes B without counting. - expected.push_back(IntroStep::ADVANCE_WITH_B); - expected.push_back(IntroStep::CLEAR_WITH_B); - expected.push_back(IntroStep::CLOSE_MENU_AND_FINISH); - const size_t others = 2 * (INTRO_B_BEFORE_X + 1); - for (size_t c = 0; c < expected.size(); c++){ - const char* screen = c < others ? "other" : c == others ? "dialog" : c == others + 1 ? "other" : "menu"; - const IntroStep step = decide_intro_step(screen, progress); - record_intro_step(progress, step); - if (step != expected[c]){ - return "Screen " + std::to_string(c) + " (\"" + screen + "\"): step " + - std::to_string((int)step) + ", expected " + std::to_string((int)expected[c]) + "."; + screens.insert(screens.end(), {"dialog", "other", "menu"}); + expected.insert(expected.end(), {IntroStep::ADVANCE_WITH_B, IntroStep::CLEAR_WITH_B, IntroStep::CLOSE_MENU_AND_FINISH}); + const std::vector steps = intro_steps(progress, screens); + for (size_t c = 0; c < steps.size(); c++){ + if (steps[c] != expected[c]){ + return "Screen " + std::to_string(c) + " (\"" + screens[c] + "\"): step " + + std::to_string((int)steps[c]) + ", expected " + std::to_string((int)expected[c]) + "."; } } IntroProgress pressed = intro_progress(2); @@ -403,6 +451,7 @@ class Test_SchoolLamp : public UnitTest{ void add_tests_AutoStory_Actions(UnitTestDatabase& database){ database.add(); database.add(); + database.add(); database.add(); database.add(); database.add(); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index 9f40fc46f3..22dc791fe2 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -151,8 +151,19 @@ void record_intro_step(IntroProgress& progress, IntroStep step){ case IntroStep::ENTER_PLAYER_NAME: progress.player_named = true; return; + case IntroStep::MOVE_TO_PRESET: + // The list again: the name was not confirmed. + progress.rival_picked = false; + progress.screens_after_pick = 0; + return; case IntroStep::PICK_PRESET: - progress.rival_named = true; + progress.rival_picked = true; + progress.screens_after_pick = 0; + return; + case IntroStep::ADVANCE: + if (progress.rival_picked && ++progress.screens_after_pick >= INTRO_SCREENS_AFTER_RIVAL_PICK){ + progress.rival_named = true; + } return; case IntroStep::CLEAR_WITH_B: progress.b_presses++; @@ -370,7 +381,7 @@ std::vector intro_states(IntroRun& run){ [&run]{ act(run, "rival-list-tell"); }, {"rival-list-preset", "black", "other"}, 6}, {"rival-list-preset", [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-preset"); }, - [&run]{ act(run, "rival-list-preset"); }, {"dialog", "black", "other"}, 3}, + [&run]{ act(run, "rival-list-preset"); }, {"dialog", "black", "other"}, 6}, {"menu", [=](const ImageViewRGB32& s){ return screen_is(s, "menu"); }, [&run]{ act(run, "menu"); }, {}, 2, true}, diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h index af832034f1..9b1149eeb0 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h @@ -48,6 +48,14 @@ enum class IntroStep{ }; struct IntroProgress{ bool player_named = false; + // A preset name was picked from the rival's list, and how many screens + // have shown since without the list coming back. "Barry, is it?" and + // its Yes/No follow, and A (Yes, where the cursor starts) confirms it; + // B would answer No and bring the list back (r4, 2026-11-01). + bool rival_picked = false; + size_t screens_after_pick = 0; + // The name is final: INTRO_SCREENS_AFTER_RIVAL_PICK screens passed + // without the list. bool rival_named = false; // A screen only BDSP shows has been seen (intro_screen_confirms_bdsp()). bool game_confirmed = false; @@ -73,6 +81,11 @@ inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::secon // the user select are the Switch's. bool intro_screen_confirms_bdsp(const std::string& screen); +// Screens after the rival's name is picked, each taking A, before the name +// counts as confirmed and B takes over: "Barry, is it?", the box closing +// (Barry alone), the Yes/No, and a few of Rowan's lines after it. +inline constexpr size_t INTRO_SCREENS_AFTER_RIVAL_PICK = 8; + // Once the rival is named, the intro only has dialogue and cutscenes left // before the bedroom, and screens the classifier misses (r2: a dialogue box // without its arrow). Each unknown screen takes B, which advances dialogue, From 09e5ca1f236a13e9c02eb8ff25a0d9a0058b9b83 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 15:23:51 -0800 Subject: [PATCH 13/23] BDSP AutoStory: the X menu by its bottom bar; the intro also ends on X opening something from a still bedroom r5 reached the bedroom, but a new game's X menu (BAG and OPTIONS with a NEW tag) is not the full menu MenuDetector wants, and the intro pressed X twelve times. XMenuDetector also takes the blue bar along the bottom, there from the first menu on, for the intro and the Options steps. And when X opens something no detector knows from a screen that held still, and B puts it back, the player is in control too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../PokemonBDSP_AutoStory_XMenuDetector.cpp | 49 +++++++++++++ .../PokemonBDSP_AutoStory_XMenuDetector.h | 56 +++++++++++++++ .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 72 +++++++++++++++++++ .../PokemonBDSP_AutoStory_NewGame.cpp | 64 +++++++++++++++-- .../AutoStory/PokemonBDSP_AutoStory_NewGame.h | 16 +++++ .../PokemonBDSP_AutoStory_Settings.cpp | 14 ++-- SerialPrograms/cmake/SourceFiles.cmake | 2 + 7 files changed, 259 insertions(+), 14 deletions(-) create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp create mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp new file mode 100644 index 0000000000..507edb09c0 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp @@ -0,0 +1,49 @@ +/* BDSP AutoStory - X menu detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP_AutoStory_XMenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +bool x_menu_bar(const ImageStats& bar){ + // A flat sky blue: blue well over red, and nearly no texture. + return bar.average.b > 200 && bar.average.r < 140 && bar.average.b - bar.average.r > 90 && + bar.stddev.sum() < 45; +} + +XMenuDetector::XMenuDetector(Color color) + : m_color(color) +{} +void XMenuDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, X_MENU_BAR_BOX); +} +bool XMenuDetector::detect(const ImageViewRGB32& screen){ + if (x_menu_bar(image_stats(extract_box_reference(screen, X_MENU_BAR_BOX)))){ + return true; + } + return MenuDetector(m_color, true).detect(screen); +} + +XMenuWatcher::XMenuWatcher(Color color) + : XMenuDetector(color) + , VisualInferenceCallback("XMenuWatcher") +{} +void XMenuWatcher::make_overlays(VideoOverlaySet& items) const{ + XMenuDetector::make_overlays(items); +} +bool XMenuWatcher::process_frame(const ImageViewRGB32& frame, WallClock){ + return detect(frame); +} + + +} +} +} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h new file mode 100644 index 0000000000..ed18dc93a4 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h @@ -0,0 +1,56 @@ +/* BDSP AutoStory - X menu detector + * + * From: https://github.com/PokemonAutomation/ + * + * The X menu at every stage of the game. MenuDetector wants the full menu's + * white columns between the icons; a new game's menu has two icons (BAG, + * and OPTIONS with a "NEW" tag) and was missed (r5, 2026-11-01). The blue + * bar along the bottom ("A Confirm R Save B Close") is there from the + * first menu on. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_XMenuDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_XMenuDetector_H + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The bottom bar's left end, left of its button prompts. +inline const ImageFloatBox X_MENU_BAR_BOX(0.02, 0.965, 0.18, 0.025); +// Whether `bar` (the stats of X_MENU_BAR_BOX) is the menu's flat blue bar. +bool x_menu_bar(const ImageStats& bar); + +class XMenuDetector : public StaticScreenDetector{ +public: + XMenuDetector(Color color = COLOR_RED); + + virtual void make_overlays(VideoOverlaySet& items) const override; + // The bottom bar, or the full menu as MenuDetector (with the NEW tag + // skipped) sees it. + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; +}; + +class XMenuWatcher : public XMenuDetector, public VisualInferenceCallback{ +public: + XMenuWatcher(Color color = COLOR_RED); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool process_frame(const ImageViewRGB32& frame, WallClock timestamp) override; +}; + + +} +} +} +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp index 32c3efbc16..bdcc61a886 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -19,6 +19,7 @@ #include "Data/PokemonBDSP_AutoStory_StoryMaps.h" #include "Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_Center.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h" #include "PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP_AutoStory_Actions_Tests.h" @@ -179,6 +180,75 @@ class Test_IntroBeforeMenu : public UnitTest{ }; +// The X menu by its bottom bar at every stage: r5's new-game menu, two icons +// and a NEW tag, was missed by MenuDetector. /PokemonBDSP/ +// AutoStory/XMenu/open/*.png must read as the menu, closed/*.png (a dialogue, +// the intro, the overworld, HOME) must not. +class Test_XMenu : public UnitTest{ +public: + Test_XMenu() + : UnitTest("PokemonBDSP::AutoStory - X menu at every stage") + {} + + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + const std::string folder = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/XMenu/"; + if (!Filesystem::is_directory(Filesystem::Path(folder))){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshots not found: " + folder); + } + size_t failed = 0; + for (const char* expected : {"open", "closed"}){ + for (const Filesystem::Path& file : Filesystem::list_directory(Filesystem::Path(folder + expected))){ + if (file.extension().string() != ".png"){ + continue; + } + const ImageRGB32 image(file.string()); + const ImageStats bar = image_stats(extract_box_reference(image, X_MENU_BAR_BOX)); + const bool open = XMenuDetector().detect(image); + const bool ok = open == (std::string(expected) == "open"); + failed += ok ? 0 : 1; + logger.log(std::string("[X menu] ") + (ok ? "ok " : "FAIL ") + expected + "/" + file.filename().string() + + ": bar " + bar.average.to_string() + " sd " + bar.stddev.to_string(), ok ? COLOR_BLUE : COLOR_RED); + } + } + if (failed != 0){ + return std::to_string(failed) + " X menu screenshots misread; see the log."; + } + return true; + } +}; + + +// X from a still bedroom opening something the detector misses, closed +// again by B, is the player in control; moving screens or X doing nothing +// are not. +class Test_IntroInControl : public UnitTest{ +public: + Test_IntroInControl() + : UnitTest("PokemonBDSP::AutoStory - new-game intro: in control without the menu read") + {} + + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + struct Case{ + XProbe probe; + bool expected; + const char* what; + }; + const Case cases[] = { + {{1, 60, 2}, true, "still, X opened, B closed"}, + {{12, 60, 2}, false, "moving before X (sparkles, a cutscene)"}, + {{1, 5, 2}, false, "X changed nothing"}, + {{1, 60, 40}, false, "B did not put it back"}, + }; + for (const Case& c : cases){ + if (intro_in_control_without_menu(c.probe) != c.expected){ + return std::string("Wrong for: ") + c.what + "."; + } + } + return true; + } +}; + + // The intro gates its A presses on a BDSP screen within a time from the // first screen that is not black: past it, it is another game (r1, // 2026-11-01, launched FireRed). The boot's black screens, about 25 s of @@ -454,6 +524,8 @@ void add_tests_AutoStory_Actions(UnitTestDatabase& database){ database.add(); database.add(); database.add(); + database.add(); + database.add(); database.add(); database.add(); database.add(); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index 22dc791fe2..c4f49e3c11 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -22,7 +22,7 @@ #include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" #include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h" #include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" -#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" #include "PokemonBDSP_AutoStory_NewGame.h" @@ -75,7 +75,7 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& default: return "rival-list-preset"; } - if (MenuDetector().detect(screen)){ + if (XMenuDetector().detect(screen)){ return "menu"; } // Includes the appearance grid, whose prompt sits in a dialogue box. @@ -146,6 +146,10 @@ void note_intro_screen(IntroProgress& progress, const std::string& screen, std:: } } +bool intro_in_control_without_menu(const XProbe& probe){ + return probe.still_change <= INTRO_STILL_CHANGE && probe.x_change > INTRO_X_CHANGE && probe.b_change <= INTRO_B_RESTORE; +} + void record_intro_step(IntroProgress& progress, IntroStep step){ switch (step){ case IntroStep::ENTER_PLAYER_NAME: @@ -258,6 +262,8 @@ struct IntroRun{ WallClock start; IntroProgress progress; size_t x_without_menu = 0; + // X opened something and B closed it, from a still screen. + bool in_control = false; }; void press(IntroRun& run, Button button){ @@ -270,19 +276,59 @@ void close_menu_and_finish(IntroRun& run){ pbf_wait(run.context, 1000ms); run.context.wait_for_all_requests(); VideoSnapshot screen_now = run.env.console.video().snapshot(); - if (!screen_now || MenuDetector().detect(screen_now)){ + if (!screen_now || XMenuDetector().detect(screen_now)){ run.trace.fail("The main menu did not close."); } } -// X, then the menu or B presses again: whatever X opened instead, B closes. -void probe_for_menu(IntroRun& run){ +// The menu's area, which X changes when the menu opens. +const ImageFloatBox INTRO_MENU_AREA(0.12, 0.08, 0.76, 0.56); + +double menu_area_change(const VideoSnapshot& before, const VideoSnapshot& after){ + if (!before || !after){ + return 1000; + } + return ImageMatch::pixel_RMSD( + extract_box_reference(*before.frame, INTRO_MENU_AREA), extract_box_reference(*after.frame, INTRO_MENU_AREA) + ); +} + +enum class XProbeResult{ + MENU, // the menu showed + IN_CONTROL, // something X opened and B closed, from a still screen + NOTHING, +}; +// X from a screen that held still; B after it if no menu showed. +XProbeResult x_from_still_screen(IntroRun& run){ VideoStream& stream = run.env.console; + run.context.wait_for_all_requests(); + const VideoSnapshot first = stream.video().snapshot(); + run.context.wait_for(INTRO_STILL_TIME); + const VideoSnapshot before = stream.video().snapshot(); press(run, BUTTON_X); run.context.wait_for_all_requests(); - MenuWatcher menu; + XMenuWatcher menu; if (wait_until(stream, run.context, 3s, {{menu}}) == 0){ - return; // The next read sees the menu and finishes. + return XProbeResult::MENU; + } + const VideoSnapshot after_x = stream.video().snapshot(); + pbf_press_button(run.context, BUTTON_B, 80ms, 1000ms); + run.context.wait_for_all_requests(); + const VideoSnapshot after_b = stream.video().snapshot(); + const XProbe probe{menu_area_change(first, before), menu_area_change(before, after_x), menu_area_change(before, after_b)}; + run.trace.action("X probe: still " + std::to_string(probe.still_change) + ", X " + std::to_string(probe.x_change) + + ", after B " + std::to_string(probe.b_change) + "."); + return intro_in_control_without_menu(probe) ? XProbeResult::IN_CONTROL : XProbeResult::NOTHING; +} + +// X, then the menu or B presses again: whatever X opened instead, B closes. +// X opening something unrecognized from a still screen, which B closes +// again, is the player in control too. +void probe_for_menu(IntroRun& run){ + const XProbeResult result = x_from_still_screen(run); + run.in_control = result == XProbeResult::IN_CONTROL; + if (result != XProbeResult::NOTHING){ + return; // The next read sees the menu, or the in-control state, and finishes. } run.trace.action("X opened no menu; B again before the next X."); if (++run.x_without_menu >= INTRO_X_WITHOUT_MENU){ @@ -364,6 +410,10 @@ std::vector intro_states(IntroRun& run){ return classify_intro_screen(&run.env.console, screen) == name; }; return { + {"in-control", + [&run](const ImageViewRGB32&){ return run.in_control; }, + [&run]{ run.trace.action("X opened and B closed the menu from a still bedroom: the player is in control."); }, + {}, 1, true}, {"user-select", [=](const ImageViewRGB32& s){ return screen_is(s, "user-select"); }, [&run]{ act(run, "user-select"); }, {}, 3}, diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h index 9b1149eeb0..632e2026cb 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h @@ -100,6 +100,22 @@ inline constexpr size_t INTRO_X_WITHOUT_MENU = 20; // game is confirmed on a BDSP screen; black screens leave the clock alone. void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start); IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); + +// When X opens nothing the menu detector knows, the player is still in +// control if the screen held still for INTRO_STILL_TIME before X (no +// dialogue, no fade, no intro sparkles), X changed the menu's area by more +// than INTRO_X_CHANGE, and B put it back within INTRO_B_RESTORE: the menu +// opened and closed. Changes are pixel RMSD. +inline constexpr std::chrono::milliseconds INTRO_STILL_TIME = std::chrono::seconds(2); +inline constexpr double INTRO_STILL_CHANGE = 4; +inline constexpr double INTRO_X_CHANGE = 25; +inline constexpr double INTRO_B_RESTORE = 10; +struct XProbe{ + double still_change; // over INTRO_STILL_TIME before X + double x_change; // from before X to after it + double b_change; // from before X to after the B that follows +}; +bool intro_in_control_without_menu(const XProbe& probe); // Counts `step` in `progress`. void record_intro_step(IntroProgress& progress, IntroStep step); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp index c9f5b50e24..61ff5eabb1 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp @@ -16,7 +16,7 @@ #include "CommonTools/Async/InferenceRoutines.h" #include "CommonTools/VisualDetectors/BlackScreenDetector.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" -#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h" #include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" #include "PokemonBDSP_AutoStory_Bag.h" #include "PokemonBDSP_AutoStory_Settings.h" @@ -195,7 +195,7 @@ struct OptionReferences{ // From the overworld (or the main menu): the main menu, settled. void open_main_menu(VideoStream& stream, ProControllerContext& context){ context.wait_for_all_requests(); - MenuDetector menu; + XMenuDetector menu; VideoOverlaySet detector_overlays(stream.overlay()); menu.make_overlays(detector_overlays); detector_overlays.add(COLOR_CYAN, {0.04, 0.065, 0.93, 0.70}, ""); @@ -206,7 +206,7 @@ void open_main_menu(VideoStream& stream, ProControllerContext& context){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); } - MenuWatcher menu_open; + XMenuWatcher menu_open; if (wait_until(stream, context, 5s, {{menu_open}}) != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Settings checkpoint: Main menu not detected.", stream); @@ -229,7 +229,7 @@ void open_options(VideoStream& stream, ProControllerContext& context, const Tuto } pbf_press_button(context, BUTTON_B, 80ms, 1000ms); context.wait_for_all_requests(); - MenuWatcher back_to_menu; + XMenuWatcher back_to_menu; if (wait_until(stream, context, 5s, {{back_to_menu}}) != 0){ break; } @@ -399,7 +399,7 @@ void save_options(VideoStream& stream, ProControllerContext& context, const std: void reopen_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); - MenuWatcher verify_menu; + XMenuWatcher verify_menu; if (wait_until(stream, context, 5s, {{verify_menu}}) != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Autosave verification: Main menu not detected.", stream); @@ -413,7 +413,7 @@ void reopen_options(VideoStream& stream, ProControllerContext& context, const Tu } pbf_press_button(context, BUTTON_B, 80ms, 1000ms); context.wait_for_all_requests(); - MenuWatcher return_menu; + XMenuWatcher return_menu; if (wait_until(stream, context, 5s, {{return_menu}}) != 0){ break; } @@ -490,7 +490,7 @@ void back_to_overworld(VideoStream& stream, ProControllerContext& context){ pbf_mash_button(context, BUTTON_B, 3000ms); context.wait_for_all_requests(); OverworldDetector overworld; - MenuDetector menu; + XMenuDetector menu; BlackScreenDetector black; bool recovered = false; size_t confirmations = 0; diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index eae54af4ca..c9a0ceacd2 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1627,6 +1627,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/PokemonBDSP_Settings.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h Source/PokemonBDSP/PokemonBDSP_Tests.cpp Source/PokemonBDSP/PokemonBDSP_Tests.h Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp From 2807ad4782005e2aecbe00bd58e536051c5a8848 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 15:35:00 -0800 Subject: [PATCH 14/23] BDSP detectors: MenuDetector below the NEW tag, ShortDialogDetector below a long first line; their screenshots in CommandLineTests The intro's misses were the detectors' (r2, r5), so they are fixed there: MenuDetector also looks at its white columns from below the "NEW" tags (a new game's OPTIONS has one beside the third column), and ShortDialogDetector takes the box's right end below the first line when a long first line covers it. The AutoStory-only XMenuDetector and the intro's own dialogue check go; the screenshots are registered per file from CommandLineTests' PokemonBDSP/MenuDetector and DialogDetector, and the AutoStory folder readers for them are removed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../ComputerPrograms_UnitTestRunner.cpp | 4 ++ .../Inference/PokemonBDSP_DialogDetector.cpp | 33 ++++++++- .../Inference/PokemonBDSP_DialogDetector.h | 6 ++ .../Inference/PokemonBDSP_MenuDetector.cpp | 63 ++++++++++++----- .../Inference/PokemonBDSP_MenuDetector.h | 5 ++ .../PokemonBDSP_AutoStory_XMenuDetector.cpp | 49 ------------- .../PokemonBDSP_AutoStory_XMenuDetector.h | 56 --------------- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 64 ----------------- .../PokemonBDSP_AutoStory_NewGame.cpp | 69 ++----------------- .../AutoStory/PokemonBDSP_AutoStory_NewGame.h | 4 -- .../PokemonBDSP_AutoStory_Settings.cpp | 14 ++-- SerialPrograms/cmake/SourceFiles.cmake | 2 - 12 files changed, 104 insertions(+), 265 deletions(-) delete mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp delete mode 100644 SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 7b20b845bc..6833c7c134 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -32,6 +32,8 @@ #include "Pokemon/AutoStory/Pokemon_AutoStory_Tests.h" #include "Pokemon/Navigation/Pokemon_Navigation_Tests.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" #include "PokemonBDSP/PokemonBDSP_Tests.h" #include "VideoGames/PokemonHome/PokemonHome_Tests.h" #include "VideoGames/PokemonSwSh/PokemonSwSh_Tests.h" @@ -79,6 +81,8 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ NintendoSwitch::add_tests_UpdatePopupDetector(ret); NintendoSwitch::Typewriter::add_tests_TypewriterTextMapping(ret); NintendoSwitch::PokemonBDSP::add_tests(ret); + NintendoSwitch::PokemonBDSP::add_tests_DialogDetector(ret); + NintendoSwitch::PokemonBDSP::add_tests_MenuDetector(ret); NintendoSwitch::PokemonFRLG::add_tests(ret); NintendoSwitch::PokemonRSE::add_tests(ret); NintendoSwitch::PokemonHome::add_tests(ret); diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.cpp index c8636d99cb..e0b9cd0abc 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.cpp @@ -6,6 +6,8 @@ #include "CommonFramework/VideoPipeline/VideoOverlay.h" #include "CommonTools/Images/SolidColorTest.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Tests/ImageUnitTest.h" #include "PokemonBDSP_DialogDetector.h" //#include @@ -23,6 +25,7 @@ ShortDialogDetector::ShortDialogDetector(Color color) , m_left_white(0.21, 0.835, 0.008, 0.12) , m_left(0.18, 0.835, 0.02, 0.12) , m_right_white(0.785, 0.835, 0.008, 0.12) + , m_right_white_low(0.785, 0.915, 0.008, 0.04) , m_right(0.822, 0.835, 0.02, 0.12) {} void ShortDialogDetector::make_overlays(VideoOverlaySet& items) const{ @@ -40,7 +43,9 @@ bool ShortDialogDetector::detect(const ImageViewRGB32& screen){ } ImageStats right_white = image_stats(extract_box_reference(screen, m_right_white)); // cout << right_white.average << right_white.stddev << endl; - if (!is_white(right_white)){ + // A long first line runs into m_right_white (Rowan's "Now, go on, and + // leap into the world of Pokemon!"): then the box's right end below it. + if (!is_white(right_white) && !is_white(image_stats(extract_box_reference(screen, m_right_white_low)))){ return false; } // ImageStats bottom = image_stats(extract_box_reference(screen, m_bottom)); @@ -125,6 +130,32 @@ bool ShortDialogPromptDetector::process_frame(const ImageViewRGB32& screen, Wall + + +class Test_ShortDialogDetector : public ImageUnitTest{ +public: + Test_ShortDialogDetector(const std::string& image, bool expected) + : ImageUnitTest("PokemonBDSP::ShortDialogDetector - " + image, {image}) + , m_expected(expected) + {} + + virtual UnitTestResult run_on_images( + Logger&, CancellableScope&, + const std::vector& images + ) const override{ + return ShortDialogDetector().detect(images[0]) == m_expected; + } + +private: + bool m_expected; +}; + +void add_tests_DialogDetector(UnitTestDatabase& database){ + database.add("PokemonBDSP/DialogDetector/IntroRowanNameTab_True.png", true); + database.add("PokemonBDSP/DialogDetector/IntroRowanNoArrow_True.png", true); +} + + } } } diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h index fe7c90cbb0..f1f98bcc3a 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h @@ -7,6 +7,7 @@ #ifndef PokemonAutomation_PokemonBDSP_BattleDialogDetector_H #define PokemonAutomation_PokemonBDSP_BattleDialogDetector_H +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" #include "CommonTools/VisualDetector.h" #include "PokemonBDSP_SelectionArrow.h" @@ -31,6 +32,7 @@ class ShortDialogDetector : public StaticScreenDetector{ ImageFloatBox m_left_white; ImageFloatBox m_left; ImageFloatBox m_right_white; + ImageFloatBox m_right_white_low; ImageFloatBox m_right; }; class ShortDialogWatcher : public DetectorToFinder{ @@ -79,6 +81,10 @@ class ShortDialogPromptDetector : public VisualInferenceCallback{ + +// The screenshot tests in CommandLineTests/PokemonBDSP/DialogDetector/. +void add_tests_DialogDetector(UnitTestDatabase& database); + } } } diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp index 12da43966e..57814e7e7b 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp @@ -7,6 +7,8 @@ #include "CommonFramework/ImageTools/ImageStats.h" #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" #include "CommonTools/Images/SolidColorTest.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "Tests/ImageUnitTest.h" #include "PokemonBDSP_MenuDetector.h" namespace PokemonAutomation{ @@ -41,25 +43,21 @@ void MenuDetector::make_overlays(VideoOverlaySet& items) const{ items.add(m_color, m_line4); items.add(m_color, m_cross); } -bool MenuDetector::detect(const ImageViewRGB32& screen){ - ImageStats stats0 = image_stats(extract_box_reference(screen, m_line0)); - if (!is_white(stats0)){ - return false; - } - ImageStats stats1 = image_stats(extract_box_reference(screen, m_line1)); - if (!is_white(stats1)){ - return false; - } - ImageStats stats2 = image_stats(extract_box_reference(screen, m_line2)); - if (!is_white(stats2)){ - return false; - } - ImageStats stats3 = image_stats(extract_box_reference(screen, m_line3)); - if (!is_white(stats3)){ - return false; +// Whether the five columns between the icons are white from `top` down. +static bool stripes_white(const ImageViewRGB32& screen, double top){ + for (size_t index = 0; index < 5; index++){ + const ImageFloatBox stripe(0.160 + 0.166 * (double)index, top, 0.015, STRIPE_BOTTOM - top); + if (!is_white(image_stats(extract_box_reference(screen, stripe)))){ + return false; + } } - ImageStats stats4 = image_stats(extract_box_reference(screen, m_line4)); - if (!is_white(stats4)){ + return true; +} +bool MenuDetector::detect(const ImageViewRGB32& screen){ + // A "NEW" tag above an icon (a new game's OPTIONS, next to the third + // column) breaks a column at the top; below the tags the columns are + // white either way. + if (!stripes_white(screen, m_line0.y) && !stripes_white(screen, STRIPE_TOP_BELOW_BANNER)){ return false; } ImageStats cross = image_stats(extract_box_reference(screen, m_cross)); @@ -84,6 +82,35 @@ bool MenuWatcher::process_frame(const ImageViewRGB32& frame, WallClock timestamp + +class Test_MenuDetector : public ImageUnitTest{ +public: + Test_MenuDetector(const std::string& image, bool expected) + : ImageUnitTest("PokemonBDSP::MenuDetector - " + image, {image}) + , m_expected(expected) + {} + + virtual UnitTestResult run_on_images( + Logger&, CancellableScope&, + const std::vector& images + ) const override{ + return MenuDetector().detect(images[0]) == m_expected; + } + +private: + bool m_expected; +}; + +void add_tests_MenuDetector(UnitTestDatabase& database){ + database.add("PokemonBDSP/MenuDetector/NewGameBagOptions_True.png", true); + database.add("PokemonBDSP/MenuDetector/Home_False.png", false); + database.add("PokemonBDSP/MenuDetector/OverworldSandgemMart_False.png", false); + database.add("PokemonBDSP/MenuDetector/OverworldOreburghMine_False.png", false); + database.add("PokemonBDSP/MenuDetector/IntroDialog_False.png", false); + database.add("PokemonBDSP/MenuDetector/IntroBarry_False.png", false); +} + + } } } diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h index f7e5a0517f..46ca52470a 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h @@ -11,6 +11,7 @@ #include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" #include "CommonTools/VisualDetector.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" namespace PokemonAutomation{ namespace NintendoSwitch{ @@ -46,6 +47,10 @@ class MenuWatcher : public MenuDetector, public VisualInferenceCallback{ + +// The screenshot tests in CommandLineTests/PokemonBDSP/MenuDetector/. +void add_tests_MenuDetector(UnitTestDatabase& database); + } } } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp deleted file mode 100644 index 507edb09c0..0000000000 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp +++ /dev/null @@ -1,49 +0,0 @@ -/* BDSP AutoStory - X menu detector - * - * From: https://github.com/PokemonAutomation/ - * - */ - -#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" -#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" -#include "PokemonBDSP_AutoStory_XMenuDetector.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -bool x_menu_bar(const ImageStats& bar){ - // A flat sky blue: blue well over red, and nearly no texture. - return bar.average.b > 200 && bar.average.r < 140 && bar.average.b - bar.average.r > 90 && - bar.stddev.sum() < 45; -} - -XMenuDetector::XMenuDetector(Color color) - : m_color(color) -{} -void XMenuDetector::make_overlays(VideoOverlaySet& items) const{ - items.add(m_color, X_MENU_BAR_BOX); -} -bool XMenuDetector::detect(const ImageViewRGB32& screen){ - if (x_menu_bar(image_stats(extract_box_reference(screen, X_MENU_BAR_BOX)))){ - return true; - } - return MenuDetector(m_color, true).detect(screen); -} - -XMenuWatcher::XMenuWatcher(Color color) - : XMenuDetector(color) - , VisualInferenceCallback("XMenuWatcher") -{} -void XMenuWatcher::make_overlays(VideoOverlaySet& items) const{ - XMenuDetector::make_overlays(items); -} -bool XMenuWatcher::process_frame(const ImageViewRGB32& frame, WallClock){ - return detect(frame); -} - - -} -} -} diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h deleted file mode 100644 index ed18dc93a4..0000000000 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h +++ /dev/null @@ -1,56 +0,0 @@ -/* BDSP AutoStory - X menu detector - * - * From: https://github.com/PokemonAutomation/ - * - * The X menu at every stage of the game. MenuDetector wants the full menu's - * white columns between the icons; a new game's menu has two icons (BAG, - * and OPTIONS with a "NEW" tag) and was missed (r5, 2026-11-01). The blue - * bar along the bottom ("A Confirm R Save B Close") is there from the - * first menu on. - * - */ - -#ifndef PokemonAutomation_PokemonBDSP_AutoStory_XMenuDetector_H -#define PokemonAutomation_PokemonBDSP_AutoStory_XMenuDetector_H - -#include "CommonFramework/ImageTools/ImageBoxes.h" -#include "CommonFramework/ImageTools/ImageStats.h" -#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" -#include "CommonTools/VisualDetector.h" - -namespace PokemonAutomation{ -namespace NintendoSwitch{ -namespace PokemonBDSP{ - - -// The bottom bar's left end, left of its button prompts. -inline const ImageFloatBox X_MENU_BAR_BOX(0.02, 0.965, 0.18, 0.025); -// Whether `bar` (the stats of X_MENU_BAR_BOX) is the menu's flat blue bar. -bool x_menu_bar(const ImageStats& bar); - -class XMenuDetector : public StaticScreenDetector{ -public: - XMenuDetector(Color color = COLOR_RED); - - virtual void make_overlays(VideoOverlaySet& items) const override; - // The bottom bar, or the full menu as MenuDetector (with the NEW tag - // skipped) sees it. - virtual bool detect(const ImageViewRGB32& screen) override; - -private: - Color m_color; -}; - -class XMenuWatcher : public XMenuDetector, public VisualInferenceCallback{ -public: - XMenuWatcher(Color color = COLOR_RED); - - virtual void make_overlays(VideoOverlaySet& items) const override; - virtual bool process_frame(const ImageViewRGB32& frame, WallClock timestamp) override; -}; - - -} -} -} -#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp index bdcc61a886..eedf48ac01 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -19,7 +19,6 @@ #include "Data/PokemonBDSP_AutoStory_StoryMaps.h" #include "Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_Center.h" -#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h" #include "PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP_AutoStory_Actions_Tests.h" @@ -180,44 +179,6 @@ class Test_IntroBeforeMenu : public UnitTest{ }; -// The X menu by its bottom bar at every stage: r5's new-game menu, two icons -// and a NEW tag, was missed by MenuDetector. /PokemonBDSP/ -// AutoStory/XMenu/open/*.png must read as the menu, closed/*.png (a dialogue, -// the intro, the overworld, HOME) must not. -class Test_XMenu : public UnitTest{ -public: - Test_XMenu() - : UnitTest("PokemonBDSP::AutoStory - X menu at every stage") - {} - - virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ - const std::string folder = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/XMenu/"; - if (!Filesystem::is_directory(Filesystem::Path(folder))){ - return UnitTestResult(UnitTestResult::SKIPPED, "Screenshots not found: " + folder); - } - size_t failed = 0; - for (const char* expected : {"open", "closed"}){ - for (const Filesystem::Path& file : Filesystem::list_directory(Filesystem::Path(folder + expected))){ - if (file.extension().string() != ".png"){ - continue; - } - const ImageRGB32 image(file.string()); - const ImageStats bar = image_stats(extract_box_reference(image, X_MENU_BAR_BOX)); - const bool open = XMenuDetector().detect(image); - const bool ok = open == (std::string(expected) == "open"); - failed += ok ? 0 : 1; - logger.log(std::string("[X menu] ") + (ok ? "ok " : "FAIL ") + expected + "/" + file.filename().string() + - ": bar " + bar.average.to_string() + " sd " + bar.stddev.to_string(), ok ? COLOR_BLUE : COLOR_RED); - } - } - if (failed != 0){ - return std::to_string(failed) + " X menu screenshots misread; see the log."; - } - return true; - } -}; - - // X from a still bedroom opening something the detector misses, closed // again by B, is the player in control; moving screens or X doing nothing // are not. @@ -319,29 +280,6 @@ class Test_IntroGate : public UnitTest{ } }; -// The intro's screens, classified: /PokemonBDSP/AutoStory/IntroScreens/ -// /*.png (in CommandLineTests), at 1080p and 720p. Among -// them r2's "dialog/020-other.png": Rowan's last line, a dialogue box with -// no "more" arrow. -class Test_IntroScreens : public UnitTest{ -public: - Test_IntroScreens() - : UnitTest("PokemonBDSP::AutoStory - new-game intro screens") - {} - - virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ - const std::string folder = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/IntroScreens/"; - if (!Filesystem::is_directory(Filesystem::Path(folder))){ - return UnitTestResult(UnitTestResult::SKIPPED, "Screenshots not found: " + folder); - } - const size_t failed = run_intro_screen_tests(logger, folder); - if (failed != 0){ - return std::to_string(failed) + " intro screens misread; see the log."; - } - return true; - } -}; - // The Mart's doorway on the failure screenshots of r121a and r122a, kept in // the unit test resources (the CommandLineTests checkout): where it is // relative to the player, who is in the middle of the screen. @@ -523,8 +461,6 @@ void add_tests_AutoStory_Actions(UnitTestDatabase& database){ database.add(); database.add(); database.add(); - database.add(); - database.add(); database.add(); database.add(); database.add(); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index c4f49e3c11..24bc8e56cc 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -5,11 +5,8 @@ */ #include "Common/Cpp/Exceptions.h" -#include "Common/Cpp/Filesystem/Filesystem.h" #include "CommonFramework/ImageTools/ImageBoxes.h" #include "CommonFramework/ImageTools/ImageDiff.h" -#include "CommonFramework/ImageTools/ImageStats.h" -#include "CommonTools/Images/SolidColorTest.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/VideoPipeline/VideoFeed.h" #include "CommonTools/Async/InferenceRoutines.h" @@ -22,7 +19,7 @@ #include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" #include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h" #include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" -#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" #include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" #include "PokemonBDSP_AutoStory_NewGame.h" @@ -33,29 +30,6 @@ namespace PokemonBDSP{ using namespace std::chrono_literals; -namespace{ - -// BDSP's dialogue box. ShortDialogDetector also wants something busy at the -// box's right end, which the "more text" arrow gives; a speaker's last line -// has no arrow and read as "other" (r2, 2026-11-01: Rowan's "Now, go on, and -// leap into the world of Pokemon!"). Without the arrow: white at the text -// area's left end, along its bottom and at its right end below the first -// line, and the box's edge on the left. -bool intro_dialog_showing(const ImageViewRGB32& screen){ - if (ShortDialogDetector().detect(screen)){ - return true; - } - const ImageStats left_white = image_stats(extract_box_reference(screen, ImageFloatBox(0.21, 0.835, 0.008, 0.12))); - // Below the first line's end: a long line reaches it (r2's ran to 0.79). - const ImageStats right_white = image_stats(extract_box_reference(screen, ImageFloatBox(0.785, 0.915, 0.008, 0.04))); - const ImageStats bottom = image_stats(extract_box_reference(screen, ImageFloatBox(0.50, 0.91, 0.29, 0.05))); - const ImageStats left_edge = image_stats(extract_box_reference(screen, ImageFloatBox(0.18, 0.835, 0.02, 0.12))); - return is_white(left_white) && is_white(right_white) && is_white(bottom) && left_edge.stddev.sum() >= 30; -} - -} - - std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ // The Switch's own screens come first: BDSP's dialogue detector also fires on them. if (console != nullptr && StartGameUserSelectDetector(*console).detect(screen)){ @@ -75,11 +49,11 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& default: return "rival-list-preset"; } - if (XMenuDetector().detect(screen)){ + if (MenuDetector().detect(screen)){ return "menu"; } // Includes the appearance grid, whose prompt sits in a dialogue box. - if (intro_dialog_showing(screen)){ + if (ShortDialogDetector().detect(screen)){ return "dialog"; } // The boot and fades. After the dialogue: Rowan's first lines show on black. @@ -180,39 +154,6 @@ void record_intro_step(IntroProgress& progress, IntroStep step){ } } -size_t run_intro_screen_tests(Logger& logger, const std::string& folder){ - size_t passed = 0; - size_t failed = 0; - for (const Filesystem::Path& dir : Filesystem::list_directory(Filesystem::Path(folder))){ - if (!Filesystem::is_directory(dir)){ - continue; - } - const std::string expected = dir.filename().string(); - for (const Filesystem::Path& file : Filesystem::list_directory(dir)){ - if (file.extension().string() != ".png"){ - continue; - } - const ImageRGB32 image(file.string()); - // 1080p and 720p. At 960x540 the keyboard's and the name list's checks - // do not hold (blurred key text, an arrow below the object size limit). - for (size_t width : {size_t(1920), size_t(1280)}){ - const std::string actual = classify_intro_screen(nullptr, image.scale_to(width, width * 9 / 16)); - if (actual == expected){ - passed++; - continue; - } - failed++; - logger.log("[Intro tests] FAIL " + expected + "/" + file.filename().string() + - " at " + std::to_string(width) + "px: read as " + actual, COLOR_RED); - } - } - } - logger.log("[Intro tests] " + std::to_string(passed) + " passed, " + std::to_string(failed) + " failed.", - failed == 0 ? COLOR_BLUE : COLOR_RED); - return failed; -} - - namespace{ // Waits until the keyboard is gone (or, with `gone` false, has appeared). @@ -276,7 +217,7 @@ void close_menu_and_finish(IntroRun& run){ pbf_wait(run.context, 1000ms); run.context.wait_for_all_requests(); VideoSnapshot screen_now = run.env.console.video().snapshot(); - if (!screen_now || XMenuDetector().detect(screen_now)){ + if (!screen_now || MenuDetector().detect(screen_now)){ run.trace.fail("The main menu did not close."); } } @@ -307,7 +248,7 @@ XProbeResult x_from_still_screen(IntroRun& run){ const VideoSnapshot before = stream.video().snapshot(); press(run, BUTTON_X); run.context.wait_for_all_requests(); - XMenuWatcher menu; + MenuWatcher menu; if (wait_until(stream, run.context, 3s, {{menu}}) == 0){ return XProbeResult::MENU; } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h index 632e2026cb..0bd88fa64c 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h @@ -119,10 +119,6 @@ bool intro_in_control_without_menu(const XProbe& probe); // Counts `step` in `progress`. void record_intro_step(IntroProgress& progress, IntroStep step); -// Checks classify_intro_screen() on //*.png at 1920 and -// 1280 px wide. Logs each mismatch; returns the number of them. -size_t run_intro_screen_tests(Logger& logger, const std::string& folder); - // From the title screen or any intro screen of a game with no save, with BDSP // running in front: from HOME it throws UserSetupError, since the restart would // launch whatever tile HOME's cursor is on. Restarts the game first, so it diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp index 61ff5eabb1..c9f5b50e24 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp @@ -16,7 +16,7 @@ #include "CommonTools/Async/InferenceRoutines.h" #include "CommonTools/VisualDetectors/BlackScreenDetector.h" #include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" -#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" #include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" #include "PokemonBDSP_AutoStory_Bag.h" #include "PokemonBDSP_AutoStory_Settings.h" @@ -195,7 +195,7 @@ struct OptionReferences{ // From the overworld (or the main menu): the main menu, settled. void open_main_menu(VideoStream& stream, ProControllerContext& context){ context.wait_for_all_requests(); - XMenuDetector menu; + MenuDetector menu; VideoOverlaySet detector_overlays(stream.overlay()); menu.make_overlays(detector_overlays); detector_overlays.add(COLOR_CYAN, {0.04, 0.065, 0.93, 0.70}, ""); @@ -206,7 +206,7 @@ void open_main_menu(VideoStream& stream, ProControllerContext& context){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); } - XMenuWatcher menu_open; + MenuWatcher menu_open; if (wait_until(stream, context, 5s, {{menu_open}}) != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Settings checkpoint: Main menu not detected.", stream); @@ -229,7 +229,7 @@ void open_options(VideoStream& stream, ProControllerContext& context, const Tuto } pbf_press_button(context, BUTTON_B, 80ms, 1000ms); context.wait_for_all_requests(); - XMenuWatcher back_to_menu; + MenuWatcher back_to_menu; if (wait_until(stream, context, 5s, {{back_to_menu}}) != 0){ break; } @@ -399,7 +399,7 @@ void save_options(VideoStream& stream, ProControllerContext& context, const std: void reopen_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); - XMenuWatcher verify_menu; + MenuWatcher verify_menu; if (wait_until(stream, context, 5s, {{verify_menu}}) != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Autosave verification: Main menu not detected.", stream); @@ -413,7 +413,7 @@ void reopen_options(VideoStream& stream, ProControllerContext& context, const Tu } pbf_press_button(context, BUTTON_B, 80ms, 1000ms); context.wait_for_all_requests(); - XMenuWatcher return_menu; + MenuWatcher return_menu; if (wait_until(stream, context, 5s, {{return_menu}}) != 0){ break; } @@ -490,7 +490,7 @@ void back_to_overworld(VideoStream& stream, ProControllerContext& context){ pbf_mash_button(context, BUTTON_B, 3000ms); context.wait_for_all_requests(); OverworldDetector overworld; - XMenuDetector menu; + MenuDetector menu; BlackScreenDetector black; bool recovered = false; size_t confirmations = 0; diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index c9a0ceacd2..eae54af4ca 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1627,8 +1627,6 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/PokemonBDSP_Settings.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h - Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.cpp - Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_XMenuDetector.h Source/PokemonBDSP/PokemonBDSP_Tests.cpp Source/PokemonBDSP/PokemonBDSP_Tests.h Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp From 7c2b8c4dce47effb09f828b82c42ce678c9d0488 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 16:26:23 -0800 Subject: [PATCH 15/23] BDSP AutoStory: NewGame and Settings look for the main menu below the NEW tags A new game's main menu has a "NEW" tag above OPTIONS that breaks the top of the third white column. MenuDetector already handles that with skip_new_banner = true, so every menu check in the intro and the settings checkpoint passes it, instead of MenuDetector itself changing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../AutoStory/PokemonBDSP_AutoStory_NewGame.cpp | 6 +++--- .../AutoStory/PokemonBDSP_AutoStory_Settings.cpp | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index 24bc8e56cc..929c908bcf 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -49,7 +49,7 @@ std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& default: return "rival-list-preset"; } - if (MenuDetector().detect(screen)){ + if (MenuDetector(COLOR_RED, /*skip_new_banner*/ true).detect(screen)){ return "menu"; } // Includes the appearance grid, whose prompt sits in a dialogue box. @@ -217,7 +217,7 @@ void close_menu_and_finish(IntroRun& run){ pbf_wait(run.context, 1000ms); run.context.wait_for_all_requests(); VideoSnapshot screen_now = run.env.console.video().snapshot(); - if (!screen_now || MenuDetector().detect(screen_now)){ + if (!screen_now || MenuDetector(COLOR_RED, /*skip_new_banner*/ true).detect(screen_now)){ run.trace.fail("The main menu did not close."); } } @@ -248,7 +248,7 @@ XProbeResult x_from_still_screen(IntroRun& run){ const VideoSnapshot before = stream.video().snapshot(); press(run, BUTTON_X); run.context.wait_for_all_requests(); - MenuWatcher menu; + MenuWatcher menu(COLOR_RED, /*skip_new_banner*/ true); if (wait_until(stream, run.context, 3s, {{menu}}) == 0){ return XProbeResult::MENU; } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp index c9f5b50e24..24f8c96b4f 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp @@ -195,7 +195,7 @@ struct OptionReferences{ // From the overworld (or the main menu): the main menu, settled. void open_main_menu(VideoStream& stream, ProControllerContext& context){ context.wait_for_all_requests(); - MenuDetector menu; + MenuDetector menu(COLOR_RED, /*skip_new_banner*/ true); VideoOverlaySet detector_overlays(stream.overlay()); menu.make_overlays(detector_overlays); detector_overlays.add(COLOR_CYAN, {0.04, 0.065, 0.93, 0.70}, ""); @@ -206,7 +206,7 @@ void open_main_menu(VideoStream& stream, ProControllerContext& context){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); } - MenuWatcher menu_open; + MenuWatcher menu_open(COLOR_RED, /*skip_new_banner*/ true); if (wait_until(stream, context, 5s, {{menu_open}}) != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Settings checkpoint: Main menu not detected.", stream); @@ -229,7 +229,7 @@ void open_options(VideoStream& stream, ProControllerContext& context, const Tuto } pbf_press_button(context, BUTTON_B, 80ms, 1000ms); context.wait_for_all_requests(); - MenuWatcher back_to_menu; + MenuWatcher back_to_menu(COLOR_RED, /*skip_new_banner*/ true); if (wait_until(stream, context, 5s, {{back_to_menu}}) != 0){ break; } @@ -399,7 +399,7 @@ void save_options(VideoStream& stream, ProControllerContext& context, const std: void reopen_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ pbf_press_button(context, BUTTON_X, 80ms, 1000ms); context.wait_for_all_requests(); - MenuWatcher verify_menu; + MenuWatcher verify_menu(COLOR_RED, /*skip_new_banner*/ true); if (wait_until(stream, context, 5s, {{verify_menu}}) != 0){ OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Autosave verification: Main menu not detected.", stream); @@ -413,7 +413,7 @@ void reopen_options(VideoStream& stream, ProControllerContext& context, const Tu } pbf_press_button(context, BUTTON_B, 80ms, 1000ms); context.wait_for_all_requests(); - MenuWatcher return_menu; + MenuWatcher return_menu(COLOR_RED, /*skip_new_banner*/ true); if (wait_until(stream, context, 5s, {{return_menu}}) != 0){ break; } @@ -490,7 +490,7 @@ void back_to_overworld(VideoStream& stream, ProControllerContext& context){ pbf_mash_button(context, BUTTON_B, 3000ms); context.wait_for_all_requests(); OverworldDetector overworld; - MenuDetector menu; + MenuDetector menu(COLOR_RED, /*skip_new_banner*/ true); BlackScreenDetector black; bool recovered = false; size_t confirmations = 0; From 615805b6bb0798647b75cac2d44595d1a81718c7 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 18:18:47 -0800 Subject: [PATCH 16/23] BDSP AutoStory: the save prompt reads the menu past its NEW tags Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../AutoStory/PokemonBDSP_AutoStory_Actions.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp index 870a989c4d..e8dc995ceb 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp @@ -37,7 +37,16 @@ namespace{ // shows the play time and badges, a record of where each checkpoint save // stands. Returns the map its card names. std::string open_save_prompt(VideoStream& stream, ProControllerContext& context){ - overworld_to_menu(stream, context); + // overworld_to_menu() reads the menu with the NEW tags' columns, which a new + // Pokemon in the party breaks (r19: three misses, then the blind save). + pbf_press_button(context, BUTTON_X, 160ms, 840ms); + context.wait_for_all_requests(); + MenuWatcher menu(COLOR_RED, true); + if (wait_until(stream, context, std::chrono::seconds(10), {{menu}}) < 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Menu not detected after 10 seconds.", stream); + } + context.wait_for(100ms); pbf_press_button(context, BUTTON_R, 80ms, 2000ms); context.wait_for_all_requests(); VideoSnapshot screen = stream.video().snapshot(); From 5c8cde5e2fb32c3fd60fd45919df2b1ec1275dfb Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 20:33:41 -0800 Subject: [PATCH 17/23] BDSP AutoStory: run from wild battles on the menu check that holds for any party Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp index 9a6f01695f..361868fc84 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp @@ -413,8 +413,11 @@ void escape_wild_battle(VideoStream& stream, stream); } +// STARTER's check of the menu (its Battle button) holds for any party: +// STANDARD also reads the party's balls, which a lone starter does not +// match, so Route 201's first wild battle never saw its menu (r34). void run_from_wild_battle(VideoStream& stream, ProControllerContext& context){ - escape_wild_battle(stream, context, BattleType::STANDARD); + escape_wild_battle(stream, context, BattleType::STARTER); } From 016f86cdbb83904cba82e86154de75d78a0b6c57 Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 22:34:01 -0800 Subject: [PATCH 18/23] BDSP AutoStory actions: heal from inside the Center's door heal_inside_pokemon_center() is heal_at_pokemon_center() after the door, for a caller that walks in from a known tile. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp | 6 ++++++ .../Programs/AutoStory/PokemonBDSP_AutoStory_Center.h | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp index e4b20bd0e0..bebf46d788 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp @@ -417,7 +417,13 @@ void heal_at_pokemon_center( const std::string& label ){ enter_lit_door(stream, context, label); + heal_inside_pokemon_center(stream, context, label); +} +void heal_inside_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ // The nurse's counter is straight ahead of the door. route_status(stream, label + ": talking to the nurse."); if (walk_to_talk(stream, context, run_from_wild_battle, {0, 1}, 6000ms, label + ": to the nurse").stop != WalkStop::DIALOG){ diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h index ec94028fbe..ed1afae6af 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h @@ -110,6 +110,14 @@ void heal_at_pokemon_center( const std::string& label ); +// heal_at_pokemon_center() from just inside the door: talks to the nurse, +// accepts, and walks back out. Ends on the tile below the door. For a caller +// that walked in on its own (from a known tile, straight north). +void heal_inside_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + } From 44f25ed8ac56f21fc50492563c32ea034cf4603d Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 23:01:03 -0800 Subject: [PATCH 19/23] BDSP AutoStory actions: talk_to_nurse() without the walk out For a caller that leaves the Center its own way: the walk out's indoor check read "not a room" a step from the counter (r50), so 00.04b ended inside. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../AutoStory/PokemonBDSP_AutoStory_Center.cpp | 12 +++++++++--- .../AutoStory/PokemonBDSP_AutoStory_Center.h | 8 ++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp index bebf46d788..c04ef05234 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp @@ -423,6 +423,15 @@ void heal_at_pokemon_center( void heal_inside_pokemon_center( VideoStream& stream, ProControllerContext& context, const std::string& label +){ + talk_to_nurse(stream, context, label); + route_status(stream, label + ": healed; walking out."); + leave_door_south(stream, context, run_from_wild_battle, label + ": out the door"); +} + +void talk_to_nurse( + VideoStream& stream, ProControllerContext& context, + const std::string& label ){ // The nurse's counter is straight ahead of the door. route_status(stream, label + ": talking to the nurse."); @@ -443,9 +452,6 @@ void heal_inside_pokemon_center( } } context.wait_for_all_requests(); - - route_status(stream, label + ": healed; walking out."); - leave_door_south(stream, context, run_from_wild_battle, label + ": out the door"); } diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h index ed1afae6af..189f89f28a 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h @@ -118,6 +118,14 @@ void heal_inside_pokemon_center( const std::string& label ); +// From just inside a Pokemon Center's door: walks up to the nurse, talks to +// her and accepts. Ends at her counter, for a caller that walks out its own +// way. +void talk_to_nurse( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + } From 50cb0d9766291120a8f11f4670c2e565014497ec Mon Sep 17 00:00:00 2001 From: dayfine Date: Sun, 1 Nov 2026 23:07:33 -0800 Subject: [PATCH 20/23] BDSP AutoStory actions: open the menu for the Bag the way the save does The Bag check used overworld_to_menu(), whose menu read misses the NEW tags (r52: the menu was open, "Menu not detected after 10 seconds"). One opener, with the tags allowed, for both. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../PokemonBDSP_AutoStory_Actions.cpp | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp index e8dc995ceb..a2604400f5 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp @@ -33,12 +33,11 @@ using namespace std::chrono_literals; namespace{ -// Opens the save prompt from the overworld and keeps a screenshot of it: it -// shows the play time and badges, a record of where each checkpoint save -// stands. Returns the map its card names. -std::string open_save_prompt(VideoStream& stream, ProControllerContext& context){ - // overworld_to_menu() reads the menu with the NEW tags' columns, which a new - // Pokemon in the party breaks (r19: three misses, then the blind save). +// X from the overworld, then waits up to 10 s for the menu. Reads it with +// the NEW tags allowed: overworld_to_menu() reads it with the tags' columns, +// which a new Pokemon in the party breaks (r19: three misses, then the blind +// save; r52: the Bag check after the Mart lady's Potions). +void open_menu_from_overworld(VideoStream& stream, ProControllerContext& context){ pbf_press_button(context, BUTTON_X, 160ms, 840ms); context.wait_for_all_requests(); MenuWatcher menu(COLOR_RED, true); @@ -46,6 +45,13 @@ std::string open_save_prompt(VideoStream& stream, ProControllerContext& context) OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, "Menu not detected after 10 seconds.", stream); } +} + +// Opens the save prompt from the overworld and keeps a screenshot of it: it +// shows the play time and badges, a record of where each checkpoint save +// stands. Returns the map its card names. +std::string open_save_prompt(VideoStream& stream, ProControllerContext& context){ + open_menu_from_overworld(stream, context); context.wait_for(100ms); pbf_press_button(context, BUTTON_R, 80ms, 2000ms); context.wait_for_all_requests(); @@ -200,9 +206,9 @@ MenuCursorPosition read_menu_cursor(VideoStream& stream, ProControllerContext& c // Opens the menu unless it is already open: a late-opening menu may show // after a timeout, and X would close it. void open_menu_unless_open(VideoStream& stream, ProControllerContext& context){ - MenuWatcher already_open; + MenuWatcher already_open(COLOR_RED, true); if (wait_until(stream, context, 300ms, {{already_open}}) < 0){ - overworld_to_menu(stream, context); + open_menu_from_overworld(stream, context); }else{ stream.log("[Open menu] Menu already visible; skipping X."); } From 53f077a016467944e91eaa9a7348f7df0dc87010 Mon Sep 17 00:00:00 2001 From: dayfine Date: Thu, 5 Nov 2026 22:29:56 -0800 Subject: [PATCH 21/23] BDSP AutoStory actions: align with PR #231 lint conventions --- .../NintendoSwitch_SystemKeyboardDetector.cpp | 6 +- .../NintendoSwitch_SystemKeyboardDetector.h | 5 +- .../PokemonBDSP_NameListDetector.cpp | 8 +- .../Inference/PokemonBDSP_NameListDetector.h | 7 +- ...PokemonBDSP_AutoStory_BattleMenuReader.cpp | 8 +- .../PokemonBDSP_AutoStory_BattleMenuReader.h | 7 +- ...kemonBDSP_AutoStory_MenuCursorDetector.cpp | 8 +- ...PokemonBDSP_AutoStory_MenuCursorDetector.h | 10 +- .../PokemonBDSP_AutoStory_Actions.cpp | 8 +- .../AutoStory/PokemonBDSP_AutoStory_Actions.h | 6 +- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 712 +++++++++--------- .../PokemonBDSP_AutoStory_Actions_Tests.h | 6 +- .../AutoStory/PokemonBDSP_AutoStory_Bag.cpp | 8 +- .../AutoStory/PokemonBDSP_AutoStory_Bag.h | 8 +- .../PokemonBDSP_AutoStory_Battle.cpp | 8 +- .../AutoStory/PokemonBDSP_AutoStory_Battle.h | 6 +- .../PokemonBDSP_AutoStory_BattleMoves.cpp | 8 +- .../PokemonBDSP_AutoStory_BattleMoves.h | 6 +- .../PokemonBDSP_AutoStory_Center.cpp | 10 +- .../AutoStory/PokemonBDSP_AutoStory_Center.h | 6 +- .../PokemonBDSP_AutoStory_NewGame.cpp | 8 +- .../AutoStory/PokemonBDSP_AutoStory_NewGame.h | 13 +- .../PokemonBDSP_AutoStory_Settings.cpp | 10 +- .../PokemonBDSP_AutoStory_Settings.h | 6 +- .../PokemonBDSP_AutoStory_WildBattle.cpp | 10 +- .../PokemonBDSP_AutoStory_WildBattle.h | 6 +- 26 files changed, 463 insertions(+), 441 deletions(-) diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp index 7403cfac90..2bab1c3dbe 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -128,7 +128,7 @@ bool move_keyboard_cursor(VideoStream& stream, ProControllerContext& context, in return false; } -} +} // namespace void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text){ @@ -167,5 +167,5 @@ void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, -} -} +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h index c29b21931e..6091d9a14f 100644 --- a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h @@ -37,6 +37,7 @@ class SystemKeyboardDetector : public StaticScreenDetector{ ImageFloatBox m_bottom_bar; ImageFloatBox m_ok_button; }; +// Periodic watcher for the Switch system on-screen keyboard. class SystemKeyboardWatcher : public DetectorToFinder{ public: SystemKeyboardWatcher(Color color = COLOR_CYAN) @@ -62,6 +63,6 @@ void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, -} -} +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp index f9bf85a2a6..0ee0a55f55 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp @@ -49,7 +49,7 @@ std::vector find_name_list_arrows(const ImageViewRGB32& image){ } return ret; } -} +} // namespace NameListDetector::NameListDetector(Color color) @@ -91,6 +91,6 @@ int NameListDetector::selected_row(const ImageViewRGB32& screen) const{ } -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h index f247f1da74..9e40fb77d0 100644 --- a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h @@ -17,6 +17,7 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ +// Detects the rival preset name selection menu and reads the selected row. class NameListDetector : public StaticScreenDetector{ public: NameListDetector(Color color = COLOR_MAGENTA); @@ -35,7 +36,7 @@ class NameListDetector : public StaticScreenDetector{ }; -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp index 7cb7d4f296..d5f91e6914 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp @@ -124,7 +124,7 @@ bool move_button_selected(const ImageViewRGB32& screen, double top, double botto return false; } -} +} // namespace BattleMenuOption battle_menu_selected_option(BattleMenuDetector& menu, const ImageViewRGB32& screen){ @@ -177,6 +177,6 @@ BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen){ -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h index 07d5c68eab..1e053c4953 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h @@ -44,11 +44,12 @@ struct BattleMoveMenuState{ size_t selected_slot = 0; std::array buttons; }; +// Detects the populated move slots and the currently selected slot from the battle move menu. BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen); -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp index 47db816a50..48bb59b8d5 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp @@ -185,7 +185,7 @@ double template_match_rmsd(const ImageViewRGB32& screen, const ImageFloatBox& bo return min_rmsd; } -} // anonymous namespace +} // namespace MenuCursorPosition MenuCursorDetector::detect_dark_pixels(const ImageViewRGB32& screen) const { @@ -281,6 +281,6 @@ MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& } -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h index ca855d8e79..cadbf1e4e4 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h @@ -58,6 +58,7 @@ namespace NintendoSwitch{ namespace PokemonBDSP{ +// Possible icon positions for the BDSP pause/X menu cursor. enum class MenuCursorPosition { UNKNOWN = -1, POKEDEX = 0, @@ -70,11 +71,14 @@ enum class MenuCursorPosition { MYSTERYGIFT = 7, }; +// Total number of icons in the pause menu layout. static constexpr int MENU_CURSOR_NUM_ICONS = 8; +// Returns the debug name of a MenuCursorPosition. std::string menu_cursor_to_string(MenuCursorPosition pos); +// Detects which icon in the BDSP pause menu is currently focused by the cursor. class MenuCursorDetector { public: // ── Approach 1: dark pixel detection ───────────────────────────────────── @@ -142,7 +146,7 @@ class MenuCursorDetector { MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& context); -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp index a2604400f5..acbc234da3 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp @@ -265,7 +265,7 @@ void move_menu_cursor(VideoStream& stream, ProControllerContext& context, MenuCu } } -} +} // namespace std::string read_location_from_save_screen(VideoStream& stream, ProControllerContext& context){ const std::string location = open_save_prompt(stream, context); @@ -352,6 +352,6 @@ void open_menu(VideoStream& stream, ProControllerContext& context, MenuCursorPos -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h index a2fa75786e..afeb466640 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h @@ -57,7 +57,7 @@ void open_menu(VideoStream& stream, ProControllerContext& context, MenuCursorPos -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp index eedf48ac01..bd1476caf1 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -9,13 +9,17 @@ * */ +#include +#include #include #include #include +#include "Common/Cpp/Exceptions.h" #include "Common/Cpp/Filesystem/Filesystem.h" #include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/Logging/Logger.h" #include "Data/PokemonBDSP_AutoStory_StoryMaps.h" #include "Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_Center.h" @@ -40,52 +44,45 @@ IntroProgress intro_progress(size_t names_given){ return progress; } -class Test_IntroDecisions : public UnitTest{ -public: - Test_IntroDecisions() - : UnitTest("PokemonBDSP::AutoStory - new-game intro decisions") - {} - - virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - struct Case{ - const char* screen; - IntroProgress progress; - IntroStep expected; - }; - const IntroProgress start = intro_progress(0); - const IntroProgress named = intro_progress(1); - const IntroProgress done = intro_progress(2); - const Case cases[] = { - {"user-select", start, IntroStep::SELECT_USER}, - {"home", done, IntroStep::FAIL_NOT_IN_GAME}, - {"keyboard", start, IntroStep::ENTER_PLAYER_NAME}, - {"keyboard", named, IntroStep::CANCEL_RIVAL_KEYBOARD}, - {"rival-list-tell", named, IntroStep::MOVE_TO_PRESET}, - {"rival-list-preset", named, IntroStep::PICK_PRESET}, - {"menu", done, IntroStep::CLOSE_MENU_AND_FINISH}, - // The appearance grid has its prompt in a dialogue box. - {"dialog", start, IntroStep::ADVANCE}, - {"dialog", done, IntroStep::ADVANCE_WITH_B}, - // Logos, the title, language and the appearance preview: no X - // before the bedroom is possible. - {"other", start, IntroStep::ADVANCE}, - {"other", named, IntroStep::ADVANCE}, - {"other", done, IntroStep::CLEAR_WITH_B}, - {"black", start, IntroStep::WAIT}, - {"black", done, IntroStep::WAIT}, - }; - for (const Case& c : cases){ - const IntroStep actual = decide_intro_step(c.screen, c.progress); - if (actual != c.expected){ - return std::string("Screen \"") + c.screen + "\" (player named " + - (c.progress.player_named ? "yes" : "no") + ", rival named " + - (c.progress.rival_named ? "yes" : "no") + "): step " + - std::to_string((int)actual) + ", expected " + std::to_string((int)c.expected) + "."; - } +UnitTestResult test_intro_decisions(Logger&){ + struct Case{ + const char* screen; + IntroProgress progress; + IntroStep expected; + }; + const IntroProgress start = intro_progress(0); + const IntroProgress named = intro_progress(1); + const IntroProgress done = intro_progress(2); + const Case cases[] = { + {"user-select", start, IntroStep::SELECT_USER}, + {"home", done, IntroStep::FAIL_NOT_IN_GAME}, + {"keyboard", start, IntroStep::ENTER_PLAYER_NAME}, + {"keyboard", named, IntroStep::CANCEL_RIVAL_KEYBOARD}, + {"rival-list-tell", named, IntroStep::MOVE_TO_PRESET}, + {"rival-list-preset", named, IntroStep::PICK_PRESET}, + {"menu", done, IntroStep::CLOSE_MENU_AND_FINISH}, + // The appearance grid has its prompt in a dialogue box. + {"dialog", start, IntroStep::ADVANCE}, + {"dialog", done, IntroStep::ADVANCE_WITH_B}, + // Logos, the title, language and the appearance preview: no X + // before the bedroom is possible. + {"other", start, IntroStep::ADVANCE}, + {"other", named, IntroStep::ADVANCE}, + {"other", done, IntroStep::CLEAR_WITH_B}, + {"black", start, IntroStep::WAIT}, + {"black", done, IntroStep::WAIT}, + }; + for (const Case& c : cases){ + const IntroStep actual = decide_intro_step(c.screen, c.progress); + if (actual != c.expected){ + return std::string("Screen \"") + c.screen + "\" (player named " + + (c.progress.player_named ? "yes" : "no") + ", rival named " + + (c.progress.rival_named ? "yes" : "no") + "): step " + + std::to_string((int)actual) + ", expected " + std::to_string((int)c.expected) + "."; } - return true; } -}; + return true; +} // Runs `screens` through decide_intro_step() and record_intro_step() from @@ -102,374 +99,383 @@ std::vector intro_steps(IntroProgress& progress, const std::vector confirm = intro_steps(progress, {"rival-list-preset", "dialog", "other", "dialog"}); - const std::vector expected{IntroStep::PICK_PRESET, IntroStep::ADVANCE, IntroStep::ADVANCE, IntroStep::ADVANCE}; - if (confirm != expected){ - return "The pick and its confirmation must take A, never B."; - } - if (progress.rival_named){ - return "The name is not final until INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick."; - } - // The list back (No was answered): pick again, the count restarts. - if (intro_steps(progress, {"rival-list-tell", "rival-list-preset"}) != - std::vector{IntroStep::MOVE_TO_PRESET, IntroStep::PICK_PRESET}){ - return "The list coming back after a pick must be picked from again."; - } - for (size_t c = 0; c < INTRO_SCREENS_AFTER_RIVAL_PICK; c++){ - if (intro_steps(progress, {c % 2 ? "other" : "dialog"}) != std::vector{IntroStep::ADVANCE}){ - return "Screen " + std::to_string(c) + " after the pick must take A."; - } - } - if (!progress.rival_named){ - return "INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick, the name is final."; +UnitTestResult test_intro_rival_confirm(Logger&){ + IntroProgress progress = intro_progress(1); + // r4's frames 029 to 032: the pick, "Barry, is it?", Barry alone + // as the Yes/No comes up, the Yes/No. + const std::vector confirm = intro_steps(progress, {"rival-list-preset", "dialog", "other", "dialog"}); + const std::vector expected{IntroStep::PICK_PRESET, IntroStep::ADVANCE, IntroStep::ADVANCE, IntroStep::ADVANCE}; + if (confirm != expected){ + return "The pick and its confirmation must take A, never B."; + } + if (progress.rival_named){ + return "The name is not final until INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick."; + } + // The list back (No was answered): pick again, the count restarts. + if (intro_steps(progress, {"rival-list-tell", "rival-list-preset"}) != + std::vector{IntroStep::MOVE_TO_PRESET, IntroStep::PICK_PRESET}){ + return "The list coming back after a pick must be picked from again."; + } + for (size_t c = 0; c < INTRO_SCREENS_AFTER_RIVAL_PICK; c++){ + if (intro_steps(progress, {c % 2 ? "other" : "dialog"}) != std::vector{IntroStep::ADVANCE}){ + return "Screen " + std::to_string(c) + " after the pick must take A."; } - return true; } -}; + if (!progress.rival_named){ + return "INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick, the name is final."; + } + return true; +} // After the rival is named, unknown screens take INTRO_B_BEFORE_X B presses, // then X; only the menu ends the intro (r2: Rowan's last line, misread). -class Test_IntroBeforeMenu : public UnitTest{ -public: - Test_IntroBeforeMenu() - : UnitTest("PokemonBDSP::AutoStory - new-game intro: B on unknown screens, then X for the menu") - {} - - virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - IntroProgress progress = intro_progress(1); - std::vector screens{"rival-list-preset", "dialog", "other"}; - std::vector expected{IntroStep::PICK_PRESET, IntroStep::ADVANCE, IntroStep::ADVANCE}; - for (size_t c = 2; c < INTRO_SCREENS_AFTER_RIVAL_PICK; c++){ - screens.push_back("dialog"); - expected.push_back(IntroStep::ADVANCE); - } - for (size_t round = 0; round < 2; round++){ - screens.insert(screens.end(), INTRO_B_BEFORE_X + 1, "other"); - expected.insert(expected.end(), INTRO_B_BEFORE_X, IntroStep::CLEAR_WITH_B); - expected.push_back(IntroStep::PROBE_FOR_MENU); - } - // A dialogue line in between takes B without counting. - screens.insert(screens.end(), {"dialog", "other", "menu"}); - expected.insert(expected.end(), {IntroStep::ADVANCE_WITH_B, IntroStep::CLEAR_WITH_B, IntroStep::CLOSE_MENU_AND_FINISH}); - const std::vector steps = intro_steps(progress, screens); - for (size_t c = 0; c < steps.size(); c++){ - if (steps[c] != expected[c]){ - return "Screen " + std::to_string(c) + " (\"" + screens[c] + "\"): step " + - std::to_string((int)steps[c]) + ", expected " + std::to_string((int)expected[c]) + "."; - } - } - IntroProgress pressed = intro_progress(2); - pressed.b_presses = INTRO_B_BEFORE_X; - if (decide_intro_step("other", pressed) != IntroStep::PROBE_FOR_MENU){ - return "X must follow INTRO_B_BEFORE_X B presses."; +UnitTestResult test_intro_before_menu(Logger&){ + IntroProgress progress = intro_progress(1); + std::vector screens{"rival-list-preset", "dialog", "other"}; + std::vector expected{IntroStep::PICK_PRESET, IntroStep::ADVANCE, IntroStep::ADVANCE}; + for (size_t c = 2; c < INTRO_SCREENS_AFTER_RIVAL_PICK; c++){ + screens.push_back("dialog"); + expected.push_back(IntroStep::ADVANCE); + } + for (size_t round = 0; round < 2; round++){ + screens.insert(screens.end(), INTRO_B_BEFORE_X + 1, "other"); + expected.insert(expected.end(), INTRO_B_BEFORE_X, IntroStep::CLEAR_WITH_B); + expected.push_back(IntroStep::PROBE_FOR_MENU); + } + // A dialogue line in between takes B without counting. + screens.insert(screens.end(), {"dialog", "other", "menu"}); + expected.insert(expected.end(), {IntroStep::ADVANCE_WITH_B, IntroStep::CLEAR_WITH_B, IntroStep::CLOSE_MENU_AND_FINISH}); + const std::vector steps = intro_steps(progress, screens); + for (size_t c = 0; c < steps.size(); c++){ + if (steps[c] != expected[c]){ + return "Screen " + std::to_string(c) + " (\"" + screens[c] + "\"): step " + + std::to_string((int)steps[c]) + ", expected " + std::to_string((int)expected[c]) + "."; } - return true; } -}; + IntroProgress pressed = intro_progress(2); + pressed.b_presses = INTRO_B_BEFORE_X; + if (decide_intro_step("other", pressed) != IntroStep::PROBE_FOR_MENU){ + return "X must follow INTRO_B_BEFORE_X B presses."; + } + return true; +} // X from a still bedroom opening something the detector misses, closed // again by B, is the player in control; moving screens or X doing nothing // are not. -class Test_IntroInControl : public UnitTest{ -public: - Test_IntroInControl() - : UnitTest("PokemonBDSP::AutoStory - new-game intro: in control without the menu read") - {} +UnitTestResult test_intro_in_control(Logger&){ + struct Case{ + XProbe probe; + bool expected; + const char* what; + }; + const Case cases[] = { + {{1, 60, 2}, true, "still, X opened, B closed"}, + {{12, 60, 2}, false, "moving before X (sparkles, a cutscene)"}, + {{1, 5, 2}, false, "X changed nothing"}, + {{1, 60, 40}, false, "B did not put it back"}, + }; + for (const Case& c : cases){ + if (intro_in_control_without_menu(c.probe) != c.expected){ + return std::string("Wrong for: ") + c.what + "."; + } + } + return true; +} - virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - struct Case{ - XProbe probe; - bool expected; - const char* what; - }; - const Case cases[] = { - {{1, 60, 2}, true, "still, X opened, B closed"}, - {{12, 60, 2}, false, "moving before X (sparkles, a cutscene)"}, - {{1, 5, 2}, false, "X changed nothing"}, - {{1, 60, 40}, false, "B did not put it back"}, - }; - for (const Case& c : cases){ - if (intro_in_control_without_menu(c.probe) != c.expected){ - return std::string("Wrong for: ") + c.what + "."; - } + +// Black from the restart until the first logo, well past the limit. +std::string check_boot(IntroProgress& progress){ + using std::chrono::milliseconds; + for (milliseconds t(0); t <= 2 * INTRO_TIME_BEFORE_BDSP; t += milliseconds(500)){ + note_intro_screen(progress, "black", t); + if (decide_intro_step("black", progress) != IntroStep::WAIT){ + return "A black screen must take no press."; } - return true; } -}; + if (progress.first_shown || progress.unconfirmed_for != milliseconds(0)){ + return "Black screens must not start the wrong-game clock."; + } + return ""; +} +UnitTestResult check_confirming_screens(){ + for (const char* screen : {"dialog", "rival-list-tell", "rival-list-preset", "menu"}){ + if (!intro_screen_confirms_bdsp(screen)){ + return std::string("Expected \"") + screen + "\" to confirm BDSP."; + } + } + for (const char* screen : {"user-select", "home", "keyboard", "black", "other"}){ + if (intro_screen_confirms_bdsp(screen)){ + return std::string("Expected \"") + screen + "\" not to confirm BDSP."; + } + } + return true; +} // The intro gates its A presses on a BDSP screen within a time from the // first screen that is not black: past it, it is another game (r1, // 2026-11-01, launched FireRed). The boot's black screens, about 25 s of // them (r3), neither take presses nor count. -class Test_IntroGate : public UnitTest{ -public: - Test_IntroGate() - : UnitTest("PokemonBDSP::AutoStory - new-game intro: BDSP or stop") - {} - - virtual UnitTestResult run(Logger&, CancellableScope&) const override{ - using std::chrono::milliseconds; - IntroProgress progress; - const std::string boot = check_boot(progress); - if (!boot.empty()){ - return boot; - } - const milliseconds shown = 2 * INTRO_TIME_BEFORE_BDSP + milliseconds(500); - note_intro_screen(progress, "other", shown); - if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ - return "The first logo must take A."; - } - note_intro_screen(progress, "black", shown + INTRO_TIME_BEFORE_BDSP / 2); - note_intro_screen(progress, "other", shown + INTRO_TIME_BEFORE_BDSP - milliseconds(1)); - if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ - return "Unknown screens take A until INTRO_TIME_BEFORE_BDSP has passed."; - } - IntroProgress other_game = progress; - note_intro_screen(other_game, "other", shown + INTRO_TIME_BEFORE_BDSP); - if (decide_intro_step("other", other_game) != IntroStep::FAIL_NOT_BDSP){ - return "Past INTRO_TIME_BEFORE_BDSP of unknown screens, it is another game."; - } - note_intro_screen(progress, "dialog", shown + INTRO_TIME_BEFORE_BDSP - milliseconds(1)); - note_intro_screen(progress, "other", shown + 10 * INTRO_TIME_BEFORE_BDSP); - if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ - return "Once BDSP's dialogue has shown, the clock stops."; - } - return check_confirming_screens(); +UnitTestResult test_intro_gate(Logger&){ + using std::chrono::milliseconds; + IntroProgress progress; + const std::string boot = check_boot(progress); + if (!boot.empty()){ + return boot; } - -private: - // Black from the restart until the first logo, well past the limit. - static std::string check_boot(IntroProgress& progress){ - using std::chrono::milliseconds; - for (milliseconds t(0); t <= 2 * INTRO_TIME_BEFORE_BDSP; t += milliseconds(500)){ - note_intro_screen(progress, "black", t); - if (decide_intro_step("black", progress) != IntroStep::WAIT){ - return "A black screen must take no press."; - } - } - if (progress.first_shown || progress.unconfirmed_for != milliseconds(0)){ - return "Black screens must not start the wrong-game clock."; - } - return ""; + const milliseconds shown = 2 * INTRO_TIME_BEFORE_BDSP + milliseconds(500); + note_intro_screen(progress, "other", shown); + if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ + return "The first logo must take A."; } - static UnitTestResult check_confirming_screens(){ - for (const char* screen : {"dialog", "rival-list-tell", "rival-list-preset", "menu"}){ - if (!intro_screen_confirms_bdsp(screen)){ - return std::string("Expected \"") + screen + "\" to confirm BDSP."; - } - } - for (const char* screen : {"user-select", "home", "keyboard", "black", "other"}){ - if (intro_screen_confirms_bdsp(screen)){ - return std::string("Expected \"") + screen + "\" not to confirm BDSP."; - } - } - return true; + note_intro_screen(progress, "black", shown + INTRO_TIME_BEFORE_BDSP / 2); + note_intro_screen(progress, "other", shown + INTRO_TIME_BEFORE_BDSP - milliseconds(1)); + if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ + return "Unknown screens take A until INTRO_TIME_BEFORE_BDSP has passed."; } -}; + IntroProgress other_game = progress; + note_intro_screen(other_game, "other", shown + INTRO_TIME_BEFORE_BDSP); + if (decide_intro_step("other", other_game) != IntroStep::FAIL_NOT_BDSP){ + return "Past INTRO_TIME_BEFORE_BDSP of unknown screens, it is another game."; + } + note_intro_screen(progress, "dialog", shown + INTRO_TIME_BEFORE_BDSP - milliseconds(1)); + note_intro_screen(progress, "other", shown + 10 * INTRO_TIME_BEFORE_BDSP); + if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ + return "Once BDSP's dialogue has shown, the clock stops."; + } + return check_confirming_screens(); +} + // The Mart's doorway on the failure screenshots of r121a and r122a, kept in // the unit test resources (the CommandLineTests checkout): where it is // relative to the player, who is in the middle of the screen. -class Test_LitDoorwayScreenshots : public UnitTest{ -public: - Test_LitDoorwayScreenshots() - : UnitTest("PokemonBDSP::AutoStory - lit doorway screenshots") - {} - - virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ - struct Case{ - const char* file; - double tiles_across_min; - double tiles_across_max; - double rows_up_min; - double rows_up_max; - }; - const Case cases[] = { - // r121a: below-left of the door, which is part of a tile east, straight ahead. - {"PokemonBDSP/AutoStory/SandgemMartDoorUpRight-20261030-023443.png", 0.2, 0.7, 0.5, 1.5}, - // r122a: a tile west of the door and three or four rows south of its front. - {"PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", -1.3, -0.5, 2.5, 4.5}, - }; - for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; - if (!Filesystem::is_regular_file(path)){ - return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); - } - const ImageRGB32 screen(path); - const std::optional doorway = find_lit_doorway(screen); - if (!doorway){ - return std::string(c.file) + ": no lit doorway found."; - } - logger.log(std::string(c.file) + ": doorway at " + std::to_string(doorway->x) + ", " + - std::to_string(doorway->y) + ": " + std::to_string(doorway->tiles_across) + " tiles across, " + - std::to_string(doorway->rows_up) + " rows up."); - if (doorway->tiles_across < c.tiles_across_min || doorway->tiles_across > c.tiles_across_max){ - return std::string(c.file) + ": doorway " + std::to_string(doorway->tiles_across) + - " tiles across, expected " + std::to_string(c.tiles_across_min) + " to " + - std::to_string(c.tiles_across_max) + "."; - } - if (doorway->rows_up < c.rows_up_min || doorway->rows_up > c.rows_up_max){ - return std::string(c.file) + ": doorway " + std::to_string(doorway->rows_up) + - " rows up, expected " + std::to_string(c.rows_up_min) + " to " + - std::to_string(c.rows_up_max) + "."; - } +UnitTestResult test_lit_doorway_screenshots(Logger& logger){ + struct Case{ + const char* file; + double tiles_across_min; + double tiles_across_max; + double rows_up_min; + double rows_up_max; + }; + const Case cases[] = { + // r121a: below-left of the door, which is part of a tile east, straight ahead. + {"PokemonBDSP/AutoStory/SandgemMartDoorUpRight-20261030-023443.png", 0.2, 0.7, 0.5, 1.5}, + // r122a: a tile west of the door and three or four rows south of its front. + {"PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", -1.3, -0.5, 2.5, 4.5}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const std::optional doorway = find_lit_doorway(screen); + if (!doorway){ + return std::string(c.file) + ": no lit doorway found."; + } + logger.log(std::string(c.file) + ": doorway at " + std::to_string(doorway->x) + ", " + + std::to_string(doorway->y) + ": " + std::to_string(doorway->tiles_across) + " tiles across, " + + std::to_string(doorway->rows_up) + " rows up."); + if (doorway->tiles_across < c.tiles_across_min || doorway->tiles_across > c.tiles_across_max){ + return std::string(c.file) + ": doorway " + std::to_string(doorway->tiles_across) + + " tiles across, expected " + std::to_string(c.tiles_across_min) + " to " + + std::to_string(c.tiles_across_max) + "."; + } + if (doorway->rows_up < c.rows_up_min || doorway->rows_up > c.rows_up_max){ + return std::string(c.file) + ": doorway " + std::to_string(doorway->rows_up) + + " rows up, expected " + std::to_string(c.rows_up_min) + " to " + + std::to_string(c.rows_up_max) + "."; } - return true; } -}; + return true; +} // The save prompt r124a took beside Sandgem's Poke Mart, sure it stood on // Route 202 (in CommandLineTests under PokemonBDSP/AutoStory): its card // names Sandgem Town, which the save must read before confirming. -class Test_SaveScreenLocation : public UnitTest{ -public: - Test_SaveScreenLocation() - : UnitTest("PokemonBDSP::AutoStory - save screen location") - {} +UnitTestResult test_save_screen_location(Logger& logger){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/SaveScreenSandgemTown-20261030-034444.png"; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const std::string read = read_save_screen_location(ImageRGB32(path)); + logger.log("Save screen location read: \"" + read + "\"."); + if (!location_banner_matches(read, MAP_SANDGEM_TOWN)){ + return "Save screen location \"" + read + "\" should match Sandgem Town."; + } + if (location_banner_matches(read, MAP_ROUTE_202)){ + return "Save screen location \"" + read + "\" should not match Route 202."; + } + return true; +} - virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ - const std::string path = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/SaveScreenSandgemTown-20261030-034444.png"; + +// The two shots around r126's door walk: below the Center's sign, outside, +// and on the entrance mat inside, where the lit floor read as a doorway and +// the walk kept lining up with it (runs 2 and 3). +UnitTestResult test_pokemon_center_inside(Logger& logger){ + struct Case{ + const char* file; + bool inside; + bool doorway; + }; + const Case cases[] = { + {.file = "PokemonBDSP/AutoStory/SandgemCenterInside-20261030-053555.png", .inside = true, .doorway = false}, + // r127c: walked in by the turn north at the door, facing north. + {.file = "PokemonBDSP/AutoStory/SandgemCenterInsideFacingNorth-r127c.png", .inside = true, .doorway = false}, + {.file = "PokemonBDSP/AutoStory/SandgemCenterBelowSign-20261030-053527.png", .inside = false, .doorway = true}, + {.file = "PokemonBDSP/AutoStory/SandgemMartDoorUpRight-20261030-023443.png", .inside = false, .doorway = true}, + {.file = "PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", .inside = false, .doorway = true}, + // Jubilife (2026-10-30): inside its Center twice, at its door, and + // on the doorstep after walking out (the banner up). + {.file = "PokemonBDSP/AutoStory/JubilifeCenterInside-20261030-c6.png", .inside = true, .doorway = false}, + {.file = "PokemonBDSP/AutoStory/JubilifeCenterInside-20261030-c4.png", .inside = true, .doorway = false}, + {.file = "PokemonBDSP/AutoStory/JubilifeCenterFrontDoor-20261030-q2.png", .inside = false, .doorway = true}, + {.file = "PokemonBDSP/AutoStory/JubilifeCenterDoorstep-20261030-c5.png", .inside = false, .doorway = true}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; if (!Filesystem::is_regular_file(path)){ return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); } - const std::string read = read_save_screen_location(ImageRGB32(path)); - logger.log("Save screen location read: \"" + read + "\"."); - if (!location_banner_matches(read, MAP_SANDGEM_TOWN)){ - return "Save screen location \"" + read + "\" should match Sandgem Town."; + const ImageRGB32 screen(path); + const bool inside = pokemon_center_inside(screen); + const std::optional doorway = find_lit_doorway(screen); + logger.log(std::string(c.file) + ": inside " + std::to_string(inside) + ", doorway " + + (doorway ? std::to_string(doorway->tiles_across) + " across, " + std::to_string(doorway->rows_up) + " rows up" : "none")); + if (inside != c.inside){ + return std::string(c.file) + ": expected " + (c.inside ? "inside" : "outside") + + " a Pokemon Center, read " + (inside ? "inside" : "outside") + "."; } - if (location_banner_matches(read, MAP_ROUTE_202)){ - return "Save screen location \"" + read + "\" should not match Route 202."; + if (doorway.has_value() != c.doorway){ + return std::string(c.file) + ": expected " + (c.doorway ? "a doorway" : "no doorway") + + ", read " + (doorway ? "one" : "none") + "."; } - return true; } -}; + return true; +} -// The two shots around r126's door walk: below the Center's sign, outside, -// and on the entrance mat inside, where the lit floor read as a doorway and -// the walk kept lining up with it (runs 2 and 3). -class Test_PokemonCenterInside : public UnitTest{ -public: - Test_PokemonCenterInside() - : UnitTest("PokemonBDSP::AutoStory - Pokemon Center inside") - {} - - virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ - struct Case{ - const char* file; - bool inside; - bool doorway; - }; - const Case cases[] = { - {.file = "PokemonBDSP/AutoStory/SandgemCenterInside-20261030-053555.png", .inside = true, .doorway = false}, - // r127c: walked in by the turn north at the door, facing north. - {.file = "PokemonBDSP/AutoStory/SandgemCenterInsideFacingNorth-r127c.png", .inside = true, .doorway = false}, - {.file = "PokemonBDSP/AutoStory/SandgemCenterBelowSign-20261030-053527.png", .inside = false, .doorway = true}, - {.file = "PokemonBDSP/AutoStory/SandgemMartDoorUpRight-20261030-023443.png", .inside = false, .doorway = true}, - {.file = "PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", .inside = false, .doorway = true}, - // Jubilife (2026-10-30): inside its Center twice, at its door, and - // on the doorstep after walking out (the banner up). - {.file = "PokemonBDSP/AutoStory/JubilifeCenterInside-20261030-c6.png", .inside = true, .doorway = false}, - {.file = "PokemonBDSP/AutoStory/JubilifeCenterInside-20261030-c4.png", .inside = true, .doorway = false}, - {.file = "PokemonBDSP/AutoStory/JubilifeCenterFrontDoor-20261030-q2.png", .inside = false, .doorway = true}, - {.file = "PokemonBDSP/AutoStory/JubilifeCenterDoorstep-20261030-c5.png", .inside = false, .doorway = true}, - }; - for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; - if (!Filesystem::is_regular_file(path)){ - return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); - } - const ImageRGB32 screen(path); - const bool inside = pokemon_center_inside(screen); - const std::optional doorway = find_lit_doorway(screen); - logger.log(std::string(c.file) + ": inside " + std::to_string(inside) + ", doorway " + - (doorway ? std::to_string(doorway->tiles_across) + " across, " + std::to_string(doorway->rows_up) + " rows up" : "none")); - if (inside != c.inside){ - return std::string(c.file) + ": expected " + (c.inside ? "inside" : "outside") + - " a Pokemon Center, read " + (inside ? "inside" : "outside") + "."; - } - if (doorway.has_value() != c.doorway){ - return std::string(c.file) + ": expected " + (c.doorway ? "a doorway" : "no doorway") + - ", read " + (doorway ? "one" : "none") + "."; - } +// The School's lamp at night, from its doorsteps' row (2026-10-30): where +// the door took us in, the lamp reads within half a tile of the door; a +// tile either side, more. +UnitTestResult test_school_lamp(Logger& logger){ + const ImageFloatBox box(0.55, 0.20, 0.30, 0.10); + struct Case{ const char* file; double min; double max; }; + const Case cases[] = { + {"PokemonBDSP/AutoStory/JubilifeSchoolRightOfDoor-20261030.png", -1.5, -0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolOneTileShort-20261030.png", -1.5, -0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolAtDoor-20261030.png", -0.5, 0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolAtDoor2-20261030.png", -0.5, 0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolDoorstep-20261030.png", -0.5, 0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolOneTilePast-20261030.png", 0.5, 1.5}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const std::optional lamp = find_lit_lamp_x(screen, box); + if (!lamp){ + return std::string(c.file) + ": no lit lamp found."; + } + const double across = tiles_across_at(*lamp) - tiles_across_at(0.67); + logger.log(std::string(c.file) + ": lamp at " + std::to_string(*lamp) + ", door " + std::to_string(across) + " tiles across."); + if (across < c.min || across > c.max){ + return std::string(c.file) + ": door " + std::to_string(across) + " tiles across, expected " + + std::to_string(c.min) + " to " + std::to_string(c.max) + "."; } - return true; } + return true; +} + + +struct AutoStoryActionsInput{ + std::string name; + std::function body; }; +struct AutoStoryActionsExpectation{ +}; -// The School's lamp at night, from its doorsteps' row (2026-10-30): where -// the door took us in, the lamp reads within half a tile of the door; a -// tile either side, more. -class Test_SchoolLamp : public UnitTest{ +struct AutoStoryActionsCase{ + AutoStoryActionsInput input; + AutoStoryActionsExpectation expectation; +}; + +class Test_AutoStoryActions : public UnitTest{ public: - Test_SchoolLamp() - : UnitTest("PokemonBDSP::AutoStory - Jubilife School lamp") + explicit Test_AutoStoryActions(AutoStoryActionsCase test_case) + : UnitTest("PokemonBDSP::AutoStory - " + test_case.input.name) + , m_case(std::move(test_case)) {} virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ - const ImageFloatBox box(0.55, 0.20, 0.30, 0.10); - struct Case{ const char* file; double min; double max; }; - const Case cases[] = { - {"PokemonBDSP/AutoStory/JubilifeSchoolRightOfDoor-20261030.png", -1.5, -0.5}, - {"PokemonBDSP/AutoStory/JubilifeSchoolOneTileShort-20261030.png", -1.5, -0.5}, - {"PokemonBDSP/AutoStory/JubilifeSchoolAtDoor-20261030.png", -0.5, 0.5}, - {"PokemonBDSP/AutoStory/JubilifeSchoolAtDoor2-20261030.png", -0.5, 0.5}, - {"PokemonBDSP/AutoStory/JubilifeSchoolDoorstep-20261030.png", -0.5, 0.5}, - {"PokemonBDSP/AutoStory/JubilifeSchoolOneTilePast-20261030.png", 0.5, 1.5}, - }; - for (const Case& c : cases){ - const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; - if (!Filesystem::is_regular_file(path)){ - return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); - } - const ImageRGB32 screen(path); - const std::optional lamp = find_lit_lamp_x(screen, box); - if (!lamp){ - return std::string(c.file) + ": no lit lamp found."; - } - const double across = tiles_across_at(*lamp) - tiles_across_at(0.67); - logger.log(std::string(c.file) + ": lamp at " + std::to_string(*lamp) + ", door " + std::to_string(across) + " tiles across."); - if (across < c.min || across > c.max){ - return std::string(c.file) + ": door " + std::to_string(across) + " tiles across, expected " + - std::to_string(c.min) + " to " + std::to_string(c.max) + "."; - } + try{ + return m_case.input.body(logger); + }catch (Exception& e){ + return std::string(e.name()) + ": " + e.message(); } - return true; } -}; - -} +private: + AutoStoryActionsCase m_case; +}; +} // namespace + + +const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { + { + .input = {"new-game intro decisions", test_intro_decisions}, + .expectation = {}, + }, + { + .input = {"new-game intro: the rival's name confirmed with A", test_intro_rival_confirm}, + .expectation = {}, + }, + { + .input = {"new-game intro: B on unknown screens, then X for the menu", test_intro_before_menu}, + .expectation = {}, + }, + { + .input = {"new-game intro: in control without the menu read", test_intro_in_control}, + .expectation = {}, + }, + { + .input = {"new-game intro: BDSP or stop", test_intro_gate}, + .expectation = {}, + }, + { + .input = {"lit doorway screenshots", test_lit_doorway_screenshots}, + .expectation = {}, + }, + { + .input = {"save screen location", test_save_screen_location}, + .expectation = {}, + }, + { + .input = {"Pokemon Center inside", test_pokemon_center_inside}, + .expectation = {}, + }, + { + .input = {"Jubilife School lamp", test_school_lamp}, + .expectation = {}, + }, +}; void add_tests_AutoStory_Actions(UnitTestDatabase& database){ - database.add(); - database.add(); - database.add(); - database.add(); - database.add(); - database.add(); - database.add(); - database.add(); - database.add(); + for (const auto& item : AUTO_STORY_ACTIONS_CASES){ + database.add(item); + } } - -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h index b6b09c7d32..b92b104fd1 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h @@ -19,7 +19,7 @@ void add_tests_AutoStory_Actions(UnitTestDatabase& database); -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp index 7de60c3a36..0cf1365c69 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp @@ -253,7 +253,7 @@ Dose potion_dose(VideoStream& stream, ProControllerContext& context, size_t dose return after >= 0.98 ? Dose::HEALED : Dose::GIVEN; } -} +} // namespace int bag_pocket_position(const ImageViewRGB32& frame){ if (!frame){ @@ -422,6 +422,6 @@ void use_potion_first_pokemon(VideoStream& stream, ProControllerContext& context -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h index 8e0e73695e..abb2d8e653 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h @@ -30,12 +30,14 @@ bool bag_list_is_empty(const ImageViewRGB32& frame); // The cursor's row, 1-7, or 0 if unknown. int bag_cursor_position(const ImageViewRGB32& frame); +// Navigation action between bag pocket tabs. enum class BagPocketAction{ UNKNOWN, CONFIRM, LEFT, RIGHT, }; +// Plan to navigate from one bag pocket tab to another. struct BagPocketMovePlan{ BagPocketAction action = BagPocketAction::UNKNOWN; size_t presses = 0; @@ -85,7 +87,7 @@ void use_potion_first_pokemon( -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp index 6b9a49ea24..568247aba4 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp @@ -374,7 +374,7 @@ void fight_with_best_move(VideoStream& stream, ProControllerContext& context, co } } -} +} // namespace void confirm_battle_overworld(VideoStream& stream, ProControllerContext& context, const std::string& label){ @@ -426,6 +426,6 @@ void fight_wild_battle(VideoStream& stream, ProControllerContext& context, const } -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h index c038cbadbe..13dfe78cf5 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h @@ -55,7 +55,7 @@ void fight_wild_battle(VideoStream& stream, ProControllerContext& context, const -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp index d96cc7da5d..be262d9cef 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp @@ -224,7 +224,7 @@ std::pair identify_move(const ImageViewRGB32& scr return {name, known}; } -} +} // namespace size_t pick_best_move(VideoStream& stream, const ImageViewRGB32& screen, @@ -271,6 +271,6 @@ size_t pick_best_move(VideoStream& stream, const ImageViewRGB32& screen, -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h index e5618b329f..2f9b1fa674 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h @@ -31,7 +31,7 @@ size_t pick_best_move(VideoStream& stream, const ImageViewRGB32& screen, const B -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp index c04ef05234..cfeae1e1b6 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp @@ -203,7 +203,7 @@ void log_doorway(VideoStream& stream, const std::string& label, const LitDoorway std::to_string(doorway.x) + ", " + std::to_string(doorway.y) + " of the screen)."); } -} +} // namespace bool pokemon_center_inside(const ImageViewRGB32& screen){ @@ -364,7 +364,7 @@ void confirm_inside_without_doorway(VideoStream& stream, ProControllerContext& c pbf_wait(context, 1500ms); } -} +} // namespace void enter_lit_door( VideoStream& stream, ProControllerContext& context, @@ -461,6 +461,6 @@ void move_across(VideoStream& stream, ProControllerContext& context, double tile } -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h index 189f89f28a..ded333a567 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h @@ -128,7 +128,7 @@ void talk_to_nurse( -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp index 929c908bcf..5bd6a47b09 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp @@ -389,7 +389,7 @@ std::vector intro_states(IntroRun& run){ }; } -} +} // namespace @@ -431,6 +431,6 @@ void play_new_game_intro( -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h index 0bd88fa64c..35fb8d8ca9 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h @@ -46,6 +46,7 @@ enum class IntroStep{ PROBE_FOR_MENU, // X, to see whether the player is in control WAIT, // nothing: a black screen (the boot, a fade) }; +// Progress tracking state for the new-game introductory sequence. struct IntroProgress{ bool player_named = false; // A preset name was picked from the rival's list, and how many screens @@ -99,6 +100,7 @@ inline constexpr size_t INTRO_X_WITHOUT_MENU = 20; // Notes `screen`, read `since_start` into the intro, in `progress`: the // game is confirmed on a BDSP screen; black screens leave the clock alone. void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start); +// Decides the next controller action based on the current screen and intro progress. IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); // When X opens nothing the menu detector knows, the player is still in @@ -107,14 +109,19 @@ IntroStep decide_intro_step(const std::string& screen, const IntroProgress& prog // than INTRO_X_CHANGE, and B put it back within INTRO_B_RESTORE: the menu // opened and closed. Changes are pixel RMSD. inline constexpr std::chrono::milliseconds INTRO_STILL_TIME = std::chrono::seconds(2); +// Maximum pixel RMSD threshold for considering the screen still before probing menu. inline constexpr double INTRO_STILL_CHANGE = 4; +// Minimum pixel RMSD change indicating menu opened after pressing X. inline constexpr double INTRO_X_CHANGE = 25; +// Maximum pixel RMSD difference between pre-X screen and post-B screen. inline constexpr double INTRO_B_RESTORE = 10; +// Probe metrics for detecting menu presence via screen changes from X and B. struct XProbe{ double still_change; // over INTRO_STILL_TIME before X double x_change; // from before X to after it double b_change; // from before X to after the B that follows }; +// Evaluates whether probe metrics indicate player is in control without menu detection. bool intro_in_control_without_menu(const XProbe& probe); // Counts `step` in `progress`. void record_intro_step(IntroProgress& progress, IntroStep step); @@ -135,7 +142,7 @@ void play_new_game_intro( ); -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp index 24f8c96b4f..e1f71ad647 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp @@ -446,7 +446,7 @@ void check_saved_autosave( context.wait_for_all_requests(); } stream.log("[Tutorial settings] Autosave = Off: confirmed after reopening saved Options (3 frames)."); -} +} // namespace void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& context){ open_main_menu(stream, context); @@ -514,7 +514,7 @@ void back_to_overworld(VideoStream& stream, ProControllerContext& context){ context.wait_for_all_requests(); } -} +} // namespace void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context){ // Settings-only recovery: initial attempt plus three retries. @@ -538,6 +538,6 @@ void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h index 2fa2d27641..206237de07 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h @@ -25,7 +25,7 @@ void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp index 361868fc84..8be35a67dd 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp @@ -258,7 +258,7 @@ void mash_to_overworld(VideoStream& stream, ProControllerContext& context, const } } -} +} // namespace void fight_starly(VideoStream& stream, ProControllerContext& context){ stream.log("[AutoStory] Starting Starly battle...", COLOR_BLUE); @@ -351,7 +351,7 @@ bool throw_until_caught(VideoStream& stream, ProControllerContext& context, EndB "Catch failed: Exhausted ten ball attempts.", stream); } -} +} // namespace void catch_1_pokemon( VideoStream& stream, @@ -422,6 +422,6 @@ void run_from_wild_battle(VideoStream& stream, ProControllerContext& context){ -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h index 5377a3c1b4..6ef1b1b8fa 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h @@ -50,7 +50,7 @@ void run_from_wild_battle(VideoStream& stream, ProControllerContext& context); -} -} -} +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation #endif From 14a916a78ae16d69dd975700cfaec5185a57609c Mon Sep 17 00:00:00 2001 From: dayfine Date: Sat, 7 Nov 2026 14:07:30 -0800 Subject: [PATCH 22/23] BDSP AutoStory: move action files into Actions/ subdirectory --- .../Source/PokemonBDSP/PokemonBDSP_Tests.cpp | 2 +- .../PokemonBDSP_AutoStory_Actions.cpp | 0 .../PokemonBDSP_AutoStory_Actions.h | 0 .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 0 .../PokemonBDSP_AutoStory_Actions_Tests.h | 0 .../PokemonBDSP_AutoStory_Bag.cpp | 0 .../{ => Actions}/PokemonBDSP_AutoStory_Bag.h | 0 .../PokemonBDSP_AutoStory_Battle.cpp | 0 .../PokemonBDSP_AutoStory_Battle.h | 0 .../PokemonBDSP_AutoStory_BattleMoves.cpp | 0 .../PokemonBDSP_AutoStory_BattleMoves.h | 0 .../PokemonBDSP_AutoStory_Center.cpp | 0 .../PokemonBDSP_AutoStory_Center.h | 0 .../PokemonBDSP_AutoStory_NewGame.cpp | 0 .../PokemonBDSP_AutoStory_NewGame.h | 0 .../PokemonBDSP_AutoStory_Settings.cpp | 0 .../PokemonBDSP_AutoStory_Settings.h | 0 .../PokemonBDSP_AutoStory_WildBattle.cpp | 0 .../PokemonBDSP_AutoStory_WildBattle.h | 0 SerialPrograms/cmake/SourceFiles.cmake | 36 +++++++++---------- 20 files changed, 19 insertions(+), 19 deletions(-) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Actions.cpp (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Actions.h (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Actions_Tests.cpp (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Actions_Tests.h (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Bag.cpp (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Bag.h (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Battle.cpp (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Battle.h (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_BattleMoves.cpp (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_BattleMoves.h (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Center.cpp (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Center.h (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_NewGame.cpp (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_NewGame.h (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Settings.cpp (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_Settings.h (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_WildBattle.cpp (100%) rename SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/{ => Actions}/PokemonBDSP_AutoStory_WildBattle.h (100%) diff --git a/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp index a53516ad5f..ff991df1bb 100644 --- a/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp @@ -5,7 +5,7 @@ */ #include "Common/Cpp/TestRunners/UnitTestDatabase.h" -#include "Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h" +#include "Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h" #include "Programs/AutoStory/PokemonBDSP_AutoStory_Maps_Tests.h" #include "Programs/AutoStory/PokemonBDSP_AutoStory_PlannedRoute_Tests.h" #include "Programs/AutoStory/PokemonBDSP_AutoStory_RoutePlanner_Tests.h" diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.cpp similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.cpp diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.h similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.h similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.cpp similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.cpp diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.h similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.h similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.cpp similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.cpp diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.h similarity index 100% rename from SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h rename to SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.h diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 6ed34f2e3f..6963ce0697 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1640,24 +1640,24 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h Source/PokemonBDSP/PokemonBDSP_Tests.cpp Source/PokemonBDSP/PokemonBDSP_Tests.h - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.cpp - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions.h - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.cpp - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Actions_Tests.h - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.cpp - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Bag.h - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.cpp - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Battle.h - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.cpp - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_BattleMoves.h - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.cpp - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Center.h - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.cpp - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_NewGame.h - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.cpp - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_Settings.h - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.cpp - Source/PokemonBDSP/Programs/AutoStory/PokemonBDSP_AutoStory_WildBattle.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp From 90bb79d74aa62994a7ebf2c7ed2365d630da1983 Mon Sep 17 00:00:00 2001 From: dayfine Date: Mon, 9 Nov 2026 00:32:49 -0800 Subject: [PATCH 23/23] BDSP: fold #226 fixes into the center and bag actions Moves #226's versions of these files into the PR that owns them. #226 commits: b7b74431c BDSP AutoStory: running out of Potions ends the doses, not the run e5388ccc5 BDSP AutoStory: a Pokemon Center's inside reads at the nurse's counter too 4d2c7a544 BDSP AutoStory: red tests for Oreburgh's Center read from its counter fc56a720b BDSP AutoStory: test real Oreburgh Mine corridor runs to and back in simulation Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_013VUubhHeYSMGSSDXZkKZss --- .../PokemonBDSP_AutoStory_Actions_Tests.cpp | 43 +++++++++++++++++- .../Actions/PokemonBDSP_AutoStory_Bag.cpp | 22 ++++----- .../Actions/PokemonBDSP_AutoStory_Center.cpp | 45 ++++++++++++++----- .../Actions/PokemonBDSP_AutoStory_Center.h | 9 ++-- 4 files changed, 92 insertions(+), 27 deletions(-) diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp index bd1476caf1..49a0dd2313 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -20,8 +20,8 @@ #include "CommonFramework/GlobalAutoPaths.h" #include "CommonFramework/ImageTypes/ImageRGB32.h" #include "CommonFramework/Logging/Logger.h" -#include "Data/PokemonBDSP_AutoStory_StoryMaps.h" -#include "Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryMaps.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" #include "PokemonBDSP_AutoStory_Center.h" #include "PokemonBDSP_AutoStory_NewGame.h" #include "PokemonBDSP_AutoStory_Actions_Tests.h" @@ -361,6 +361,41 @@ UnitTestResult test_pokemon_center_inside(Logger& logger){ } +// Oreburgh's Center, lab-B 2026-11-08 (in CommandLineTests under +// PokemonBDSP/AutoStory, skipped without them): 01.03's run in saw no fade +// and stopped at the nurse's counter, the camera in close and the lit floor +// down to the screen's bottom; on the exit mat; and outside on the doorstep +// at sunset, the ground as orange as the Center's back wall. +UnitTestResult test_pokemon_center_counter(Logger& logger){ + struct Case{ + const char* file; + bool inside; + }; + const Case cases[] = { + {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .inside = true}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterExitMat-20261108.png", .inside = true}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .inside = false}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const bool inside = pokemon_center_inside(screen); + logger.log(std::string(c.file) + ": inside " + std::to_string(inside)); + if (inside != c.inside){ + return std::string(c.file) + ": expected " + (c.inside ? "inside" : "outside") + + " a Pokemon Center, read " + (inside ? "inside" : "outside") + "."; + } + if (inside && find_lit_doorway(screen)){ + return std::string(c.file) + ": inside, yet a lit doorway was read."; + } + } + return true; +} + + // The School's lamp at night, from its doorsteps' row (2026-10-30): where // the door took us in, the lamp reads within half a tile of the door; a // tile either side, more. @@ -463,6 +498,10 @@ const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { .input = {"Pokemon Center inside", test_pokemon_center_inside}, .expectation = {}, }, + { + .input = {"Pokemon Center inside: Oreburgh's counter, exit mat and doorstep", test_pokemon_center_counter}, + .expectation = {}, + }, { .input = {"Jubilife School lamp", test_school_lamp}, .expectation = {}, diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp index 0cf1365c69..6230464190 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp @@ -203,7 +203,7 @@ void bag_back_to_overworld(VideoStream& stream, ProControllerContext& context, c } } -enum class Dose{ HEALED, GIVEN }; +enum class Dose{ HEALED, GIVEN, NONE_LEFT }; // One Potion on the lead, from the overworld, its HP read before and after // in the Bag. HEALED when no dose is needed (HP full, or above @@ -212,14 +212,11 @@ enum class Dose{ HEALED, GIVEN }; Dose potion_dose(VideoStream& stream, ProControllerContext& context, size_t dose, double heal_below){ stream.log("[Potion] Finding Bag/Medicine/Potion; dose " + std::to_string(dose + 1) + "/3."); if (!find_bag_item_by_name(stream, context, "potion", 1, 5)){ - if (dose == 0 && heal_below < 1.0){ - // Only a check: without Potions there is nothing to do here. - stream.log("[Potion] No Potion in the Bag; skipping the HP check.", COLOR_ORANGE); - return Dose::HEALED; - } - OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, - "Potion: Could not verify Potion in the Medicine pocket.", - stream); + // No Potion left is not a failure: the next Center heals. After Roark + // the one Potion went on the first dose and the second found the + // Medicine pocket empty (lab-B, 2026-11-08 15:26). + stream.log("[Potion] No Potion left in the Bag after " + std::to_string(dose) + " dose(s); going on.", COLOR_ORANGE); + return Dose::NONE_LEFT; } const double before = confirmed_potion_hp(stream, context); stream.log("[Potion] Slot 1 HP bar before use=" + std::to_string(before * 100) + "%."); @@ -405,7 +402,12 @@ void use_potion_first_pokemon(VideoStream& stream, ProControllerContext& context OverlayBoxScope hp_overlay(stream.overlay(), COLOR_GREEN, {0.137, 0.160, 0.202, 0.016}, ""); bool healed = false; for (size_t dose = 0; dose < 3 && !healed; dose++){ - healed = potion_dose(stream, context, dose, heal_below) == Dose::HEALED; + const Dose result = potion_dose(stream, context, dose, heal_below); + if (result == Dose::NONE_LEFT){ + bag_back_to_overworld(stream, context, "Potion: No overworld after finding no Potion."); + return; + } + healed = result == Dose::HEALED; if (!healed){ pbf_mash_button(context, BUTTON_B, 2000ms); context.wait_for_all_requests(); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp index cfeae1e1b6..9a58559284 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp @@ -4,6 +4,7 @@ * */ +#include #include #include #include @@ -203,6 +204,38 @@ void log_doorway(VideoStream& stream, const std::string& label, const LitDoorway std::to_string(doorway.x) + ", " + std::to_string(doorway.y) + " of the screen)."); } +// Near the entrance: black below the floor, either side of the step out +// of the door, and the floor lit between them: not a fade (349 in +// Jubilife at night). +bool center_entrance_view(const ImageViewRGB32& screen){ + for (const ImageFloatBox& box : {ImageFloatBox(0.02, 0.95, 0.30, 0.04), ImageFloatBox(0.65, 0.95, 0.33, 0.04)}){ + if (image_stats(extract_box_reference(screen, box)).average.sum() > 90){ + return false; + } + } + return image_stats(extract_box_reference(screen, ImageFloatBox(0.30, 0.55, 0.40, 0.15))).average.sum() > 250; +} + +// At the nurse's counter, the camera in close and the lit floor down to +// the screen's bottom (lab-B 2026-11-08: a run in whose fade went unseen +// stopped there): either side of us, the counter's red top (r 246, g 84) +// over its white front (about 200 each). Outside Oreburgh's Center at +// sunset the ground there is orange (r 218, g 134), with nothing white. +bool center_counter_view(const ImageViewRGB32& screen){ + for (double x : {0.15, 0.56}){ + const FloatPixel top = image_stats(extract_box_reference(screen, ImageFloatBox(x, 0.58, 0.12, 0.03))).average; + const FloatPixel front = image_stats(extract_box_reference(screen, ImageFloatBox(x, 0.66, 0.12, 0.03))).average; + const double front_min = std::min({front.r, front.g, front.b}); + const double front_max = std::max({front.r, front.g, front.b}); + const bool red = top.r >= 120 && top.r > 2 * top.g && top.g > top.b; + const bool white = front_min >= 120 && front_max - front_min <= 40; + if (!red || !white){ + return false; + } + } + return true; +} + } // namespace @@ -216,17 +249,7 @@ bool pokemon_center_inside(const ImageViewRGB32& screen){ const double r = box.average.r, g = box.average.g, b = box.average.b; wall += r >= 140 && r > g + 40 && g > b + 40; } - if (wall < 2){ - return false; - } - // Black below the floor, either side of the step out of the door. - for (const ImageFloatBox& box : {ImageFloatBox(0.02, 0.95, 0.30, 0.04), ImageFloatBox(0.65, 0.95, 0.33, 0.04)}){ - if (image_stats(extract_box_reference(screen, box)).average.sum() > 90){ - return false; - } - } - // And the floor lit between them: not a fade (349 in Jubilife at night). - return image_stats(extract_box_reference(screen, ImageFloatBox(0.30, 0.55, 0.40, 0.15))).average.sum() > 250; + return wall >= 2 && (center_entrance_view(screen) || center_counter_view(screen)); } void fail_if_blacked_out(VideoStream& stream, ProControllerContext& context, const std::string& label){ diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h index ded333a567..96a8288af9 100644 --- a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h @@ -31,10 +31,11 @@ struct LitDoorway{ double rows_up; // rows north of the player's tile, roughly: 1 straight ahead }; -// Whether `screen` is the inside of a Pokemon Center seen from its door: -// the orange back wall at the top left, and the black below the entrance -// mat along the bottom edge. Its lit floor reads as a doorway -// (find_lit_doorway()), so this is what says we are in (r126 runs 2, 3). +// Whether `screen` is the inside of a Pokemon Center: the orange back wall +// along the top, and either the black below the entrance mat along the +// bottom edge (seen from the door) or the nurse's counter close up. Its lit +// floor reads as a doorway (find_lit_doorway()), so this is what says we +// are in (r126 runs 2, 3). bool pokemon_center_inside(const ImageViewRGB32& screen); // Finds a lit doorway (a Pokemon Center's or a Poke Mart's) in the upper