diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp new file mode 100644 index 0000000000..2bab1c3dbe --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp @@ -0,0 +1,171 @@ +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch_SystemKeyboardDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + +using namespace std::chrono_literals; + + +// Key grid of the QWERTY layout and where each key sits: rows "1".."z", columns "1".."-". +static const char* const KEY_ROWS[] = {"1234567890-", "qwertyuiop/", "asdfghjkl:'", "zxcvbnm,.?!"}; +static constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770}; +static constexpr double KEY_COL_X0 = 0.077; +static constexpr double KEY_COL_STEP = 0.075; + + +SystemKeyboardDetector::SystemKeyboardDetector(Color color) + : m_color(color) + , m_key_panel(0.15, 0.555, 0.65, 0.020) + , m_bottom_bar(0.10, 0.935, 0.35, 0.030) + , m_ok_button(0.875, 0.760, 0.045, 0.020) +{} +void SystemKeyboardDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_key_panel); + items.add(m_color, m_bottom_bar); + items.add(m_color, m_ok_button); +} +bool SystemKeyboardDetector::detect(const ImageViewRGB32& screen){ + if (!is_white(extract_box_reference(screen, m_key_panel)) || + !is_white(extract_box_reference(screen, m_bottom_bar)) + ){ + return false; + } + // OK is blue, or flat grey while the field is empty (disabled). + const ImageStats ok = image_stats(extract_box_reference(screen, m_ok_button)); + return is_solid(ok, {0.172, 0.276, 0.552}) || is_grey(ok, 600, 690); +} + + +bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col){ + // The cursor is a cyan border around one key. Neighbouring keys overlap that border, + // so take the centroid of all cyan pixels and map it to the nearest key. + const ImageFloatBox area(0.02, 0.44, 0.84, 0.38); + ImageViewRGB32 keys = extract_box_reference(screen, area); + double count = 0, sum_x = 0, sum_y = 0; + for (size_t y = 0; y < keys.height(); y++){ + for (size_t x = 0; x < keys.width(); x++){ + const uint32_t pixel = keys.pixel(x, y); + const int red = (pixel >> 16) & 0xff; + const int green = (pixel >> 8) & 0xff; + const int blue = pixel & 0xff; + if (red < 150 && green > 200 && blue > 190){ + count++; + sum_x += (double)x; + sum_y += (double)y; + } + } + } + // The border is about 0.15% of the screen; a few stray pixels are not a cursor. + if (count < 0.0005 * screen.width() * screen.height()){ + return false; + } + const double cx = area.x + sum_x / count / keys.width() * area.width; + const double cy = area.y + sum_y / count / keys.height() * area.height; + col = (int)std::lround((cx - KEY_COL_X0) / KEY_COL_STEP); + row = 0; + for (int r = 1; r < 4; r++){ + if (std::abs(KEY_ROW_Y[r] - cy) < std::abs(KEY_ROW_Y[row] - cy)){ + row = r; + } + } + return 0 <= col && col < 11; +} + +bool system_keyboard_field_has_text(const ImageViewRGB32& screen){ + // Left part of the field, after the caret's position on an empty field. + const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.11, 0.235, 0.19, 0.030))); + return stats.stddev.sum() > 30; +} + + +namespace{ + +// Moves the keyboard's cursor onto (target_row, target_col). A button +// pressed while the keyboard slides in is ignored, so read the cursor again +// after each move until it is on the target. Whether it got there. +bool move_keyboard_cursor(VideoStream& stream, ProControllerContext& context, int target_row, int target_col){ + for (size_t attempt = 0; attempt < 10; attempt++){ + context.wait_for_all_requests(); + int row = -1; + int col = -1; + VideoSnapshot screen = stream.video().snapshot(); + if (!screen || !find_system_keyboard_cursor(screen, row, col)){ + pbf_wait(context, 300ms); + continue; + } + if (row == target_row && col == target_col){ + return true; + } + for (; row < target_row; row++){ + pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms); + } + for (; row > target_row; row--){ + pbf_press_dpad(context, DPAD_UP, 100ms, 200ms); + } + for (; col < target_col; col++){ + pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms); + } + for (; col > target_col; col--){ + pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms); + } + pbf_wait(context, 300ms); + } + return false; +} + +} // namespace + + +void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text){ + for (char ch : text){ + const char key = (char)std::tolower((unsigned char)ch); + int target_row = -1; + int target_col = -1; + for (int r = 0; r < 4 && target_row < 0; r++){ + const char* p = std::strchr(KEY_ROWS[r], key); + if (p != nullptr && key != '\0'){ + target_row = r; + target_col = (int)(p - KEY_ROWS[r]); + } + } + if (target_row < 0){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::string("System keyboard: no key for '") + ch + "'.", + stream + ); + } + if (!move_keyboard_cursor(stream, context, target_row, target_col)){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + std::string("System keyboard: the cursor did not reach '") + key + "'.", + stream + ); + } + if (std::isupper((unsigned char)ch)){ + pbf_press_button(context, BUTTON_LCLICK, 100ms, 250ms); + } + pbf_press_button(context, BUTTON_A, 80ms, 300ms); + } + context.wait_for_all_requests(); +} + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h new file mode 100644 index 0000000000..6091d9a14f --- /dev/null +++ b/SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h @@ -0,0 +1,68 @@ +/* System Keyboard Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The Switch's own software keyboard, which games open for names (e.g. the + * player's name in BDSP): detect it, find the key under its cursor, and type + * a string by moving the cursor and checking where it landed. + * + * Measured on 1920x1080 captures of the QWERTY layout. + * + */ + +#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H +#define PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H + +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonTools/VisualDetector.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ + + +// White key panel and bottom bar, and the blue OK button. +class SystemKeyboardDetector : public StaticScreenDetector{ +public: + SystemKeyboardDetector(Color color = COLOR_CYAN); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + +private: + Color m_color; + ImageFloatBox m_key_panel; + ImageFloatBox m_bottom_bar; + ImageFloatBox m_ok_button; +}; +// Periodic watcher for the Switch system on-screen keyboard. +class SystemKeyboardWatcher : public DetectorToFinder{ +public: + SystemKeyboardWatcher(Color color = COLOR_CYAN) + : DetectorToFinder("SystemKeyboardWatcher", std::chrono::milliseconds(250), color) + {} +}; + + +// The key under the cursor, as row (0: digits .. 3: "zxcv") and column (0..10). +// The keyboard remembers the cursor between uses, so read it, don't assume it. +// Returns false if no cursor border is visible. +bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col); + +// Whether the text field holds any text. Pressing B on an empty field closes the +// keyboard, so delete only while this is true. +bool system_keyboard_field_has_text(const ImageViewRGB32& screen); + +// Types `text` (letters, digits and the symbols on the QWERTY grid). For each character, +// reads the cursor, moves it with the D-pad and checks it arrived before pressing A; an +// uppercase letter is typed with Shift (left stick click). Throws, with a screenshot, if +// the cursor can't be read or doesn't arrive. +void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text); + + + +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp new file mode 100644 index 0000000000..0ee0a55f55 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp @@ -0,0 +1,96 @@ +/* Name List Detector + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include "Kernels/Waterfill/Kernels_Waterfill.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/Images/BinaryImage_FilterRgb32.h" +#include "CommonTools/ImageMatch/ExactImageMatcher.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Images/SolidColorTest.h" +#include "PokemonBDSP_NameListDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ +// Row centers, as a fraction of the screen height (1080p: 307, 370, 433, ...). +const double FIRST_ROW_Y = 0.284; +const double ROW_STEP_Y = 0.0587; +const int ROWS = 5; + +// The menu's selection arrows in `image`, as PokemonBDSP_SelectionArrow.cpp +// finds them: that file keeps its finder out of its header, so the same +// waterfill and template match are repeated here rather than changing it. +const ImageMatch::ExactImageMatcher& name_list_arrow(){ + static ImageMatch::ExactImageMatcher matcher(RESOURCE_PATH() + "PokemonBDSP/SelectionArrow.png"); + return matcher; +} +std::vector find_name_list_arrows(const ImageViewRGB32& image){ + using namespace Kernels; + using namespace Kernels::Waterfill; + PackedBinaryMatrix matrix = compress_rgb32_to_binary_max(image, 200, 200, 200); + std::vector objects = find_objects_inplace(matrix, 200); + std::vector ret; + for (const WaterfillObject& object : objects){ + if (object.width() > object.height() || object.height() > 3 * object.width()){ + continue; + } + if (name_list_arrow().rmsd(extract_box_reference(image, object)) <= 90){ + ret.emplace_back(object.min_x, object.min_y, object.max_x, object.max_y); + } + } + return ret; +} +} // namespace + + +NameListDetector::NameListDetector(Color color) + : m_color(color) + , m_arrows(0.137, 0.245, 0.030, 0.310) + , m_panel(0.140, 0.245, 0.140, 0.300) +{} +void NameListDetector::make_overlays(VideoOverlaySet& items) const{ + items.add(m_color, m_arrows); + items.add(m_color, m_panel); +} +bool NameListDetector::detect(const ImageViewRGB32& screen){ + return selected_row(screen) >= 0; +} +int NameListDetector::selected_row(const ImageViewRGB32& screen) const{ + // The list is a white panel with a little dark text. A thin margin strip is not + // enough: the border's antialiasing reaches into it on some frames. + const ImageViewRGB32 panel = extract_box_reference(screen, m_panel); + size_t white = 0; + for (size_t y = 0; y < panel.height(); y += 2){ + for (size_t x = 0; x < panel.width(); x += 2){ + const uint32_t p = panel.pixel(x, y); + white += ((p >> 16) & 0xff) > 220 && ((p >> 8) & 0xff) > 220 && (p & 0xff) > 220; + } + } + const size_t samples = ((panel.width() + 1) / 2) * ((panel.height() + 1) / 2); + if (samples == 0 || white < samples * 6 / 10){ + return -1; + } + ImageViewRGB32 area = extract_box_reference(screen, m_arrows); + std::vector arrows = find_name_list_arrows(area); + if (arrows.size() != 1){ + return -1; + } + const double center_y = m_arrows.y + + m_arrows.height * (arrows[0].min_y + arrows[0].max_y) / 2.0 / (double)area.height(); + const int row = (int)std::lround((center_y - FIRST_ROW_Y) / ROW_STEP_Y); + return row >= 0 && row < ROWS ? row : -1; +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h new file mode 100644 index 0000000000..9e40fb77d0 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h @@ -0,0 +1,42 @@ +/* Name List Detector + * + * From: https://github.com/PokemonAutomation/ + * + * The list of preset names shown for the rival: "I'll tell you...", then four names. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_NameListDetector_H +#define PokemonAutomation_PokemonBDSP_NameListDetector_H + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonTools/VisualDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Detects the rival preset name selection menu and reads the selected row. +class NameListDetector : public StaticScreenDetector{ +public: + NameListDetector(Color color = COLOR_MAGENTA); + + virtual void make_overlays(VideoOverlaySet& items) const override; + virtual bool detect(const ImageViewRGB32& screen) override; + + // The row the arrow is on: 0 is "I'll tell you...", 1 the first preset name. + // -1 if the list is not showing. + int selected_row(const ImageViewRGB32& screen) const; + +private: + Color m_color; + ImageFloatBox m_arrows; + ImageFloatBox m_panel; +}; + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp index 7f5bca1843..22208e80f8 100644 --- a/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp +++ b/SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp @@ -7,6 +7,7 @@ #include "Common/Cpp/TestRunners/UnitTestDatabase.h" #include "Inference/PokemonBDSP_DialogDetector.h" #include "Inference/PokemonBDSP_MenuDetector.h" +#include "Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h" #include "Programs/AutoStory/PokemonBDSP_AutoStory_Maps_Tests.h" #include "Programs/AutoStory/PokemonBDSP_AutoStory_PlannedRoute_Tests.h" #include "Programs/AutoStory/PokemonBDSP_AutoStory_RoutePlanner_Tests.h" @@ -25,6 +26,7 @@ namespace PokemonBDSP{ void add_tests(UnitTestDatabase& database){ add_tests_DialogDetector(database); add_tests_MenuDetector(database); + add_tests_AutoStory_Actions(database); add_tests_AutoStory_Maps(database); add_tests_AutoStory_StoryWorld(database); add_tests_AutoStory_RoutePlanner(database); diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.cpp new file mode 100644 index 0000000000..acbc234da3 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.cpp @@ -0,0 +1,357 @@ +/* BDSP AutoStory - Actions + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/Exceptions/OperationFailedException.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/Globals.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Programs/PokemonBDSP_GameNavigation.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_Actions.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +namespace{ + +// X from the overworld, then waits up to 10 s for the menu. Reads it with +// the NEW tags allowed: overworld_to_menu() reads it with the tags' columns, +// which a new Pokemon in the party breaks (r19: three misses, then the blind +// save; r52: the Bag check after the Mart lady's Potions). +void open_menu_from_overworld(VideoStream& stream, ProControllerContext& context){ + pbf_press_button(context, BUTTON_X, 160ms, 840ms); + context.wait_for_all_requests(); + MenuWatcher menu(COLOR_RED, true); + if (wait_until(stream, context, std::chrono::seconds(10), {{menu}}) < 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Menu not detected after 10 seconds.", stream); + } +} + +// Opens the save prompt from the overworld and keeps a screenshot of it: it +// shows the play time and badges, a record of where each checkpoint save +// stands. Returns the map its card names. +std::string open_save_prompt(VideoStream& stream, ProControllerContext& context){ + open_menu_from_overworld(stream, context); + context.wait_for(100ms); + pbf_press_button(context, BUTTON_R, 80ms, 2000ms); + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + return ""; + } + const std::string folder = SCREENSHOTS_PATH() + "AutoStory/saves/"; + Filesystem::create_directories(Filesystem::Path(folder)); + const std::string path = folder + now_to_filestring() + ".png"; + screen.frame->save(path); + stream.log("Save prompt screenshot: " + path); + const std::string location = read_save_screen_location(*screen.frame); + stream.log("Save screen location: \"" + location + "\"."); + return location; +} + +// Backs out of the save prompt and its menu to the overworld. +void leave_save_prompt(VideoStream& stream, ProControllerContext& context){ + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + OverworldWatcher overworld; + if (wait_until(stream, context, std::chrono::seconds(15), {overworld}) < 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Failed to detect the overworld after leaving the save prompt.", stream); + } +} + +struct IconGrid { + int row, col; +}; + +IconGrid icon_to_grid(MenuCursorPosition pos, int num_icons){ + switch (pos){ + case MenuCursorPosition::POKEDEX: + return {0, 0}; + case MenuCursorPosition::POKEMON: + return {0, 1}; + case MenuCursorPosition::BAG: + return {0, 2}; + case MenuCursorPosition::CARD: + return {0, 3}; + case MenuCursorPosition::OPTIONS: + return {1, num_icons <= 5 ? 0 : 2}; + case MenuCursorPosition::MAP: + return {1, 0}; + case MenuCursorPosition::CAPSULES: + return {1, 1}; + case MenuCursorPosition::MYSTERYGIFT: + return {1, 3}; + default: + return {-1, -1}; + } +} + +enum class SaveOutcome{ SAVED, WRONG_PLACE, FAILED }; +struct SaveAttempt{ + SaveOutcome outcome = SaveOutcome::FAILED; + std::string wrong_place; +}; + +// One feedback-checked save: the save prompt, its card checked against +// `expected_location` if given, then the save and back to the overworld. +SaveAttempt save_once( + SingleSwitchProgramEnvironment& env, ProControllerContext& context, + const std::string& expected_location, size_t attempt +){ + try{ + env.console.log("Checkpoint save attempt: " + std::to_string(attempt + 1)); + + // Only mash B on retries - no need to on first attempt + // if already cleanly in the overworld. + if (attempt > 0){ + pbf_mash_button(context, BUTTON_B, 1000ms); + } + + // The feedback-based save of save_game(VideoStream&, ...), split at the + // save prompt to check where it stands. + const std::string location = open_save_prompt(env.console, context); + if (!expected_location.empty() && !location_banner_matches(location, expected_location)){ + SaveAttempt wrong{SaveOutcome::WRONG_PLACE, + "About to save in \"" + location + "\", expected " + expected_location + "; not saving."}; + env.console.log(wrong.wrong_place, COLOR_RED); + leave_save_prompt(env.console, context); + return wrong; + } + pbf_press_button(context, BUTTON_ZL, 80ms, 5000ms); + + // Return to overworld cleanly. + pbf_mash_button(context, BUTTON_B, 1000ms); + + // Drain R/confirm/B before detection; an earlier overworld frame + // must not report success while saving commands are still queued. + context.wait_for_all_requests(); + + // Verify overworld. Generous timeout: save animation + fade can be slow. + OverworldWatcher overworld; + if (wait_until(env.console, context, std::chrono::seconds(15), {overworld}) < 0){ + throw OperationFailedExceptionWithScreenshot(ErrorReportMode::SEND_ERROR_REPORT, + "Failed to detect overworld after save.", + env.console); + } + + env.console.log("Checkpoint save verified."); + return {SaveOutcome::SAVED, ""}; + }catch (OperationFailedException& e){ + env.console.log("Checkpoint save failed: " + e.message(), COLOR_RED); + context.wait_for(1000ms); + return {SaveOutcome::FAILED, ""}; + } +} + +// The X menu's icons as hovered, where packaged: the cursor detector's +// template match. Without them the dark-pixel detector still works. +void load_menu_icon_references(MenuCursorDetector& detector){ + static const char* ICON_NAMES[MENU_CURSOR_NUM_ICONS] = { + "POKEDEX", + "POKEMON", + "BAG", + "CARD", + "OPTIONS", + "MAP", + "CAPSULES", + "MYSTERYGIFT", + }; + const std::string ref_dir = RESOURCE_PATH() + "PokemonBDSP/Autostory"; + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + const std::string path = ref_dir + "/" + ICON_NAMES[i] + ".png"; + if (!Filesystem::is_regular_file(path)){ + continue; + } + ImageRGB32 img(path); + if (img){ + detector.set_icon_reference(static_cast(i), std::move(img)); + } + } +} + +// The cursor on the open menu, from up to ten frames; UNKNOWN if none reads. +MenuCursorPosition read_menu_cursor(VideoStream& stream, ProControllerContext& context, const MenuCursorDetector& detector){ + MenuCursorPosition cur = MenuCursorPosition::UNKNOWN; + for (int frame = 0; frame < 10 && cur == MenuCursorPosition::UNKNOWN; frame++){ + context.wait_for(50ms); + const VideoSnapshot snapshot = stream.video().snapshot(); + if (snapshot){ + cur = detector.detect(snapshot); + } + } + return cur; +} + +// Opens the menu unless it is already open: a late-opening menu may show +// after a timeout, and X would close it. +void open_menu_unless_open(VideoStream& stream, ProControllerContext& context){ + MenuWatcher already_open(COLOR_RED, true); + if (wait_until(stream, context, 300ms, {{already_open}}) < 0){ + open_menu_from_overworld(stream, context); + }else{ + stream.log("[Open menu] Menu already visible; skipping X."); + } +} + +// Opens the menu and reads its cursor, in three attempts. Once the menu +// is confirmed open, retries only look: no input that could close it. +MenuCursorPosition open_menu_and_find_cursor(VideoStream& stream, ProControllerContext& context, const MenuCursorDetector& detector){ + MenuCursorPosition cur = MenuCursorPosition::UNKNOWN; + bool menu_confirmed_open = false; + for (size_t attempt = 1; attempt <= 3 && cur == MenuCursorPosition::UNKNOWN; attempt++){ + stream.log("[Open menu] Open and detect attempt " + std::to_string(attempt) + "/3."); + try{ + if (!menu_confirmed_open){ + open_menu_unless_open(stream, context); + menu_confirmed_open = true; + }else{ + stream.log("[Open menu] Retrying cursor detection in the open menu."); + } + pbf_wait(context, 1000ms); + context.wait_for_all_requests(); + cur = read_menu_cursor(stream, context, detector); + stream.log("[Open menu] Menu cursor " + std::string(cur == MenuCursorPosition::UNKNOWN ? "not detected" : "detected") + + " on attempt " + std::to_string(attempt) + ".", cur == MenuCursorPosition::UNKNOWN ? COLOR_ORANGE : COLOR_GREEN); + }catch (OperationFailedException& error){ + stream.log("[Open menu] Attempt " + std::to_string(attempt) + " failed: " + error.message(), COLOR_ORANGE); + } + } + if (cur == MenuCursorPosition::UNKNOWN){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Menu or cursor detection failed after 3 opening attempts.", + stream); + } + return cur; +} + +// From the icon `cur` to `target`: a row up or down, then across. +void move_menu_cursor(VideoStream& stream, ProControllerContext& context, MenuCursorPosition cur, MenuCursorPosition target, int num_icons){ + const IconGrid cur_g = icon_to_grid(cur, num_icons); + const IconGrid tgt_g = icon_to_grid(target, num_icons); + if (tgt_g.row == -1){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Unknown target menu icon.", + stream); + } + if (cur_g.row != tgt_g.row){ + pbf_press_dpad(context, cur_g.row < tgt_g.row ? DPAD_DOWN : DPAD_UP, 80ms, 300ms); + context.wait_for_all_requests(); + } + const int col_diff = tgt_g.col - cur_g.col; + for (int i = 0; i < std::abs(col_diff); i++){ + pbf_press_dpad(context, col_diff > 0 ? DPAD_RIGHT : DPAD_LEFT, 80ms, 300ms); + context.wait_for_all_requests(); + } +} + +} // namespace + +std::string read_location_from_save_screen(VideoStream& stream, ProControllerContext& context){ + const std::string location = open_save_prompt(stream, context); + leave_save_prompt(stream, context); + return location; +} + +bool save_game( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& expected_location +){ + static constexpr size_t MAX_RETRIES = 3; + + for (size_t attempt = 0; attempt < MAX_RETRIES; attempt++){ + const SaveAttempt result = save_once(env, context, expected_location, attempt); + if (result.outcome == SaveOutcome::SAVED){ + return true; + } + // No blind fallback for the wrong place: it would save there. + if (result.outcome == SaveOutcome::WRONG_PLACE){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, result.wrong_place, env.console); + } + } + + // Blind fallback: reuse the non-feedback save_game(context) from GameNavigation + // rather than duplicating the button sequence manually. + env.console.log("All save retries failed. Using blind fallback.", COLOR_ORANGE); + pbf_mash_button(context, BUTTON_B, 1000ms); + save_game(context); // PokemonBDSP_GameNavigation.h - non-feedback overload + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + + return false; +} + +void wait_for_dialogue(VideoStream& stream, + ProControllerContext& context, + const std::string& label, + Milliseconds timeout){ + auto log_detected = [&] { + if (!label.empty()){ + stream.log(label + " dialogue detected."); + } + }; + + ShortDialogWatcher watcher_cyan(COLOR_CYAN); + int ret = run_until(stream, + context, + [timeout](ProControllerContext& context){ + context.wait_for(timeout); + }, + {{watcher_cyan}}); + if (ret == 0){ + log_detected(); + return; + } + + ShortDialogWatcher watcher_red(COLOR_RED); + ret = run_until(stream, + context, + [timeout](ProControllerContext& context){ + context.wait_for(timeout); + }, + {{watcher_red}}); + if (ret == 0){ + log_detected(); + return; + } + + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + (label.empty() ? "wait_for_dialogue" : label) + ": dialogue not detected within timeout.", + stream); +} + +void open_menu(VideoStream& stream, ProControllerContext& context, MenuCursorPosition target, int num_icons){ + context.wait_for_all_requests(); + MenuCursorDetector detector; + load_menu_icon_references(detector); + const MenuCursorPosition cur = open_menu_and_find_cursor(stream, context, detector); + move_menu_cursor(stream, context, cur, target, num_icons); + pbf_press_button(context, BUTTON_A, 80ms, 200ms); +} + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.h new file mode 100644 index 0000000000..afeb466640 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.h @@ -0,0 +1,63 @@ +/* BDSP AutoStory - Actions + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * The game's menus as the story uses them: saving, opening the X menu, and + * waiting for a dialogue. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Actions_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Actions_H + +#include +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Save the game from the overworld with retries. Returns true if the save was +// verified via overworld detection, false if the blind fallback was used. +// With `expected_location`, the map named on the save screen's card must be +// it (compared as a location banner is), or the save is backed out of and +// the function fails with a screenshot: r124a saved in Sandgem Town after a +// walk that was sure it stood on Route 202. +bool save_game( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& expected_location = "" +); + +// Opens the save screen from the overworld, reads where the game says we +// are (read_save_screen_location() in the Location module), and backs out +// without saving. +std::string read_location_from_save_screen(VideoStream& stream, ProControllerContext& context); + +// Wait (without pressing any buttons) until a short dialogue box appears. +// Logs `label + " dialogue detected."` when found (omit label to skip logging). +// Throws OperationFailedException if no dialogue appears within `timeout`. +void wait_for_dialogue( + VideoStream& stream, + ProControllerContext& context, + const std::string& label = "", + Milliseconds timeout = std::chrono::seconds(60) +); + +// Opens the X-menu from the overworld, navigates the cursor to `target`, and presses A. +// num_icons: 5 (early-game layout) or 8 (late-game layout). +void open_menu(VideoStream& stream, ProControllerContext& context, MenuCursorPosition target, int num_icons); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp new file mode 100644 index 0000000000..49a0dd2313 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp @@ -0,0 +1,520 @@ +/* BDSP AutoStory - Actions Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The new game's intro decisions, and the screenshot checks behind the + * Pokemon Center, the save card and the Jubilife School's lamp. Screenshots + * are in the unit test resources (the CommandLineTests checkout); a test + * whose screenshot is missing is skipped. + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/Exceptions.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/Logging/Logger.h" +#include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryMaps.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_Center.h" +#include "PokemonBDSP_AutoStory_NewGame.h" +#include "PokemonBDSP_AutoStory_Actions_Tests.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + + +// How far the intro has got, with BDSP confirmed: 1, the player named; +// 2, the rival too. +IntroProgress intro_progress(size_t names_given){ + IntroProgress progress; + progress.player_named = names_given >= 1; + progress.rival_named = names_given >= 2; + progress.game_confirmed = true; + return progress; +} + +UnitTestResult test_intro_decisions(Logger&){ + struct Case{ + const char* screen; + IntroProgress progress; + IntroStep expected; + }; + const IntroProgress start = intro_progress(0); + const IntroProgress named = intro_progress(1); + const IntroProgress done = intro_progress(2); + const Case cases[] = { + {"user-select", start, IntroStep::SELECT_USER}, + {"home", done, IntroStep::FAIL_NOT_IN_GAME}, + {"keyboard", start, IntroStep::ENTER_PLAYER_NAME}, + {"keyboard", named, IntroStep::CANCEL_RIVAL_KEYBOARD}, + {"rival-list-tell", named, IntroStep::MOVE_TO_PRESET}, + {"rival-list-preset", named, IntroStep::PICK_PRESET}, + {"menu", done, IntroStep::CLOSE_MENU_AND_FINISH}, + // The appearance grid has its prompt in a dialogue box. + {"dialog", start, IntroStep::ADVANCE}, + {"dialog", done, IntroStep::ADVANCE_WITH_B}, + // Logos, the title, language and the appearance preview: no X + // before the bedroom is possible. + {"other", start, IntroStep::ADVANCE}, + {"other", named, IntroStep::ADVANCE}, + {"other", done, IntroStep::CLEAR_WITH_B}, + {"black", start, IntroStep::WAIT}, + {"black", done, IntroStep::WAIT}, + }; + for (const Case& c : cases){ + const IntroStep actual = decide_intro_step(c.screen, c.progress); + if (actual != c.expected){ + return std::string("Screen \"") + c.screen + "\" (player named " + + (c.progress.player_named ? "yes" : "no") + ", rival named " + + (c.progress.rival_named ? "yes" : "no") + "): step " + + std::to_string((int)actual) + ", expected " + std::to_string((int)c.expected) + "."; + } + } + return true; +} + + +// Runs `screens` through decide_intro_step() and record_intro_step() from +// `progress`; the steps taken. +std::vector intro_steps(IntroProgress& progress, const std::vector& screens){ + std::vector steps; + for (const std::string& screen : screens){ + steps.push_back(decide_intro_step(screen, progress)); + record_intro_step(progress, steps.back()); + } + return steps; +} + +// The rival's preset name is confirmed with A on "Barry, is it?" and its +// Yes/No; B there answers No and the list comes back (r4). Only +// INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick does B take over. +UnitTestResult test_intro_rival_confirm(Logger&){ + IntroProgress progress = intro_progress(1); + // r4's frames 029 to 032: the pick, "Barry, is it?", Barry alone + // as the Yes/No comes up, the Yes/No. + const std::vector confirm = intro_steps(progress, {"rival-list-preset", "dialog", "other", "dialog"}); + const std::vector expected{IntroStep::PICK_PRESET, IntroStep::ADVANCE, IntroStep::ADVANCE, IntroStep::ADVANCE}; + if (confirm != expected){ + return "The pick and its confirmation must take A, never B."; + } + if (progress.rival_named){ + return "The name is not final until INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick."; + } + // The list back (No was answered): pick again, the count restarts. + if (intro_steps(progress, {"rival-list-tell", "rival-list-preset"}) != + std::vector{IntroStep::MOVE_TO_PRESET, IntroStep::PICK_PRESET}){ + return "The list coming back after a pick must be picked from again."; + } + for (size_t c = 0; c < INTRO_SCREENS_AFTER_RIVAL_PICK; c++){ + if (intro_steps(progress, {c % 2 ? "other" : "dialog"}) != std::vector{IntroStep::ADVANCE}){ + return "Screen " + std::to_string(c) + " after the pick must take A."; + } + } + if (!progress.rival_named){ + return "INTRO_SCREENS_AFTER_RIVAL_PICK screens after the pick, the name is final."; + } + return true; +} + + +// After the rival is named, unknown screens take INTRO_B_BEFORE_X B presses, +// then X; only the menu ends the intro (r2: Rowan's last line, misread). +UnitTestResult test_intro_before_menu(Logger&){ + IntroProgress progress = intro_progress(1); + std::vector screens{"rival-list-preset", "dialog", "other"}; + std::vector expected{IntroStep::PICK_PRESET, IntroStep::ADVANCE, IntroStep::ADVANCE}; + for (size_t c = 2; c < INTRO_SCREENS_AFTER_RIVAL_PICK; c++){ + screens.push_back("dialog"); + expected.push_back(IntroStep::ADVANCE); + } + for (size_t round = 0; round < 2; round++){ + screens.insert(screens.end(), INTRO_B_BEFORE_X + 1, "other"); + expected.insert(expected.end(), INTRO_B_BEFORE_X, IntroStep::CLEAR_WITH_B); + expected.push_back(IntroStep::PROBE_FOR_MENU); + } + // A dialogue line in between takes B without counting. + screens.insert(screens.end(), {"dialog", "other", "menu"}); + expected.insert(expected.end(), {IntroStep::ADVANCE_WITH_B, IntroStep::CLEAR_WITH_B, IntroStep::CLOSE_MENU_AND_FINISH}); + const std::vector steps = intro_steps(progress, screens); + for (size_t c = 0; c < steps.size(); c++){ + if (steps[c] != expected[c]){ + return "Screen " + std::to_string(c) + " (\"" + screens[c] + "\"): step " + + std::to_string((int)steps[c]) + ", expected " + std::to_string((int)expected[c]) + "."; + } + } + IntroProgress pressed = intro_progress(2); + pressed.b_presses = INTRO_B_BEFORE_X; + if (decide_intro_step("other", pressed) != IntroStep::PROBE_FOR_MENU){ + return "X must follow INTRO_B_BEFORE_X B presses."; + } + return true; +} + + +// X from a still bedroom opening something the detector misses, closed +// again by B, is the player in control; moving screens or X doing nothing +// are not. +UnitTestResult test_intro_in_control(Logger&){ + struct Case{ + XProbe probe; + bool expected; + const char* what; + }; + const Case cases[] = { + {{1, 60, 2}, true, "still, X opened, B closed"}, + {{12, 60, 2}, false, "moving before X (sparkles, a cutscene)"}, + {{1, 5, 2}, false, "X changed nothing"}, + {{1, 60, 40}, false, "B did not put it back"}, + }; + for (const Case& c : cases){ + if (intro_in_control_without_menu(c.probe) != c.expected){ + return std::string("Wrong for: ") + c.what + "."; + } + } + return true; +} + + +// Black from the restart until the first logo, well past the limit. +std::string check_boot(IntroProgress& progress){ + using std::chrono::milliseconds; + for (milliseconds t(0); t <= 2 * INTRO_TIME_BEFORE_BDSP; t += milliseconds(500)){ + note_intro_screen(progress, "black", t); + if (decide_intro_step("black", progress) != IntroStep::WAIT){ + return "A black screen must take no press."; + } + } + if (progress.first_shown || progress.unconfirmed_for != milliseconds(0)){ + return "Black screens must not start the wrong-game clock."; + } + return ""; +} + +UnitTestResult check_confirming_screens(){ + for (const char* screen : {"dialog", "rival-list-tell", "rival-list-preset", "menu"}){ + if (!intro_screen_confirms_bdsp(screen)){ + return std::string("Expected \"") + screen + "\" to confirm BDSP."; + } + } + for (const char* screen : {"user-select", "home", "keyboard", "black", "other"}){ + if (intro_screen_confirms_bdsp(screen)){ + return std::string("Expected \"") + screen + "\" not to confirm BDSP."; + } + } + return true; +} + +// The intro gates its A presses on a BDSP screen within a time from the +// first screen that is not black: past it, it is another game (r1, +// 2026-11-01, launched FireRed). The boot's black screens, about 25 s of +// them (r3), neither take presses nor count. +UnitTestResult test_intro_gate(Logger&){ + using std::chrono::milliseconds; + IntroProgress progress; + const std::string boot = check_boot(progress); + if (!boot.empty()){ + return boot; + } + const milliseconds shown = 2 * INTRO_TIME_BEFORE_BDSP + milliseconds(500); + note_intro_screen(progress, "other", shown); + if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ + return "The first logo must take A."; + } + note_intro_screen(progress, "black", shown + INTRO_TIME_BEFORE_BDSP / 2); + note_intro_screen(progress, "other", shown + INTRO_TIME_BEFORE_BDSP - milliseconds(1)); + if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ + return "Unknown screens take A until INTRO_TIME_BEFORE_BDSP has passed."; + } + IntroProgress other_game = progress; + note_intro_screen(other_game, "other", shown + INTRO_TIME_BEFORE_BDSP); + if (decide_intro_step("other", other_game) != IntroStep::FAIL_NOT_BDSP){ + return "Past INTRO_TIME_BEFORE_BDSP of unknown screens, it is another game."; + } + note_intro_screen(progress, "dialog", shown + INTRO_TIME_BEFORE_BDSP - milliseconds(1)); + note_intro_screen(progress, "other", shown + 10 * INTRO_TIME_BEFORE_BDSP); + if (decide_intro_step("other", progress) != IntroStep::ADVANCE){ + return "Once BDSP's dialogue has shown, the clock stops."; + } + return check_confirming_screens(); +} + + +// The Mart's doorway on the failure screenshots of r121a and r122a, kept in +// the unit test resources (the CommandLineTests checkout): where it is +// relative to the player, who is in the middle of the screen. +UnitTestResult test_lit_doorway_screenshots(Logger& logger){ + struct Case{ + const char* file; + double tiles_across_min; + double tiles_across_max; + double rows_up_min; + double rows_up_max; + }; + const Case cases[] = { + // r121a: below-left of the door, which is part of a tile east, straight ahead. + {"PokemonBDSP/AutoStory/SandgemMartDoorUpRight-20261030-023443.png", 0.2, 0.7, 0.5, 1.5}, + // r122a: a tile west of the door and three or four rows south of its front. + {"PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", -1.3, -0.5, 2.5, 4.5}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const std::optional doorway = find_lit_doorway(screen); + if (!doorway){ + return std::string(c.file) + ": no lit doorway found."; + } + logger.log(std::string(c.file) + ": doorway at " + std::to_string(doorway->x) + ", " + + std::to_string(doorway->y) + ": " + std::to_string(doorway->tiles_across) + " tiles across, " + + std::to_string(doorway->rows_up) + " rows up."); + if (doorway->tiles_across < c.tiles_across_min || doorway->tiles_across > c.tiles_across_max){ + return std::string(c.file) + ": doorway " + std::to_string(doorway->tiles_across) + + " tiles across, expected " + std::to_string(c.tiles_across_min) + " to " + + std::to_string(c.tiles_across_max) + "."; + } + if (doorway->rows_up < c.rows_up_min || doorway->rows_up > c.rows_up_max){ + return std::string(c.file) + ": doorway " + std::to_string(doorway->rows_up) + + " rows up, expected " + std::to_string(c.rows_up_min) + " to " + + std::to_string(c.rows_up_max) + "."; + } + } + return true; +} + + +// The save prompt r124a took beside Sandgem's Poke Mart, sure it stood on +// Route 202 (in CommandLineTests under PokemonBDSP/AutoStory): its card +// names Sandgem Town, which the save must read before confirming. +UnitTestResult test_save_screen_location(Logger& logger){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + "PokemonBDSP/AutoStory/SaveScreenSandgemTown-20261030-034444.png"; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const std::string read = read_save_screen_location(ImageRGB32(path)); + logger.log("Save screen location read: \"" + read + "\"."); + if (!location_banner_matches(read, MAP_SANDGEM_TOWN)){ + return "Save screen location \"" + read + "\" should match Sandgem Town."; + } + if (location_banner_matches(read, MAP_ROUTE_202)){ + return "Save screen location \"" + read + "\" should not match Route 202."; + } + return true; +} + + +// The two shots around r126's door walk: below the Center's sign, outside, +// and on the entrance mat inside, where the lit floor read as a doorway and +// the walk kept lining up with it (runs 2 and 3). +UnitTestResult test_pokemon_center_inside(Logger& logger){ + struct Case{ + const char* file; + bool inside; + bool doorway; + }; + const Case cases[] = { + {.file = "PokemonBDSP/AutoStory/SandgemCenterInside-20261030-053555.png", .inside = true, .doorway = false}, + // r127c: walked in by the turn north at the door, facing north. + {.file = "PokemonBDSP/AutoStory/SandgemCenterInsideFacingNorth-r127c.png", .inside = true, .doorway = false}, + {.file = "PokemonBDSP/AutoStory/SandgemCenterBelowSign-20261030-053527.png", .inside = false, .doorway = true}, + {.file = "PokemonBDSP/AutoStory/SandgemMartDoorUpRight-20261030-023443.png", .inside = false, .doorway = true}, + {.file = "PokemonBDSP/AutoStory/SandgemMartDoorFarLeft-20261030-024648.png", .inside = false, .doorway = true}, + // Jubilife (2026-10-30): inside its Center twice, at its door, and + // on the doorstep after walking out (the banner up). + {.file = "PokemonBDSP/AutoStory/JubilifeCenterInside-20261030-c6.png", .inside = true, .doorway = false}, + {.file = "PokemonBDSP/AutoStory/JubilifeCenterInside-20261030-c4.png", .inside = true, .doorway = false}, + {.file = "PokemonBDSP/AutoStory/JubilifeCenterFrontDoor-20261030-q2.png", .inside = false, .doorway = true}, + {.file = "PokemonBDSP/AutoStory/JubilifeCenterDoorstep-20261030-c5.png", .inside = false, .doorway = true}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const bool inside = pokemon_center_inside(screen); + const std::optional doorway = find_lit_doorway(screen); + logger.log(std::string(c.file) + ": inside " + std::to_string(inside) + ", doorway " + + (doorway ? std::to_string(doorway->tiles_across) + " across, " + std::to_string(doorway->rows_up) + " rows up" : "none")); + if (inside != c.inside){ + return std::string(c.file) + ": expected " + (c.inside ? "inside" : "outside") + + " a Pokemon Center, read " + (inside ? "inside" : "outside") + "."; + } + if (doorway.has_value() != c.doorway){ + return std::string(c.file) + ": expected " + (c.doorway ? "a doorway" : "no doorway") + + ", read " + (doorway ? "one" : "none") + "."; + } + } + return true; +} + + +// Oreburgh's Center, lab-B 2026-11-08 (in CommandLineTests under +// PokemonBDSP/AutoStory, skipped without them): 01.03's run in saw no fade +// and stopped at the nurse's counter, the camera in close and the lit floor +// down to the screen's bottom; on the exit mat; and outside on the doorstep +// at sunset, the ground as orange as the Center's back wall. +UnitTestResult test_pokemon_center_counter(Logger& logger){ + struct Case{ + const char* file; + bool inside; + }; + const Case cases[] = { + {.file = "PokemonBDSP/AutoStory/OreburghCenterCounter-20261108-094735.png", .inside = true}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterExitMat-20261108.png", .inside = true}, + {.file = "PokemonBDSP/AutoStory/OreburghCenterDoorstep-20261108.png", .inside = false}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const bool inside = pokemon_center_inside(screen); + logger.log(std::string(c.file) + ": inside " + std::to_string(inside)); + if (inside != c.inside){ + return std::string(c.file) + ": expected " + (c.inside ? "inside" : "outside") + + " a Pokemon Center, read " + (inside ? "inside" : "outside") + "."; + } + if (inside && find_lit_doorway(screen)){ + return std::string(c.file) + ": inside, yet a lit doorway was read."; + } + } + return true; +} + + +// The School's lamp at night, from its doorsteps' row (2026-10-30): where +// the door took us in, the lamp reads within half a tile of the door; a +// tile either side, more. +UnitTestResult test_school_lamp(Logger& logger){ + const ImageFloatBox box(0.55, 0.20, 0.30, 0.10); + struct Case{ const char* file; double min; double max; }; + const Case cases[] = { + {"PokemonBDSP/AutoStory/JubilifeSchoolRightOfDoor-20261030.png", -1.5, -0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolOneTileShort-20261030.png", -1.5, -0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolAtDoor-20261030.png", -0.5, 0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolAtDoor2-20261030.png", -0.5, 0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolDoorstep-20261030.png", -0.5, 0.5}, + {"PokemonBDSP/AutoStory/JubilifeSchoolOneTilePast-20261030.png", 0.5, 1.5}, + }; + for (const Case& c : cases){ + const std::string path = UNIT_TEST_RESOURCE_PATH() + c.file; + if (!Filesystem::is_regular_file(path)){ + return UnitTestResult(UnitTestResult::SKIPPED, "Screenshot not found: " + path); + } + const ImageRGB32 screen(path); + const std::optional lamp = find_lit_lamp_x(screen, box); + if (!lamp){ + return std::string(c.file) + ": no lit lamp found."; + } + const double across = tiles_across_at(*lamp) - tiles_across_at(0.67); + logger.log(std::string(c.file) + ": lamp at " + std::to_string(*lamp) + ", door " + std::to_string(across) + " tiles across."); + if (across < c.min || across > c.max){ + return std::string(c.file) + ": door " + std::to_string(across) + " tiles across, expected " + + std::to_string(c.min) + " to " + std::to_string(c.max) + "."; + } + } + return true; +} + + +struct AutoStoryActionsInput{ + std::string name; + std::function body; +}; + +struct AutoStoryActionsExpectation{ +}; + +struct AutoStoryActionsCase{ + AutoStoryActionsInput input; + AutoStoryActionsExpectation expectation; +}; + +class Test_AutoStoryActions : public UnitTest{ +public: + explicit Test_AutoStoryActions(AutoStoryActionsCase test_case) + : UnitTest("PokemonBDSP::AutoStory - " + test_case.input.name) + , m_case(std::move(test_case)) + {} + virtual UnitTestResult run(Logger& logger, CancellableScope&) const override{ + try{ + return m_case.input.body(logger); + }catch (Exception& e){ + return std::string(e.name()) + ": " + e.message(); + } + } + +private: + AutoStoryActionsCase m_case; +}; + +} // namespace + + +const AutoStoryActionsCase AUTO_STORY_ACTIONS_CASES[] = { + { + .input = {"new-game intro decisions", test_intro_decisions}, + .expectation = {}, + }, + { + .input = {"new-game intro: the rival's name confirmed with A", test_intro_rival_confirm}, + .expectation = {}, + }, + { + .input = {"new-game intro: B on unknown screens, then X for the menu", test_intro_before_menu}, + .expectation = {}, + }, + { + .input = {"new-game intro: in control without the menu read", test_intro_in_control}, + .expectation = {}, + }, + { + .input = {"new-game intro: BDSP or stop", test_intro_gate}, + .expectation = {}, + }, + { + .input = {"lit doorway screenshots", test_lit_doorway_screenshots}, + .expectation = {}, + }, + { + .input = {"save screen location", test_save_screen_location}, + .expectation = {}, + }, + { + .input = {"Pokemon Center inside", test_pokemon_center_inside}, + .expectation = {}, + }, + { + .input = {"Pokemon Center inside: Oreburgh's counter, exit mat and doorstep", test_pokemon_center_counter}, + .expectation = {}, + }, + { + .input = {"Jubilife School lamp", test_school_lamp}, + .expectation = {}, + }, +}; + +void add_tests_AutoStory_Actions(UnitTestDatabase& database){ + for (const auto& item : AUTO_STORY_ACTIONS_CASES){ + database.add(item); + } +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h new file mode 100644 index 0000000000..b92b104fd1 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h @@ -0,0 +1,25 @@ +/* BDSP AutoStory - Actions Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Actions_Tests_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Actions_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Tests that need no console: the new game's intro decisions, and the screenshot checks of the Pokemon Center, the save card and the School's lamp. +void add_tests_AutoStory_Actions(UnitTestDatabase& database); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp new file mode 100644 index 0000000000..6230464190 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp @@ -0,0 +1,429 @@ +/* BDSP AutoStory - Bag + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/OCR/OCR_RawTesseractOCR.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_Actions.h" +#include "PokemonBDSP_AutoStory_Bag.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +namespace{ + +ImageFloatBox bag_list_name_box(size_t row){ + // Previous height included the faint lower selection border in rows 2/6. + // return ImageFloatBox(0.492, 0.213 + (row - 1) * 0.075, 0.335, 0.052); + return ImageFloatBox(0.492, 0.213 + (row - 1) * 0.075, 0.335, 0.050); +} + +int bag_confirm_pocket(VideoStream& stream, ProControllerContext& context){ + int previous = 0; + size_t matches = 0; + for (size_t check = 0; check < 10; check++){ + context.wait_for_all_requests(); + const auto frame = stream.video().snapshot(); + const int pocket = frame ? bag_pocket_position(frame) : 0; + matches = pocket != 0 && pocket == previous ? matches + 1 : pocket != 0 ? 1 : 0; + previous = pocket; + if (matches >= 3){ + return pocket; + } + pbf_wait(context, 100ms); + } + return 0; +} + +bool bag_navigate_to_pocket(VideoStream& stream, ProControllerContext& context, int target){ + int current = bag_confirm_pocket(stream, context); + for (size_t attempt = 0; attempt < 3; attempt++){ + const auto plan = plan_bag_pocket_move(current, target); + if (plan.action == BagPocketAction::CONFIRM){ + return true; + } + if (plan.action == BagPocketAction::UNKNOWN){ + stream.log("[Bag search] Selected pocket not detected.", COLOR_ORANGE); + return false; + } + stream.log("[Bag search] Pocket " + std::to_string(current) + " to " + std::to_string(target) + ": " + + (plan.action == BagPocketAction::RIGHT ? "right " : "left ") + std::to_string(plan.presses) + + " presses, attempt " + std::to_string(attempt + 1) + "/3."); + for (size_t press = 0; press < plan.presses; press++){ + pbf_press_dpad(context, plan.action == BagPocketAction::RIGHT ? DPAD_RIGHT : DPAD_LEFT, 80ms, 150ms); + } + pbf_wait(context, 350ms); + current = bag_confirm_pocket(stream, context); + stream.log( + "[Bag search] Final pocket=" + std::to_string(current) + ", expected=" + std::to_string(target) + "."); + if (current == target){ + return true; + } + } + stream.log("[Bag search] Target pocket not reached after 3 movement batches.", COLOR_ORANGE); + return false; +} + +// Slot 1's colored HP bar in the Bag. Read fill position rather than OCR. +double potion_hp_fraction(const ImageViewRGB32& frame){ + if (bag_pocket_position(frame) == 0){ + return -1; + } + const ImageViewRGB32 bar = extract_box_reference(frame, ImageFloatBox{0.137, 0.160, 0.202, 0.016}); + size_t filled = 0; + for (size_t x = 0; x < bar.width(); x++){ + size_t color = 0; + for (size_t y = 0; y < bar.height(); y++){ + uint32_t p = bar.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + const bool green = g > r + 30 && g > b + 20; + const bool yellow = r > 150 && g > 120 && b < 100; + const bool red = r > 150 && r > g + 50 && r > b + 50; + color += green || yellow || red; + } + filled += color > bar.height() / 4; + } + return bar.width() ? (double)filled / bar.width() : -1; +} + +double confirmed_potion_hp(VideoStream& stream, ProControllerContext& context){ + double previous = -1; + size_t stable = 0; + for (size_t check = 0; check < 20; check++){ + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + double hp = frame ? potion_hp_fraction(frame) : -1; + if (hp >= 0 && previous >= 0 && std::abs(hp - previous) <= 0.015){ + if (++stable >= 3){ + return hp; + } + }else{ + stable = 0; + } + previous = hp; + pbf_wait(context, 150ms); + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Slot 1 HP bar could not be confirmed.", + stream); +} + +// A walk down a pocket's list, reading the cursor's row. +struct BagScan{ + DpadPosition direction = DPAD_DOWN; + std::string previous; + size_t same = 0; + size_t confirmed = 0; + size_t unknown_cursor = 0; +}; +enum class BagLook{ GO_ON, FOUND, NOT_FOUND }; + +// The name on the cursor's row `cursor`, as lower-case letters. +std::string read_bag_row(const ImageViewRGB32& frame, int cursor){ + const std::string text = OCR::tesseract_ocr_read(Language::English, + extract_box_reference(frame, bag_list_name_box(cursor))); + std::string normalized; + for (unsigned char ch : text){ + if (std::isalpha(ch)){ + normalized.push_back((char)std::tolower(ch)); + } + } + return normalized; +} + +// One look at the list: the row under the cursor is `name` (twice in a +// row), or the list ran out both ways, or the cursor moves on. +BagLook bag_scan_step(VideoStream& stream, ProControllerContext& context, BagScan& scan, const std::string& name){ + context.wait_for_all_requests(); + const VideoSnapshot frame = stream.video().snapshot(); + if (!frame || bag_list_is_empty(*frame.frame)){ + stream.log("[Bag search] " + name + ": pocket is empty.", COLOR_ORANGE); + return BagLook::NOT_FOUND; + } + const int cursor = bag_cursor_position(*frame.frame); + if (cursor == 0){ + if (++scan.unknown_cursor >= 3){ + stream.log("[Bag search] " + name + ": cursor not found.", COLOR_ORANGE); + return BagLook::NOT_FOUND; + } + pbf_wait(context, 200ms); + return BagLook::GO_ON; + } + scan.unknown_cursor = 0; + const std::string normalized = read_bag_row(*frame.frame, cursor); + stream.log("[Bag search] Row " + std::to_string(cursor) + " reads \"" + normalized + "\"."); + if (normalized == name){ + if (++scan.confirmed >= 2){ + stream.log("[Bag search] " + name + " confirmed in row " + std::to_string(cursor) + ".", COLOR_GREEN); + return BagLook::FOUND; + } + pbf_wait(context, 150ms); + return BagLook::GO_ON; + } + scan.confirmed = 0; + scan.same = normalized == scan.previous ? scan.same + 1 : 0; + scan.previous = normalized; + if (scan.same >= 2){ + if (scan.direction == DPAD_UP){ + stream.log("[Bag search] " + name + " not found.", COLOR_ORANGE); + return BagLook::NOT_FOUND; + } + scan.direction = DPAD_UP; + scan.same = 0; + } + pbf_press_dpad(context, scan.direction, 80ms, 300ms); + return BagLook::GO_ON; +} + +// B out of the Bag to the overworld; fails with `failure` if it does not show. +void bag_back_to_overworld(VideoStream& stream, ProControllerContext& context, const std::string& failure){ + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + OverworldWatcher overworld; + if (wait_until(stream, context, std::chrono::seconds(10), {{overworld}}) != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, failure, stream); + } +} + +enum class Dose{ HEALED, GIVEN, NONE_LEFT }; + +// One Potion on the lead, from the overworld, its HP read before and after +// in the Bag. HEALED when no dose is needed (HP full, or above +// `heal_below` on the first look, or no Potion when only checking) or the +// dose filled it; GIVEN when HP rose but is not full yet. +Dose potion_dose(VideoStream& stream, ProControllerContext& context, size_t dose, double heal_below){ + stream.log("[Potion] Finding Bag/Medicine/Potion; dose " + std::to_string(dose + 1) + "/3."); + if (!find_bag_item_by_name(stream, context, "potion", 1, 5)){ + // No Potion left is not a failure: the next Center heals. After Roark + // the one Potion went on the first dose and the second found the + // Medicine pocket empty (lab-B, 2026-11-08 15:26). + stream.log("[Potion] No Potion left in the Bag after " + std::to_string(dose) + " dose(s); going on.", COLOR_ORANGE); + return Dose::NONE_LEFT; + } + const double before = confirmed_potion_hp(stream, context); + stream.log("[Potion] Slot 1 HP bar before use=" + std::to_string(before * 100) + "%."); + if (before >= 0.98){ + return Dose::HEALED; + } + if (dose == 0 && before >= heal_below){ + stream.log("[Potion] Slot 1 HP is at least " + std::to_string(heal_below * 100) + "%; no Potion needed."); + return Dose::HEALED; + } + if (before <= 0.005){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Slot 1 appears fainted; a Potion cannot revive it.", + stream); + } + pbf_press_button(context, BUTTON_A, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + bag_back_to_overworld(stream, context, "Potion: Did not return to the overworld after use."); + // Reopen Bag to verify the result, including when the last Potion was used. + open_menu(stream, context, MenuCursorPosition::BAG, 5); + pbf_wait(context, 750ms); + context.wait_for_all_requests(); + const double after = confirmed_potion_hp(stream, context); + stream.log("[Potion] Slot 1 HP bar after use=" + std::to_string(after * 100) + "%."); + if (after <= before + 0.005){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Slot 1 HP did not increase; refusing to continue movement.", + stream); + } + return after >= 0.98 ? Dose::HEALED : Dose::GIVEN; +} + +} // namespace + +int bag_pocket_position(const ImageViewRGB32& frame){ + if (!frame){ + return 0; + } + int selected = 0; + for (int pocket = 1; pocket <= 8; pocket++){ + // The lower left edge of each tab is below the red NEW badge. + // This strip includes both the normal outline and enlarged selection. + const auto edge = + extract_box_reference(frame, ImageFloatBox(0.570 + (pocket - 1) * 0.0421, 0.098, 0.0045, 0.021)); + size_t colored = 0; + for (size_t y = 0; y < edge.height(); y++){ + for (size_t x = 0; x < edge.width(); x++){ + const uint32_t pixel = edge.pixel(x, y); + const int r = (pixel >> 16) & 255; + const int g = (pixel >> 8) & 255; + const int b = pixel & 255; + // Red when focused, blue when the item list has focus. + colored += (r > 160 && r > g + 60 && r > b + 60) || (b > 160 && b > r + 40 && b > g + 20); + } + } + const double pixels = (double)edge.width() * edge.height(); + if (pixels > 0 && colored / pixels >= 0.18){ + if (selected != 0){ + return 0; + } + selected = pocket; + } + } + return selected; +} + +// One narrow strip on the left side of the red selection border. Divide its +// height into the seven row bands; the item's art and NEW badge are outside it. +int bag_cursor_position(const ImageViewRGB32& frame){ + if (!frame){ + return 0; + } + const ImageViewRGB32 strip = extract_box_reference(frame, ImageFloatBox(0.431, 0.229, 0.005, 0.470)); + double best = 0.0; + double second = 0.0; + int position = 0; + for (int row = 0; row < 7; row++){ + const size_t top = static_cast(strip.height() * (row * 0.075 / 0.470)); + const size_t bottom = static_cast(strip.height() * ((row * 0.075 + 0.020) / 0.470)); + size_t red = 0; + size_t total = 0; + for (size_t y = top; y < bottom && y < strip.height(); y++){ + for (size_t x = 0; x < strip.width(); x++){ + const uint32_t pixel = strip.pixel(x, y); + const unsigned r = (pixel >> 16) & 255; + const unsigned g = (pixel >> 8) & 255; + const unsigned b = pixel & 255; + red += r > 180 && g < 120 && b < 120; + total++; + } + } + const double score = total == 0 ? 0.0 : static_cast(red) / total; + if (score > best){ + second = best; + best = score; + position = row + 1; + }else if (score > second){ + second = score; + } + } + return best >= 0.15 && best - second >= 0.10 ? position : 0; +} + +bool bag_list_is_empty(const ImageViewRGB32& frame){ + if (!frame || bag_pocket_position(frame) == 0 || bag_cursor_position(frame) != 0){ + return false; + } + // Empty pockets have no item icons. The central Empty label is outside + // this column; counting dark pixels in the name area included that label. + const auto icons = extract_box_reference(frame, ImageFloatBox(0.449, 0.213, 0.032, 0.500)); + const auto stats = image_stats(icons); + return stats.average.r > 220 && stats.average.g > 220 && stats.average.b > 220 && stats.stddev.sum() < 20; +} + +BagPocketMovePlan plan_bag_pocket_move(int current, int target){ + if (current < 1 || current > 8 || target < 1 || target > 8){ + return {}; + } + const int right = (target - current + 8) % 8; + const int left = (current - target + 8) % 8; + if (right == 0){ + return {BagPocketAction::CONFIRM, 0}; + } + return right <= left ? BagPocketMovePlan{BagPocketAction::RIGHT, static_cast(right)} + : BagPocketMovePlan{BagPocketAction::LEFT, static_cast(left)}; +} + +bool find_bag_item_by_name( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket, + int num_icons +){ + open_menu(stream, context, MenuCursorPosition::BAG, num_icons); + pbf_wait(context, 1000ms); + return find_bag_item_in_open_bag(stream, context, name, target_pocket); +} + +bool find_bag_item_in_open_bag( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket +){ + context.wait_for_all_requests(); + if (!bag_navigate_to_pocket(stream, context, target_pocket)){ + return false; + } + + // No reader for the game's language: the cursor's row, the pocket's + // first, if the pocket has one. Early in the story the pocket holds only + // what was bought or given (Potions in Medicine). + if (!story_reads_text()){ + const VideoSnapshot frame = stream.video().snapshot(); + const bool any = frame && !bag_list_is_empty(*frame.frame) && bag_cursor_position(*frame.frame) != 0; + stream.log("[Bag search] " + name + ": no text reader for the game's language; " + + (any ? "taking the pocket's cursor row." : "the pocket is empty."), COLOR_ORANGE); + return any; + } + + // Walk down the list reading the cursor row. The list does not wrap, so + // turn around once the cursor stops moving. + BagScan scan; + for (size_t look = 0; look < 128; look++){ + const BagLook result = bag_scan_step(stream, context, scan, name); + if (result != BagLook::GO_ON){ + return result == BagLook::FOUND; + } + } + stream.log("[Bag search] " + name + " not found.", COLOR_ORANGE); + return false; +} + +bool bag_has_item(VideoStream& stream, ProControllerContext& context, const std::string& name, int target_pocket){ + const bool found = find_bag_item_by_name(stream, context, name, target_pocket, 5); + bag_back_to_overworld(stream, context, "Bag: Did not return to the overworld after looking for " + name + "."); + return found; +} + +void use_potion_first_pokemon(VideoStream& stream, ProControllerContext& context, double heal_below){ + OverlayBoxScope hp_overlay(stream.overlay(), COLOR_GREEN, {0.137, 0.160, 0.202, 0.016}, ""); + bool healed = false; + for (size_t dose = 0; dose < 3 && !healed; dose++){ + const Dose result = potion_dose(stream, context, dose, heal_below); + if (result == Dose::NONE_LEFT){ + bag_back_to_overworld(stream, context, "Potion: No overworld after finding no Potion."); + return; + } + healed = result == Dose::HEALED; + if (!healed){ + pbf_mash_button(context, BUTTON_B, 2000ms); + context.wait_for_all_requests(); + } + } + if (!healed){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Potion: Full HP not confirmed after three uses.", + stream); + } + bag_back_to_overworld(stream, context, "Potion: Final overworld return not confirmed."); + stream.log("[Potion] Slot 1 HP bar full and overworld confirmed."); +} + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.h new file mode 100644 index 0000000000..abb2d8e653 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.h @@ -0,0 +1,93 @@ +/* BDSP AutoStory - Bag + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Finding an item in the Bag, and Potions on the lead. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Bag_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Bag_H + +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Pocket positions 1-8: Medicine, Balls, Battle Items, Berries, Other Items, +// TMs, Treasures, Key Items. Zero means the selected tab is uncertain. +int bag_pocket_position(const ImageViewRGB32& frame); +// Whether the open pocket shows no item. +bool bag_list_is_empty(const ImageViewRGB32& frame); +// The cursor's row, 1-7, or 0 if unknown. +int bag_cursor_position(const ImageViewRGB32& frame); + +// Navigation action between bag pocket tabs. +enum class BagPocketAction{ + UNKNOWN, + CONFIRM, + LEFT, + RIGHT, +}; +// Plan to navigate from one bag pocket tab to another. +struct BagPocketMovePlan{ + BagPocketAction action = BagPocketAction::UNKNOWN; + size_t presses = 0; +}; +// Circular eight-pocket route; ties prefer Right. Invalid positions send no input. +BagPocketMovePlan plan_bag_pocket_move(int current, int target); + +// Starts in the overworld, opens the Bag on `target_pocket`, and moves the cursor +// onto the item whose name reads `name` (letters only, lower case, e.g. "potion"). +// Reads the cursor row's name with OCR, so it does not depend on the capture +// card's colors or sharpness the way a template match does. +// Returns false if the pocket is empty or the name is not found. +bool find_bag_item_by_name( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket, + int num_icons = 8 +); +// The same search in a Bag that is already open, such as the one a battle's +// Bag command opens. +bool find_bag_item_in_open_bag( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket +); + +// Open the Bag from the overworld, look for `name` in the pocket, and close +// it again. True if the item is there. +bool bag_has_item( + VideoStream& stream, + ProControllerContext& context, + const std::string& name, + int target_pocket +); + +// Read the lead's HP bar in the Bag and, if it is below `heal_below` (a +// fraction of full), use Potions until it is full. heal_below = 1 always +// heals; 0.5 only checks and heals when the lead is under half, and is +// skipped when the Bag has no Potion. +void use_potion_first_pokemon( + VideoStream& stream, + ProControllerContext& context, + double heal_below = 1.0 +); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.cpp new file mode 100644 index 0000000000..568247aba4 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.cpp @@ -0,0 +1,431 @@ +/* BDSP AutoStory - Battle + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/OCR/OCR_RawTesseractOCR.h" +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_EndBattleDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_SelectionArrow.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_Bag.h" +#include "PokemonBDSP_AutoStory_BattleMoves.h" +#include "PokemonBDSP_AutoStory_Battle.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +BattleDialogWatcher::BattleDialogWatcher(Color color) + : DetectorToFinder( + "BattleDialogWatcher", + std::chrono::milliseconds(250), + color + ) +{ +} + +namespace{ + +// On the battle menu: the cursor onto Battle (the command list's first +// row), confirmed on three frames, then A. +void choose_battle_command(VideoStream& stream, ProControllerContext& context, BattleType battle_type, const std::string& label){ + BattleMenuDetector commands(battle_type); + VideoOverlaySet overlays(stream.overlay()); + commands.make_overlays(overlays); + overlays.add(COLOR_GREEN, {0.90, 0.48, 0.075, 0.505}); + size_t confirmed = 0; + for (size_t check = 0; check < 30 && confirmed < 3; check++){ + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + BattleMenuOption selected = frame ? battle_menu_selected_option(commands, frame) : BattleMenuOption::UNKNOWN; + stream.log("[" + label + "] Command cursor=" + std::to_string((int)selected + 1) + "; target Battle=1."); + if (selected == BattleMenuOption::BATTLE){ + if (++confirmed < 3){ + pbf_wait(context, 100ms); + } + }else if (selected == BattleMenuOption::UNKNOWN){ + confirmed = 0; + pbf_wait(context, 150ms); + }else{ + confirmed = 0; + pbf_press_dpad(context, DPAD_UP, 80ms, 250ms); + } + } + if (confirmed < 3){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Could not confirm Battle command; A not pressed.", stream); + } + pbf_press_button(context, BUTTON_A, 80ms, 500ms); +} + +// Watching the move menu settle: how many frames in a row showed the same +// number of moves, and how many had the cursor on the target. +struct MoveMenuWatch{ + size_t previous_move_count = 0; + size_t stable_move_count = 0; + size_t confirmed = 0; + // The move to use, 1-based; 0 until the menu has settled and the best is picked. + size_t target = 0; +}; + +// One look at the move menu. Returns whether the target is confirmed +// under the cursor; otherwise moves the cursor toward it or waits. +bool move_menu_step(VideoStream& stream, ProControllerContext& context, MoveMenuWatch& watch, const std::string& label){ + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + const BattleMoveMenuState moves = frame ? detect_battle_move_menu(frame) : BattleMoveMenuState{}; + const int slot = (int)moves.selected_slot; + VideoOverlaySet move_overlays(stream.overlay()); + for (size_t i = 0; i < moves.move_count; i++){ + move_overlays.add(moves.selected_slot == i + 1 ? COLOR_GREEN : COLOR_YELLOW, moves.buttons[i]); + } + stream.log("[" + label + "] Available moves=" + std::to_string(moves.move_count) + + "; selected slot=" + std::to_string(slot) + "/4; target=" + std::to_string(watch.target) + "."); + if (moves.move_count == watch.previous_move_count && moves.move_count > 0){ + watch.stable_move_count++; + }else{ + watch.previous_move_count = moves.move_count; + watch.stable_move_count = 1; + watch.confirmed = 0; + } + // Pick the best move once the menu has settled. + if (watch.target == 0 && watch.stable_move_count >= 3 && frame){ + watch.target = pick_best_move(stream, *frame.frame, moves, label); + } + if (moves.move_count > 0 && moves.move_count < watch.target && watch.stable_move_count >= 3){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Move slot " + std::to_string(watch.target) + " is absent; no move confirmed.", stream); + } + if (slot == (int)watch.target && watch.stable_move_count >= 3){ + if (++watch.confirmed >= 3){ + return true; + } + pbf_wait(context, 100ms); + }else{ + watch.confirmed = 0; + if (slot == 0 || slot == (int)watch.target){ + pbf_wait(context, 150ms); + }else{ + pbf_press_dpad(context, slot < (int)watch.target ? DPAD_DOWN : DPAD_UP, 80ms, 250ms); + } + } + context.wait_for_all_requests(); + return false; +} + +// On the battle menu: Battle, then the best move on the move menu once +// it has settled, confirmed under the cursor, then A. +void use_best_move(VideoStream& stream, ProControllerContext& context, BattleType battle_type, const std::string& label){ + choose_battle_command(stream, context, battle_type, label); + MoveMenuWatch watch; + for (size_t check = 0; check < 30; check++){ + if (move_menu_step(stream, context, watch, label)){ + stream.log("[" + label + "] Move " + std::to_string(watch.target) + " confirmed; using it."); + pbf_press_button(context, BUTTON_A, 80ms, 3000ms); + return; + } + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Could not confirm move " + std::to_string(watch.target) + "; A not pressed.", stream); +} + +// Our HP bar in the battle's bottom-left panel, as a fraction; -1 if +// no bar is there. Measured on 1920x1080 frames: the bar runs from +// x 88 to 466 on rows 961 to 965. +double battle_player_hp(const ImageViewRGB32& frame){ + const ImageViewRGB32 bar = extract_box_reference(frame, ImageFloatBox{0.046, 0.890, 0.197, 0.004}); + size_t filled = 0; + for (size_t x = 0; x < bar.width(); x++){ + size_t color = 0; + for (size_t y = 0; y < bar.height(); y++){ + const uint32_t p = bar.pixel(x, y); + const int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + const bool green = g > r + 30 && g > b + 20; + const bool yellow = r > 150 && g > 120 && b < 100; + const bool red = r > 150 && r > g + 50 && r > b + 50; + color += green || yellow || red; + } + filled += color > bar.height() / 2; + } + return bar.width() && filled ? (double)filled / bar.width() : -1; +} + +// Our HP at the battle menu, from up to five looks; -1 if unread. +double read_battle_hp(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + double hp = -1; + for (size_t check = 0; check < 5 && hp < 0; check++){ + const auto frame = stream.video().snapshot(); + if (frame){ + hp = battle_player_hp(*frame.frame); + } + if (hp < 0){ + pbf_wait(context, 150ms); + } + context.wait_for_all_requests(); + } + return hp; +} + +// On the battle menu: the cursor onto Bag. Whether it got there. +bool cursor_to_bag(VideoStream& stream, ProControllerContext& context, BattleType battle_type){ + BattleMenuDetector commands(battle_type); + for (size_t check = 0; check < 30; check++){ + context.wait_for_all_requests(); + const auto frame = stream.video().snapshot(); + const BattleMenuOption selected = frame ? battle_menu_selected_option(commands, frame) : BattleMenuOption::UNKNOWN; + if (selected == BattleMenuOption::BAG){ + return true; + } + if (selected == BattleMenuOption::UNKNOWN){ + pbf_wait(context, 150ms); + }else{ + pbf_press_dpad(context, (int)selected < (int)BattleMenuOption::BAG ? DPAD_DOWN : DPAD_UP, 80ms, 300ms); + } + } + return false; +} + +enum class BattleHeal{ NOT_NEEDED, USED, NONE_LEFT }; + +// At the battle menu: if our HP is under `heal_below`, use a Potion +// from the Bag on the lead, which takes the turn. Otherwise the +// battle menu is left up. +BattleHeal battle_heal_if_low(VideoStream& stream, ProControllerContext& context, + BattleType battle_type, double heal_below, const std::string& label){ + const double hp = read_battle_hp(stream, context); + stream.log("[" + label + "] Our HP bar reads " + std::to_string((int)(hp * 100)) + "%."); + if (hp < 0 || hp >= heal_below){ + return BattleHeal::NOT_NEEDED; + } + if (!cursor_to_bag(stream, context, battle_type)){ + stream.log("[" + label + "] Could not reach the Bag command; fighting on.", COLOR_ORANGE); + return BattleHeal::NONE_LEFT; + } + pbf_press_button(context, BUTTON_A, 80ms, 2000ms); + if (!find_bag_item_in_open_bag(stream, context, "potion", 1)){ + stream.log("[" + label + "] No Potion left; fighting on.", COLOR_ORANGE); + pbf_press_button(context, BUTTON_B, 80ms, 1500ms); + pbf_press_button(context, BUTTON_B, 80ms, 1500ms); + context.wait_for_all_requests(); + return BattleHeal::NONE_LEFT; + } + // Potion, then the lead in the party list, then Use. + stream.log("[" + label + "] Using a Potion on the lead.", COLOR_BLUE); + pbf_press_button(context, BUTTON_A, 80ms, 1500ms); + pbf_press_button(context, BUTTON_A, 80ms, 1500ms); + pbf_press_button(context, BUTTON_A, 80ms, 2500ms); + context.wait_for_all_requests(); + return BattleHeal::USED; +} + +// How a battle is fought: every battle here takes the best move each turn. +struct BestMoveBattle{ + BattleType battle_type = BattleType::TRAINER; + BattleFormat format = BattleFormat::SINGLES; + // After a faint the game shows the party: send out the next Pokemon. + // Without, a quiet two minutes is a timeout. + bool switch_on_faint = false; +}; + +// What a battle has done so far, across turns. +struct BattleProgress{ + bool battle_menu_seen = false; + bool out_of_potions = false; + size_t potions_used = 0; + size_t turns = 0; + size_t faint_switches = 0; +}; + +// One turn at the battle menu: a Potion when a trainer's battle has left +// our HP low, else the best move (and the default target in doubles). +void take_turn( + VideoStream& stream, ProControllerContext& context, + const BestMoveBattle& battle, BattleProgress& progress, const std::string& label +){ + progress.battle_menu_seen = true; + // Early battles take a few turns; this many means the chosen + // move is not being used (no PP, a misread menu). + if (++progress.turns > 25){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": still at the battle menu after 25 turns.", stream); + } + stream.log("[AutoStory] Battle menu detected.", COLOR_BLUE); + // A trainer battle we may lose: a Potion when HP is low. + if (battle.battle_type == BattleType::TRAINER && !progress.out_of_potions && progress.potions_used < 4){ + const BattleHeal heal = battle_heal_if_low(stream, context, battle.battle_type, 0.35, label); + if (heal == BattleHeal::USED){ + progress.potions_used++; + return; + } + progress.out_of_potions = heal == BattleHeal::NONE_LEFT; + } + stream.log("[" + label + "] Move list position 1: use move 0."); + use_best_move(stream, context, battle.battle_type, label); + // A double battle asks for the target next; B would + // back out of it. The default target is an opponent. + if (battle.format == BattleFormat::DOUBLES){ + pbf_wait(context, 1200ms); + pbf_press_button(context, BUTTON_A, 80ms, 1500ms); + } +} + +// The battle has ended: its text to the overworld, which must stay +// dialogue-free for four seconds. +void finish_battle(VideoStream& stream, ProControllerContext& context, const std::string& label){ + stream.log("[AutoStory] " + label + " battle complete.", COLOR_GREEN); + OverworldWatcher overworld; + const int cleanup_ret = run_until(stream, context, + [](ProControllerContext& ctx){ + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, {{overworld}}); + if (cleanup_ret != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Post-battle cleanup timed out.", stream); + } + context.wait_for_all_requests(); + confirm_battle_overworld(stream, context, label + " cleanup"); +} + +// B through the battle's text until its menu, its end, or a move-learning +// prompt shows: 0, 1 or 2; -1 after the wait. +int wait_for_battle_screen( + VideoStream& stream, ProControllerContext& context, const BestMoveBattle& battle, + const BattleProgress& progress, EndBattleWatcher& end_battle +){ + context.wait_for_all_requests(); + BattleMenuWatcher battle_menu(battle.battle_type); + // Reuse the farming handlers' move-learning prompt recovery. + SelectionArrowFinder learn_move(stream.overlay(), {0.50, 0.62, 0.40, 0.18}, COLOR_YELLOW); + const bool watch_learn_move = battle.format == BattleFormat::DOUBLES || battle.switch_on_faint; + return run_until( + stream, + context, + [&](ProControllerContext& context){ + // Clear narration with B while watching for commands. + // A here can open Fight before selection begins. + // With faint switching, a quiet 30 s means the party + // screen after a faint, which B cannot leave. + pbf_mash_button(context, BUTTON_B, battle.switch_on_faint ? 30000ms : 120000ms); + }, + { + {battle_menu}, + progress.battle_menu_seen ? PeriodicInferenceCallback{end_battle} : PeriodicInferenceCallback{}, + watch_learn_move ? PeriodicInferenceCallback{learn_move} : PeriodicInferenceCallback{}, + } + ); +} + +// Fights the battle on screen to its end and the overworld after it. +void fight_with_best_move(VideoStream& stream, ProControllerContext& context, const BestMoveBattle& battle, const std::string& label){ + stream.log("[AutoStory] Starting " + label + " battle...", COLOR_BLUE); + // Preserve the existing early-game transition delay. + pbf_wait(context, 5000ms); + BattleProgress progress; + EndBattleWatcher end_battle; + while (true){ + const int ret = wait_for_battle_screen(stream, context, battle, progress, end_battle); + if (ret == 0){ + take_turn(stream, context, battle, progress, label); + }else if (ret == 1){ + finish_battle(stream, context, label); + return; + }else if (ret == 2){ + stream.log("[" + label + "] Declining move learning; preserving scripted move positions."); + pbf_move_right_joystick(context, {0, -1}, 160ms, 840ms); + pbf_press_button(context, BUTTON_ZL, 160ms, 840ms); + }else if (battle.switch_on_faint && progress.faint_switches < 5){ + // After a faint the game shows the party: Down to the next + // Pokemon, A, then A on "Send out" (by hand, 2026-10-29 against Roark). + progress.faint_switches++; + stream.log("[" + label + "] No battle menu for 30 s: a faint; sending out the next Pokemon.", COLOR_ORANGE); + pbf_press_dpad(context, DPAD_DOWN, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 1200ms); + pbf_press_button(context, BUTTON_A, 80ms, 2000ms); + }else{ + stream.log("[AutoStory] " + label + " battle timeout.", COLOR_RED); + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "[AutoStory] " + label + " battle timeout.", stream); + } + } +} + +} // namespace + +void confirm_battle_overworld(VideoStream& stream, ProControllerContext& context, + const std::string& label){ + OverworldDetector overworld; + ShortDialogDetector dialogue(COLOR_YELLOW); + BlackScreenDetector black; + VideoOverlaySet overlays(stream.overlay()); + overworld.make_overlays(overlays); + dialogue.make_overlays(overlays); + black.make_overlays(overlays); + const auto deadline = std::chrono::steady_clock::now() + 180s; + auto quiet_since = std::chrono::steady_clock::now(); + bool finished = false; + while (std::chrono::steady_clock::now() < deadline){ + const auto frame = stream.video().snapshot(); + if (!frame){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": No video frame.", stream); + } + const auto now = std::chrono::steady_clock::now(); + if (dialogue.detect(frame) || black.detect(frame) || !overworld.detect(frame)){ + quiet_since = now; + }else if (now - quiet_since >= 4s){ + finished = true; + break; + } + pbf_mash_button(context, BUTTON_B, 100ms); + context.wait_for_all_requests(); + } + if (!finished){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": Dialogue-free overworld not confirmed for four seconds.", stream); + } + context.wait_for_all_requests(); +} + +void fight_story_battle(VideoStream& stream, ProControllerContext& context, const std::string& label, BattleFormat format){ + // The best move each turn; the next Pokemon after a faint; no new moves. + fight_with_best_move(stream, context, {.battle_type = BattleType::TRAINER, .format = format, .switch_on_faint = true}, label); +} + +void fight_trainer(VideoStream& stream, ProControllerContext& context, const std::string& label){ + // The best move on the menu each turn, whatever the starter. + fight_with_best_move(stream, context, {.battle_type = BattleType::TRAINER, .format = BattleFormat::SINGLES, .switch_on_faint = false}, label); +} + +void fight_wild_battle(VideoStream& stream, ProControllerContext& context, const std::string& label){ + fight_with_best_move(stream, context, {.battle_type = BattleType::STANDARD, .format = BattleFormat::SINGLES, .switch_on_faint = false}, label); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.h new file mode 100644 index 0000000000..13dfe78cf5 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.h @@ -0,0 +1,61 @@ +/* BDSP AutoStory - Battle + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Fighting a battle with the best move on the menu: the move names are read + * and scored by power, so it works whatever the starter knows. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Battle_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Battle_H + +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonTools/VisualDetector.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// A battle's text box, as a watcher. +class BattleDialogWatcher : public DetectorToFinder{ +public: + BattleDialogWatcher(Color color = COLOR_RED); +}; + +// One opponent, or two (a tag or twin battle, which asks for a target). +enum class BattleFormat{ + SINGLES, + DOUBLES, +}; + +// Mashes B until the overworld has shown with no dialogue for four seconds. +// Fails with a screenshot after three minutes. +void confirm_battle_overworld(VideoStream& stream, ProControllerContext& context, const std::string& label); + +// A story battle that may go badly: the best move each turn, Potions +// under 35 %, the next Pokemon when one faints, and "Keep old moves" +// on level-up prompts. `doubles` confirms the default target each +// turn in DOUBLES. Returns once the battle has ended, +// won or lost; after a loss we stand in the last Pokemon Center. +void fight_story_battle(VideoStream& stream, ProControllerContext& context, const std::string& label, BattleFormat format); + +// Any trainer battle, whatever the starter: the best move on the menu each +// turn until the battle ends, then the text after it. +void fight_trainer(VideoStream& stream, ProControllerContext& context, const std::string& label); + +// A wild battle at its battle menu, fought with the best move until it ends. +void fight_wild_battle(VideoStream& stream, ProControllerContext& context, const std::string& label); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp new file mode 100644 index 0000000000..be262d9cef --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp @@ -0,0 +1,276 @@ +/* BDSP AutoStory - Battle moves + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonTools/OCR/OCR_RawTesseractOCR.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP_AutoStory_BattleMoves.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// Early moves the starters and their foes have, as type, power and +// most PP. A status move has power 0. +struct KnownMove{ + const char* name; + const char* type; + int power; + int pp; +}; +const KnownMove KNOWN_MOVES[] = { + {"pound", "normal", 40, 35}, {"tackle", "normal", 40, 35}, {"scratch", "normal", 40, 35}, + {"quickattack", "normal", 40, 30}, {"headbutt", "normal", 70, 15}, + {"watergun", "water", 40, 25}, {"bubble", "water", 40, 30}, {"bubblebeam", "water", 65, 20}, + {"ember", "fire", 40, 25}, {"flamewheel", "fire", 60, 25}, {"machpunch", "fighting", 40, 30}, + {"absorb", "grass", 20, 25}, {"razorleaf", "grass", 55, 25}, {"bite", "dark", 60, 25}, + {"peck", "flying", 35, 35}, {"wingattack", "flying", 60, 35}, {"metalclaw", "steel", 50, 35}, + {"growl", "normal", 0, 40}, {"leer", "normal", 0, 30}, {"tailwhip", "normal", 0, 30}, + {"withdraw", "water", 0, 40}, {"taunt", "dark", 0, 20}, +}; + +// The box's text, whatever differs from its median colour by more than +// 120, as black on white with an 8-pixel margin. +ImageRGB32 pp_text_image(const ImageViewRGB32& box){ + std::vector pixels; + for (size_t y = 0; y < box.height(); y++){ + for (size_t x = 0; x < box.width(); x++){ + pixels.push_back(box.pixel(x, y)); + } + } + auto brightness = [](uint32_t p){ return ((p >> 16) & 255) + ((p >> 8) & 255) + (p & 255); }; + std::nth_element(pixels.begin(), pixels.begin() + pixels.size() / 2, pixels.end(), + [&](uint32_t a, uint32_t b){ return brightness(a) < brightness(b); }); + const uint32_t background = pixels[pixels.size() / 2]; + const size_t pad = 8; + ImageRGB32 text(box.width() + 2 * pad, box.height() + 2 * pad); + text.fill(0xffffffff); + for (size_t y = 0; y < box.height(); y++){ + for (size_t x = 0; x < box.width(); x++){ + const uint32_t p = box.pixel(x, y); + int distance = 0; + for (int shift : {16, 8, 0}){ + distance += std::abs((int)((p >> shift) & 255) - (int)((background >> shift) & 255)); + } + if (distance > 120){ + text.pixel(x + pad, y + pad) = 0xff000000; + } + } + } + return text; +} + +// A move button's PP left, "8/25" at its right end; -1 if it cannot be +// read. The text is dark, or yellow to red as PP runs low, so text is +// whatever differs from the button's own color. `max_pp` helps when +// the OCR reads the slash as a digit. +int read_move_pp(const ImageViewRGB32& screen, const ImageFloatBox& button, int max_pp){ + const ImageViewRGB32 box = extract_box_reference(screen, ImageFloatBox{ + button.x + button.width * 0.75, button.y + button.height * 0.10, + button.width * 0.21, button.height * 0.80}); + if (box.width() == 0 || box.height() == 0){ + return -1; + } + const std::string read = OCR::tesseract_ocr_read(Language::English, pp_text_image(box)); + const size_t slash = read.find('/'); + std::string digits; + for (size_t i = 0; i < read.size() && i < slash; i++){ + if (std::isdigit((unsigned char)read[i])){ + digits.push_back(read[i]); + } + } + if (slash == std::string::npos && max_pp > 0){ + // "29135" for 29/35: drop the most PP and the misread slash. + const std::string max = std::to_string(max_pp); + if (digits.size() <= max.size() + 1 || digits.compare(digits.size() - max.size(), max.size(), max) != 0){ + return -1; + } + digits.resize(digits.size() - max.size() - 1); + } + if (digits.empty() || digits.size() > 2){ + return -1; + } + return std::stoi(digits); +} + +// A box's gray levels, row by row, and the cut between text and background: +// halfway between the background and the darkest pixel (a fixed cut thins +// the strokes on the blue button). +struct GrayBox{ + std::vector gray; + size_t width = 0; + size_t height = 0; + int threshold = 0; + bool text_in_row(size_t row) const{ + for (size_t x = 0; x < width; x++){ + if (gray[row * width + x] < threshold){ + return true; + } + } + return false; + } +}; +GrayBox gray_box(const ImageViewRGB32& box){ + GrayBox ret{std::vector(box.width() * box.height()), box.width(), box.height(), 0}; + for (size_t y = 0; y < box.height(); y++){ + for (size_t x = 0; x < box.width(); x++){ + const uint32_t pixel = box.pixel(x, y); + ret.gray[y * box.width() + x] = (int)(((pixel >> 16) & 0xff) + ((pixel >> 8) & 0xff) + (pixel & 0xff)) / 3; + } + } + std::vector sorted = ret.gray; + std::nth_element(sorted.begin(), sorted.begin() + sorted.size() / 2, sorted.end()); + const int background = sorted[sorted.size() / 2]; + const int darkest = *std::min_element(ret.gray.begin(), ret.gray.end()); + ret.threshold = (background + darkest) / 2; + return ret; +} + +// The tallest band of text rows, [top, bottom): the name; "Effective" is smaller. +std::pair tallest_text_band(const GrayBox& box){ + size_t band_top = 0, band_bottom = 0; + for (size_t y = 0; y < box.height;){ + if (!box.text_in_row(y)){ + y++; + continue; + } + size_t end = y; + while (end < box.height && box.text_in_row(end)){ + end++; + } + if (end - y > band_bottom - band_top){ + band_top = y; + band_bottom = end; + } + y = end; + } + return {band_top, band_bottom}; +} + +// The name is dark text on a light button (grey, or light blue under the +// cursor). Take the box right of the type icon and left of the PP, tall +// enough for a status move's name, which sits lower: it has no "Effective" +// line under it. Read as lower-case letters; nothing where there is no +// reader for the game's language (an unread move counts as a plain attack). +std::string read_move_name(const ImageViewRGB32& screen, const ImageFloatBox& button){ + const ImageFloatBox name_box{button.x + button.width * 0.17, button.y + button.height * 0.08, + button.width * 0.53, button.height * 0.80}; + const GrayBox box = gray_box(extract_box_reference(screen, name_box)); + const auto [band_top, band_bottom] = tallest_text_band(box); + // Black text on white, with a margin, for the OCR. + const size_t pad = 6; + const size_t top = band_top >= 4 ? band_top - 4 : 0; + const size_t bottom = std::min(box.height, band_bottom + 4); + ImageRGB32 text(box.width + 2 * pad, bottom - top + 2 * pad); + text.fill(0xffffffff); + for (size_t y = top; y < bottom; y++){ + for (size_t x = 0; x < box.width; x++){ + if (box.gray[y * box.width + x] < box.threshold){ + text.pixel(x + pad, y - top + pad) = 0xff000000; + } + } + } + std::string name; + for (unsigned char ch : story_reads_text() ? OCR::tesseract_ocr_read(story_language(), text) : std::string()){ + if (std::isalpha(ch)){ + name.push_back((char)std::tolower(ch)); + } + } + return name; +} + +// The longest known move whose name appears in `name`: "bubblebeam" over "bubble". +const KnownMove* known_move(const std::string& name){ + const KnownMove* known = nullptr; + for (const KnownMove& move : KNOWN_MOVES){ + if (name.find(move.name) != std::string::npos && + (!known || std::string(move.name).size() > std::string(known->name).size())){ + known = &move; + } + } + return known; +} + +// The move in slot `i`: its read name, and what it is if known. A slot +// read well once keeps its name when a later read fails. +std::pair identify_move(const ImageViewRGB32& screen, const ImageFloatBox& button, size_t i){ + static std::array last_read; + std::string name = read_move_name(screen, button); + const KnownMove* known = known_move(name); + if (!known){ + name = last_read[i]; + known = known_move(name); + } + if (known){ + last_read[i] = known->name; + } + return {name, known}; +} + +} // namespace + + +size_t pick_best_move(VideoStream& stream, const ImageViewRGB32& screen, + const BattleMoveMenuState& moves, const std::string& label){ + // Keep the menu as it looked, to check the reads against. + const std::string folder = SCREENSHOTS_PATH() + "AutoStory/moves/"; + Filesystem::create_directories(Filesystem::Path(folder)); + screen.save(folder + now_to_filestring() + "-" + label + ".png"); + size_t best = 1; + int best_score = -1; + bool any_attack = false; + size_t last_not_status = 1; + for (size_t i = 0; i < moves.move_count; i++){ + const auto [name, known] = identify_move(screen, moves.buttons[i], i); + // A move with no PP left cannot be chosen; the game says so and + // keeps the menu up. + const int pp = read_move_pp(screen, moves.buttons[i], known ? known->pp : 0); + stream.log("[" + label + "] Move " + std::to_string(i + 1) + " PP left: " + (pp < 0 ? std::string("unread") : std::to_string(pp)) + "."); + if (pp == 0){ + continue; + } + // An unreadable move counts as a plain attack, ahead of known + // status moves. + const int score = known ? known->power * 2 + (std::string(known->type) != "normal" ? 1 : 0) : 40 * 2 - 1; + stream.log("[" + label + "] Move " + std::to_string(i + 1) + " reads \"" + name + "\"" + + (known ? std::string(" = ") + known->name + " (" + known->type + ", " + std::to_string(known->power) + ")" : " (unknown)") + "."); + if (score > best_score){ + best_score = score; + best = i + 1; + } + any_attack = any_attack || (known && known->power > 0); + if (!known || known->power > 0){ + last_not_status = i + 1; + } + } + // No attack read: the newest move is the starter's own-type attack + // (Water Gun, Ember, Absorb), unless it is a status move. + if (!any_attack){ + best = last_not_status; + } + stream.log("[" + label + "] Best move: " + std::to_string(best) + "."); + return best; +} + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.h new file mode 100644 index 0000000000..2f9b1fa674 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.h @@ -0,0 +1,37 @@ +/* BDSP AutoStory - Battle moves + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Which move to use: the move menu's names and PP read with OCR, scored by + * the power of the early moves the starters and their foes have. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_BattleMoves_H +#define PokemonAutomation_PokemonBDSP_AutoStory_BattleMoves_H + +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The best move on the move menu `moves` of `screen`, 1-based: the most +// powerful one we know, and on a tie one that is not Normal (an early +// starter's own type). A move with no PP left is passed over; with no +// attack read, the newest move that is not a status move. Keeps a +// screenshot of the menu, labelled `label`. +size_t pick_best_move(VideoStream& stream, const ImageViewRGB32& screen, const BattleMoveMenuState& moves, const std::string& label); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp new file mode 100644 index 0000000000..9a58559284 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp @@ -0,0 +1,489 @@ +/* BDSP AutoStory - Pokemon Center + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include +#include +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageStats.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_RouteWalk.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Walk.h" +#include "PokemonBDSP_AutoStory_WildBattle.h" +#include "PokemonBDSP_AutoStory_Center.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; +using namespace Pokemon; + + +namespace{ + +// How many tiles the 1080p view spans across: a tile is about 125 px. +constexpr double SCREEN_WIDTH_TILES = 1920.0 / 125; +// The camera keeps the player in the middle of the screen across, and the +// centre of his tile a little above the middle down (r121a and r122a: his +// feet at y = 590 of 1080). +constexpr double PLAYER_X = 0.5; +constexpr double PLAYER_Y = 0.505; +// How much of the screen's height one row of tiles takes near the player: +// the doorway straight ahead of him (r121a) is centred this far above his +// tile's centre. Rows further north look shorter, so this undercounts them. +constexpr double SCREEN_ROW_HEIGHT = 0.12; +// Doorways are looked for down to here, 0 to 1: a door the player can walk +// into is north of him. +constexpr double DOORWAY_SEARCH_BOTTOM = 0.67; +// A doorway found within this many rows of straight ahead is at the front +// row; further, the walk to the door ended short. +constexpr double DOORWAY_FRONT_ROWS = 1.5; + +// The doorway is looked for on a view this size: 4 px of 1080p per pixel. +constexpr size_t SMALL_WIDTH = 480; +constexpr size_t SMALL_HEIGHT = 270; +// A doorway's lit columns run at least this many small rows (the Mart's 12, +// the Center's 15); the buildings' lit roof bands 3 to 5. +constexpr size_t DOORWAY_COLUMN_ROWS = 8; +// A doorway has this many lit small pixels (the Mart's 260 to 350); a roof +// band's run 60 to 75. +constexpr double DOORWAY_MIN_MASS = 80; + +// Whether a pixel is the pale yellow-white of a lit doorway: the Mart's is +// (235, 243, 205), the Center's (255, 255, 185). The buildings' white roof +// bands are bluer, their orange bands and the light they throw on the +// ground outside are darker. +bool lit_doorway_pixel(uint32_t pixel){ + const int r = (pixel >> 16) & 0xff; + const int g = (pixel >> 8) & 0xff; + const int b = pixel & 0xff; + return r >= 215 && g >= 215 && b >= 150 && b <= g - 15 && r >= g - 25; +} + +// Each column of the small view: how many lit doorway pixels it has in the +// search band, and the sum of their rows. +struct LitColumns{ + std::vector count; + std::vector row_sum; +}; +LitColumns lit_columns(const ImageViewRGB32& screen){ + const ImageRGB32 small = screen.scale_to(SMALL_WIDTH, SMALL_HEIGHT); + LitColumns columns{std::vector(small.width(), 0), std::vector(small.width(), 0)}; + const size_t bottom = (size_t)(DOORWAY_SEARCH_BOTTOM * (double)small.height()); + for (size_t y = 0; y < bottom; y++){ + for (size_t x = 0; x < small.width(); x++){ + if (lit_doorway_pixel(small.pixel(x, y))){ + columns.count[x]++; + columns.row_sum[x] += (double)y; + } + } + } + return columns; +} + +// A run of lit columns: its lit pixels, and the sums of their columns and rows. +struct LitRun{ + double mass = 0; + double column_sum = 0; + double row_sum = 0; +}; +// The run of columns with at least DOORWAY_COLUMN_ROWS lit that starts at +// `x`, allowing gaps of up to 3 columns (the door's frame). Moves `x` past it. +LitRun lit_run(const LitColumns& columns, size_t& x){ + LitRun run; + size_t gap = 0; + for (; x < columns.count.size() && gap <= 3; x++){ + if (columns.count[x] < DOORWAY_COLUMN_ROWS){ + gap++; + continue; + } + run.mass += (double)columns.count[x]; + run.column_sum += (double)columns.count[x] * (double)x; + run.row_sum += columns.row_sum[x]; + gap = 0; + } + return run; +} + +// Where a building's lit doorway is across and down the screen, 0 to 1: the +// biggest run of lit columns within 0.35 of the middle (a second building's +// door may show further off). Nothing if no doorway shows. +std::optional> lit_doorway_position(const ImageViewRGB32& screen){ + const LitColumns columns = lit_columns(screen); + const double width = (double)columns.count.size(); + std::optional> best; + double best_mass = 0; + for (size_t x = 0; x < columns.count.size(); ){ + if (columns.count[x] < DOORWAY_COLUMN_ROWS){ + x++; + continue; + } + const LitRun run = lit_run(columns, x); + const double center = run.column_sum / run.mass / width; + if (run.mass < DOORWAY_MIN_MASS || std::abs(center - PLAYER_X) >= 0.35 || run.mass <= best_mass){ + continue; + } + best = {center, run.row_sum / run.mass / (double)SMALL_HEIGHT}; + best_mass = run.mass; + } + return best; +} + +// Whether the whole view is a different scene from "before". +bool scene_changed_since(VideoStream& stream, ProControllerContext& context, const VideoSnapshot& before){ + context.wait_for_all_requests(); + VideoSnapshot now = stream.video().snapshot(); + return before && now && walk_screen_change(*before.frame, *now.frame) > WALK_SCENE_CHANGE; +} + +// Moves across by "tiles", east if positive, west if negative. +void step_across(VideoStream& stream, ProControllerContext& context, double tiles){ + const Direction side = tiles > 0 ? Direction::EAST : Direction::WEST; + route_turn(stream, context, side); + const Milliseconds hold = std::max(Milliseconds(40), + Milliseconds((int64_t)(std::abs(tiles) * ROUTE_MS_PER_TILE_ACROSS.count()))); + pbf_move_left_joystick(context, {tiles > 0 ? 1.0 : -1.0, 0}, hold, 300ms); +} + +// Whether the screen now shows a Pokemon Center's inside. +bool center_inside_now(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + VideoSnapshot now = stream.video().snapshot(); + return now && pokemon_center_inside(*now.frame); +} + +// Walks north into the door straight ahead, `doorway` away. Returns whether +// we are inside. +bool walk_into_door( + VideoStream& stream, ProControllerContext& context, + const VideoSnapshot& outside, const LitDoorway& doorway, const std::string& label +){ + // Time for the rows to the door (none when it is straight ahead), the + // door opening, and a start delay. + const double rows = std::max(0.0, doorway.rows_up - 1); + const Milliseconds limit = 1500ms + Milliseconds((int64_t)(rows * ROUTE_MS_PER_TILE_UP_DOWN.count())); + // The Center's fade is too light to read as black (r126 runs 2 and 3: a + // screen change of 49 at 400 ms), so its inside, once it shows, stops + // the walk too. + const WalkStop stop = walk_until(stream, context, run_from_wild_battle, {0, 1}, limit, label + ": into the door", + [](const ImageViewRGB32& screen){ return pokemon_center_inside(screen); }).stop; + if (stop == WalkStop::TRANSITION || stop == WalkStop::ARRIVED){ + pbf_wait(context, 2000ms); + return true; + } + // The door slides open before it takes us in, and its fade is not always + // black enough to read as a transition, and may finish after the walk; a + // whole new scene says we are in. Not after a walk of several rows, which + // scrolls the view: the next look decides. + pbf_wait(context, 2000ms); + if (center_inside_now(stream, context)){ + stream.log("[Route] " + label + ": a Pokemon Center's inside shows; inside."); + return true; + } + if (doorway.rows_up > DOORWAY_FRONT_ROWS || !scene_changed_since(stream, context, outside)){ + return false; + } + stream.log("[Route] " + label + ": the scene changed; inside."); + return true; +} + +// Where the doorway is on screen, in the log. +void log_doorway(VideoStream& stream, const std::string& label, const LitDoorway& doorway){ + stream.log("[Route] " + label + ": doorway " + std::to_string(doorway.tiles_across) + + " tiles across, " + std::to_string(doorway.rows_up) + " rows up (at " + + std::to_string(doorway.x) + ", " + std::to_string(doorway.y) + " of the screen)."); +} + +// Near the entrance: black below the floor, either side of the step out +// of the door, and the floor lit between them: not a fade (349 in +// Jubilife at night). +bool center_entrance_view(const ImageViewRGB32& screen){ + for (const ImageFloatBox& box : {ImageFloatBox(0.02, 0.95, 0.30, 0.04), ImageFloatBox(0.65, 0.95, 0.33, 0.04)}){ + if (image_stats(extract_box_reference(screen, box)).average.sum() > 90){ + return false; + } + } + return image_stats(extract_box_reference(screen, ImageFloatBox(0.30, 0.55, 0.40, 0.15))).average.sum() > 250; +} + +// At the nurse's counter, the camera in close and the lit floor down to +// the screen's bottom (lab-B 2026-11-08: a run in whose fade went unseen +// stopped there): either side of us, the counter's red top (r 246, g 84) +// over its white front (about 200 each). Outside Oreburgh's Center at +// sunset the ground there is orange (r 218, g 134), with nothing white. +bool center_counter_view(const ImageViewRGB32& screen){ + for (double x : {0.15, 0.56}){ + const FloatPixel top = image_stats(extract_box_reference(screen, ImageFloatBox(x, 0.58, 0.12, 0.03))).average; + const FloatPixel front = image_stats(extract_box_reference(screen, ImageFloatBox(x, 0.66, 0.12, 0.03))).average; + const double front_min = std::min({front.r, front.g, front.b}); + const double front_max = std::max({front.r, front.g, front.b}); + const bool red = top.r >= 120 && top.r > 2 * top.g && top.g > top.b; + const bool white = front_min >= 120 && front_max - front_min <= 40; + if (!red || !white){ + return false; + } + } + return true; +} + +} // namespace + + +bool pokemon_center_inside(const ImageViewRGB32& screen){ + // The orange back wall along the top, in two of three places: a plant + // or a bookcase can stand in one (Jubilife's top left), and at night + // it is darker (r 160..200 in Jubilife, 254 in Sandgem by day). + size_t wall = 0; + for (double x : {0.02, 0.25, 0.65}){ + const ImageStats box = image_stats(extract_box_reference(screen, ImageFloatBox(x, 0.03, 0.08, 0.06))); + const double r = box.average.r, g = box.average.g, b = box.average.b; + wall += r >= 140 && r > g + 40 && g > b + 40; + } + return wall >= 2 && (center_entrance_view(screen) || center_counter_view(screen)); +} + +void fail_if_blacked_out(VideoStream& stream, ProControllerContext& context, const std::string& label){ + for (size_t look = 0; look < 10; look++){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + if (screen && indoors(*screen.frame)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": the party fainted; the game took us home (a room shows after the battle).", stream); + } + // The blackout's text and fade are dark; the grass is not. + if (screen && image_stats(*screen.frame).average.sum() > 150){ + return; + } + pbf_wait(context, 1000ms); + } +} + +std::optional find_lit_lamp_x(const ImageViewRGB32& screen, const ImageFloatBox& box){ + const size_t x0 = (size_t)(box.x * screen.width()), x1 = (size_t)((box.x + box.width) * screen.width()); + const size_t y0 = (size_t)(box.y * screen.height()), y1 = (size_t)((box.y + box.height) * screen.height()); + double sum = 0; + size_t count = 0; + for (size_t y = y0; y < y1 && y < screen.height(); y++){ + for (size_t x = x0; x < x1 && x < screen.width(); x++){ + const Color pixel(screen.pixel(x, y)); + if (pixel.red() > 235 && pixel.green() > 220 && pixel.blue() > 150){ + sum += x; + count++; + } + } + } + // Jubilife's lamps at night: 50 to 100 such pixels at 480 px across. + if (count * 480 * 480 < 30 * screen.width() * screen.width()){ + return std::nullopt; + } + return sum / count / screen.width(); +} + +double tiles_across_at(double screen_x){ + return (screen_x - PLAYER_X) * SCREEN_WIDTH_TILES; +} + +std::optional find_lit_doorway(const ImageViewRGB32& screen){ + // A Center's lit floor has the doorway's colour. + if (pokemon_center_inside(screen)){ + return std::nullopt; + } + const std::optional> position = lit_doorway_position(screen); + if (!position){ + return std::nullopt; + } + return LitDoorway{ + position->first, + position->second, + (position->first - PLAYER_X) * SCREEN_WIDTH_TILES, + (PLAYER_Y - position->second) / SCREEN_ROW_HEIGHT, + }; +} + +std::vector tiles_short_of_door( + VideoStream& stream, ProControllerContext& context, + const StoryWorld& grid, const TilePos& door, const std::string& label +){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + const std::optional doorway = screen ? find_lit_doorway(*screen.frame) : std::nullopt; + if (!doorway){ + stream.log("[Route] " + label + ": no lit doorway on screen; not walking again."); + return {}; + } + log_doorway(stream, label, *doorway); + if (doorway->rows_up <= DOORWAY_FRONT_ROWS){ + return {}; + } + return tiles_by_lit_doorway(*doorway, grid, door); +} + +std::vector tiles_by_lit_doorway(const LitDoorway& doorway, const StoryWorld& grid, const TilePos& door){ + const TilePos guess{door.x - (int)std::lround(doorway.tiles_across), door.y + (int)std::lround(doorway.rows_up)}; + std::vector tiles; + for (int y = guess.y - 1; y <= guess.y + 1; y++){ + for (int x = guess.x - 1; x <= guess.x + 1; x++){ + if (standable(grid, TilePos{x, y})){ + tiles.push_back(TilePos{x, y}); + } + } + } + return tiles; +} + +bool line_up_with_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + route_status(stream, label + ": lining up with the door."); + for (size_t tries = 0; tries < 6; tries++){ + context.wait_for_all_requests(); + VideoSnapshot screen = stream.video().snapshot(); + if (screen && pokemon_center_inside(*screen.frame)){ + stream.log("[Route] " + label + ": a Pokemon Center's inside shows; not in front of the door."); + return false; + } + const std::optional doorway = screen ? find_lit_doorway(*screen.frame) : std::nullopt; + if (!doorway){ + stream.log("[Route] " + label + ": no lit doorway on screen."); + return false; + } + log_doorway(stream, label, *doorway); + // Steps overshoot: within a quarter tile, r128 measured 0.29, -0.28, + // 0.48, -0.67 and 0.48 tiles in turn and never settled. The door + // still takes us in from 0.4 (enter_lit_door()). + if (std::abs(doorway->tiles_across) <= 0.4){ + return true; + } + step_across(stream, context, doorway->tiles_across); + // Let a door that took us in show its inside before looking again. + pbf_wait(context, 1000ms); + } + stream.log("[Route] " + label + ": not lined up after 6 steps."); + return false; +} + +namespace{ + +// No lit doorway shows: the door may have taken us in after all, more +// slowly than we watched. Fails unless the scene changed since `last_outside`. +void confirm_inside_without_doorway(VideoStream& stream, ProControllerContext& context, const VideoSnapshot& last_outside, const std::string& label){ + pbf_wait(context, 1500ms); + if (!scene_changed_since(stream, context, last_outside)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": no lit doorway on screen.", stream); + } + stream.log("[Route] " + label + ": the scene changed; inside."); + pbf_wait(context, 1500ms); +} + +} // namespace + +void enter_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + // Movement is not locked to tiles, and the door takes us in only when we + // are lined up with it. Its doorway is lit: find it on screen, where the + // camera keeps the player in the middle, move across by its offset, and + // walk in once it is straight ahead, looking again after every move. + route_status(stream, label + ": lining up with the door."); + VideoSnapshot last_outside; + for (size_t tries = 0; tries < 12; tries++){ + context.wait_for_all_requests(); + VideoSnapshot outside = stream.video().snapshot(); + if (!outside){ + continue; + } + // Inside a Center, its lit floor would read as another doorway. + if (pokemon_center_inside(*outside.frame)){ + stream.log("[Route] " + label + ": a Pokemon Center's inside shows; inside."); + return; + } + const std::optional doorway = find_lit_doorway(*outside.frame); + if (!doorway){ + confirm_inside_without_doorway(stream, context, last_outside, label); + return; + } + last_outside = outside; + log_doorway(stream, label, *doorway); + // We stand on tiles, so within half a tile is the door's column; + // a step moves about 0.7 of a reading, and at 0.25 r144a stepped + // between 0.3 and -0.4 twelve times. Try the door there, and step + // toward it if it does not take us. + if (std::abs(doorway->tiles_across) > 0.5){ + step_across(stream, context, doorway->tiles_across); + continue; + } + if (walk_into_door(stream, context, outside, *doorway, label)){ + pbf_wait(context, 1500ms); + return; + } + step_across(stream, context, doorway->tiles_across); + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": not through the door after 12 tries.", stream); +} + +void heal_at_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + enter_lit_door(stream, context, label); + heal_inside_pokemon_center(stream, context, label); +} + +void heal_inside_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + talk_to_nurse(stream, context, label); + route_status(stream, label + ": healed; walking out."); + leave_door_south(stream, context, run_from_wild_battle, label + ": out the door"); +} + +void talk_to_nurse( + VideoStream& stream, ProControllerContext& context, + const std::string& label +){ + // The nurse's counter is straight ahead of the door. + route_status(stream, label + ": talking to the nurse."); + if (walk_to_talk(stream, context, run_from_wild_battle, {0, 1}, 6000ms, label + ": to the nurse").stop != WalkStop::DIALOG){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + label + ": nurse not reached.", stream); + } + // A through her lines; the cursor starts on "Yes". Stop once no dialogue + // has shown for a while, so A does not start the talk over. + size_t quiet = 0; + for (size_t presses = 0; presses < 30 && quiet < 2; presses++){ + ShortDialogWatcher dialog(COLOR_YELLOW); + if (wait_until(stream, context, 1500ms, {{dialog}}) == 0){ + quiet = 0; + pbf_press_button(context, BUTTON_A, 80ms, 700ms); + }else{ + quiet++; + } + } + context.wait_for_all_requests(); +} + + + +void move_across(VideoStream& stream, ProControllerContext& context, double tiles){ + step_across(stream, context, tiles); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h new file mode 100644 index 0000000000..96a8288af9 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h @@ -0,0 +1,135 @@ +/* BDSP AutoStory - Pokemon Center + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Center_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Center_H + +#include +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_StoryWorld.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Indoors.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// A building's lit doorway seen on the overworld, relative to the player, +// whom the camera keeps in the middle of the screen. +struct LitDoorway{ + double x; // centre across the screen, 0 to 1 + double y; // centre down the screen, 0 to 1 + double tiles_across; // east of the player if positive, west if negative + double rows_up; // rows north of the player's tile, roughly: 1 straight ahead +}; + +// Whether `screen` is the inside of a Pokemon Center: the orange back wall +// along the top, and either the black below the entrance mat along the +// bottom edge (seen from the door) or the nurse's counter close up. Its lit +// floor reads as a doorway (find_lit_doorway()), so this is what says we +// are in (r126 runs 2, 3). +bool pokemon_center_inside(const ImageViewRGB32& screen); + +// Finds a lit doorway (a Pokemon Center's or a Poke Mart's) in the upper +// two thirds of `screen`, the pale yellow-white of its open door. Nothing if +// none shows there, or inside a Pokemon Center, whose floor is that colour. +std::optional find_lit_doorway(const ImageViewRGB32& screen); + +// After a wild battle out of doors: fails with a screenshot if the screen +// turns into a room within 10 s, which means the party fainted and the +// game took us home (r258 lost its third training battle in Route 202's +// grass, woke in Twinleaf, and walked the Sandgem plan there). Returns +// once the outside shows, not black and not a room. +void fail_if_blacked_out(VideoStream& stream, ProControllerContext& context, const std::string& label); + +// A lit street lamp's head inside `box` (0 to 1 of the screen): the mean +// x of its white core, or nothing if fewer than a lamp's worth of white +// pixels show there (by day the lamps are not lit). +std::optional find_lit_lamp_x(const ImageViewRGB32& screen, const ImageFloatBox& box); + +// How many tiles a thing seen at screen x is across from the player +// (positive: east). +double tiles_across_at(double screen_x); + +// Moves across by `tiles`, east if positive, west if negative, facing that +// way first. +void move_across(VideoStream& stream, ProControllerContext& context, double tiles); + +// The tiles a player who sees `doorway` (the door on world tile `door`) may +// stand on: the one its offset says, give or take one each way, since the +// rows are a rough count. Only standable tiles of `grid`. +std::vector tiles_by_lit_doorway( + const LitDoorway& doorway, const StoryWorld& grid, const Pokemon::TilePos& door +); + +// For a player who walked to the row below the door on world tile `door`: +// if its lit doorway shows further up the screen than straight ahead, the +// walk ended short (someone in the way stops an open leg, and the walk +// cannot tell), and the tiles he may stand on instead, read off the +// doorway's offset, are returned to walk from again. Empty when he is at +// the front row, or no doorway shows. +std::vector tiles_short_of_door( + VideoStream& stream, ProControllerContext& context, + const StoryWorld& grid, const Pokemon::TilePos& door, const std::string& label +); + +// For a player on the row below a building's door, its lit doorway on +// screen: steps across by the doorway's offset, looking again after each +// step, until it is straight ahead. Does not go in, and does not turn north +// to face it: a turn there walked us in (r127c). Returns false if no +// doorway shows, a Center's inside shows, or after 6 steps. +bool line_up_with_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + +// Goes in through a building's door (a Pokemon Center's or a Poke Mart's). +// Starts with its lit doorway on screen, within a few tiles: lines up with +// the doorway by sight and walks in, looking again after every move. Whole- +// tile steps sideways miss it, since movement is not locked to tiles, and +// its fade may not read as black: a new scene also counts as inside. Fails +// with a screenshot when no doorway shows, or after 12 moves. +void enter_lit_door( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + +// Heals the party at a Pokemon Center. Starts in front of the Center with its +// lit doorway on screen, within a few tiles of it: lines up with the doorway +// by sight, walks in, talks to the nurse, accepts, and walks back out. Ends +// on the tile below the door. +void heal_at_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + +// heal_at_pokemon_center() from just inside the door: talks to the nurse, +// accepts, and walks back out. Ends on the tile below the door. For a caller +// that walked in on its own (from a known tile, straight north). +void heal_inside_pokemon_center( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + +// From just inside a Pokemon Center's door: walks up to the nurse, talks to +// her and accepts. Ends at her counter, for a caller that walks out its own +// way. +void talk_to_nurse( + VideoStream& stream, ProControllerContext& context, + const std::string& label +); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp new file mode 100644 index 0000000000..5bd6a47b09 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp @@ -0,0 +1,436 @@ +/* BDSP AutoStory - New Game + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Common/Cpp/Exceptions.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/ScreenMachine/ScreenMachine.h" +#include "CommonTools/ScreenMachine/StepTrace.h" +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "NintendoSwitch/Inference/NintendoSwitch_HomeMenuDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.h" +#include "NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h" +#include "NintendoSwitch/Programs/NintendoSwitch_GameEntry.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h" +#include "PokemonBDSP_AutoStory_NewGame.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen){ + // The Switch's own screens come first: BDSP's dialogue detector also fires on them. + if (console != nullptr && StartGameUserSelectDetector(*console).detect(screen)){ + return "user-select"; + } + if (console != nullptr && HomeMenuDetector(*console, COLOR_RED).detect(screen)){ + return "home"; + } + if (SystemKeyboardDetector().detect(screen)){ + return "keyboard"; + } + switch (NameListDetector().selected_row(screen)){ + case -1: + break; + case 0: + return "rival-list-tell"; + default: + return "rival-list-preset"; + } + if (MenuDetector(COLOR_RED, /*skip_new_banner*/ true).detect(screen)){ + return "menu"; + } + // Includes the appearance grid, whose prompt sits in a dialogue box. + if (ShortDialogDetector().detect(screen)){ + return "dialog"; + } + // The boot and fades. After the dialogue: Rowan's first lines show on black. + if (BlackScreenDetector().detect(screen)){ + return "black"; + } + // Logos, the title, language, cutscenes, the bedroom with no menu open. + return "other"; +} + +IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress){ + if (screen == "user-select"){ + return IntroStep::SELECT_USER; + } + if (screen == "home"){ + return IntroStep::FAIL_NOT_IN_GAME; + } + if (screen == "keyboard"){ + return progress.player_named ? IntroStep::CANCEL_RIVAL_KEYBOARD : IntroStep::ENTER_PLAYER_NAME; + } + if (screen == "rival-list-tell"){ + return IntroStep::MOVE_TO_PRESET; + } + if (screen == "rival-list-preset"){ + return IntroStep::PICK_PRESET; + } + if (screen == "menu"){ + return IntroStep::CLOSE_MENU_AND_FINISH; + } + if (screen == "black"){ + return IntroStep::WAIT; + } + if (screen == "dialog"){ + return progress.rival_named ? IntroStep::ADVANCE_WITH_B : IntroStep::ADVANCE; + } + // The bedroom comes only after the rival is named. Before that, every unknown + // screen (logos, the title, language, the appearance preview, cutscenes) + // takes A, but only for so long before a screen only BDSP shows: past that + // it is another game. + if (progress.rival_named){ + return progress.b_presses < INTRO_B_BEFORE_X ? IntroStep::CLEAR_WITH_B : IntroStep::PROBE_FOR_MENU; + } + if (!progress.game_confirmed && progress.unconfirmed_for >= INTRO_TIME_BEFORE_BDSP){ + return IntroStep::FAIL_NOT_BDSP; + } + return IntroStep::ADVANCE; +} +bool intro_screen_confirms_bdsp(const std::string& screen){ + return screen == "dialog" || screen == "rival-list-tell" || screen == "rival-list-preset" || screen == "menu"; +} + +void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start){ + if (intro_screen_confirms_bdsp(screen)){ + progress.game_confirmed = true; + } + if (screen == "black"){ + return; + } + if (!progress.first_shown){ + progress.first_shown = since_start; + } + if (!progress.game_confirmed){ + progress.unconfirmed_for = since_start - *progress.first_shown; + } +} + +bool intro_in_control_without_menu(const XProbe& probe){ + return probe.still_change <= INTRO_STILL_CHANGE && probe.x_change > INTRO_X_CHANGE && probe.b_change <= INTRO_B_RESTORE; +} + +void record_intro_step(IntroProgress& progress, IntroStep step){ + switch (step){ + case IntroStep::ENTER_PLAYER_NAME: + progress.player_named = true; + return; + case IntroStep::MOVE_TO_PRESET: + // The list again: the name was not confirmed. + progress.rival_picked = false; + progress.screens_after_pick = 0; + return; + case IntroStep::PICK_PRESET: + progress.rival_picked = true; + progress.screens_after_pick = 0; + return; + case IntroStep::ADVANCE: + if (progress.rival_picked && ++progress.screens_after_pick >= INTRO_SCREENS_AFTER_RIVAL_PICK){ + progress.rival_named = true; + } + return; + case IntroStep::CLEAR_WITH_B: + progress.b_presses++; + return; + case IntroStep::PROBE_FOR_MENU: + progress.b_presses = 0; + return; + default: + return; + } +} + +namespace{ + +// Waits until the keyboard is gone (or, with `gone` false, has appeared). +bool wait_for_keyboard(VideoStream& stream, ProControllerContext& context, bool gone, Milliseconds timeout){ + context.wait_for_all_requests(); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + VideoSnapshot screen = stream.video().snapshot(); + if (screen && SystemKeyboardDetector().detect(screen) != gone){ + return true; + } + context.wait_for(100ms); + } + return false; +} + +void enter_player_name(VideoStream& stream, ProControllerContext& context, StepTrace& trace, const std::string& name){ + // Let the keyboard finish sliding in: inputs during that are dropped. + pbf_wait(context, 800ms); + context.wait_for_all_requests(); + // B deletes a character, and closes the keyboard on an empty field, so delete only + // while there is text (e.g. after a retry). + for (size_t c = 0; c < 14; c++){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen || !system_keyboard_field_has_text(screen)){ + break; + } + trace.action("Name field holds text; deleting a character."); + pbf_press_button(context, BUTTON_B, 60ms, 250ms); + context.wait_for_all_requests(); + } + trace.action("Typing the player's name: " + name); + type_on_system_keyboard(stream, context, name); + pbf_press_button(context, BUTTON_PLUS, 80ms, 0ms); + if (!wait_for_keyboard(stream, context, true, 5s)){ + trace.fail("The keyboard stayed open after PLUS (OK)."); + } + trace.action("Name confirmed."); +} + +// A run of the intro: what its steps act on and how far it has got. +struct IntroRun{ + SingleSwitchProgramEnvironment& env; + ProControllerContext& context; + StepTrace& trace; + const std::string& player_name; + WallClock start; + IntroProgress progress; + size_t x_without_menu = 0; + // X opened something and B closed it, from a still screen. + bool in_control = false; +}; + +void press(IntroRun& run, Button button){ + pbf_press_button(run.context, button, 80ms, 0ms); +} + +void close_menu_and_finish(IntroRun& run){ + run.trace.action("Main menu is open: the player is in control in the bedroom. Closing it."); + press(run, BUTTON_B); + pbf_wait(run.context, 1000ms); + run.context.wait_for_all_requests(); + VideoSnapshot screen_now = run.env.console.video().snapshot(); + if (!screen_now || MenuDetector(COLOR_RED, /*skip_new_banner*/ true).detect(screen_now)){ + run.trace.fail("The main menu did not close."); + } +} + +// The menu's area, which X changes when the menu opens. +const ImageFloatBox INTRO_MENU_AREA(0.12, 0.08, 0.76, 0.56); + +double menu_area_change(const VideoSnapshot& before, const VideoSnapshot& after){ + if (!before || !after){ + return 1000; + } + return ImageMatch::pixel_RMSD( + extract_box_reference(*before.frame, INTRO_MENU_AREA), extract_box_reference(*after.frame, INTRO_MENU_AREA) + ); +} + +enum class XProbeResult{ + MENU, // the menu showed + IN_CONTROL, // something X opened and B closed, from a still screen + NOTHING, +}; +// X from a screen that held still; B after it if no menu showed. +XProbeResult x_from_still_screen(IntroRun& run){ + VideoStream& stream = run.env.console; + run.context.wait_for_all_requests(); + const VideoSnapshot first = stream.video().snapshot(); + run.context.wait_for(INTRO_STILL_TIME); + const VideoSnapshot before = stream.video().snapshot(); + press(run, BUTTON_X); + run.context.wait_for_all_requests(); + MenuWatcher menu(COLOR_RED, /*skip_new_banner*/ true); + if (wait_until(stream, run.context, 3s, {{menu}}) == 0){ + return XProbeResult::MENU; + } + const VideoSnapshot after_x = stream.video().snapshot(); + pbf_press_button(run.context, BUTTON_B, 80ms, 1000ms); + run.context.wait_for_all_requests(); + const VideoSnapshot after_b = stream.video().snapshot(); + const XProbe probe{menu_area_change(first, before), menu_area_change(before, after_x), menu_area_change(before, after_b)}; + run.trace.action("X probe: still " + std::to_string(probe.still_change) + ", X " + std::to_string(probe.x_change) + + ", after B " + std::to_string(probe.b_change) + "."); + return intro_in_control_without_menu(probe) ? XProbeResult::IN_CONTROL : XProbeResult::NOTHING; +} + +// X, then the menu or B presses again: whatever X opened instead, B closes. +// X opening something unrecognized from a still screen, which B closes +// again, is the player in control too. +void probe_for_menu(IntroRun& run){ + const XProbeResult result = x_from_still_screen(run); + run.in_control = result == XProbeResult::IN_CONTROL; + if (result != XProbeResult::NOTHING){ + return; // The next read sees the menu, or the in-control state, and finishes. + } + run.trace.action("X opened no menu; B again before the next X."); + if (++run.x_without_menu >= INTRO_X_WITHOUT_MENU){ + run.trace.fail("No menu after " + std::to_string(run.x_without_menu) + " X presses: the player never got control."); + } +} + +// Carries out what decide_intro_step() chose for `screen`. +void act(IntroRun& run, const char* screen){ + note_intro_screen(run.progress, screen, + std::chrono::duration_cast(current_time() - run.start)); + const IntroStep step = decide_intro_step(screen, run.progress); + record_intro_step(run.progress, step); + switch (step){ + case IntroStep::SELECT_USER: + // The cursor starts on the user that launched the game. + press(run, BUTTON_A); + pbf_wait(run.context, 1000ms); + return; + case IntroStep::FAIL_NOT_IN_GAME: + run.trace.fail("The Switch HOME menu is showing: the game is not running in front."); + case IntroStep::FAIL_NOT_BDSP: + run.trace.fail("The game in front is not BDSP: no BDSP screen in " + + std::to_string(run.progress.unconfirmed_for.count() / 1000) + " s of unknown screens."); + case IntroStep::ENTER_PLAYER_NAME: + enter_player_name(run.env.console, run.context, run.trace, run.player_name); + return; + case IntroStep::CANCEL_RIVAL_KEYBOARD: + // Opened from "I'll tell you..."; inputs during the slide-in are dropped. + pbf_wait(run.context, 800ms); + press(run, BUTTON_X); + return; + case IntroStep::MOVE_TO_PRESET: + pbf_press_dpad(run.context, DPAD_DOWN, 100ms, 300ms); + return; + case IntroStep::PICK_PRESET: + press(run, BUTTON_A); + pbf_wait(run.context, 500ms); + return; + case IntroStep::CLOSE_MENU_AND_FINISH: + close_menu_and_finish(run); + return; + case IntroStep::ADVANCE: + pbf_press_button(run.context, BUTTON_A, 60ms, 400ms); + return; + case IntroStep::ADVANCE_WITH_B: + case IntroStep::CLEAR_WITH_B: + pbf_press_button(run.context, BUTTON_B, 60ms, 400ms); + return; + case IntroStep::PROBE_FOR_MENU: + probe_for_menu(run); + return; + case IntroStep::WAIT: + pbf_wait(run.context, 500ms); + return; + } +} + +// After the restart BDSP boots on black for about 25 s (r3); presses then +// would count against the wrong-game guard, so none until it ends. +void wait_out_boot(VideoStream& stream, ProControllerContext& context, StepTrace& trace){ + context.wait_for_all_requests(); + const WallClock deadline = current_time() + INTRO_BOOT_LIMIT; + while (current_time() < deadline){ + VideoSnapshot screen = stream.video().snapshot(); + if (screen && !BlackScreenDetector().detect(screen)){ + trace.action("The boot's black screen ended."); + return; + } + context.wait_for(500ms); + } + trace.fail("The screen stayed black for " + std::to_string(INTRO_BOOT_LIMIT.count() / 1000) + " s after the restart."); +} + +// The machine's states: the screens classify_intro_screen() tells apart, with +// the keyboard split by whose name it asks for. Each state's step is act(). +std::vector intro_states(IntroRun& run){ + auto screen_is = [&run](const ImageViewRGB32& screen, const char* name){ + return classify_intro_screen(&run.env.console, screen) == name; + }; + return { + {"in-control", + [&run](const ImageViewRGB32&){ return run.in_control; }, + [&run]{ run.trace.action("X opened and B closed the menu from a still bedroom: the player is in control."); }, + {}, 1, true}, + {"user-select", + [=](const ImageViewRGB32& s){ return screen_is(s, "user-select"); }, + [&run]{ act(run, "user-select"); }, {}, 3}, + {"home", + [=](const ImageViewRGB32& s){ return screen_is(s, "home"); }, + [&run]{ act(run, "home"); }, {}, 1}, + {"player-keyboard", + [=, &run](const ImageViewRGB32& s){ return !run.progress.player_named && screen_is(s, "keyboard"); }, + [&run]{ act(run, "keyboard"); }, {"dialog", "black", "other"}, 1}, + {"rival-keyboard", + [=, &run](const ImageViewRGB32& s){ return run.progress.player_named && screen_is(s, "keyboard"); }, + [&run]{ act(run, "keyboard"); }, {"rival-list-tell", "dialog", "black", "other"}, 3}, + {"rival-list-tell", + [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-tell"); }, + [&run]{ act(run, "rival-list-tell"); }, {"rival-list-preset", "black", "other"}, 6}, + {"rival-list-preset", + [=](const ImageViewRGB32& s){ return screen_is(s, "rival-list-preset"); }, + [&run]{ act(run, "rival-list-preset"); }, {"dialog", "black", "other"}, 6}, + {"menu", + [=](const ImageViewRGB32& s){ return screen_is(s, "menu"); }, + [&run]{ act(run, "menu"); }, {}, 2, true}, + {"dialog", + [=](const ImageViewRGB32& s){ return screen_is(s, "dialog"); }, + [&run]{ act(run, "dialog"); }, {}, 400}, + // The boot is waited out before the machine starts; this is fades. + {"black", + [=](const ImageViewRGB32& s){ return screen_is(s, "black"); }, + [&run]{ act(run, "black"); }, {}, 300, false, 60}, + {"other", + [](const ImageViewRGB32&){ return true; }, + [&run]{ act(run, "other"); }, {}, 300, false, 60}, + }; +} + +} // namespace + + + +void play_new_game_intro( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& player_name +){ + VideoStream& stream = env.console; + StepTrace trace(stream, "Intro"); + + // The restart below reopens the game HOME's cursor is on, which is the + // game running in front only while it runs: with none running (r1, + // 2026-11-01), closing it fails, HOME is pressed again, the cursor goes + // to the first tile, and another game was launched. So BDSP must be + // running in front when the intro starts. + context.wait_for_all_requests(); + VideoSnapshot start = stream.video().snapshot(); + if (start && HomeMenuDetector(env.console, COLOR_RED).detect(start)){ + throw UserSetupError(env.console, + "Start a new game with BDSP running in front, not from the Switch's HOME menu: " + "the intro restarts the game in front."); + } + + // Only close and reopen the game: the states below take it from the user select + // on. reset_game_from_home() waits for a loaded save's black screen to end, and a + // new game's first screen is a dialog on black, so it would sit there until its + // timeout. + trace.action("Restarting the game."); + go_home(env.console, context); + from_home_close_and_reopen_game(env.console, context, true); + + wait_out_boot(stream, context, trace); + IntroRun run{env, context, trace, player_name, current_time(), {}, 0}; + const std::vector states = intro_states(run); + run_screen_machine(stream, context, trace, states, std::chrono::minutes(15)); + trace.action("Intro done."); +} + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h new file mode 100644 index 0000000000..35fb8d8ca9 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h @@ -0,0 +1,148 @@ +/* BDSP AutoStory - New Game + * + * From: https://github.com/PokemonAutomation/ + * + * Plays a new game's intro (language, Professor Rowan, appearance, names) up to + * the bedroom with the player in control, which is where the Tutorial starts. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H +#define PokemonAutomation_PokemonBDSP_AutoStory_NewGame_H + +#include +#include +#include +#include +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "NintendoSwitch/NintendoSwitch_SingleSwitchProgram.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Which intro screen `screen` shows, as the intro's steps tell them apart: +// "user-select", "home", "keyboard", "rival-list-tell" (the rival's name list on +// "I'll tell you..."), "rival-list-preset" (on a preset name), "menu", "dialog" or +// "black" (the boot, fades) or "other". The Switch's own screens need +// `console`; without it they read as whatever else they look like. +std::string classify_intro_screen(ConsoleHandle* console, const ImageViewRGB32& screen); + +// What the intro does on a screen. Decided from the screen and how far the intro has +// got only, so it is tested without a console. +enum class IntroStep{ + SELECT_USER, // A on the Switch's user select + FAIL_NOT_IN_GAME, // HOME is showing + FAIL_NOT_BDSP, // no BDSP screen within INTRO_TIME_BEFORE_BDSP + ENTER_PLAYER_NAME, // type the name on the first keyboard + CANCEL_RIVAL_KEYBOARD, // X: back to the rival's name list + MOVE_TO_PRESET, // Down: off "I'll tell you..." + PICK_PRESET, // A on a preset name + CLOSE_MENU_AND_FINISH, // B: the menu opened in the bedroom, the intro is over + ADVANCE, // A: dialogue, language, appearance, cutscenes + ADVANCE_WITH_B, // B: dialogue once the rival is named + CLEAR_WITH_B, // B: an unknown screen once the rival is named + PROBE_FOR_MENU, // X, to see whether the player is in control + WAIT, // nothing: a black screen (the boot, a fade) +}; +// Progress tracking state for the new-game introductory sequence. +struct IntroProgress{ + bool player_named = false; + // A preset name was picked from the rival's list, and how many screens + // have shown since without the list coming back. "Barry, is it?" and + // its Yes/No follow, and A (Yes, where the cursor starts) confirms it; + // B would answer No and bring the list back (r4, 2026-11-01). + bool rival_picked = false; + size_t screens_after_pick = 0; + // The name is final: INTRO_SCREENS_AFTER_RIVAL_PICK screens passed + // without the list. + bool rival_named = false; + // A screen only BDSP shows has been seen (intro_screen_confirms_bdsp()). + bool game_confirmed = false; + // When the first screen that is not black showed, since the intro + // started; none yet while the game boots. + std::optional first_shown = std::nullopt; + // How long screens have shown without one only BDSP shows: the logos, + // the title, the language select. + std::chrono::milliseconds unconfirmed_for = std::chrono::milliseconds(0); + // B presses on unknown screens since the last X. + size_t b_presses = 0; +}; +// How long the intro presses A on unknown screens, from the first screen +// that is not black, before a BDSP screen must show: the logos, the title +// and the language select. Another game launched by mistake (r1, +// 2026-11-01: FireRed) fails here instead of being played. Black screens +// take no presses: BDSP boots on black for about 25 s (r3, 2026-11-01). +inline constexpr std::chrono::milliseconds INTRO_TIME_BEFORE_BDSP = std::chrono::seconds(60); +// How long the boot may stay black after the restart. +inline constexpr std::chrono::milliseconds INTRO_BOOT_LIMIT = std::chrono::seconds(60); +// Whether `screen` (classify_intro_screen()) is one only BDSP's intro shows: +// its dialogue box, the rival's name list, its main menu. The keyboard and +// the user select are the Switch's. +bool intro_screen_confirms_bdsp(const std::string& screen); + +// Screens after the rival's name is picked, each taking A, before the name +// counts as confirmed and B takes over: "Barry, is it?", the box closing +// (Barry alone), the Yes/No, and a few of Rowan's lines after it. +inline constexpr size_t INTRO_SCREENS_AFTER_RIVAL_PICK = 8; + +// Once the rival is named, the intro only has dialogue and cutscenes left +// before the bedroom, and screens the classifier misses (r2: a dialogue box +// without its arrow). Each unknown screen takes B, which advances dialogue, +// never starts a talk and does nothing in the bedroom; after this many, X +// checks for the menu, and only the menu ends the intro. +inline constexpr size_t INTRO_B_BEFORE_X = 5; +// X presses that opened no menu before the intro gives up: with +// INTRO_B_BEFORE_X B presses between them, about two minutes. +inline constexpr size_t INTRO_X_WITHOUT_MENU = 20; + +// Notes `screen`, read `since_start` into the intro, in `progress`: the +// game is confirmed on a BDSP screen; black screens leave the clock alone. +void note_intro_screen(IntroProgress& progress, const std::string& screen, std::chrono::milliseconds since_start); +// Decides the next controller action based on the current screen and intro progress. +IntroStep decide_intro_step(const std::string& screen, const IntroProgress& progress); + +// When X opens nothing the menu detector knows, the player is still in +// control if the screen held still for INTRO_STILL_TIME before X (no +// dialogue, no fade, no intro sparkles), X changed the menu's area by more +// than INTRO_X_CHANGE, and B put it back within INTRO_B_RESTORE: the menu +// opened and closed. Changes are pixel RMSD. +inline constexpr std::chrono::milliseconds INTRO_STILL_TIME = std::chrono::seconds(2); +// Maximum pixel RMSD threshold for considering the screen still before probing menu. +inline constexpr double INTRO_STILL_CHANGE = 4; +// Minimum pixel RMSD change indicating menu opened after pressing X. +inline constexpr double INTRO_X_CHANGE = 25; +// Maximum pixel RMSD difference between pre-X screen and post-B screen. +inline constexpr double INTRO_B_RESTORE = 10; +// Probe metrics for detecting menu presence via screen changes from X and B. +struct XProbe{ + double still_change; // over INTRO_STILL_TIME before X + double x_change; // from before X to after it + double b_change; // from before X to after the B that follows +}; +// Evaluates whether probe metrics indicate player is in control without menu detection. +bool intro_in_control_without_menu(const XProbe& probe); +// Counts `step` in `progress`. +void record_intro_step(IntroProgress& progress, IntroStep step); + +// From the title screen or any intro screen of a game with no save, with BDSP +// running in front: from HOME it throws UserSetupError, since the restart would +// launch whatever tile HOME's cursor is on. Restarts the game first, so it +// always starts from the title, waits out the black boot (up to +// INTRO_BOOT_LIMIT) without pressing anything, and fails if no BDSP screen shows within +// INTRO_TIME_BEFORE_BDSP of screens. Names the player `player_name` +// and picks the rival's default name (Barry) from the list. Each step reads the +// screen and waits for the expected next one; an unexpected screen fails with a +// screenshot and the recent screens. +void play_new_game_intro( + SingleSwitchProgramEnvironment& env, + ProControllerContext& context, + const std::string& player_name +); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp new file mode 100644 index 0000000000..e1f71ad647 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp @@ -0,0 +1,543 @@ +/* BDSP AutoStory - Settings + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP_AutoStory_Bag.h" +#include "PokemonBDSP_AutoStory_Settings.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +namespace{ + +// Options chrome and position detectors are independent of setting values. +class TutorialOptionsDetector { +public: + bool detect(const ImageViewRGB32& screen) const { + ImageViewRGB32 panel = extract_box_reference(screen, ImageFloatBox{0.04, 0.065, 0.93, 0.70}); + size_t white = 0; + for (size_t y = 0; y < panel.height(); y += 4) + for (size_t x = 0; x < panel.width(); x += 4){ + uint32_t p = panel.pixel(x, y); + if (((p >> 16) & 255) > 225 && ((p >> 8) & 255) > 225 && (p & 255) > 225){ + white++; + } + } + double samples = (double)(((panel.width() + 3) / 4) * ((panel.height() + 3) / 4)); + // Distinguish Options from the icon menu and bag using its + // blue heading and full-width white settings panel. + ImageViewRGB32 heading = extract_box_reference(screen, ImageFloatBox{0.01, 0.005, 0.20, 0.025}); + size_t blue = 0, total = 0; + for (size_t y = 0; y < heading.height(); y++) + for (size_t x = 0; x < heading.width(); x++){ + uint32_t p = heading.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + blue += b > r + 20 && g > r + 10; + total++; + } + return samples > 0 && white / samples > 0.70 && total > 0 && blue > total / 4; + } +}; + +// Options fades in after A on its icon: poll instead of judging one frame. +bool wait_for_options_screen(VideoStream& stream, ProControllerContext& context, + const TutorialOptionsDetector& options_screen, Milliseconds timeout){ + context.wait_for_all_requests(); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + auto frame = stream.video().snapshot(); + if (frame && options_screen.detect(frame)){ + return true; + } + context.wait_for(100ms); + } + return false; +} + +class TutorialOptionPositionDetector { +public: + // Locate the two long red horizontal lines surrounding the + // selected row. Scrolling changes their position automatically. + double selected_row(const ImageViewRGB32& screen) const { + int first = -1, last = -1; + for (size_t y = (size_t)(screen.height() * 0.08); y < screen.height() * 0.75; y++){ + size_t red = 0, total = 0; + for (size_t x = (size_t)(screen.width() * 0.08); x < screen.width() * 0.90; x += 2){ + uint32_t p = screen.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + red += r > 150 && r > g + 40 && r > b + 40; + total++; + } + if (total && red > total * 0.70){ + if (first < 0){ + first = (int)y; + } + last = (int)y; + } + } + if (first < 0 || last - first < screen.height() * 0.04 || last - first > screen.height() * 0.12){ + return -1; + } + return (first + last) / (2.0 * screen.height()); + } + // Locate selected red value text separately from the row border. + bool selected_value(const ImageViewRGB32& screen, double y) const { + ImageViewRGB32 value = extract_box_reference(screen, ImageFloatBox{0.33, y - 0.025, 0.54, 0.05}); + size_t red = 0; + for (size_t py = 0; py < value.height(); py++) + for (size_t px = 0; px < value.width(); px++){ + uint32_t p = value.pixel(px, py); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + red += r > g + 20 && r > b + 20; + } + return red > value.width() * value.height() * 0.0005; + } +}; + + +// Compare binary visual shapes with small alignment tolerance. +// No text recognition is needed for fixed English option labels. +double option_image_score(const ImageViewRGB32& screen, + const ImageFloatBox& box, const ImageRGB32& reference, bool red_only){ + ImageRGB32 actual = extract_box_reference(screen, box).scale_to(384, 32); + ImageRGB32 expected = reference.scale_to(384, 32); + auto ink = [red_only](uint32_t p){ + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + return red_only ? r > g + 20 && r > b + 20 : r + g + b < 500; + }; + double best = 0; + for (int dy = -2; dy <= 2; dy++){ + for (int dx = -2; dx <= 2; dx++){ + size_t overlap = 0, count = 0; + for (int y = 2; y < 30; y++){ + for (int x = 2; x < 382; x++){ + bool a = ink(actual.pixel(x + dx, y + dy)); + bool b = ink(expected.pixel(x, y)); + overlap += a && b; + count += a + b; + } + } + if (count){ + best = std::max(best, 2.0 * overlap / count); + } + } + } + return best; +} + +bool autosave_off_highlight(const ImageViewRGB32& screen, double y){ + auto red_count = [&](const ImageFloatBox& box){ + ImageViewRGB32 image = extract_box_reference(screen, box); + size_t red = 0; + for (size_t py = 0; py < image.height(); py++) + for (size_t px = 0; px < image.width(); px++){ + uint32_t p = image.pixel(px, py); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + red += r > g + 30 && r > b + 30; + } + return red; + }; + size_t off = red_count({0.63, y - 0.022, 0.055, 0.044}); + size_t on = red_count({0.34, y - 0.022, 0.055, 0.044}); + // Require a clear Off highlight and reject any competing On highlight. + return off >= 10 && on * 10 < off; +} + +// The options the story needs, as each reference image names them: the +// row's label and the value to select. +struct Setting{ + const char* label; + const char* value; +}; +const Setting SETTINGS[] = { + {"textspeed", "fast"}, {"changewindow", "type1"}, + {"battleeffects", "off"}, {"battlestyle", "set"}, + {"sendtoboxes", "automatic"}, {"givenicknames", "dontgive"}, + {"gyroscope", "use"}, {"autosave", "off"}, + {"shownicknames", "show"}, {"backgroundmusic", "8"}, + {"soundeffects", "8"}, +}; +constexpr size_t SETTING_COUNT = sizeof(SETTINGS) / sizeof(SETTINGS[0]); +constexpr size_t AUTOSAVE_ROW = 7; +constexpr size_t FIRST_SLIDER_ROW = 9; + +// The Options screen and its selected row, as one reader. +struct OptionsScreen{ + TutorialOptionsDetector options_screen; + TutorialOptionPositionDetector position; +}; +// A row's label and target value, as the reference images show them. +struct OptionReferences{ + ImageRGB32 label; + ImageRGB32 value; +}; + +// From the overworld (or the main menu): the main menu, settled. +void open_main_menu(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + MenuDetector menu(COLOR_RED, /*skip_new_banner*/ true); + VideoOverlaySet detector_overlays(stream.overlay()); + menu.make_overlays(detector_overlays); + detector_overlays.add(COLOR_CYAN, {0.04, 0.065, 0.93, 0.70}, ""); + detector_overlays.add(COLOR_BLUE, {0.01, 0.005, 0.20, 0.025}, ""); + detector_overlays.add(COLOR_RED, {0.08, 0.08, 0.82, 0.67}, ""); + auto initial = stream.video().snapshot(); + if (!initial || !menu.detect(initial)){ + pbf_press_button(context, BUTTON_X, 80ms, 1000ms); + context.wait_for_all_requests(); + } + MenuWatcher menu_open(COLOR_RED, /*skip_new_banner*/ true); + if (wait_until(stream, context, 5s, {{menu_open}}) != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Main menu not detected.", stream); + } + stream.log("[Tutorial settings] Main menu detected; settling for 500ms before Options navigation."); + pbf_wait(context, 500ms); + context.wait_for_all_requests(); +} + +// From the main menu: Options. Only two icons are available at this +// initial checkpoint; verify the main menu before trying the other. +void open_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ + bool options_open = false; + for (size_t attempt = 0; attempt < 2; attempt++){ + pbf_press_dpad(context, attempt == 0 ? DPAD_RIGHT : DPAD_LEFT, 80ms, 350ms); + pbf_press_button(context, BUTTON_A, 80ms, 0ms); + if (wait_for_options_screen(stream, context, options_screen, 5s)){ + options_open = true; + break; + } + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + context.wait_for_all_requests(); + MenuWatcher back_to_menu(COLOR_RED, /*skip_new_banner*/ true); + if (wait_until(stream, context, 5s, {{back_to_menu}}) != 0){ + break; + } + } + if (!options_open){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Options screen not detected.", stream); + } + stream.log("[Tutorial settings] Options screen detected."); + pbf_wait(context, 500ms); + context.wait_for_all_requests(); +} + +// How well the selected row's value matches the reference. +double option_value_score(const ImageViewRGB32& frame, double y, size_t row, const ImageRGB32& value_reference){ + if (row >= FIRST_SLIDER_ROW){ + // Compare the numeric value rather than the long slider: + // neighboring slider levels otherwise look too similar. + ImageViewRGB32 number = value_reference.sub_image( + value_reference.width() * 82 / 100, 0, + value_reference.width() - value_reference.width() * 82 / 100, + value_reference.height()); + ImageRGB32 number_reference = number.scale_to(96, 32); + return option_image_score(frame, + {0.7728, y - 0.025, 0.0972, 0.05}, number_reference, true); + } + return option_image_score(frame, + {0.33, y - 0.025, 0.54, 0.05}, value_reference, true); +} + +// Whether Options shows Autosave selected and set to Off: a frame's look. +bool autosave_off_on_screen( + const ImageViewRGB32& check, const TutorialOptionsDetector& options_screen, + const TutorialOptionPositionDetector& position, const ImageRGB32& autosave_label +){ + const double check_y = check ? position.selected_row(check) : -1; + return check && options_screen.detect(check) && check_y >= 0 && + option_image_score(check, {0.08, check_y - 0.025, 0.24, 0.05}, autosave_label, false) >= 0.80 && + autosave_off_highlight(check, check_y); +} + +// Autosave Off, on three more frames 200 ms apart. +void confirm_autosave_off( + VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen, + const TutorialOptionPositionDetector& position, const ImageRGB32& label_reference +){ + for (size_t confirmation = 0; confirmation < 3; confirmation++){ + pbf_wait(context, 200ms); + context.wait_for_all_requests(); + auto check = stream.video().snapshot(); + if (!autosave_off_on_screen(check, options_screen, position, label_reference)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Autosave Off could not be confirmed. Stopping before movement.", stream); + } + } +} + +// One press toward the target value. Sliders clamp at zero: establish a +// known endpoint, then increment; the target image is verified afterward. +void correct_option(ProControllerContext& context, size_t row, bool& slider_reset){ + if (row >= FIRST_SLIDER_ROW && !slider_reset){ + for (size_t step = 0; step < 12; step++){ + pbf_press_dpad(context, DPAD_LEFT, 80ms, 100ms); + } + slider_reset = true; + }else{ + pbf_press_dpad(context, DPAD_RIGHT, 80ms, 350ms); + } + context.wait_for_all_requests(); +} + +// The selected row on screen, if it is the expected one with its value +// highlighted; -1 otherwise. +double expected_row_y( + VideoStream& stream, const ImageViewRGB32& frame, const TutorialOptionsDetector& options_screen, + const TutorialOptionPositionDetector& position, size_t row, const ImageRGB32& label_reference +){ + const double y = frame && options_screen.detect(frame) ? position.selected_row(frame) : -1; + const bool value_visible = y >= 0 && position.selected_value(frame, y); + const double label_score = y >= 0 ? option_image_score(frame, + {0.08, y - 0.025, 0.24, 0.05}, label_reference, false) : 0; + stream.log("[Tutorial settings] Expected row=" + std::string(SETTINGS[row].label) + + " row_y=" + std::to_string(y) + " highlighted_value=" + std::to_string(value_visible) + + " label_match=" + std::to_string(label_score)); + return y >= 0 && value_visible && label_score >= 0.80 ? y : -1; +} + +// Whether the selected row (at `y`) shows the target value: checked on +// the frame, with the row's boxes on the overlay. +bool option_verified( + VideoStream& stream, ProControllerContext& context, const ImageViewRGB32& frame, double y, size_t row, + const OptionsScreen& screen, const OptionReferences& references +){ + const ImageRGB32& value_reference = references.value; + // These are regular video-overlay boxes, so the existing overlay + // display control can show/hide every active ROI. + OverlayBoxScope row_overlay(stream.overlay(), COLOR_RED, {0.07, y - 0.04, 0.84, 0.08}, ""); + OverlayBoxScope label_overlay(stream.overlay(), COLOR_YELLOW, {0.08, y - 0.025, 0.24, 0.05}, ""); + OverlayBoxScope value_overlay(stream.overlay(), COLOR_GREEN, {0.33, y - 0.025, 0.54, 0.05}, ""); + VideoOverlaySet autosave_rois(stream.overlay()); + if (row == AUTOSAVE_ROW){ + autosave_rois.add(COLOR_GREEN, {0.63, y - 0.022, 0.055, 0.044}); + autosave_rois.add(COLOR_RED, {0.34, y - 0.022, 0.055, 0.044}); + } + const double score = option_value_score(frame, y, row, value_reference); + stream.log("[Tutorial settings] " + std::string(SETTINGS[row].label) + " target match=" + std::to_string(score)); + // The reference recording uses inactive pale-red text; the focused live + // row has a thicker dark-red glyph mask. Verified Fast scores 0.734 + // while Slow/Normal score 0. + if (score < 0.65 || (row == AUTOSAVE_ROW && !autosave_off_highlight(frame, y))){ + return false; + } + if (row == AUTOSAVE_ROW){ + confirm_autosave_off(stream, context, screen.options_screen, screen.position, references.label); + } + return true; +} + +// Sets the option on `row`, the row selected now, to its target value. +void set_option(VideoStream& stream, ProControllerContext& context, size_t row, const OptionsScreen& screen){ + const std::string base = RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/" + SETTINGS[row].label; + const OptionReferences references{ImageRGB32(base + "_label.png"), ImageRGB32(base + "_value.png")}; + bool slider_reset = false; + for (size_t attempt = 0; attempt < 30; attempt++){ + // Detect the selected row afresh after every navigation/correction. + context.wait_for_all_requests(); + auto frame = stream.video().snapshot(); + const double y = expected_row_y(stream, frame, screen.options_screen, screen.position, row, references.label); + if (y < 0){ + // A menu can be detected while its row/text is still fading + // in. Re-read without directional inputs first. + if (attempt < 4){ + pbf_wait(context, 200ms); + context.wait_for_all_requests(); + continue; + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Unable to confirm expected row " + std::string(SETTINGS[row].label), stream); + } + if (option_verified(stream, context, frame, y, row, screen, references)){ + stream.log("[Tutorial settings] Verified " + std::string(SETTINGS[row].label) + "=" + SETTINGS[row].value); + return; + } + stream.log("[Tutorial settings] Correcting " + std::string(SETTINGS[row].label)); + correct_option(context, row, slider_reset); + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Settings checkpoint: Could not apply " + std::string(SETTINGS[row].label), stream); +} + +// Confirms the settings and leaves Options (A A A A B B), back to the overworld. +void save_options(VideoStream& stream, ProControllerContext& context, const std::string& failure){ + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + pbf_press_button(context, BUTTON_A, 80ms, 1200ms); + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + context.wait_for_all_requests(); + OverworldWatcher overworld; + if (wait_until(stream, context, 10s, {{overworld}}) != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, failure, stream); + } +} + +// From the overworld: the saved Options screen again. +void reopen_options(VideoStream& stream, ProControllerContext& context, const TutorialOptionsDetector& options_screen){ + pbf_press_button(context, BUTTON_X, 80ms, 1000ms); + context.wait_for_all_requests(); + MenuWatcher verify_menu(COLOR_RED, /*skip_new_banner*/ true); + if (wait_until(stream, context, 5s, {{verify_menu}}) != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Autosave verification: Main menu not detected.", stream); + } + bool reopened = false; + for (size_t attempt = 0; attempt < 2; attempt++){ + pbf_press_button(context, BUTTON_A, 80ms, 0ms); + if (wait_for_options_screen(stream, context, options_screen, 5s)){ + reopened = true; + break; + } + pbf_press_button(context, BUTTON_B, 80ms, 1000ms); + context.wait_for_all_requests(); + MenuWatcher return_menu(COLOR_RED, /*skip_new_banner*/ true); + if (wait_until(stream, context, 5s, {{return_menu}}) != 0){ + break; + } + pbf_press_dpad(context, DPAD_RIGHT, 80ms, 350ms); + } + if (!reopened){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Autosave verification: Could not reopen saved Options.", stream); + } +} + +// In the saved Options: Autosave is Off on three frames. Options starts +// on Text Speed; Autosave is seven rows below it. +void check_saved_autosave( + VideoStream& stream, ProControllerContext& context, + const TutorialOptionsDetector& options_screen, const TutorialOptionPositionDetector& position +){ + for (size_t step = 0; step < AUTOSAVE_ROW; step++){ + pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + } + context.wait_for_all_requests(); + const ImageRGB32 autosave_label(RESOURCE_PATH() + "PokemonBDSP/Autostory/Options/autosave_label.png"); + for (size_t confirmation = 0; confirmation < 3; confirmation++){ + auto check = stream.video().snapshot(); + if (!autosave_off_on_screen(check, options_screen, position, autosave_label)){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Saved Autosave setting is not verified Off. Stopping before movement.", stream); + } + pbf_wait(context, 200ms); + context.wait_for_all_requests(); + } + stream.log("[Tutorial settings] Autosave = Off: confirmed after reopening saved Options (3 frames)."); +} // namespace + +void verify_tutorial_settings_once(VideoStream& stream, ProControllerContext& context){ + open_main_menu(stream, context); + OptionsScreen screen; + open_options(stream, context, screen.options_screen); + for (size_t row = 0; row < SETTING_COUNT; row++){ + set_option(stream, context, row, screen); + if (row + 1 < SETTING_COUNT){ + pbf_press_dpad(context, DPAD_DOWN, 80ms, 350ms); + } + } + save_options(stream, context, "Settings checkpoint: Did not return to the overworld after saving options."); + // Reopen the saved Options screen and independently verify Autosave. + // No route movement is permitted until this persisted-value check passes. + reopen_options(stream, context, screen.options_screen); + check_saved_autosave(stream, context, screen.options_screen, screen.position); + save_options(stream, context, "Autosave verification: Could not return to the overworld."); + // Report inferred values explicitly: these are visual matches, + // allowing the user to compare our conclusions with the screen. + stream.log("[Tutorial settings] Completed. Values believed applied (visual matches before saving):"); + stream.log("[Tutorial settings] Text Speed = Fast; Change Window = Type 1; Battle Effects = Off; Battle Style = Set."); + stream.log("[Tutorial settings] Send to Boxes = Automatic; Give Nicknames = Don't give; Gyroscope = Use."); + stream.log("[Tutorial settings] Autosave = Off; Show Nicknames = Show; Background Music = 8; Sound Effects = 8."); +} + +// After a failed attempt: back out of Options, save prompts, the Bag or +// the menu without moving the player or resetting/loading the game. +void back_to_overworld(VideoStream& stream, ProControllerContext& context){ + stream.log("[Tutorial settings] Returning to overworld before retry."); + context.wait_for_all_requests(); + TutorialOptionsDetector options_screen; + const auto failed_frame = stream.video().snapshot(); + if (failed_frame && options_screen.detect(failed_frame)){ + // Finish the known Options save/exit sequence before observing + // state; never cancel it on an Options frame misclassified as + // overworld by the generic detector. + for (size_t press = 0; press < 4; press++){ + pbf_press_button(context, BUTTON_A, 80ms, 1000ms); + } + } + pbf_mash_button(context, BUTTON_B, 3000ms); + context.wait_for_all_requests(); + OverworldDetector overworld; + MenuDetector menu(COLOR_RED, /*skip_new_banner*/ true); + BlackScreenDetector black; + bool recovered = false; + size_t confirmations = 0; + for (size_t check = 0; check < 15 && !recovered; check++){ + const auto frame = stream.video().snapshot(); + const bool clear = frame && !options_screen.detect(frame) && + !menu.detect(frame) && !black.detect(frame) && bag_pocket_position(frame) == 0 && overworld.detect(frame); + if (clear && ++confirmations >= 3){ + recovered = true; + }else if (clear){ + pbf_wait(context, 200ms); + }else{ + confirmations = 0; + pbf_mash_button(context, BUTTON_B, 500ms); + } + context.wait_for_all_requests(); + } + stream.log(recovered ? "[Tutorial settings] Recovery confirmed outside Options/menu; retrying." + : "[Tutorial settings] Recovery not confirmed; next attempt will recheck the screen."); + pbf_wait(context, 500ms); + context.wait_for_all_requests(); +} + +} // namespace + +void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context){ + // Settings-only recovery: initial attempt plus three retries. + // Never permit route movement unless the complete check succeeds. + constexpr size_t MAX_RETRIES = 3; + for (size_t attempt = 0; attempt <= MAX_RETRIES; attempt++){ + stream.log("[Tutorial settings] Attempt " + std::to_string(attempt + 1) + "/4."); + try{ + verify_tutorial_settings_once(stream, context); + return; + }catch (OperationFailedException& error){ + stream.log("[Tutorial settings] Attempt failed: " + error.message(), COLOR_ORANGE); + if (attempt == MAX_RETRIES){ + stream.log("[Tutorial settings] All three retries exhausted; stopping before movement.", COLOR_RED); + throw; + } + back_to_overworld(stream, context); + } + } +} + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h new file mode 100644 index 0000000000..206237de07 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h @@ -0,0 +1,31 @@ +/* BDSP AutoStory - Settings + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_Settings_H +#define PokemonAutomation_PokemonBDSP_AutoStory_Settings_H + +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// From the overworld: opens Options and sets what the story needs (fast +// text, Send to Boxes automatic, no nickname prompt, Autosave off), saves +// them and returns to the overworld. Tries four times, backing out to the +// overworld between tries, and fails with a screenshot before any movement. +void verify_tutorial_settings(VideoStream& stream, ProControllerContext& context); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.cpp new file mode 100644 index 0000000000..8be35a67dd --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.cpp @@ -0,0 +1,427 @@ +/* BDSP AutoStory - Wild battles + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include +#include +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "Common/Cpp/PrettyPrint.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/OCR/OCR_RawTesseractOCR.h" +#include "CommonTools/VisualDetectors/BlackScreenDetector.h" +#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_EndBattleDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_DialogDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_SelectionArrow.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h" +#include "PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_Location.h" +#include "PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h" +#include "PokemonBDSP/Inference/PokemonBDSP_ReceivePokemonDetector.h" +#include "PokemonBDSP/Programs/AutoStory/Utils/PokemonBDSP_AutoStory_RouteWalk.h" +#include "PokemonBDSP_AutoStory_Battle.h" +#include "PokemonBDSP_AutoStory_WildBattle.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + +using namespace std::chrono_literals; + + +namespace{ + +// Two grass rows at 240 ms each, with room to spare: the wall stops us. +constexpr Milliseconds CATCH_GRASS_WALK = 900ms; + +// Whether a wild battle has started: its black screen, menu or text. +bool wild_battle_started(VideoStream& stream, ProControllerContext& context){ + BlackScreenWatcher battle_start; + BattleMenuWatcher battle_menu(BattleType::STANDARD, COLOR_YELLOW); + BattleDialogWatcher battle_dialog(COLOR_YELLOW); + const int ret = wait_until( + stream, + context, + 1500ms, + { + {battle_start}, + {battle_menu}, + {battle_dialog}, + } + ); + return ret >= 0; +} + +// One walk across the catch grass `direction`, to the wall; whether a +// battle started. +bool walk_catch_grass(VideoStream& stream, ProControllerContext& context, Pokemon::Direction direction){ + route_turn(stream, context, direction); + pbf_move_left_joystick(context, {0, direction == Pokemon::Direction::NORTH ? 1.0 : -1.0}, CATCH_GRASS_WALK, 200ms); + context.wait_for_all_requests(); + return wild_battle_started(stream, context); +} + +void encounter_one_wild_pokemon(VideoStream& stream, ProControllerContext& context){ + context.wait_for_all_requests(); + pbf_wait(context, 1000ms); + + bool encounter_found = false; + const auto deadline = std::chrono::steady_clock::now() + 180s; + + // Walk up and down the catch grass until a battle starts. Its three rows + // sit between tree walls, so each walk ends at a wall, on the same tile + // every time, after two new grass tiles. Short sideways steps drift and + // can rock across one tile edge without entering new grass. + while (!encounter_found && std::chrono::steady_clock::now() < deadline){ + encounter_found = walk_catch_grass(stream, context, Pokemon::Direction::NORTH) || + walk_catch_grass(stream, context, Pokemon::Direction::SOUTH); + } + + if (!encounter_found){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Wild encounter: walking up and down the grass found no battle within three minutes.", stream); + } + stream.log("[AutoStory] Wild battle detected!", COLOR_GREEN); + + // Wait until the actual battle menu appears. + BattleMenuWatcher confirm_menu(BattleType::STANDARD); + if (wait_until(stream, context, 15s, {{confirm_menu}}) < 0){ + stream.log("[AutoStory] Failed to detect battle menu.", COLOR_RED); + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "[AutoStory] Failed to detect battle menu.", stream); + } + + stream.log("[AutoStory] Battle menu detected.", COLOR_GREEN); + + context.wait_for_all_requests(); + pbf_wait(context, 500ms); +} + +// With the game's main menu open after a battle: B until it closes, the +// overworld or a screen without it. Whether it was seen closed. +bool close_menu_after_battle(VideoStream& stream, ProControllerContext& context){ + OverworldWatcher overworld; + MenuDetector menu_visible; + for (size_t close = 0; close < 3; close++){ + pbf_press_button(context, BUTTON_B, 80ms, 500ms); + context.wait_for_all_requests(); + if (wait_until(stream, context, 1s, {{overworld}}) == 0){ + return true; + } + BlackScreenWatcher visible(COLOR_YELLOW, + {0.1, 0.1, 0.8, 0.8}, + 100, + 10, + BlackScreenWatcher::FinderType::GONE, + 500ms); + if (wait_until(stream, context, 2s, {{visible}}) != 0){ + continue; + } + const VideoSnapshot frame = stream.video().snapshot(); + if (frame && !menu_visible.detect(frame)){ + stream.log("[Mine escape] Confirmed game menu closed; returning to navigation.", COLOR_GREEN); + return true; + } + } + stream.log("[Mine escape] Main menu detected but could not confirm it closed.", COLOR_ORANGE); + return false; +} + +bool mine_confirm_escape_via_menu(VideoStream& stream, ProControllerContext& context, BattleType battle_type){ + MenuWatcher game_menu; + BattleMenuWatcher battle_menu(battle_type, COLOR_YELLOW); + for (size_t attempt = 1; attempt <= 3; attempt++){ + // Do not accept the absence-based overworld detector before menu proof. + const int state = wait_until(stream, context, 300ms, {{game_menu}, {battle_menu}}); + if (state == 1){ + stream.log("[Mine escape] Battle menu still present; retrying Run."); + return false; + } + int detected = state; + if (state != 0){ + stream.log("[Mine escape] Main menu fallback attempt " + std::to_string(attempt) + "/3."); + detected = run_until(stream, + context, + [](ProControllerContext& controller){ + pbf_press_button(controller, BUTTON_X, 160ms, 840ms); + pbf_wait(controller, 3s); + controller.wait_for_all_requests(); + }, + {game_menu, battle_menu}); + if (detected == 1){ + return false; + } + } + if (detected != 0){ + continue; + } + stream.log("[Mine escape] Main game menu confirmed; battle has ended.", COLOR_GREEN); + // Close the confirmed menu before returning control to navigation. + return close_menu_after_battle(stream, context); + } + return false; +} + +bool mine_select_run(VideoStream& stream, ProControllerContext& context, BattleType battle_type){ + BattleMenuDetector cursor(battle_type, COLOR_YELLOW); + size_t matching_frames = 0; + for (size_t check = 0; check < 16; check++){ + context.wait_for_all_requests(); + const VideoSnapshot frame = stream.video().snapshot(); + if (!frame){ + return false; + } + const BattleMenuOption selected = battle_menu_selected_option(cursor, frame); + stream.log( + "[Mine escape] Battle cursor row=" + std::to_string(static_cast(selected) + 1) + "; target Run=4."); + if (selected == BattleMenuOption::RUN){ + if (++matching_frames >= 3){ + stream.log("[Mine escape] Run cursor confirmed.", COLOR_GREEN); + return true; + } + pbf_wait(context, 100ms); + }else{ + matching_frames = 0; + if (selected == BattleMenuOption::UNKNOWN){ + // Do not send a direction or A without a known cursor. + pbf_wait(context, 150ms); + }else{ + const DpadPosition direction = selected == BattleMenuOption::BATTLE ? DPAD_UP : DPAD_DOWN; + pbf_press_dpad(context, direction, 80ms, 250ms); + } + } + } + stream.log("[Mine escape] Could not confirm Run selection; A was not pressed.", COLOR_ORANGE); + return false; +} + +// Run on the battle menu, until the battle's exit shows or `timeout` +// runs out. Whether the exit was seen. +bool run_until_escaped( + VideoStream& stream, ProControllerContext& context, BattleType battle_type, + std::chrono::seconds timeout, EndBattleWatcher& battle_end +){ + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (std::chrono::steady_clock::now() < deadline){ + pbf_press_button(context, BUTTON_B, 40ms, 200ms); + context.wait_for_all_requests(); + BattleMenuWatcher menu_ready(battle_type, COLOR_YELLOW); + const int ready = wait_until(stream, context, 3s, {{battle_end}, {menu_ready}}); + if (ready == 0){ + return true; + } + if (ready != 1 || !mine_select_run(stream, context, battle_type)){ + continue; + } + stream.log("[Mine escape] Run input sent; watching for black-screen exit."); + const int result = run_until(stream, + context, + [](ProControllerContext& controller){ + pbf_press_button(controller, BUTTON_A, 60ms, 0ms); + pbf_wait(controller, 4s); + controller.wait_for_all_requests(); + }, + {battle_end}); + if (result == 0){ + return true; + } + } + return false; +} + +// B until the overworld shows; fails with `failure` after two minutes. +void mash_to_overworld(VideoStream& stream, ProControllerContext& context, const std::string& failure){ + // Mash through XP / level-up / dialog screens back to overworld. + OverworldWatcher overworld; + const int cleanup_ret = run_until( + stream, context, + [](ProControllerContext& ctx){ + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, + {{overworld}} + ); + if (cleanup_ret != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, failure, stream); + } +} + +} // namespace + +void fight_starly(VideoStream& stream, ProControllerContext& context){ + stream.log("[AutoStory] Starting Starly battle...", COLOR_BLUE); + + // Declare OUTSIDE the loop so accumulated detector state + // (trigger counts, black-screen transitions) is preserved across frames. + // STARTER type: player has no Pokeballs, so ball-count areas won't + // be white. STARTER skips that check and only verifies status bars + // and the Fight button color. + BattleMenuWatcher battle_menu(BattleType::STARTER, COLOR_YELLOW); + EndBattleWatcher end_battle; + + while (true){ + context.wait_for_all_requests(); + + // Phase 1: wait for either battle menu or battle end. + const int ret = run_until( + stream, context, + [](ProControllerContext& ctx){ + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, + {{battle_menu}, {end_battle}} + ); + if (ret == 1){ + stream.log("[AutoStory] Starly battle complete.", COLOR_GREEN); + mash_to_overworld(stream, context, "Starly battle: Overworld cleanup timed out."); + return; + } + if (ret != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Starly battle timed out.", stream); + } + stream.log("[AutoStory] Battle menu detected. Selecting move...", COLOR_BLUE); + // Select Fight, then move slot 1; loop back and mash B through the + // animations until the next menu or the end. + pbf_press_button(context, BUTTON_A, 80ms, 500ms); + pbf_press_button(context, BUTTON_A, 80ms, 3000ms); + } +} + +void battle_1_pokemon(VideoStream& stream, ProControllerContext& context){ + stream.log("[Training] Finding one wild Pokemon using the catch encounter wiggle.", COLOR_BLUE); + encounter_one_wild_pokemon(stream, context); + // The best move on the menu, whatever the starter. + fight_wild_battle(stream, context, "Training"); + context.wait_for_all_requests(); + stream.log("[Training] One wild battle completed.", COLOR_GREEN); +} + +namespace{ + +// Throws the first ball in the Bag at the wild Pokemon, up to ten times, +// until its receipt shows. Returns whether the battle's exit was seen. +bool throw_until_caught(VideoStream& stream, ProControllerContext& context, EndBattleWatcher& battle_ended){ + ReceivePokemonDetector received_pokemon; + for (int throw_attempt = 0; throw_attempt < 10; throw_attempt++){ + // Open bag from battle menu (X), confirm first slot (ZL). + pbf_press_button(context, BUTTON_X, 160ms, 840ms); + context.wait_for_all_requests(); + pbf_press_button(context, BUTTON_ZL, 160ms, 1000ms); + context.wait_for_all_requests(); + + // Wait to see if the battle menu comes back (ball failed) + // or if the battle ends (caught or fainted). + BattleMenuWatcher menu_back(BattleType::STANDARD, COLOR_YELLOW); + const int throw_ret = run_until( + stream, context, + [](ProControllerContext& ctx){ + pbf_mash_button(ctx, BUTTON_B, 120000ms); + }, + {{menu_back}, {received_pokemon}, {battle_ended}} + ); + + if (throw_ret == 1 || (throw_ret == 2 && received_pokemon.received())){ + stream.log("[AutoStory] Pokemon receipt verified.", COLOR_GREEN); + context.wait_for_all_requests(); + return throw_ret == 2; + } + if (throw_ret < 0 || throw_ret == 2){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Catch failed: No verified Pokemon receipt before battle ended or timed out.", stream); + } + stream.log( + "[AutoStory] Ball failed, retrying (attempt " + std::to_string(throw_attempt + 2) + ")...", + COLOR_ORANGE + ); + } + context.wait_for_all_requests(); + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Catch failed: Exhausted ten ball attempts.", stream); +} + +} // namespace + +void catch_1_pokemon( + VideoStream& stream, + ProControllerContext& context +){ + stream.log("[AutoStory] Starting catch_1_pokemon helper...", COLOR_BLUE); + + encounter_one_wild_pokemon(stream, context); + + // Catch the pokemon. + stream.log("[AutoStory] Throwing Poke Ball (manual bag open)...", COLOR_BLUE); + // Preserve exit-transition state from the ball outcome through + // receipt/experience cleanup; do not recreate it between phases. + EndBattleWatcher battle_ended; + const bool exit_seen = throw_until_caught(stream, context, battle_ended); + // Receipt alone is not an overworld return: clear experience, + // automatic move learning, dex registration and nickname screens + // until the same completed-black-transition watcher used in battles fires. + if (!exit_seen){ + stream.log("[Catch cleanup] Receipt confirmed; waiting for battle exit transition."); + const int cleanup_ret = run_until(stream, context, + [](ProControllerContext& ctx){ pbf_mash_button(ctx, BUTTON_B, 120000ms); }, + {{battle_ended}}); + if (cleanup_ret != 0){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "Catch cleanup: Battle exit transition did not finish.", stream); + } + } + context.wait_for_all_requests(); + stream.log("[Catch cleanup] Battle exit transition confirmed; checking stable overworld."); + confirm_battle_overworld(stream, context, "Catch cleanup"); + stream.log("[Catch cleanup] Verified receipt, completed battle exit, and four seconds of clear overworld.", COLOR_GREEN); +} + +void escape_wild_battle(VideoStream& stream, + ProControllerContext& context, + BattleType battle_type, + std::chrono::seconds timeout){ + EndBattleWatcher battle_end; + bool exit_transition = run_until_escaped(stream, context, battle_type, timeout, battle_end); + if (!exit_transition && wait_until(stream, context, 10s, {{battle_end}}) == 0){ + exit_transition = true; + } + if (exit_transition){ + // EndBattleWatcher requires black followed by a visible screen. + stream.log("[Mine escape] Black-screen exit transition confirmed.", COLOR_GREEN); + pbf_wait(context, 800ms); + context.wait_for_all_requests(); + return; + } + // Only probe X after normal escape attempts and the final transition wait. + // Absence of battle UI by itself is not evidence of an overworld return. + stream.log("[Mine escape] Exit transition missed; trying late main-menu fallback."); + if (mine_confirm_escape_via_menu(stream, context, battle_type)){ + return; + } + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, + "escape_wild_battle: Exit transition and late main-menu fallback could not confirm escape.", + stream); +} + +// STARTER's check of the menu (its Battle button) holds for any party: +// STANDARD also reads the party's balls, which a lone starter does not +// match, so Route 201's first wild battle never saw its menu (r34). +void run_from_wild_battle(VideoStream& stream, ProControllerContext& context){ + escape_wild_battle(stream, context, BattleType::STARTER); +} + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.h new file mode 100644 index 0000000000..6ef1b1b8fa --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.h @@ -0,0 +1,56 @@ +/* BDSP AutoStory - Wild battles + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * The story's wild battles: the Starly the starter is chosen against, one + * found in the grass to fight or to catch, and running from one that + * interrupts a walk. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_WildBattle_H +#define PokemonAutomation_PokemonBDSP_AutoStory_WildBattle_H + +#include +#include "CommonFramework/Tools/VideoStream.h" +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// The Starly battle at Lake Verity, from its first text to the overworld. +void fight_starly(VideoStream& stream, ProControllerContext& context); + +// Finds one wild Pokemon in the grass by walking up and down a column +// between two tree walls, and fights it with the best move. Does not heal. +void battle_1_pokemon(VideoStream& stream, ProControllerContext& context); + +// Finds one wild Pokemon the same way and catches it with the first ball +// in the Bag, up to ten throws; ends on the overworld. +void catch_1_pokemon(VideoStream& stream, ProControllerContext& context); + +// Runs from the wild battle on screen: Run on the battle menu until the +// exit's black screen shows. Fails with a screenshot after `timeout`. +void escape_wild_battle( + VideoStream& stream, + ProControllerContext& context, + BattleType battle_type = BattleType::STANDARD, + std::chrono::seconds timeout = std::chrono::seconds(30) +); + +// escape_wild_battle() for a standard wild battle, as the walker takes it +// (WildBattleEscape): how the story's walks leave the battles that +// interrupt them. +void run_from_wild_battle(VideoStream& stream, ProControllerContext& context); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp new file mode 100644 index 0000000000..d5f91e6914 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp @@ -0,0 +1,182 @@ +/* BDSP AutoStory - Battle Menu Reader + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include +#include +#include +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "PokemonBDSP_AutoStory_BattleMenuReader.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +namespace{ + +// Whether at least 15% of the box's pixels are white (min channel over +// `low`, spread under `spread`): a cursor bracket's corner. +bool white_corner(const ImageViewRGB32& screen, const ImageFloatBox& box, unsigned low_min, unsigned spread_max){ + const ImageViewRGB32 patch = extract_box_reference(screen, box); + size_t white = 0; + const size_t total = patch.width() * patch.height(); + for (size_t y = 0; y < patch.height(); y++){ + for (size_t x = 0; x < patch.width(); x++){ + const uint32_t pixel = patch.pixel(x, y); + const unsigned r = (pixel >> 16) & 255; + const unsigned g = (pixel >> 8) & 255; + const unsigned b = pixel & 255; + const unsigned low = std::min(r, std::min(g, b)); + const unsigned high = std::max(r, std::max(g, b)); + white += low > low_min && high - low < spread_max; + } + } + return total != 0 && static_cast(white) / total >= 0.15; +} + +// Whether the command menu's row `row` has the cursor's corner brackets. +// They wiggle; all four must agree on one small offset. +bool row_has_brackets(const ImageViewRGB32& screen, int row){ + const double offset = row * 0.1075; + const double offsets[] = {0.0, -0.003, 0.003, -0.006, 0.006}; + for (double dy : offsets){ + for (double dx : offsets){ + const ImageFloatBox corners[] = { + {0.793 + dx, 0.562 + offset + dy, 0.018, 0.023}, + {0.976 + dx, 0.562 + offset + dy, 0.018, 0.023}, + {0.793 + dx, 0.652 + offset + dy, 0.018, 0.023}, + {0.976 + dx, 0.652 + offset + dy, 0.018, 0.023}, + }; + if (std::all_of(std::begin(corners), std::end(corners), [&](const ImageFloatBox& box){ + return white_corner(screen, box, 210, 35); + })){ + return true; + } + } + } + return false; +} + +// The white PP panels in `patch` (the search box), top to bottom: nothing +// for no panel, more than four, or geometry that is not a move list. +std::vector find_pp_panels(const ImageViewRGB32& screen, const ImageViewRGB32& patch){ + cv::Mat white((int)patch.height(), (int)patch.width(), CV_8UC1); + for (size_t y = 0; y < patch.height(); y++){ + for (size_t x = 0; x < patch.width(); x++){ + uint32_t p = patch.pixel(x, y); + int r = (p >> 16) & 255, g = (p >> 8) & 255, b = p & 255; + white.at((int)y, (int)x) = std::min(r, std::min(g, b)) > 225 && + std::max(r, std::max(g, b)) - std::min(r, std::min(g, b)) < 30 ? 255 : 0; + } + } + cv::Mat labels, stats, centers; + int components = cv::connectedComponentsWithStats(white, labels, stats, centers); + std::vector panels; + for (int component = 1; component < components; component++){ + cv::Rect box(stats.at(component, cv::CC_STAT_LEFT), stats.at(component, cv::CC_STAT_TOP), + stats.at(component, cv::CC_STAT_WIDTH), stats.at(component, cv::CC_STAT_HEIGHT)); + const double area = (double)stats.at(component, cv::CC_STAT_AREA); + if (box.width < patch.width() * 0.60 || box.height < screen.height() * 0.055 || + box.height > screen.height() * 0.13 || area / box.area() < 0.55){ + continue; + } + panels.push_back(box); + } + if (panels.empty() || panels.size() > 4){ + return {}; + } + std::sort(panels.begin(), panels.end(), [](const cv::Rect& a, const cv::Rect& b){ return a.y < b.y; }); + // Reject inconsistent/overlapping geometry rather than count noise as moves. + for (size_t i = 1; i < panels.size(); i++){ + if (panels[i].y <= panels[i - 1].y + panels[i - 1].height || + std::abs(panels[i].height - panels[0].height) > screen.height() * 0.025){ + return {}; + } + } + return panels; +} + +// Whether the move button from `top` to `bottom` (of the screen) has the +// cursor's four corners, at a small offset. +bool move_button_selected(const ImageViewRGB32& screen, double top, double bottom){ + auto is_corner = [&](double x, double y){ + if (x < 0 || y < 0 || x + 0.018 > 1 || y + 0.023 > 1){ + return false; + } + return white_corner(screen, ImageFloatBox{x, y, 0.018, 0.023}, 210, 35); + }; + for (double dy : {0.0, -0.003, 0.003, -0.006, 0.006, -0.010, 0.010}){ + for (double dx : {0.0, -0.003, 0.003}){ + if (is_corner(0.663 + dx, top - 0.010 + dy) && + is_corner(0.976 + dx, top - 0.010 + dy) && + is_corner(0.663 + dx, bottom - 0.014 + dy) && + is_corner(0.976 + dx, bottom - 0.014 + dy)){ + return true; + } + } + } + return false; +} + +} // namespace + + +BattleMenuOption battle_menu_selected_option(BattleMenuDetector& menu, const ImageViewRGB32& screen){ + if (!screen || !menu.detect(screen)){ + return BattleMenuOption::UNKNOWN; + } + int selected = -1; + for (int row = 0; row < 4; row++){ + if (!row_has_brackets(screen, row)){ + continue; + } + if (selected >= 0){ + return BattleMenuOption::UNKNOWN; + } + selected = row; + } + return static_cast(selected); +} + +BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen){ + BattleMoveMenuState result; + if (!screen){ + return result; + } + // Every populated move has a white PP panel, irrespective of move type or + // selected row. Find connected panels rather than assume a 4-row layout. + const ImageFloatBox search{0.90, 0.48, 0.075, 0.505}; + const std::vector panels = find_pp_panels(screen, extract_box_reference(screen, search)); + if (panels.empty()){ + return result; + } + result.move_count = panels.size(); + for (size_t slot = 0; slot < panels.size(); slot++){ + double top = search.y + (double)panels[slot].y / screen.height(); + double bottom = top + (double)panels[slot].height / screen.height(); + result.buttons[slot] = ImageFloatBox{0.665, std::max(0.0, top - 0.01), 0.333, + std::min(1.0, bottom + 0.01) - std::max(0.0, top - 0.01)}; + if (!move_button_selected(screen, top, bottom)){ + continue; + } + if (result.selected_slot){ + result.selected_slot = 0; + return result; + } + result.selected_slot = slot + 1; + } + return result; +} + + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h new file mode 100644 index 0000000000..1e053c4953 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h @@ -0,0 +1,55 @@ +/* BDSP AutoStory - Battle Menu Reader + * + * From: https://github.com/PokemonAutomation/ + * + * Which battle command and which move the cursor is on. Adapted from + * PokemonAutomation/Arduino-Source#1717 by Developer-Butters, which added + * them to PokemonBDSP_BattleMenuDetector; here they sit beside it, so the + * upstream detector stays as it is. + * + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_BattleMenuReader_H +#define PokemonAutomation_PokemonBDSP_AutoStory_BattleMenuReader_H + +#include +#include +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "PokemonBDSP/Inference/Battles/PokemonBDSP_BattleMenuDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Vertical command order in the BDSP battle menu. +enum class BattleMenuOption{ + UNKNOWN = -1, + BATTLE = 0, + POKEMON = 1, + BAG = 2, + RUN = 3, +}; + +// The command under the cursor: `menu` must see its battle menu, and white +// selection brackets must sit at all four corners of exactly one command. +// Missing or ambiguous brackets read UNKNOWN; the cursor is never guessed. +BattleMenuOption battle_menu_selected_option(BattleMenuDetector& menu, const ImageViewRGB32& screen); + +// Logical move slots are ordered top to bottom among the populated move +// buttons. `selected_slot` is 1 to 4, or 0 when missing or ambiguous. +struct BattleMoveMenuState{ + size_t move_count = 0; + size_t selected_slot = 0; + std::array buttons; +}; +// Detects the populated move slots and the currently selected slot from the battle move menu. +BattleMoveMenuState detect_battle_move_menu(const ImageViewRGB32& screen); + + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp new file mode 100644 index 0000000000..48bb59b8d5 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp @@ -0,0 +1,286 @@ +/* BDSP AutoStory - Menu Cursor Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + */ + +#include +#include + +#include "CommonFramework/VideoPipeline/VideoFeed.h" + +#include "CommonTools/Images/ImageFilter.h" + +#include "PokemonBDSP_AutoStory_MenuCursorDetector.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +std::string menu_cursor_to_string(MenuCursorPosition pos){ + switch (pos){ + case MenuCursorPosition::POKEDEX: + return "POKEDEX"; + case MenuCursorPosition::POKEMON: + return "POKEMON"; + case MenuCursorPosition::BAG: + return "BAG"; + case MenuCursorPosition::CARD: + return "CARD"; + case MenuCursorPosition::OPTIONS: + return "OPTIONS"; + case MenuCursorPosition::MAP: + return "MAP"; + case MenuCursorPosition::CAPSULES: + return "CAPSULES"; + case MenuCursorPosition::MYSTERYGIFT: + return "MYSTERYGIFT"; + default: + return "UNKNOWN"; + } +} + + +MenuCursorDetector::MenuCursorDetector(Color color) : + m_color(color) + // Top row, left to right. + , m_pokedex_box{ + ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), + }, + m_pokemon_box{ + ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), + }, + m_bag_box{ + ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), + }, + m_card_box{ + ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), + } + // Bottom row — left to right + // 5-icon layout: OPTIONS is the only bottom icon (far-left position). + // 8-icon layout: MAP, CAPSULES, OPTIONS, MYSTERYGIFT from left to right. + // OPTIONS sits at a different x-position in each layout — use a separate + // MenuCursorDetector instance if you need to detect OPTIONS in both. + , m_options_box{ + ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), + }, + m_map_box{ + ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), + }, + m_capsules_box{ + ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), + }, + m_mysterygift_box{ + ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), + } + // Approach 2 search boxes — one per icon, same area as the dark-pixel box. + // The template PNG is slid across this area at its native size. + , m_icon_match_box{ + ImageFloatBox(0.180000, 0.115000, 0.135000, 0.195000), // 0 POKEDEX + ImageFloatBox(0.347500, 0.115000, 0.135000, 0.195000), // 1 POKEMON + ImageFloatBox(0.515000, 0.115000, 0.135000, 0.195000), // 2 BAG + ImageFloatBox(0.682500, 0.115000, 0.135000, 0.195000), // 3 CARD + ImageFloatBox(0.515000, 0.364000, 0.135000, 0.195000), // 4 OPTIONS (8-icon layout) + ImageFloatBox(0.180000, 0.364000, 0.135000, 0.195000), // 5 MAP + ImageFloatBox(0.347500, 0.364000, 0.135000, 0.195000), // 6 CAPSULES + ImageFloatBox(0.682500, 0.364000, 0.135000, 0.195000), // 7 MYSTERYGIFT + } { +} + + +void MenuCursorDetector::make_overlays(VideoOverlaySet& items) const { + for (const ImageFloatBox* icon_boxes : { + m_pokedex_box, + m_pokemon_box, + m_bag_box, + m_card_box, + m_options_box, + m_map_box, + m_capsules_box, + m_mysterygift_box, + }){ + for (int i = 0; i < BOXES_PER_ICON; i++){ + if (icon_boxes[i].width > 0.0 && icon_boxes[i].height > 0.0){ + items.add(m_color, icon_boxes[i]); + } + } + } + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + if (m_icon_match_box[i].width > 0.0 && m_icon_match_box[i].height > 0.0){ + items.add(m_color, m_icon_match_box[i]); + } + } +} + + +namespace{ + +// Count how many of `n` boxes contain >= MIN_DARK_PIXEL_COUNT pixels with +// R, G, B all <= BLACK_CHANNEL_MAX. Boxes with zero width are skipped. +int count_dark_boxes(const ImageViewRGB32& screen, const ImageFloatBox* boxes, int n){ + const uint32_t ch = uint32_t(MenuCursorDetector::BLACK_CHANNEL_MAX); + const uint32_t maxs = 0xff000000u | (ch << 16) | (ch << 8) | ch; + + int count = 0; + for (int i = 0; i < n; i++){ + if (boxes[i].width == 0.0){ + continue; + } + size_t dark_pixels = 0; + filter_rgb32_range(dark_pixels, + extract_box_reference(screen, boxes[i]), + 0xff000000u, + maxs, + Color(0xffffffff), + false); + if (dark_pixels >= MenuCursorDetector::MIN_DARK_PIXEL_COUNT){ + count++; + } + } + return count; +} + +// Pixel-wise RMSD between two same-size images over RGB channels only. +double compute_rmsd(const ImageViewRGB32& a, const ImageViewRGB32& b){ + size_t n = a.width() * a.height(); + if (n == 0){ + return 0.0; + } + double sum_sq = 0.0; + for (size_t y = 0; y < a.height(); y++){ + for (size_t x = 0; x < a.width(); x++){ + uint32_t pa = a.pixel(x, y); + uint32_t pb = b.pixel(x, y); + int dr = int((pa >> 16) & 0xff) - int((pb >> 16) & 0xff); + int dg = int((pa >> 8) & 0xff) - int((pb >> 8) & 0xff); + int db = int(pa & 0xff) - int(pb & 0xff); + sum_sq += double(dr * dr + dg * dg + db * db); + } + } + return std::sqrt(sum_sq / (3.0 * n)); +} + +// Slide `tmpl` (at its native pixel size) across the region of `screen` +// defined by `box`. Returns the minimum RMSD found over all window positions. +double template_match_rmsd(const ImageViewRGB32& screen, const ImageFloatBox& box, const ImageViewRGB32& tmpl){ + ImageViewRGB32 region = extract_box_reference(screen, box); + size_t rw = tmpl.width(), rh = tmpl.height(); + size_t lw = region.width(), lh = region.height(); + if (rw == 0 || rh == 0 || rw > lw || rh > lh){ + return 0.0; + } + + double min_rmsd = std::numeric_limits::max(); + for (size_t dy = 0; dy <= lh - rh; dy++){ + for (size_t dx = 0; dx <= lw - rw; dx++){ + double r = compute_rmsd(region.sub_image(dx, dy, rw, rh), tmpl); + if (r < min_rmsd){ + min_rmsd = r; + } + } + } + return min_rmsd; +} + +} // namespace + + +MenuCursorPosition MenuCursorDetector::detect_dark_pixels(const ImageViewRGB32& screen) const { + struct IconEntry { + MenuCursorPosition position; + const ImageFloatBox* boxes; + }; + const IconEntry icons[] = { + {MenuCursorPosition::POKEDEX, m_pokedex_box}, + {MenuCursorPosition::POKEMON, m_pokemon_box}, + {MenuCursorPosition::BAG, m_bag_box}, + {MenuCursorPosition::CARD, m_card_box}, + {MenuCursorPosition::OPTIONS, m_options_box}, + {MenuCursorPosition::MAP, m_map_box}, + {MenuCursorPosition::CAPSULES, m_capsules_box}, + {MenuCursorPosition::MYSTERYGIFT, m_mysterygift_box}, + }; + + MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; + int best_count = 0; + + for (const auto& entry : icons){ + int count = count_dark_boxes(screen, entry.boxes, BOXES_PER_ICON); + if (count > best_count){ + best_count = count; + best_pos = entry.position; + } + } + + return (best_count >= REQUIRED_VOTES) ? best_pos : MenuCursorPosition::UNKNOWN; +} + + +MenuCursorPosition MenuCursorDetector::detect_image_match(const ImageViewRGB32& screen) const { + MenuCursorPosition best_pos = MenuCursorPosition::UNKNOWN; + double best_rmsd = MATCH_RMSD_THRESHOLD; // must exceed this + + for (int i = 0; i < MENU_CURSOR_NUM_ICONS; i++){ + if (!m_icon_references[i].has_value()){ + continue; + } + if (m_icon_match_box[i].width == 0.0){ + continue; + } + double r = template_match_rmsd(screen, m_icon_match_box[i], *m_icon_references[i]); + if (r > best_rmsd){ + best_rmsd = r; + best_pos = static_cast(i); + } + } + return best_pos; +} + + +MenuCursorPosition MenuCursorDetector::detect(const ImageViewRGB32& screen) const { + MenuCursorPosition p1 = detect_dark_pixels(screen); + MenuCursorPosition p2 = detect_image_match(screen); + + if (p1 == p2){ + return p1; // both agree (or both UNKNOWN) + } + if (p1 == MenuCursorPosition::UNKNOWN){ + return p2; // only approach 2 fired + } + if (p2 == MenuCursorPosition::UNKNOWN){ + return p1; // only approach 1 fired + } + return p1; // conflict: prefer approach 1 +} + + +void MenuCursorDetector::set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference){ + m_icon_references[static_cast(icon)].emplace(std::move(reference)); +} + + +double MenuCursorDetector::icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const { + int idx = static_cast(icon); + if (!m_icon_references[idx].has_value() || m_icon_match_box[idx].width == 0.0){ + return -1.0; + } + return template_match_rmsd(screen, m_icon_match_box[idx], *m_icon_references[idx]); +} + + +MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& /*context*/){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + return MenuCursorPosition::UNKNOWN; + } + MenuCursorDetector detector; + return detector.detect(screen); +} + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation diff --git a/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h new file mode 100644 index 0000000000..cadbf1e4e4 --- /dev/null +++ b/SerialPrograms/Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h @@ -0,0 +1,152 @@ +/* BDSP AutoStory - Menu Cursor Detector + * + * From: https://github.com/PokemonAutomation/ + * + * Adapted from PokemonAutomation/Arduino-Source#1717 by Developer-Butters. + * + * Detects which icon the cursor is hovering over in the X-menu. + * + * Two independent detection approaches are provided for testing: + * + * Approach 1 — dark pixel count (detect_dark_pixels): + * Each icon has BOXES_PER_ICON small boxes placed over a region that gains + * dark pixels when hovered. A box fires if >= MIN_DARK_PIXEL_COUNT pixels + * in it have R, G, B all <= BLACK_CHANNEL_MAX. An icon is selected when its + * fire count reaches REQUIRED_VOTES. + * Tune BLACK_CHANNEL_MAX upward if real dark pixels are being missed. + * Tune it downward (or raise MIN_DARK_PIXEL_COUNT) to reduce false positives. + * + * Approach 2 — template matching (detect_image_match): + * Each icon has one larger search box and a hovered-state reference PNG loaded + * at runtime via set_icon_reference(). The reference PNG is slid (at its native + * pixel size, without resizing) across the search box and the minimum RMSD over + * all window positions is computed. The icon with the lowest min-RMSD that is + * also <= MATCH_RMSD_THRESHOLD is considered selected. + * Tune MATCH_RMSD_THRESHOLD downward to require a closer match. + * + * detect() is the production method and currently delegates to approach 1. + * It will become a combined vote once both approaches are validated. + * + * Menu layouts: + * 5-icon (early game): POKEDEX POKEMON BAG CARD / OPTIONS + * 8-icon (late game): POKEDEX POKEMON BAG CARD / MAP CAPSULES OPTIONS MYSTERYGIFT + * + * Note: OPTIONS sits at a different screen position in each layout. + * Use separate MenuCursorDetector instances with different box coordinates + * if you need to detect OPTIONS reliably in both layouts. + */ + +#ifndef PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H +#define PokemonAutomation_PokemonBDSP_AutoStory_MenuCursorDetector_H + +#include +#include +#include + +#include "Common/Cpp/Color.h" + +#include "CommonFramework/ImageTools/ImageBoxes.h" +#include "CommonFramework/ImageTypes/ImageRGB32.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h" + +#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h" + +namespace PokemonAutomation{ +namespace NintendoSwitch{ +namespace PokemonBDSP{ + + +// Possible icon positions for the BDSP pause/X menu cursor. +enum class MenuCursorPosition { + UNKNOWN = -1, + POKEDEX = 0, + POKEMON = 1, + BAG = 2, + CARD = 3, + OPTIONS = 4, + MAP = 5, + CAPSULES = 6, + MYSTERYGIFT = 7, +}; + +// Total number of icons in the pause menu layout. +static constexpr int MENU_CURSOR_NUM_ICONS = 8; + +// Returns the debug name of a MenuCursorPosition. +std::string menu_cursor_to_string(MenuCursorPosition pos); + + +// Detects which icon in the BDSP pause menu is currently focused by the cursor. +class MenuCursorDetector { +public: + // ── Approach 1: dark pixel detection ───────────────────────────────────── + static constexpr int BOXES_PER_ICON = 3; + static constexpr int REQUIRED_VOTES = 1; // out of BOXES_PER_ICON + static constexpr uint8_t BLACK_CHANNEL_MAX = 50; // R,G,B <= this → dark pixel + static constexpr size_t MIN_DARK_PIXEL_COUNT = 1; // dark pixels per box to fire + + // ── Approach 2: template match ─────────────────────────────────────────── + // Reference PNGs are the UN-hovered state of each icon. When hovered, the + // icon gains dark pixels and diverges from the reference, so RMSD increases. + // detect_image_match() selects the icon with the HIGHEST RMSD >= threshold. + // Tune upward to reduce false positives; downward if the correct icon is missed. + static constexpr double MATCH_RMSD_THRESHOLD = 53.0; + + MenuCursorDetector(Color color = COLOR_RED); + + void make_overlays(VideoOverlaySet& items) const; + + // Approach 1: votes across dark-pixel boxes, returns icon with highest + // vote count >= REQUIRED_VOTES, or UNKNOWN. + MenuCursorPosition detect_dark_pixels(const ImageViewRGB32& screen) const; + + // Approach 2: returns the icon with the highest RMSD against its reference, + // provided that RMSD >= MATCH_RMSD_THRESHOLD (high RMSD = hovered). + // Returns UNKNOWN if no references are loaded or nothing exceeds threshold. + MenuCursorPosition detect_image_match(const ImageViewRGB32& screen) const; + + // Production method — currently delegates to detect_dark_pixels(). + MenuCursorPosition detect(const ImageViewRGB32& screen) const; + + // Load a hovered-state reference image for one icon (enables approach 2 for + // that icon). The match box for that icon must already be set in the + // constructor (m_icon_match_box); this method just attaches the reference. + void set_icon_reference(MenuCursorPosition icon, ImageRGB32 reference); + + // Returns the raw RMSD of `icon` against its reference on the given frame. + // Returns -1.0 if no reference is loaded for that icon. + double icon_rmsd(const ImageViewRGB32& screen, MenuCursorPosition icon) const; + +private: + Color m_color; + + // ── Approach 1 boxes ───────────────────────────────────────────────────── + // 3 boxes per icon targeting regions that gain dark pixels when hovered. + // All initialized to (0,0,0,0) — replace with real normalized coordinates. + ImageFloatBox m_pokedex_box[BOXES_PER_ICON]; + ImageFloatBox m_pokemon_box[BOXES_PER_ICON]; + ImageFloatBox m_bag_box[BOXES_PER_ICON]; + ImageFloatBox m_card_box[BOXES_PER_ICON]; + ImageFloatBox m_options_box[BOXES_PER_ICON]; + ImageFloatBox m_map_box[BOXES_PER_ICON]; + ImageFloatBox m_capsules_box[BOXES_PER_ICON]; + ImageFloatBox m_mysterygift_box[BOXES_PER_ICON]; + + // ── Approach 2 search boxes + references ──────────────────────────────── + // One search box per icon (the area to slide the template across). + // References are populated via set_icon_reference() at runtime. + ImageFloatBox m_icon_match_box[MENU_CURSOR_NUM_ICONS]; + std::array, MENU_CURSOR_NUM_ICONS> m_icon_references; +}; + + +// Captures the current frame from `stream` and runs detect() on it. +MenuCursorPosition detect_menu_cursor(VideoStream& stream, ProControllerContext& context); + + +} // namespace PokemonBDSP +} // namespace NintendoSwitch +} // namespace PokemonAutomation +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 1aba91efd9..b833d85199 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1357,6 +1357,8 @@ file(GLOB LIBRARY_SOURCES Source/NintendoSwitch/Inference/NintendoSwitch_SelectedSettingDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_StartGameUserSelectDetector.h + Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp + Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.cpp Source/NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h Source/NintendoSwitch/NintendoSwitch_ConsoleHandle.cpp @@ -1613,6 +1615,8 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Inference/PokemonBDSP_TransitionWatchers.h Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.h + Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp + Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h Source/PokemonBDSP/Inference/PokemonBDSP_PokeballSpriteMatcher.cpp Source/PokemonBDSP/Inference/PokemonBDSP_PokeballSpriteMatcher.h Source/PokemonBDSP/Inference/PokemonBDSP_ReceivePokemonDetector.cpp @@ -1665,6 +1669,28 @@ file(GLOB LIBRARY_SOURCES Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_OverworldDetector.h Source/PokemonBDSP/PokemonBDSP_Tests.cpp Source/PokemonBDSP/PokemonBDSP_Tests.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Bag.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Battle.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_BattleMoves.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Center.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_NewGame.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Settings.h + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.cpp + Source/PokemonBDSP/Programs/AutoStory/Actions/PokemonBDSP_AutoStory_WildBattle.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_BattleMenuReader.h + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.cpp + Source/PokemonBDSP/Programs/AutoStory/Detect/PokemonBDSP_AutoStory_MenuCursorDetector.h Source/PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_Maps.cpp Source/PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_Maps.h Source/PokemonBDSP/Programs/AutoStory/Data/PokemonBDSP_AutoStory_PlanCosts.cpp