BDSP AutoStory MVP: new game to Floaroma on #1717 and the shared core

SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp

@@ -0,0 +1,156 @@
+/*  System Keyboard Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <cctype>
+#include <cmath>
+#include <cstring>
+#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h"
+#include "CommonFramework/ImageTools/ImageStats.h"
+#include "CommonFramework/VideoPipeline/VideoFeed.h"
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+#include "CommonTools/Images/SolidColorTest.h"
+#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h"
+#include "NintendoSwitch_SystemKeyboardDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+
+using namespace std::chrono_literals;
+
+
+//  Key grid of the QWERTY layout and where each key sits: rows "1".."z", columns "1".."-".
+static const char* const KEY_ROWS[] = {"1234567890-", "qwertyuiop/", "asdfghjkl:'", "zxcvbnm,.?!"};
+static constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770};
+static constexpr double KEY_COL_X0 = 0.077;
+static constexpr double KEY_COL_STEP = 0.075;
+
+
+SystemKeyboardDetector::SystemKeyboardDetector(Color color)
+    : m_color(color)
+    , m_key_panel(0.15, 0.555, 0.65, 0.020)
+    , m_bottom_bar(0.10, 0.935, 0.35, 0.030)
+    , m_ok_button(0.875, 0.760, 0.045, 0.020)
+{}
+void SystemKeyboardDetector::make_overlays(VideoOverlaySet& items) const{
+    items.add(m_color, m_key_panel);
+    items.add(m_color, m_bottom_bar);
+    items.add(m_color, m_ok_button);
+}
+bool SystemKeyboardDetector::detect(const ImageViewRGB32& screen){
+    if (!is_white(extract_box_reference(screen, m_key_panel)) ||
+        !is_white(extract_box_reference(screen, m_bottom_bar))
+    ){
+        return false;
+    }
+    //  OK is blue, or flat grey while the field is empty (disabled).
+    const ImageStats ok = image_stats(extract_box_reference(screen, m_ok_button));
+    return is_solid(ok, {0.172, 0.276, 0.552}) || is_grey(ok, 600, 690);
+}
+
+
+bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col){
+    //  The cursor is a cyan border around one key. Neighbouring keys overlap that border,
+    //  so take the centroid of all cyan pixels and map it to the nearest key.
+    const ImageFloatBox area(0.02, 0.44, 0.84, 0.38);
+    ImageViewRGB32 keys = extract_box_reference(screen, area);
+    double count = 0, sum_x = 0, sum_y = 0;
+    for (size_t y = 0; y < keys.height(); y++){
+        for (size_t x = 0; x < keys.width(); x++){
+            const uint32_t pixel = keys.pixel(x, y);
+            const int red = (pixel >> 16) & 0xff;
+            const int green = (pixel >> 8) & 0xff;
+            const int blue = pixel & 0xff;
+            if (red < 150 && green > 200 && blue > 190){
+                count++;
+                sum_x += (double)x;
+                sum_y += (double)y;
+            }
+        }
+    }
+    //  The border is about 0.15% of the screen; a few stray pixels are not a cursor.
+    if (count < 0.0005 * screen.width() * screen.height()){
+        return false;
+    }
+    const double cx = area.x + sum_x / count / keys.width() * area.width;
+    const double cy = area.y + sum_y / count / keys.height() * area.height;
+    col = (int)std::lround((cx - KEY_COL_X0) / KEY_COL_STEP);
+    row = 0;
+    for (int r = 1; r < 4; r++){
+        if (std::abs(KEY_ROW_Y[r] - cy) < std::abs(KEY_ROW_Y[row] - cy)){
+            row = r;
+        }
+    }
+    return 0 <= col && col < 11;
+}
+
+bool system_keyboard_field_has_text(const ImageViewRGB32& screen){
+    //  Left part of the field, after the caret's position on an empty field.
+    const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.11, 0.235, 0.19, 0.030)));
+    return stats.stddev.sum() > 30;
+}
+
+
+void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text){
+    for (char ch : text){
+        const char key = (char)std::tolower((unsigned char)ch);
+        int target_row = -1;
+        int target_col = -1;
+        for (int r = 0; r < 4 && target_row < 0; r++){
+            const char* p = std::strchr(KEY_ROWS[r], key);
+            if (p != nullptr && key != '\0'){
+                target_row = r;
+                target_col = (int)(p - KEY_ROWS[r]);
+            }
+        }
+        if (target_row < 0){
+            OperationFailedExceptionWithScreenshot::fire(
+                ErrorReportMode::SEND_ERROR_REPORT,
+                std::string("System keyboard: no key for '") + ch + "'.",
+                stream
+            );
+        }
+
+        //  A button pressed while the keyboard slides in is ignored, so read the cursor
+        //  again after each move until it is on the target.
+        bool arrived = false;
+        for (size_t attempt = 0; attempt < 10 && !arrived; attempt++){
+            context.wait_for_all_requests();
+            int row = -1;
+            int col = -1;
+            VideoSnapshot screen = stream.video().snapshot();
+            if (!screen || !find_system_keyboard_cursor(screen, row, col)){
+                pbf_wait(context, 300ms);
+                continue;
+            }
+            if (row == target_row && col == target_col){
+                arrived = true;
+                break;
+            }
+            for (; row < target_row; row++) pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms);
+            for (; row > target_row; row--) pbf_press_dpad(context, DPAD_UP, 100ms, 200ms);
+            for (; col < target_col; col++) pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms);
+            for (; col > target_col; col--) pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms);
+            pbf_wait(context, 300ms);
+        }
+        if (!arrived){
+            OperationFailedExceptionWithScreenshot::fire(
+                ErrorReportMode::SEND_ERROR_REPORT,
+                std::string("System keyboard: the cursor did not reach '") + key + "'.",
+                stream
+            );
+        }
+        if (std::isupper((unsigned char)ch)){
+            pbf_press_button(context, BUTTON_LCLICK, 100ms, 250ms);
+        }
+        pbf_press_button(context, BUTTON_A, 80ms, 300ms);
+    }
+    context.wait_for_all_requests();
+}
+
+
+
+}
+}

SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h

@@ -0,0 +1,67 @@
+/*  System Keyboard Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ *  The Switch's own software keyboard, which games open for names (e.g. the
+ *  player's name in BDSP): detect it, find the key under its cursor, and type
+ *  a string by moving the cursor and checking where it landed.
+ *
+ *  Measured on 1920x1080 captures of the QWERTY layout.
+ *
+ */
+
+#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H
+#define PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H
+
+#include <string>
+#include "CommonFramework/ImageTools/ImageBoxes.h"
+#include "CommonFramework/Tools/VideoStream.h"
+#include "CommonTools/VisualDetector.h"
+#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+
+
+//  White key panel and bottom bar, and the blue OK button.
+class SystemKeyboardDetector : public StaticScreenDetector{
+public:
+    SystemKeyboardDetector(Color color = COLOR_CYAN);
+
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+    virtual bool detect(const ImageViewRGB32& screen) override;
+
+private:
+    Color m_color;
+    ImageFloatBox m_key_panel;
+    ImageFloatBox m_bottom_bar;
+    ImageFloatBox m_ok_button;
+};
+class SystemKeyboardWatcher : public DetectorToFinder<SystemKeyboardDetector>{
+public:
+    SystemKeyboardWatcher(Color color = COLOR_CYAN)
+        : DetectorToFinder("SystemKeyboardWatcher", std::chrono::milliseconds(250), color)
+    {}
+};
+
+
+//  The key under the cursor, as row (0: digits .. 3: "zxcv") and column (0..10).
+//  The keyboard remembers the cursor between uses, so read it, don't assume it.
+//  Returns false if no cursor border is visible.
+bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col);
+
+//  Whether the text field holds any text. Pressing B on an empty field closes the
+//  keyboard, so delete only while this is true.
+bool system_keyboard_field_has_text(const ImageViewRGB32& screen);
+
+//  Types `text` (letters, digits and the symbols on the QWERTY grid). For each character,
+//  reads the cursor, moves it with the D-pad and checks it arrived before pressing A; an
+//  uppercase letter is typed with Shift (left stick click). Throws, with a screenshot, if
+//  the cursor can't be read or doesn't arrive.
+void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text);
+
+
+
+}
+}
+#endif

SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_MenuDetector.cpp

@@ -40,8 +40,8 @@
     , m_line4(menu_stripe(4, skip_new_banner))
     , m_cross(0.20, 0.15, 0.60, 0.37)
     , m_panel_top(0.150, 0.100, 0.700, 0.015)
-    , m_panel_left(0.150, 0.125, 0.020, 0.460)
-    , m_panel_right(0.835, 0.125, 0.015, 0.460)
+    , m_panel_left(0.156, 0.125, 0.014, 0.460)
+    , m_panel_right(0.835, 0.125, 0.009, 0.460)
     , m_footer(0.050, 0.963, 0.600, 0.025)
     , m_confirm_button(0.736, 0.960, 0.022, 0.038)
     , m_confirm_button_with_swap(0.653, 0.960, 0.022, 0.038)
@@ -94,8 +94,9 @@
     return pixels > 0 && dark / pixels >= 0.15 && light / pixels >= 0.08;
 }
 
 bool MenuDetector::detect_menu_chrome(const ImageViewRGB32& screen) const{
-    // The surrounding panel remains the same with six or eight unlocked icons.
+    // The surrounding panel remains the same with two, six or eight unlocked icons.
+    // Its edges are blue strips, so the side boxes stay inside the white.
     for (const ImageFloatBox& box : {m_panel_top, m_panel_left, m_panel_right}){
         if (!is_white(image_stats(extract_box_reference(screen, box)))){
             return false;

SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp

@@ -0,0 +1,67 @@
+/*  Name List Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <cmath>
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+#include "CommonTools/Images/SolidColorTest.h"
+#include "PokemonBDSP_SelectionArrow.h"
+#include "PokemonBDSP_NameListDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonBDSP{
+
+
+namespace{
+//  Row centers, as a fraction of the screen height (1080p: 307, 370, 433, ...).
+const double FIRST_ROW_Y = 0.284;
+const double ROW_STEP_Y  = 0.0587;
+const int ROWS = 5;
+}
+
+
+NameListDetector::NameListDetector(Color color)
+    : m_color(color)
+    , m_arrows(0.137, 0.245, 0.030, 0.310)
+    , m_panel(0.140, 0.245, 0.140, 0.300)
+{}
+void NameListDetector::make_overlays(VideoOverlaySet& items) const{
+    items.add(m_color, m_arrows);
+    items.add(m_color, m_panel);
+}
+bool NameListDetector::detect(const ImageViewRGB32& screen){
+    return selected_row(screen) >= 0;
+}
+int NameListDetector::selected_row(const ImageViewRGB32& screen) const{
+    //  The list is a white panel with a little dark text. A thin margin strip is not
+    //  enough: the border's antialiasing reaches into it on some frames.
+    const ImageViewRGB32 panel = extract_box_reference(screen, m_panel);
+    size_t white = 0;
+    for (size_t y = 0; y < panel.height(); y += 2){
+        for (size_t x = 0; x < panel.width(); x += 2){
+            const uint32_t p = panel.pixel(x, y);
+            white += ((p >> 16) & 0xff) > 220 && ((p >> 8) & 0xff) > 220 && (p & 0xff) > 220;
+        }
+    }
+    const size_t samples = ((panel.width() + 1) / 2) * ((panel.height() + 1) / 2);
+    if (samples == 0 || white < samples * 6 / 10){
+        return -1;
+    }
+    ImageViewRGB32 area = extract_box_reference(screen, m_arrows);
+    std::vector<ImagePixelBox> arrows = find_selection_arrows(area);
+    if (arrows.size() != 1){
+        return -1;
+    }
+    const double center_y = m_arrows.y +
+        m_arrows.height * (arrows[0].min_y + arrows[0].max_y) / 2.0 / (double)area.height();
+    const int row = (int)std::lround((center_y - FIRST_ROW_Y) / ROW_STEP_Y);
+    return row >= 0 && row < ROWS ? row : -1;
+}
+
+
+}
+}
+}

SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h

@@ -0,0 +1,41 @@
+/*  Name List Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ *  The list of preset names shown for the rival: "I'll tell you...", then four names.
+ *
+ */
+
+#ifndef PokemonAutomation_PokemonBDSP_NameListDetector_H
+#define PokemonAutomation_PokemonBDSP_NameListDetector_H
+
+#include "CommonFramework/ImageTools/ImageBoxes.h"
+#include "CommonTools/VisualDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonBDSP{
+
+
+class NameListDetector : public StaticScreenDetector{
+public:
+    NameListDetector(Color color = COLOR_MAGENTA);
+
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+    virtual bool detect(const ImageViewRGB32& screen) override;
+
+    //  The row the arrow is on: 0 is "I'll tell you...", 1 the first preset name.
+    //  -1 if the list is not showing.
+    int selected_row(const ImageViewRGB32& screen) const;
+
+private:
+    Color m_color;
+    ImageFloatBox m_arrows;
+    ImageFloatBox m_panel;
+};
+
+
+}
+}
+}
+#endif

SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_SelectionArrow.h

@@ -8,6 +8,8 @@
 #define PokemonAutomation_PokemonBDSP_SelectionArrow_H
 
 #include <deque>
+#include <vector>
+#include "CommonFramework/ImageTools/ImageBoxes.h"
 #include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
 #include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h"
 
@@ -16,6 +18,9 @@ namespace NintendoSwitch{
 namespace PokemonBDSP{
 
 
+//  The menu selection arrows in `image`, in its pixel coordinates.
+std::vector<ImagePixelBox> find_selection_arrows(const ImageViewRGB32& image);
+
 class SelectionArrowFinder : public VisualInferenceCallback{
 public:
     SelectionArrowFinder(

SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp

@@ -18,6 +18,7 @@
 #include "Programs/AutoStory/PokemonBDSP_AutoStoryTools.h"
 #include "PokemonBDSP_Tests.h"
 #include "Inference/PokemonBDSP_MenuDetector.h"
+#include "Programs/AutoStory/PokemonBDSP_AutoStory_Tests.h"
 
 namespace PokemonAutomation{
 namespace NintendoSwitch{
@@ -288,6 +289,7 @@ bool run_bag_screenshot_tests(Logger& logger, const std::string& folder){
 void add_tests(UnitTestDatabase& database){
     database.add<Test_BagScreenshots>();
     add_tests_MenuDetector(database);
+    add_tests_AutoStory(database);
 }
 
 }