BDSP AutoStory v4: game actions, battles and their detectors

SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.cpp

@@ -0,0 +1,171 @@
+/*  System Keyboard Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <cctype>
+#include <cmath>
+#include <cstring>
+#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h"
+#include "CommonFramework/ImageTools/ImageStats.h"
+#include "CommonFramework/VideoPipeline/VideoFeed.h"
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+#include "CommonTools/Images/SolidColorTest.h"
+#include "NintendoSwitch/Commands/NintendoSwitch_Commands_PushButtons.h"
+#include "NintendoSwitch_SystemKeyboardDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+
+using namespace std::chrono_literals;
+
+
+//  Key grid of the QWERTY layout and where each key sits: rows "1".."z", columns "1".."-".
+static const char* const KEY_ROWS[] = {"1234567890-", "qwertyuiop/", "asdfghjkl:'", "zxcvbnm,.?!"};
+static constexpr double KEY_ROW_Y[] = {0.500, 0.593, 0.681, 0.770};
+static constexpr double KEY_COL_X0 = 0.077;
+static constexpr double KEY_COL_STEP = 0.075;
+
+
+SystemKeyboardDetector::SystemKeyboardDetector(Color color)
+    : m_color(color)
+    , m_key_panel(0.15, 0.555, 0.65, 0.020)
+    , m_bottom_bar(0.10, 0.935, 0.35, 0.030)
+    , m_ok_button(0.875, 0.760, 0.045, 0.020)
+{}
+void SystemKeyboardDetector::make_overlays(VideoOverlaySet& items) const{
+    items.add(m_color, m_key_panel);
+    items.add(m_color, m_bottom_bar);
+    items.add(m_color, m_ok_button);
+}
+bool SystemKeyboardDetector::detect(const ImageViewRGB32& screen){
+    if (!is_white(extract_box_reference(screen, m_key_panel)) ||
+        !is_white(extract_box_reference(screen, m_bottom_bar))
+    ){
+        return false;
+    }
+    //  OK is blue, or flat grey while the field is empty (disabled).
+    const ImageStats ok = image_stats(extract_box_reference(screen, m_ok_button));
+    return is_solid(ok, {0.172, 0.276, 0.552}) || is_grey(ok, 600, 690);
+}
+
+
+bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col){
+    //  The cursor is a cyan border around one key. Neighbouring keys overlap that border,
+    //  so take the centroid of all cyan pixels and map it to the nearest key.
+    const ImageFloatBox area(0.02, 0.44, 0.84, 0.38);
+    ImageViewRGB32 keys = extract_box_reference(screen, area);
+    double count = 0, sum_x = 0, sum_y = 0;
+    for (size_t y = 0; y < keys.height(); y++){
+        for (size_t x = 0; x < keys.width(); x++){
+            const uint32_t pixel = keys.pixel(x, y);
+            const int red = (pixel >> 16) & 0xff;
+            const int green = (pixel >> 8) & 0xff;
+            const int blue = pixel & 0xff;
+            if (red < 150 && green > 200 && blue > 190){
+                count++;
+                sum_x += (double)x;
+                sum_y += (double)y;
+            }
+        }
+    }
+    //  The border is about 0.15% of the screen; a few stray pixels are not a cursor.
+    if (count < 0.0005 * screen.width() * screen.height()){
+        return false;
+    }
+    const double cx = area.x + sum_x / count / keys.width() * area.width;
+    const double cy = area.y + sum_y / count / keys.height() * area.height;
+    col = (int)std::lround((cx - KEY_COL_X0) / KEY_COL_STEP);
+    row = 0;
+    for (int r = 1; r < 4; r++){
+        if (std::abs(KEY_ROW_Y[r] - cy) < std::abs(KEY_ROW_Y[row] - cy)){
+            row = r;
+        }
+    }
+    return 0 <= col && col < 11;
+}
+
+bool system_keyboard_field_has_text(const ImageViewRGB32& screen){
+    //  Left part of the field, after the caret's position on an empty field.
+    const ImageStats stats = image_stats(extract_box_reference(screen, ImageFloatBox(0.11, 0.235, 0.19, 0.030)));
+    return stats.stddev.sum() > 30;
+}
+
+
+namespace{
+
+//  Moves the keyboard's cursor onto (target_row, target_col). A button
+//  pressed while the keyboard slides in is ignored, so read the cursor again
+//  after each move until it is on the target. Whether it got there.
+bool move_keyboard_cursor(VideoStream& stream, ProControllerContext& context, int target_row, int target_col){
+    for (size_t attempt = 0; attempt < 10; attempt++){
+        context.wait_for_all_requests();
+        int row = -1;
+        int col = -1;
+        VideoSnapshot screen = stream.video().snapshot();
+        if (!screen || !find_system_keyboard_cursor(screen, row, col)){
+            pbf_wait(context, 300ms);
+            continue;
+        }
+        if (row == target_row && col == target_col){
+            return true;
+        }
+        for (; row < target_row; row++){
+            pbf_press_dpad(context, DPAD_DOWN, 100ms, 200ms);
+        }
+        for (; row > target_row; row--){
+            pbf_press_dpad(context, DPAD_UP, 100ms, 200ms);
+        }
+        for (; col < target_col; col++){
+            pbf_press_dpad(context, DPAD_RIGHT, 100ms, 200ms);
+        }
+        for (; col > target_col; col--){
+            pbf_press_dpad(context, DPAD_LEFT, 100ms, 200ms);
+        }
+        pbf_wait(context, 300ms);
+    }
+    return false;
+}
+
+} // namespace
+
+
+void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text){
+    for (char ch : text){
+        const char key = (char)std::tolower((unsigned char)ch);
+        int target_row = -1;
+        int target_col = -1;
+        for (int r = 0; r < 4 && target_row < 0; r++){
+            const char* p = std::strchr(KEY_ROWS[r], key);
+            if (p != nullptr && key != '\0'){
+                target_row = r;
+                target_col = (int)(p - KEY_ROWS[r]);
+            }
+        }
+        if (target_row < 0){
+            OperationFailedExceptionWithScreenshot::fire(
+                ErrorReportMode::SEND_ERROR_REPORT,
+                std::string("System keyboard: no key for '") + ch + "'.",
+                stream
+            );
+        }
+        if (!move_keyboard_cursor(stream, context, target_row, target_col)){
+            OperationFailedExceptionWithScreenshot::fire(
+                ErrorReportMode::SEND_ERROR_REPORT,
+                std::string("System keyboard: the cursor did not reach '") + key + "'.",
+                stream
+            );
+        }
+        if (std::isupper((unsigned char)ch)){
+            pbf_press_button(context, BUTTON_LCLICK, 100ms, 250ms);
+        }
+        pbf_press_button(context, BUTTON_A, 80ms, 300ms);
+    }
+    context.wait_for_all_requests();
+}
+
+
+
+} // namespace NintendoSwitch
+} // namespace PokemonAutomation

SerialPrograms/Source/NintendoSwitch/Inference/NintendoSwitch_SystemKeyboardDetector.h

@@ -0,0 +1,68 @@
+/*  System Keyboard Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ *  The Switch's own software keyboard, which games open for names (e.g. the
+ *  player's name in BDSP): detect it, find the key under its cursor, and type
+ *  a string by moving the cursor and checking where it landed.
+ *
+ *  Measured on 1920x1080 captures of the QWERTY layout.
+ *
+ */
+
+#ifndef PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H
+#define PokemonAutomation_NintendoSwitch_SystemKeyboardDetector_H
+
+#include <string>
+#include "CommonFramework/ImageTools/ImageBoxes.h"
+#include "CommonFramework/Tools/VideoStream.h"
+#include "CommonTools/VisualDetector.h"
+#include "NintendoSwitch/Controllers/Procon/NintendoSwitch_ProController.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+
+
+//  White key panel and bottom bar, and the blue OK button.
+class SystemKeyboardDetector : public StaticScreenDetector{
+public:
+    SystemKeyboardDetector(Color color = COLOR_CYAN);
+
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+    virtual bool detect(const ImageViewRGB32& screen) override;
+
+private:
+    Color m_color;
+    ImageFloatBox m_key_panel;
+    ImageFloatBox m_bottom_bar;
+    ImageFloatBox m_ok_button;
+};
+//  Periodic watcher for the Switch system on-screen keyboard.
+class SystemKeyboardWatcher : public DetectorToFinder<SystemKeyboardDetector>{
+public:
+    SystemKeyboardWatcher(Color color = COLOR_CYAN)
+        : DetectorToFinder("SystemKeyboardWatcher", std::chrono::milliseconds(250), color)
+    {}
+};
+
+
+//  The key under the cursor, as row (0: digits .. 3: "zxcv") and column (0..10).
+//  The keyboard remembers the cursor between uses, so read it, don't assume it.
+//  Returns false if no cursor border is visible.
+bool find_system_keyboard_cursor(const ImageViewRGB32& screen, int& row, int& col);
+
+//  Whether the text field holds any text. Pressing B on an empty field closes the
+//  keyboard, so delete only while this is true.
+bool system_keyboard_field_has_text(const ImageViewRGB32& screen);
+
+//  Types `text` (letters, digits and the symbols on the QWERTY grid). For each character,
+//  reads the cursor, moves it with the D-pad and checks it arrived before pressing A; an
+//  uppercase letter is typed with Shift (left stick click). Throws, with a screenshot, if
+//  the cursor can't be read or doesn't arrive.
+void type_on_system_keyboard(VideoStream& stream, ProControllerContext& context, const std::string& text);
+
+
+
+} // namespace NintendoSwitch
+} // namespace PokemonAutomation
+#endif

SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.cpp

@@ -0,0 +1,96 @@
+/*  Name List Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <cmath>
+#include <vector>
+#include "Kernels/Waterfill/Kernels_Waterfill.h"
+#include "CommonFramework/GlobalAutoPaths.h"
+#include "CommonFramework/ImageTools/ImageBoxes.h"
+#include "CommonTools/Images/BinaryImage_FilterRgb32.h"
+#include "CommonTools/ImageMatch/ExactImageMatcher.h"
+#include "CommonFramework/VideoPipeline/VideoOverlayScopes.h"
+#include "CommonTools/Images/SolidColorTest.h"
+#include "PokemonBDSP_NameListDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonBDSP{
+
+
+namespace{
+//  Row centers, as a fraction of the screen height (1080p: 307, 370, 433, ...).
+const double FIRST_ROW_Y = 0.284;
+const double ROW_STEP_Y  = 0.0587;
+const int ROWS = 5;
+
+//  The menu's selection arrows in `image`, as PokemonBDSP_SelectionArrow.cpp
+//  finds them: that file keeps its finder out of its header, so the same
+//  waterfill and template match are repeated here rather than changing it.
+const ImageMatch::ExactImageMatcher& name_list_arrow(){
+    static ImageMatch::ExactImageMatcher matcher(RESOURCE_PATH() + "PokemonBDSP/SelectionArrow.png");
+    return matcher;
+}
+std::vector<ImagePixelBox> find_name_list_arrows(const ImageViewRGB32& image){
+    using namespace Kernels;
+    using namespace Kernels::Waterfill;
+    PackedBinaryMatrix matrix = compress_rgb32_to_binary_max(image, 200, 200, 200);
+    std::vector<WaterfillObject> objects = find_objects_inplace(matrix, 200);
+    std::vector<ImagePixelBox> ret;
+    for (const WaterfillObject& object : objects){
+        if (object.width() > object.height() || object.height() > 3 * object.width()){
+            continue;
+        }
+        if (name_list_arrow().rmsd(extract_box_reference(image, object)) <= 90){
+            ret.emplace_back(object.min_x, object.min_y, object.max_x, object.max_y);
+        }
+    }
+    return ret;
+}
+} // namespace
+
+
+NameListDetector::NameListDetector(Color color)
+    : m_color(color)
+    , m_arrows(0.137, 0.245, 0.030, 0.310)
+    , m_panel(0.140, 0.245, 0.140, 0.300)
+{}
+void NameListDetector::make_overlays(VideoOverlaySet& items) const{
+    items.add(m_color, m_arrows);
+    items.add(m_color, m_panel);
+}
+bool NameListDetector::detect(const ImageViewRGB32& screen){
+    return selected_row(screen) >= 0;
+}
+int NameListDetector::selected_row(const ImageViewRGB32& screen) const{
+    //  The list is a white panel with a little dark text. A thin margin strip is not
+    //  enough: the border's antialiasing reaches into it on some frames.
+    const ImageViewRGB32 panel = extract_box_reference(screen, m_panel);
+    size_t white = 0;
+    for (size_t y = 0; y < panel.height(); y += 2){
+        for (size_t x = 0; x < panel.width(); x += 2){
+            const uint32_t p = panel.pixel(x, y);
+            white += ((p >> 16) & 0xff) > 220 && ((p >> 8) & 0xff) > 220 && (p & 0xff) > 220;
+        }
+    }
+    const size_t samples = ((panel.width() + 1) / 2) * ((panel.height() + 1) / 2);
+    if (samples == 0 || white < samples * 6 / 10){
+        return -1;
+    }
+    ImageViewRGB32 area = extract_box_reference(screen, m_arrows);
+    std::vector<ImagePixelBox> arrows = find_name_list_arrows(area);
+    if (arrows.size() != 1){
+        return -1;
+    }
+    const double center_y = m_arrows.y +
+        m_arrows.height * (arrows[0].min_y + arrows[0].max_y) / 2.0 / (double)area.height();
+    const int row = (int)std::lround((center_y - FIRST_ROW_Y) / ROW_STEP_Y);
+    return row >= 0 && row < ROWS ? row : -1;
+}
+
+
+} // namespace PokemonBDSP
+} // namespace NintendoSwitch
+} // namespace PokemonAutomation

SerialPrograms/Source/PokemonBDSP/Inference/PokemonBDSP_NameListDetector.h

@@ -0,0 +1,42 @@
+/*  Name List Detector
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ *  The list of preset names shown for the rival: "I'll tell you...", then four names.
+ *
+ */
+
+#ifndef PokemonAutomation_PokemonBDSP_NameListDetector_H
+#define PokemonAutomation_PokemonBDSP_NameListDetector_H
+
+#include "CommonFramework/ImageTools/ImageBoxes.h"
+#include "CommonTools/VisualDetector.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonBDSP{
+
+
+//  Detects the rival preset name selection menu and reads the selected row.
+class NameListDetector : public StaticScreenDetector{
+public:
+    NameListDetector(Color color = COLOR_MAGENTA);
+
+    virtual void make_overlays(VideoOverlaySet& items) const override;
+    virtual bool detect(const ImageViewRGB32& screen) override;
+
+    //  The row the arrow is on: 0 is "I'll tell you...", 1 the first preset name.
+    //  -1 if the list is not showing.
+    int selected_row(const ImageViewRGB32& screen) const;
+
+private:
+    Color m_color;
+    ImageFloatBox m_arrows;
+    ImageFloatBox m_panel;
+};
+
+
+} // namespace PokemonBDSP
+} // namespace NintendoSwitch
+} // namespace PokemonAutomation
+#endif

SerialPrograms/Source/PokemonBDSP/PokemonBDSP_Tests.cpp

@@ -7,6 +7,7 @@
 #include "Common/Cpp/TestRunners/UnitTestDatabase.h"
 #include "Inference/PokemonBDSP_DialogDetector.h"
 #include "Inference/PokemonBDSP_MenuDetector.h"
+#include "Programs/AutoStory/Actions/PokemonBDSP_AutoStory_Actions_Tests.h"
 #include "Programs/AutoStory/PokemonBDSP_AutoStory_Maps_Tests.h"
 #include "Programs/AutoStory/PokemonBDSP_AutoStory_PlannedRoute_Tests.h"
 #include "Programs/AutoStory/PokemonBDSP_AutoStory_RoutePlanner_Tests.h"
@@ -25,6 +26,7 @@ namespace PokemonBDSP{
 void add_tests(UnitTestDatabase& database){
     add_tests_DialogDetector(database);
     add_tests_MenuDetector(database);
+    add_tests_AutoStory_Actions(database);
     add_tests_AutoStory_Maps(database);
     add_tests_AutoStory_StoryWorld(database);
     add_tests_AutoStory_RoutePlanner(database);