From 70e78a1ffc1ba62d4f2091193e28d9625564fc05 Mon Sep 17 00:00:00 2001 From: dayfine Date: Thu, 29 Oct 2026 17:38:29 -0700 Subject: [PATCH 01/10] Pokemon navigation: grid types and a leg planner over a game's step model The planner searches for the cheapest legs (hold one direction, then the next) and prefers legs that end against a wall. Each game supplies what one press of a direction does; the planner doesn't know any tile legend. Tested on a small made-up game. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../ComputerPrograms_UnitTestRunner.cpp | 2 + .../Pokemon/Navigation/Pokemon_Grid.cpp | 49 +++ .../Source/Pokemon/Navigation/Pokemon_Grid.h | 57 ++++ .../Pokemon/Navigation/Pokemon_LegPlanner.h | 285 ++++++++++++++++++ .../Navigation/Pokemon_LegPlanner_Tests.cpp | 259 ++++++++++++++++ .../Navigation/Pokemon_LegPlanner_Tests.h | 22 ++ SerialPrograms/cmake/SourceFiles.cmake | 5 + 7 files changed, 679 insertions(+) create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_Grid.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_Grid.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.h diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 1c19573466..0024cd1902 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -27,6 +27,7 @@ #include "NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h" #include "NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.h" +#include "Pokemon/Navigation/Pokemon_LegPlanner_Tests.h" #include "VideoGames/PokemonHome/PokemonHome_Tests.h" #include "VideoGames/PokemonSwSh/PokemonSwSh_Tests.h" #include "VideoGames/PokemonLA/PokemonLA_Tests.h" @@ -58,6 +59,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ add_tests_AudioFileLoader(ret); OCR::add_tests(ret); Kernels::add_tests(ret); + Pokemon::add_tests_LegPlanner(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); NintendoSwitch::add_tests_UpdatePopupDetector(ret); diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_Grid.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_Grid.cpp new file mode 100644 index 0000000000..fb4d5fb037 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_Grid.cpp @@ -0,0 +1,49 @@ +/* Grid + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Pokemon_Grid.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +const char* direction_name(Direction direction){ + switch (direction){ + case Direction::NORTH: + return "north"; + case Direction::EAST: + return "east"; + case Direction::SOUTH: + return "south"; + case Direction::WEST: + return "west"; + } + return "?"; +} + + +TilePos TilePos::next(Direction direction) const{ + switch (direction){ + case Direction::NORTH: + return {x, y - 1}; + case Direction::EAST: + return {x + 1, y}; + case Direction::SOUTH: + return {x, y + 1}; + case Direction::WEST: + return {x - 1, y}; + } + return *this; +} + +std::string to_string(const TilePos& pos){ + return "(" + std::to_string(pos.x) + ", " + std::to_string(pos.y) + ")"; +} + + + +} +} diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_Grid.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_Grid.h new file mode 100644 index 0000000000..409e6fe06e --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_Grid.h @@ -0,0 +1,57 @@ +/* Grid + * + * From: https://github.com/PokemonAutomation/ + * + * Tiles and directions on an overworld grid, shared by the games that + * move the player a tile at a time. + * + */ + +#ifndef PokemonAutomation_Pokemon_Grid_H +#define PokemonAutomation_Pokemon_Grid_H + +#include +#include +#include + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Row 0 is the top of a map, so SOUTH increases y. +enum class Direction{ + NORTH, + EAST, + SOUTH, + WEST, +}; +constexpr Direction ALL_DIRECTIONS[] = { + Direction::NORTH, Direction::EAST, Direction::SOUTH, Direction::WEST, +}; +const char* direction_name(Direction direction); + + +struct TilePos{ + int x = 0; + int y = 0; + + TilePos next(Direction direction) const; + bool operator==(const TilePos& other) const = default; +}; +std::string to_string(const TilePos& pos); + + + +} +} + + +template <> +struct std::hash{ + size_t operator()(const PokemonAutomation::Pokemon::TilePos& pos) const{ + return std::hash()(((long long)pos.x << 32) ^ (unsigned)pos.y); + } +}; + + +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner.h new file mode 100644 index 0000000000..5fbde90341 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner.h @@ -0,0 +1,285 @@ +/* Leg Planner + * + * From: https://github.com/PokemonAutomation/ + * + * Plans a walk as straight legs: hold one direction, then the next. The + * game decides what a single press does through its step model; the + * planner searches, joins steps into legs, and prefers legs that end + * against a wall, because the player stops there however long the button + * is held, while a leg that ends in the open depends on timing. + * + * A step model is a class with: + * + * using State = ...; // where the player is and what moves them: + * // comparable with == + * using StateHash = ...; // a hash of State, e.g. std::hash + * std::optional> press(const State& state, Direction direction) const; + * bool held_at(const State& state, Direction direction) const; + * + * press() is what pressing `direction` once does, or nothing if the player + * can't go that way (a wall, or something the plan must not do). held_at() + * is whether holding `direction` keeps the player on this tile: a wall, but + * not a person who may walk off or an edge to a map that isn't loaded. + * + */ + +#ifndef PokemonAutomation_Pokemon_LegPlanner_H +#define PokemonAutomation_Pokemon_LegPlanner_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include "Pokemon_Grid.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// What one press does. +template +struct Step{ + State next; + // 0 for a plain step. Anything else is a step of the game's own kind + // (e.g. starting to Surf), planned as a leg on its own. + int action = 0; + // The step can't continue the leg before it, e.g. leaving the water, + // after which the player walks at another speed. + bool starts_leg = false; + // No later step continues this leg: the game stops the player, e.g. + // at a warp. + bool ends_leg = false; + // The step takes a warp: a door, a ladder, a cave entrance. + bool warped = false; + // Which way the player faces afterwards. Usually the pressed direction; + // some warps turn the player. + Direction facing = Direction::NORTH; + // Cost on top of LegCosts::per_tile: tall grass, a warp's fade, a dialog. + int extra_cost = 0; +}; + + +enum class LegKind{ + WALK, // hold `direction` for `tiles` steps + ACTION, // one step of the game's own kind, `action` + FACE, // turn to `direction` without moving +}; + +template +struct Leg{ + LegKind kind = LegKind::WALK; + int action = 0; + Direction direction = Direction::NORTH; + int tiles = 0; + // WALK: the last step takes a warp. + bool warps = false; + // WALK: holding on keeps the player on the last tile, so the leg can be + // held for longer than it takes and walked again after an interruption. + bool to_wall = false; + Direction facing_after = Direction::NORTH; + State start; + State end; +}; + + +struct LegCosts{ + int per_tile = 1; + // Extra for a step in a new direction. + int per_turn = 2; + // Extra for ending a leg where the player could keep going: it has to + // be timed, and where an interruption leaves the player is a guess. + int per_open_stop = 8; +}; + + +// The cheapest legs from `start` to a state where `at_goal` holds, facing +// `goal_facing` if given, or nothing if there is none. `start_facing` is +// the way the player faces at the start, if known. +template +std::optional>> plan_legs( + const Model& model, + const typename Model::State& start, std::optional start_facing, + const std::function& at_goal, + std::optional goal_facing, + const LegCosts& costs = LegCosts() +); + + + +namespace LegPlannerInternal{ + +// A search node: the player, the way they face (4: not known), and whether +// they are in the middle of a leg that could go on. +template +struct Node{ + State state; + int facing = 4; + bool moving = false; + + bool operator==(const Node& other) const = default; +}; + +template +struct NodeHash{ + size_t operator()(const Node& node) const{ + return StateHash()(node.state) * 10 + node.facing * 2 + (node.moving ? 1 : 0); + } +}; + +template +struct Move{ + Node from; + Direction direction; + Step step; +}; + +template +int stop_cost(const Model& model, const Node& node, const LegCosts& costs){ + if (!node.moving || model.held_at(node.state, (Direction)node.facing)){ + return 0; + } + return costs.per_open_stop; +} + +template +std::vector> join_legs(const Model& model, const State& start, const std::vector>& moves){ + std::vector> legs; + bool can_extend = false; + State before = start; + for (const Move& move : moves){ + const Step& step = move.step; + const bool extends = can_extend && step.action == 0 && !step.starts_leg + && legs.back().kind == LegKind::WALK && legs.back().direction == move.direction; + if (extends){ + legs.back().tiles++; + }else{ + Leg leg; + leg.kind = step.action == 0 ? LegKind::WALK : LegKind::ACTION; + leg.action = step.action; + leg.direction = move.direction; + leg.tiles = 1; + leg.start = before; + legs.push_back(leg); + } + Leg& leg = legs.back(); + leg.warps = step.warped; + leg.to_wall = leg.kind == LegKind::WALK && !step.warped && model.held_at(step.next, move.direction); + leg.facing_after = step.facing; + leg.end = step.next; + can_extend = step.action == 0 && !step.ends_leg; + before = step.next; + } + return legs; +} + +} + + + +template +std::optional>> plan_legs( + const Model& model, + const typename Model::State& start, std::optional start_facing, + const std::function& at_goal, + std::optional goal_facing, + const LegCosts& costs +){ + using State = typename Model::State; + using Node = LegPlannerInternal::Node; + using Move = LegPlannerInternal::Move; + using Hash = LegPlannerInternal::NodeHash; + const int INF = std::numeric_limits::max(); + + std::unordered_map distance; + std::unordered_map came_from; + auto distance_of = [&](const Node& node){ + auto iter = distance.find(node); + return iter == distance.end() ? INF : iter->second; + }; + auto arrival_cost = [&](const Node& node){ + int cost = LegPlannerInternal::stop_cost(model, node, costs); + if (goal_facing && node.facing != (int)*goal_facing){ + cost += costs.per_turn; + } + return cost; + }; + + const Node first{start, start_facing ? (int)*start_facing : 4, false}; + distance[first] = 0; + + // Nodes are kept in a list and queued by index, so State needs no ordering. + std::vector nodes{first}; + using Entry = std::pair; + std::priority_queue, std::greater> queue; + queue.emplace(0, 0); + + std::optional best_end; + int best_total = INF; + while (!queue.empty()){ + const auto [cost, index] = queue.top(); + queue.pop(); + const Node current = nodes[index]; + if (cost != distance_of(current)){ + continue; + } + if (cost >= best_total){ + break; + } + if (at_goal(current.state)){ + const int total = cost + arrival_cost(current); + if (total < best_total){ + best_total = total; + best_end = current; + } + continue; + } + for (Direction direction : ALL_DIRECTIONS){ + std::optional> step = model.press(current.state, direction); + if (!step){ + continue; + } + const bool continues = step->action == 0 && !step->ends_leg; + const Node next{step->next, (int)step->facing, continues}; + int step_cost = costs.per_tile + step->extra_cost; + if (current.facing != (int)direction){ + step_cost += costs.per_turn + LegPlannerInternal::stop_cost(model, current, costs); + } + if (cost + step_cost < distance_of(next)){ + distance[next] = cost + step_cost; + came_from.insert_or_assign(next, Move{current, direction, *step}); + nodes.push_back(next); + queue.emplace(cost + step_cost, nodes.size() - 1); + } + } + } + if (!best_end){ + return std::nullopt; + } + + std::vector moves; + for (Node node = *best_end; !(node == first); node = came_from.at(node).from){ + moves.push_back(came_from.at(node)); + } + std::reverse(moves.begin(), moves.end()); + std::vector> legs = LegPlannerInternal::join_legs(model, start, moves); + if (goal_facing && best_end->facing != (int)*goal_facing){ + Leg face; + face.kind = LegKind::FACE; + face.direction = *goal_facing; + face.facing_after = *goal_facing; + face.start = best_end->state; + face.end = best_end->state; + legs.push_back(face); + } + return legs; +} + + + +} +} +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp new file mode 100644 index 0000000000..e239c2274f --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp @@ -0,0 +1,259 @@ +/* Leg Planner Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The planner on a small made-up game, with the kinds of tiles the real + * games have: walls, people, tall grass, a one-way tile and a warp. + * + */ + +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Pokemon_LegPlanner.h" +#include "Pokemon_LegPlanner_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +// '#' wall, '.' floor, '"' tall grass, 'p' a person (blocks, but may walk +// off, so not a wall to stop against), 'v' entered only going south, 'W' +// a warp to the tile marked 'X'. +class ToyGame{ +public: + using State = TilePos; + using StateHash = std::hash; + + explicit ToyGame(std::vector rows) + : m_rows(std::move(rows)) + { + for (int y = 0; y < (int)m_rows.size(); y++){ + for (int x = 0; x < (int)m_rows[y].size(); x++){ + if (m_rows[y][x] == 'X'){ + m_warp_to = {x, y}; + } + } + } + } + + std::optional> press(const State& state, Direction direction) const{ + const TilePos next = state.next(direction); + const char tile = at(next); + if (tile == '#' || tile == 'p' || (tile == 'v' && direction != Direction::SOUTH)){ + return std::nullopt; + } + Step step; + step.next = next; + step.facing = direction; + if (tile == '"'){ + step.extra_cost = 5; + } + if (tile == 'W'){ + step.next = m_warp_to; + step.warped = true; + step.ends_leg = true; + step.extra_cost = 10; + } + return step; + } + bool held_at(const State& state, Direction direction) const{ + const char tile = at(state.next(direction)); + return tile == '#' || (tile == 'v' && direction != Direction::SOUTH); + } + +private: + char at(const TilePos& pos) const{ + if (pos.y < 0 || pos.y >= (int)m_rows.size() || pos.x < 0 || pos.x >= (int)m_rows[pos.y].size()){ + return '#'; + } + return m_rows[pos.y][pos.x]; + } + + std::vector m_rows; + TilePos m_warp_to; +}; + + +std::string describe(const std::vector>& legs){ + std::string text; + for (const Leg& leg : legs){ + text += "\n "; + text += leg.kind == LegKind::FACE ? "face " : "walk "; + text += direction_name(leg.direction); + if (leg.kind == LegKind::WALK){ + text += " " + std::to_string(leg.tiles); + } + if (leg.to_wall){ + text += " (to wall)"; + } + if (leg.warps){ + text += " (warp)"; + } + text += " -> " + to_string(leg.end); + } + return text; +} + +std::optional>> plan( + const ToyGame& game, TilePos from, TilePos to, + std::optional start_facing = std::nullopt, + std::optional goal_facing = std::nullopt +){ + return plan_legs(game, from, start_facing, std::function([to](const TilePos& pos){ return pos == to; }), goal_facing); +} + + +UnitTestResult test_prefers_wall_legs(){ + // Two ways of equal length: east then south ends both legs in the open; + // south then east ends the last one against the pillar. + const ToyGame game({ + "......", + "...#..", + "......", + }); + const auto legs = plan(game, {0, 0}, {2, 1}); + if (!legs || legs->size() != 2){ + return "Expected 2 legs:" + (legs ? describe(*legs) : std::string(" none")); + } + if ((*legs)[1].direction != Direction::EAST || !(*legs)[1].to_wall){ + return "Expected to end going east into the pillar:" + describe(*legs); + } + return true; +} + +UnitTestResult test_person_is_not_a_wall(){ + // The only way ends next to a person, which isn't a wall to stop against. + const ToyGame game({ + "#####", + "#..p#", + "#####", + }); + const auto legs = plan(game, {1, 1}, {2, 1}); + if (!legs || legs->size() != 1 || (*legs)[0].to_wall){ + return "Expected one open leg:" + (legs ? describe(*legs) : std::string(" none")); + } + return true; +} + +UnitTestResult test_avoids_grass(){ + // Straight through the grass is 4 tiles; around it is 8, cheaper with + // 5 extra per grass tile. + const ToyGame game({ + ".....", + ".\"\"\".", + ".....", + }); + const auto legs = plan(game, {0, 1}, {4, 1}); + if (!legs){ + return "No route."; + } + for (const Leg& leg : *legs){ + if (leg.end.y == 1 && leg.end.x > 0 && leg.end.x < 4){ + return "Walked into the grass:" + describe(*legs); + } + } + return true; +} + +UnitTestResult test_one_way_tile(){ + // The 'v' can only be entered going south, so going back up must go around. + const ToyGame game({ + "#.#.#", + "#v#.#", + "#...#", + "#####", + }); + if (!plan(game, {1, 0}, {1, 2})){ + return "No route down through the one-way tile."; + } + const auto up = plan(game, {1, 2}, {1, 0}); + if (up){ + return "Went up through the one-way tile:" + describe(*up); + } + return true; +} + +UnitTestResult test_warp_ends_leg(){ + // The warp on the left takes the player to X on the right. + const ToyGame game({ + "#######", + "#W.#X.#", + "#..#..#", + "#######", + }); + const auto legs = plan(game, {2, 2}, {5, 2}); + if (!legs){ + return "No route."; + } + bool warped = false; + for (const Leg& leg : *legs){ + if (leg.warps){ + warped = true; + if (!(leg.end == TilePos{4, 1})){ + return "The warp leg ends at " + to_string(leg.end) + ":" + describe(*legs); + } + } + } + if (!warped){ + return "Expected a warp leg:" + describe(*legs); + } + return true; +} + +UnitTestResult test_goal_facing(){ + const ToyGame game({ + "....", + }); + const auto legs = plan(game, {0, 0}, {2, 0}, std::nullopt, Direction::NORTH); + if (!legs || legs->back().kind != LegKind::FACE || legs->back().direction != Direction::NORTH){ + return "Expected to end facing north:" + (legs ? describe(*legs) : std::string(" none")); + } + return true; +} + +UnitTestResult test_unreachable(){ + const ToyGame game({ + ".#.", + }); + if (plan(game, {0, 0}, {2, 0})){ + return "Planned through a wall."; + } + return true; +} + + +class Test_LegPlanner : public UnitTest{ +public: + Test_LegPlanner(std::string name, std::function body) + : UnitTest("Pokemon::LegPlanner - " + std::move(name)) + , m_body(std::move(body)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_body(); + } + +private: + std::function m_body; +}; + +} + + +void add_tests_LegPlanner(UnitTestDatabase& database){ + database.add("prefers legs that end at a wall", test_prefers_wall_legs); + database.add("a person is not a wall", test_person_is_not_a_wall); + database.add("avoids tall grass", test_avoids_grass); + database.add("one-way tile", test_one_way_tile); + database.add("a warp ends the leg", test_warp_ends_leg); + database.add("turns to face the goal", test_goal_facing); + database.add("nothing through a wall", test_unreachable); +} + + + +} +} diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.h new file mode 100644 index 0000000000..537de3309f --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.h @@ -0,0 +1,22 @@ +/* Leg Planner Tests + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#ifndef PokemonAutomation_Pokemon_LegPlanner_Tests_H +#define PokemonAutomation_Pokemon_LegPlanner_Tests_H + +#include "Common/Cpp/TestRunners/UnitTestDatabase.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +void add_tests_LegPlanner(UnitTestDatabase& database); + + + +} +} +#endif diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 9e4e0895bb..8fe4f292f7 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1425,6 +1425,11 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/Inference/Pokemon_TrainPokemonOCR.h Source/Pokemon/Inference/Pokemon_TypeReader.cpp Source/Pokemon/Inference/Pokemon_TypeReader.h + Source/Pokemon/Navigation/Pokemon_Grid.cpp + Source/Pokemon/Navigation/Pokemon_Grid.h + Source/Pokemon/Navigation/Pokemon_LegPlanner.h + Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp + Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.h Source/Pokemon/Options/Pokemon_BoxSortingTable.cpp Source/Pokemon/Options/Pokemon_BoxSortingTable.h Source/Pokemon/Options/Pokemon_EncounterBotOptions.h From 3bd531e64b28ca9251f502b3a6e1a689fb19aea2 Mon Sep 17 00:00:00 2001 From: dayfine Date: Thu, 29 Oct 2026 17:41:35 -0700 Subject: [PATCH 02/10] Pokemon navigation: leg walker with game-supplied movers and interrupts Holds one leg while watching for what can stop it part-way. On an interrupt, a leg to a wall is held again and an open leg walks what is left, estimated from when the interrupt was seen. The bookkeeping is a plain function, tested with scripted holds. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../ComputerPrograms_UnitTestRunner.cpp | 3 +- .../Navigation/Pokemon_LegPlanner_Tests.cpp | 2 +- .../Pokemon/Navigation/Pokemon_LegWalker.cpp | 70 ++++++ .../Pokemon/Navigation/Pokemon_LegWalker.h | 212 ++++++++++++++++++ .../Navigation/Pokemon_LegWalker_Tests.cpp | 176 +++++++++++++++ ...ner_Tests.h => Pokemon_Navigation_Tests.h} | 7 +- SerialPrograms/cmake/SourceFiles.cmake | 5 +- 7 files changed, 469 insertions(+), 6 deletions(-) create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker.cpp create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker.h create mode 100644 SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker_Tests.cpp rename SerialPrograms/Source/Pokemon/Navigation/{Pokemon_LegPlanner_Tests.h => Pokemon_Navigation_Tests.h} (53%) diff --git a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp index 0024cd1902..6c957de605 100644 --- a/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp +++ b/SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp @@ -27,7 +27,7 @@ #include "NintendoSwitch/Inference/NintendoSwitch_UpdatePopupDetector.h" #include "NintendoSwitch/Programs/Typewriter/NintendoSwitch_Typewriter_Tests.h" -#include "Pokemon/Navigation/Pokemon_LegPlanner_Tests.h" +#include "Pokemon/Navigation/Pokemon_Navigation_Tests.h" #include "VideoGames/PokemonHome/PokemonHome_Tests.h" #include "VideoGames/PokemonSwSh/PokemonSwSh_Tests.h" #include "VideoGames/PokemonLA/PokemonLA_Tests.h" @@ -60,6 +60,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){ OCR::add_tests(ret); Kernels::add_tests(ret); Pokemon::add_tests_LegPlanner(ret); + Pokemon::add_tests_LegWalker(ret); NintendoSwitch::add_tests_CheckOnlineDetector(ret); NintendoSwitch::add_tests_FailedToConnectDetector(ret); NintendoSwitch::add_tests_UpdatePopupDetector(ret); diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp index e239c2274f..136cb9d28c 100644 --- a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp @@ -12,7 +12,7 @@ #include #include "Common/Cpp/TestRunners/UnitTest.h" #include "Pokemon_LegPlanner.h" -#include "Pokemon_LegPlanner_Tests.h" +#include "Pokemon_Navigation_Tests.h" namespace PokemonAutomation{ namespace Pokemon{ diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker.cpp new file mode 100644 index 0000000000..40f082e2f8 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker.cpp @@ -0,0 +1,70 @@ +/* Leg Walker + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include "Pokemon_LegWalker.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +std::string to_string(const LegSpec& leg){ + return std::string(direction_name(leg.direction)) + " " + std::to_string(leg.tiles) + + (leg.to_wall ? " (to wall)" : ""); +} + + +Milliseconds leg_hold(const LegSpec& leg, int tiles){ + if (leg.to_wall){ + return leg.tile_time * (tiles + 2); + } + return leg.tile_time * (tiles - 1) + leg.tile_time / 2; +} + +int tiles_before_interrupt(const LegSpec& leg, Milliseconds elapsed, Milliseconds detection_delay, int remaining){ + const Milliseconds since_first_step = elapsed - detection_delay + leg.tile_time / 2; + if (since_first_step.count() <= 0){ + return 0; + } + return std::clamp((int)(since_first_step / leg.tile_time), 0, remaining); +} + + +LegResult walk_leg_with(const LegSpec& leg, const LegCalls& calls, size_t max_interruptions){ + LegResult result; + int remaining = leg.tiles; + while (true){ + const std::optional outcome = calls.hold_watching(leg_hold(leg, remaining)); + if (!outcome){ + result.tiles = leg.tiles; + return result; + } + result.interruptions++; + if (result.interruptions > max_interruptions){ + calls.fail("Interrupted " + std::to_string(result.interruptions) + " times on: " + to_string(leg)); + } + calls.clear(outcome->interrupt); + const std::string seen = calls.name(outcome->interrupt) + " after " + std::to_string(outcome->elapsed.count()) + "ms"; + if (leg.to_wall){ + // The wall stops the player on the leg's last tile however far + // they got, so the leg is simply held again. + calls.log(seen + ", walking to the wall again."); + continue; + } + const int walked = tiles_before_interrupt(leg, outcome->elapsed, calls.detection_delay(outcome->interrupt), remaining); + remaining -= walked; + result.exact = false; + calls.log(seen + ", about " + std::to_string(walked) + " tiles walked, " + std::to_string(remaining) + " to go."); + if (remaining <= 0){ + result.tiles = leg.tiles; + return result; + } + } +} + + + +} +} diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker.h new file mode 100644 index 0000000000..2d471b3eb9 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker.h @@ -0,0 +1,212 @@ +/* Leg Walker + * + * From: https://github.com/PokemonAutomation/ + * + * Walks one leg of a plan: holds a direction for the leg's tiles while + * watching for whatever can stop the player part-way (a wild battle, a + * message, a trainer). Each of those is a LegInterrupt the game supplies. + * When one shows, the hold stops, the game clears it, and the rest of the + * leg is walked: a leg to a wall is simply held again, since the wall stops + * the player on its tile; for an open leg, how far the player got is worked + * out from when the interrupt was seen. + * + * How a direction is held (D-pad or stick) and how long a tile takes are + * the game's too, through LegMover. Templated on the console and the + * controller context, like run_until(), so this stays free of any one + * console's types. + * + */ + +#ifndef PokemonAutomation_Pokemon_LegWalker_H +#define PokemonAutomation_Pokemon_LegWalker_H + +#include +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonTools/Async/InferenceRoutines.h" +#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h" +#include "Pokemon_Grid.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +// Something that stops the player part-way through a leg. +template +class LegInterrupt{ +public: + virtual ~LegInterrupt() = default; + + virtual std::string name() const = 0; + + // A new watcher for the screen the interruption brings up. + virtual std::unique_ptr make_watcher() const = 0; + + // How long after the step that set it off the watcher fires. The games + // only interrupt between steps, so from this the walker works out how + // many tiles of an open leg were walked. + virtual Milliseconds detection_delay() const = 0; + + // Clears it, leaving the player standing on the overworld, on the same + // tile and facing the same way. + virtual void clear(Console& console, Context& context) const = 0; +}; + +template +using LegInterrupts = std::vector>>; + + +// How the game moves the player. +template +class LegMover{ +public: + virtual ~LegMover() = default; + // Holds `direction` for `duration`, then lets go. + virtual void hold(Context& context, Direction direction, Milliseconds duration) const = 0; +}; + + +// A straight leg, for a player standing still and facing its way. +struct LegSpec{ + Direction direction = Direction::NORTH; + int tiles = 0; + // A wall stops the player after the last tile. + bool to_wall = false; + // How long one tile takes, for this leg's direction and way of moving. + Milliseconds tile_time{0}; +}; +std::string to_string(const LegSpec& leg); + +struct LegResult{ + // Tiles walked, which is `tiles` unless the leg gave up. + int tiles = 0; + // Whether `tiles` is certain: an interrupted open leg is an estimate. + bool exact = true; + size_t interruptions = 0; +}; + + +// How long to hold for `tiles` tiles: an open leg lets go half a tile before +// the last step would end, so the step under way finishes and no other +// starts; a leg to a wall is held two tiles longer than it takes. +Milliseconds leg_hold(const LegSpec& leg, int tiles); + +// How many tiles an open leg had walked when an interrupt was seen +// `elapsed` into its hold, given the interrupt's detection delay. Steps end +// a whole number of tiles into the hold, so this rounds to the nearest. +int tiles_before_interrupt(const LegSpec& leg, Milliseconds elapsed, Milliseconds detection_delay, int remaining); + + +// What one hold reported: nothing if it ran out, or which interrupt showed +// and how long into the hold. +struct HoldOutcome{ + size_t interrupt = 0; + Milliseconds elapsed{0}; +}; + +// The leg's bookkeeping, with the console side passed in, so it can be +// tested with scripted outcomes. `hold_watching` holds for the duration and +// reports the first interrupt seen; `clear` clears interrupt i; +// `detection_delay` is interrupt i's delay; `log` reports progress; `fail` +// throws, and is called after more than `max_interruptions`. +struct LegCalls{ + std::function(Milliseconds hold)> hold_watching; + std::function clear; + std::function detection_delay; + std::function name; + std::function log; + std::function fail; +}; +LegResult walk_leg_with(const LegSpec& leg, const LegCalls& calls, size_t max_interruptions); + + +// Walks `leg`. Throws OperationFailedException if it is interrupted more +// than `max_interruptions` times. +template +LegResult walk_leg( + Console& console, Context& context, + const LegMover& mover, + const LegInterrupts& interrupts, + const LegSpec& leg, + size_t max_interruptions = 5 +){ + auto hold_watching = [&](Milliseconds hold) -> std::optional{ + std::vector> watchers; + std::vector callbacks; + for (const auto& interrupt : interrupts){ + watchers.emplace_back(interrupt->make_watcher()); + callbacks.emplace_back(*watchers.back()); + } + context.wait_for_all_requests(); + WallClock start = current_time(); + const int ret = run_until( + console, context, + [&](Context& context){ + start = current_time(); + mover.hold(context, leg.direction, hold); + }, + callbacks + ); + if (ret < 0){ + return std::nullopt; + } + return HoldOutcome{(size_t)ret, std::chrono::duration_cast(current_time() - start)}; + }; + LegCalls calls; + calls.hold_watching = hold_watching; + calls.clear = [&](size_t index){ interrupts[index]->clear(console, context); }; + calls.detection_delay = [&](size_t index){ return interrupts[index]->detection_delay(); }; + calls.name = [&](size_t index){ return interrupts[index]->name(); }; + calls.log = [&](const std::string& message){ console.log("Leg: " + message, COLOR_ORANGE); }; + calls.fail = [&](const std::string& message){ + OperationFailedExceptionWithScreenshot::fire(ErrorReportMode::SEND_ERROR_REPORT, message, console); + }; + return walk_leg_with(leg, calls, max_interruptions); +} + + +// After a leg: waits `watch` for an interrupt the last step set off, which +// shows after the hold has ended, and clears it. Throws +// OperationFailedException after `max_interruptions` in a row. +template +void clear_late_interrupts( + Console& console, Context& context, + const LegInterrupts& interrupts, + Milliseconds watch, + size_t max_interruptions = 5 +){ + for (size_t count = 0; ; count++){ + std::vector> watchers; + std::vector callbacks; + for (const auto& interrupt : interrupts){ + watchers.emplace_back(interrupt->make_watcher()); + callbacks.emplace_back(*watchers.back()); + } + context.wait_for_all_requests(); + const int ret = wait_until(console, context, watch, callbacks); + if (ret < 0){ + return; + } + if (count >= max_interruptions){ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + "Interrupted " + std::to_string(count) + " times in a row after a leg.", + console + ); + } + console.log("Leg: " + interrupts[ret]->name() + " at the end of the leg.", COLOR_ORANGE); + interrupts[ret]->clear(console, context); + } +} + + + + +} +} +#endif diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker_Tests.cpp b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker_Tests.cpp new file mode 100644 index 0000000000..c0ef205267 --- /dev/null +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegWalker_Tests.cpp @@ -0,0 +1,176 @@ +/* Leg Walker Tests + * + * From: https://github.com/PokemonAutomation/ + * + * The leg walker's bookkeeping, with holds that report scripted + * interruptions instead of a console. + * + */ + +#include +#include +#include +#include +#include +#include "Common/Cpp/TestRunners/UnitTest.h" +#include "Pokemon_LegWalker.h" +#include "Pokemon_Navigation_Tests.h" + +namespace PokemonAutomation{ +namespace Pokemon{ + + +namespace{ + +using namespace std::chrono_literals; + +constexpr Milliseconds TILE = 267ms; +constexpr Milliseconds DELAY = 300ms; + +// Holds that report the scripted outcomes in order, then run out. +struct ScriptedLeg{ + std::deque> outcomes; + std::vector holds; + std::vector cleared; + + LegCalls calls(){ + LegCalls calls; + calls.hold_watching = [this](Milliseconds hold) -> std::optional{ + holds.push_back(hold); + if (outcomes.empty()){ + return std::nullopt; + } + std::optional outcome = outcomes.front(); + outcomes.pop_front(); + return outcome; + }; + calls.clear = [this](size_t index){ cleared.push_back(index); }; + calls.detection_delay = [](size_t){ return DELAY; }; + calls.name = [](size_t){ return std::string("a battle"); }; + calls.log = [](const std::string&){}; + calls.fail = [](const std::string& message){ throw std::runtime_error(message); }; + return calls; + } +}; + +// An interrupt set off by step `step`, seen DELAY after it ends. +HoldOutcome after_step(int step){ + return HoldOutcome{0, TILE * step + DELAY}; +} + + +UnitTestResult test_hold_times(){ + const LegSpec open{Direction::EAST, 4, false, TILE}; + const LegSpec wall{Direction::EAST, 4, true, TILE}; + if (leg_hold(open, 4) != TILE * 3 + TILE / 2){ + return "Open leg: " + std::to_string(leg_hold(open, 4).count()) + "ms"; + } + if (leg_hold(wall, 4) != TILE * 6){ + return "Wall leg: " + std::to_string(leg_hold(wall, 4).count()) + "ms"; + } + return true; +} + +UnitTestResult test_tiles_before_interrupt(){ + const LegSpec leg{Direction::EAST, 8, false, TILE}; + for (int step = 0; step <= 8; step++){ + for (Milliseconds jitter : {-100ms, 0ms, 100ms}){ + const int walked = tiles_before_interrupt(leg, TILE * step + DELAY + jitter, DELAY, 8); + if (walked != step){ + return "Step " + std::to_string(step) + " seen " + std::to_string(jitter.count()) + "ms off read as " + std::to_string(walked); + } + } + } + if (tiles_before_interrupt(leg, 50ms, DELAY, 8) != 0){ + return "An interrupt from before the leg should count no tiles."; + } + if (tiles_before_interrupt(leg, TILE * 20, DELAY, 3) != 3){ + return "Should not count more tiles than remain."; + } + return true; +} + +UnitTestResult test_wall_leg_walked_again(){ + ScriptedLeg script; + script.outcomes = {after_step(2)}; + const LegSpec leg{Direction::NORTH, 6, true, TILE}; + const LegResult result = walk_leg_with(leg, script.calls(), 5); + if (script.holds.size() != 2 || script.holds[1] != leg_hold(leg, 6)){ + return "Expected the whole leg held again."; + } + if (!result.exact || result.interruptions != 1 || script.cleared.size() != 1){ + return "Expected an exact result after one interruption."; + } + return true; +} + +UnitTestResult test_open_leg_rest_walked(){ + ScriptedLeg script; + script.outcomes = {after_step(2)}; + const LegSpec leg{Direction::NORTH, 6, false, TILE}; + const LegResult result = walk_leg_with(leg, script.calls(), 5); + if (script.holds.size() != 2 || script.holds[1] != leg_hold(leg, 4)){ + return "Expected the remaining 4 tiles held."; + } + if (result.exact){ + return "An interrupted open leg is an estimate."; + } + return true; +} + +UnitTestResult test_open_leg_interrupted_at_end(){ + ScriptedLeg script; + script.outcomes = {after_step(6)}; + const LegSpec leg{Direction::NORTH, 6, false, TILE}; + walk_leg_with(leg, script.calls(), 5); + if (script.holds.size() != 1){ + return "Nothing was left to walk after the last step."; + } + return true; +} + +UnitTestResult test_too_many_interruptions(){ + ScriptedLeg script; + for (int c = 0; c < 4; c++){ + script.outcomes.push_back(after_step(0)); + } + const LegSpec leg{Direction::NORTH, 6, true, TILE}; + try{ + walk_leg_with(leg, script.calls(), 3); + }catch (std::runtime_error&){ + return true; + } + return "Expected to give up after 3 interruptions."; +} + + +class Test_LegWalker : public UnitTest{ +public: + Test_LegWalker(std::string name, std::function body) + : UnitTest("Pokemon::LegWalker - " + std::move(name)) + , m_body(std::move(body)) + {} + virtual UnitTestResult run(Logger&, CancellableScope&) const override{ + return m_body(); + } + +private: + std::function m_body; +}; + +} + + +void add_tests_LegWalker(UnitTestDatabase& database){ + database.add("hold times", test_hold_times); + database.add("tiles before an interrupt", test_tiles_before_interrupt); + database.add("a wall leg is walked again", test_wall_leg_walked_again); + database.add("an open leg walks the rest", test_open_leg_rest_walked); + database.add("an open leg interrupted at its end", test_open_leg_interrupted_at_end); + database.add("gives up after too many interruptions", test_too_many_interruptions); +} + + + +} +} diff --git a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.h b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_Navigation_Tests.h similarity index 53% rename from SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.h rename to SerialPrograms/Source/Pokemon/Navigation/Pokemon_Navigation_Tests.h index 537de3309f..fa7fd8e612 100644 --- a/SerialPrograms/Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.h +++ b/SerialPrograms/Source/Pokemon/Navigation/Pokemon_Navigation_Tests.h @@ -1,11 +1,11 @@ -/* Leg Planner Tests +/* Navigation Tests * * From: https://github.com/PokemonAutomation/ * */ -#ifndef PokemonAutomation_Pokemon_LegPlanner_Tests_H -#define PokemonAutomation_Pokemon_LegPlanner_Tests_H +#ifndef PokemonAutomation_Pokemon_Navigation_Tests_H +#define PokemonAutomation_Pokemon_Navigation_Tests_H #include "Common/Cpp/TestRunners/UnitTestDatabase.h" @@ -14,6 +14,7 @@ namespace Pokemon{ void add_tests_LegPlanner(UnitTestDatabase& database); +void add_tests_LegWalker(UnitTestDatabase& database); diff --git a/SerialPrograms/cmake/SourceFiles.cmake b/SerialPrograms/cmake/SourceFiles.cmake index 8fe4f292f7..e632bd254c 100644 --- a/SerialPrograms/cmake/SourceFiles.cmake +++ b/SerialPrograms/cmake/SourceFiles.cmake @@ -1429,7 +1429,10 @@ file(GLOB LIBRARY_SOURCES Source/Pokemon/Navigation/Pokemon_Grid.h Source/Pokemon/Navigation/Pokemon_LegPlanner.h Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.cpp - Source/Pokemon/Navigation/Pokemon_LegPlanner_Tests.h + Source/Pokemon/Navigation/Pokemon_LegWalker.cpp + Source/Pokemon/Navigation/Pokemon_LegWalker.h + Source/Pokemon/Navigation/Pokemon_LegWalker_Tests.cpp + Source/Pokemon/Navigation/Pokemon_Navigation_Tests.h Source/Pokemon/Options/Pokemon_BoxSortingTable.cpp Source/Pokemon/Options/Pokemon_BoxSortingTable.h Source/Pokemon/Options/Pokemon_EncounterBotOptions.h From 0f59863af9df1dcb82b3a0fea47b17aec6acf32b Mon Sep 17 00:00:00 2001 From: dayfine Date: Wed, 28 Oct 2026 01:34:01 -0700 Subject: [PATCH 03/10] Add a screen machine and step trace for closed-loop routines Routines that mash a button or hold a direction for a fixed time break when an input drops or a stray press opens a menu, and nothing notices until much later. run_screen_machine() is the closed-loop alternative: it reads the screen (two frames must agree), takes the one step that state calls for, and fails on a screen the previous state cannot lead to, a state seen too often or too many times in a row, eight steps that change nothing, or the timeout. A capture that drops out for up to 10s is waited out. StepTrace logs each change of recognized screen with a numbered screenshot and each input, and a failure lists the screens that led to it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AKYzhrMnPK61XHWEch4a6E --- .../ScreenMachine/ScreenMachine.cpp | 97 ++++++++++++++++ .../CommonTools/ScreenMachine/ScreenMachine.h | 105 ++++++++++++++++++ .../CommonTools/ScreenMachine/StepTrace.cpp | 74 ++++++++++++ .../CommonTools/ScreenMachine/StepTrace.h | 53 +++++++++ SerialPrograms/cmake/SourceFiles.cmake | 4 + 5 files changed, 333 insertions(+) create mode 100644 SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp create mode 100644 SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h create mode 100644 SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp create mode 100644 SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp new file mode 100644 index 0000000000..8d528c4eaa --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.cpp @@ -0,0 +1,97 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "CommonFramework/ImageTools/ImageDiff.h" +#include "ScreenMachine.h" + +namespace PokemonAutomation{ + +using namespace std::chrono_literals; + + +namespace{ + +const ScreenState* classify(const std::vector& states, const ImageViewRGB32& screen){ + for (const ScreenState& state : states){ + if (state.matches(screen)){ + return &state; + } + } + return nullptr; +} + +} + + +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, StepTrace& trace, + const std::vector& states, VideoSnapshot& frame +){ + const ScreenState* last = nullptr; + WallClock video_lost = WallClock::max(); + for (size_t c = 0; c < 20; c++){ + VideoSnapshot screen = stream.video().snapshot(); + if (!screen){ + // The app resets a frozen capture by itself, which takes a second or two. + const WallClock now = current_time(); + video_lost = std::min(video_lost, now); + if (now - video_lost > std::chrono::seconds(10)){ + trace.fail("No video from the capture card for 10s."); + } + c--; + scope.wait_for(500ms); + continue; + } + video_lost = WallClock::max(); + const ScreenState* now = classify(states, screen); + frame = std::move(screen); + if (now != nullptr && now == last){ + return *now; + } + last = now; + scope.wait_for(150ms); + } + if (last == nullptr){ + trace.fail("The screen matches none of the known states."); + } + return *last; // Still changing after 3s: go with the latest read. +} + + +void ScreenMachineChecks::check(const ScreenState& state, VideoSnapshot frame){ + const bool same_size = m_last_frame && frame && + m_last_frame->width() == frame->width() && m_last_frame->height() == frame->height(); + if (same_size && ImageMatch::pixel_RMSD(m_last_frame, frame) < 2.0){ + if (++m_unchanged >= MAX_UNCHANGED){ + m_trace.fail("The screen stopped changing: " + std::to_string(MAX_UNCHANGED) + + " steps on \"" + state.name + "\" did nothing."); + } + }else{ + m_unchanged = 0; + } + m_last_frame = std::move(frame); + + if (m_previous != nullptr && m_previous != &state && !m_previous->next.empty() && + std::find(m_previous->next.begin(), m_previous->next.end(), state.name) == m_previous->next.end() + ){ + m_trace.fail("Unexpected screen after \"" + m_previous->name + "\": \"" + state.name + "\"."); + } + m_in_a_row = m_previous == &state ? m_in_a_row + 1 : 1; + if (m_in_a_row > state.max_in_a_row){ + m_trace.fail("Screen \"" + state.name + "\" came up " + std::to_string(m_in_a_row) + + " times in a row: the step for it is not getting anywhere."); + } + if (++m_visits[state.name] > state.max_visits){ + m_trace.fail("Screen \"" + state.name + "\" came up more than " + + std::to_string(state.max_visits) + " times: the step for it is not working."); + } + m_previous = &state; +} + + + +} diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h new file mode 100644 index 0000000000..f878ff709c --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/ScreenMachine.h @@ -0,0 +1,105 @@ +/* Screen Machine + * + * From: https://github.com/PokemonAutomation/ + * + * A closed loop over known screens: read the screen, take the one step that screen + * calls for, read again. Every step follows a read, a screen counts only once two + * frames agree, and each state lists the screens it may lead to, so a dropped or + * wrong input shows up as an unexpected screen or a state that keeps coming back. + * + */ + +#ifndef PokemonAutomation_CommonTools_ScreenMachine_H +#define PokemonAutomation_CommonTools_ScreenMachine_H + +#include +#include +#include +#include +#include +#include "Common/Cpp/Time.h" +#include "Common/Cpp/CancellableScope.h" +#include "CommonFramework/ImageTypes/ImageViewRGB32.h" +#include "CommonFramework/Tools/VideoStream.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +struct ScreenState{ + std::string name; + // Whether this is the screen. States are tried in order; the first match wins, so + // the last state should match anything. + std::function matches; + // The one step to take on this screen. + std::function act; + // The screens the step may lead to besides staying here. Empty: any. + std::vector next; + // Fail once the screen has been seen this many times. + size_t max_visits; + // End the run after this state's step. + bool terminal = false; + // Fail once the screen has been seen this many times in a row. + size_t max_in_a_row = SIZE_MAX; +}; + + +// The state on screen, once two frames 150ms apart agree. Waits out a video +// gap of up to 10s. Fails through `trace` if no state matches. +const ScreenState& read_screen_state( + VideoStream& stream, CancellableScope& scope, StepTrace& trace, + const std::vector& states, VideoSnapshot& frame +); + +// The checks between reads: the screen must be one the previous state can lead +// to, no state may come up too often, and steps must keep changing the screen. +class ScreenMachineChecks{ +public: + ScreenMachineChecks(StepTrace& trace) + : m_trace(trace) + {} + void check(const ScreenState& state, VideoSnapshot frame); + +private: + // A step that changes nothing on screen is a dead end (e.g. a menu the states do + // not know): fail after a few instead of running out the visit limit. + static constexpr size_t MAX_UNCHANGED = 8; + + StepTrace& m_trace; + std::map m_visits; + const ScreenState* m_previous = nullptr; + VideoSnapshot m_last_frame; + size_t m_unchanged = 0; + size_t m_in_a_row = 0; +}; + + +// Runs the states until a terminal one has acted. Returns its name. Fails through +// `trace` on an unexpected screen, a state seen too often, a stall or the timeout. +template +std::string run_screen_machine( + VideoStream& stream, ControllerContext& context, StepTrace& trace, + const std::vector& states, Milliseconds timeout +){ + ScreenMachineChecks checks(trace); + const WallClock deadline = current_time() + timeout; + while (current_time() < deadline){ + context.wait_for_all_requests(); + VideoSnapshot frame; + const ScreenState& state = read_screen_state(stream, context, trace, states, frame); + trace.screen(state.name); + checks.check(state, std::move(frame)); + state.act(); + if (state.terminal){ + context.wait_for_all_requests(); + return state.name; + } + } + trace.fail("Did not finish within " + std::to_string(timeout.count() / 1000) + "s."); +} + + + +} +#endif diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp new file mode 100644 index 0000000000..246bac04a0 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.cpp @@ -0,0 +1,74 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + */ + +#include +#include "Common/Cpp/PrettyPrint.h" +#include "Common/Cpp/Filesystem/Filesystem.h" +#include "CommonFramework/GlobalAutoPaths.h" +#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h" +#include "CommonFramework/VideoPipeline/VideoFeed.h" +#include "StepTrace.h" + +namespace PokemonAutomation{ + + +StepTrace::StepTrace(VideoStream& stream, std::string label) + : m_stream(stream) + , m_label(std::move(label)) + , m_folder(SCREENSHOTS_PATH() + "StepTrace/" + now_to_filestring() + "-" + m_label + "/") + , m_start(current_time()) +{ + Filesystem::create_directories(Filesystem::Path(m_folder)); + m_stream.log("[" + m_label + "] Trace screenshots: " + m_folder); +} + +void StepTrace::screen(const std::string& name){ + if (name == m_last){ + return; + } + m_last = name; + m_recent.push_back(name); + if (m_recent.size() > 12){ + m_recent.pop_front(); + } + m_count++; + const double seconds = std::chrono::duration(current_time() - m_start).count(); + char prefix[16]; + std::snprintf(prefix, sizeof(prefix), "%03zu-", m_count); + std::string path = m_folder + prefix + name + ".png"; + VideoSnapshot screen = m_stream.video().snapshot(); + if (screen){ + screen.frame->save(path); + }else{ + path = "(no video frame)"; + } + m_stream.log("[" + m_label + "] screen " + std::to_string(m_count) + ": " + name + + " at " + tostr_fixed(seconds, 1) + "s, " + path); +} + +void StepTrace::action(const std::string& what){ + m_stream.log("[" + m_label + "] " + what); +} + +std::string StepTrace::recent_screens() const{ + std::string text; + for (const std::string& name : m_recent){ + text += (text.empty() ? "" : " -> ") + name; + } + return text.empty() ? "(none)" : text; +} + +void StepTrace::fail(const std::string& message) const{ + OperationFailedExceptionWithScreenshot::fire( + ErrorReportMode::SEND_ERROR_REPORT, + m_label + ": " + message + " Recent screens: " + recent_screens() + ". Trace: " + m_folder, + m_stream + ); +} + + + +} diff --git a/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h new file mode 100644 index 0000000000..e0c4db09d0 --- /dev/null +++ b/SerialPrograms/Source/CommonTools/ScreenMachine/StepTrace.h @@ -0,0 +1,53 @@ +/* Step Trace + * + * From: https://github.com/PokemonAutomation/ + * + * A step log for closed-loop routines: every change of recognized screen is + * logged and saved as a numbered screenshot, every input is logged, and a + * failure reports the screens that led up to it. + * + */ + +#ifndef PokemonAutomation_CommonTools_StepTrace_H +#define PokemonAutomation_CommonTools_StepTrace_H + +#include +#include +#include "Common/Cpp/Time.h" +#include "CommonFramework/Tools/VideoStream.h" + +namespace PokemonAutomation{ + + +class StepTrace{ +public: + // Screenshots go to /StepTrace/