AutoStory: run checkpoint retries through a shared loop behind game hooks

SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStoryRetry.cpp

@@ -0,0 +1,47 @@
+/*  AutoStory Retry
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include "Pokemon_AutoStoryRetry.h"
+
+namespace PokemonAutomation{
+namespace Pokemon{
+
+
+void run_checkpoint_with_retries(
+    CheckpointRetryHooks& hooks,
+    const CheckpointRetryPolicy& policy,
+    const std::string& checkpoint_text,
+    const std::function<void(size_t attempt)>& action
+){
+    for (size_t attempt = 0;; attempt++){
+        try{
+            if (attempt == 0){
+                hooks.save_checkpoint();
+            }
+            hooks.before_attempt(attempt);
+            hooks.log(checkpoint_text + " Num attempts: " + std::to_string(attempt));
+
+            hooks.wait_for_all_requests();
+            action(attempt);
+            hooks.after_success();
+            return;
+        }catch (OperationFailedException& e){
+            if (attempt == policy.report_at_attempt){
+                hooks.report_recoverable(std::to_string(attempt) + " failed attempts. " + checkpoint_text);
+            }
+            if (attempt > policy.max_attempts){
+                hooks.fail(e);
+            }
+            hooks.wait_for_all_requests();
+            hooks.log("Resetting from checkpoint.");
+            hooks.reset_for_retry();
+        }
+    }
+}
+
+
+}
+}

SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStoryRetry.h

@@ -0,0 +1,68 @@
+/*  AutoStory Retry
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#ifndef PokemonAutomation_Pokemon_AutoStoryRetry_H
+#define PokemonAutomation_Pokemon_AutoStoryRetry_H
+
+#include <functional>
+#include <string>
+#include "CommonFramework/Exceptions/OperationFailedException.h"
+
+namespace PokemonAutomation{
+namespace Pokemon{
+
+
+//  The game steps around an AutoStory checkpoint attempt. Each game supplies its
+//  own; tests supply a scripted one.
+class CheckpointRetryHooks{
+public:
+    virtual ~CheckpointRetryHooks() = default;
+
+    virtual void log(const std::string& message) = 0;
+
+    //  Wait for the controller commands sent so far to finish.
+    virtual void wait_for_all_requests() = 0;
+
+    //  Save the game before the first attempt. A save that throws counts as a
+    //  failed attempt, and later attempts do not save again.
+    virtual void save_checkpoint() = 0;
+
+    //  Before each attempt, after the save.
+    virtual void before_attempt(size_t attempt) = 0;
+
+    //  After the checkpoint's action returns. Throwing makes the attempt fail.
+    virtual void after_success() = 0;
+
+    //  Report a checkpoint that keeps failing, without stopping.
+    virtual void report_recoverable(const std::string& message) = 0;
+
+    //  Give up on the checkpoint. Must throw.
+    [[noreturn]] virtual void fail(const OperationFailedException& e) = 0;
+
+    //  Reload the last save for the next attempt.
+    virtual void reset_for_retry() = 0;
+};
+
+struct CheckpointRetryPolicy{
+    //  Give up once the attempt number is greater than this.
+    size_t max_attempts = 100;
+    //  Report once, when this attempt fails.
+    size_t report_at_attempt = 10;
+};
+
+//  Save, then run `action` until it returns, reloading the save after each attempt
+//  that throws OperationFailedException. Other exceptions are not caught.
+void run_checkpoint_with_retries(
+    CheckpointRetryHooks& hooks,
+    const CheckpointRetryPolicy& policy,
+    const std::string& checkpoint_text,
+    const std::function<void(size_t attempt)>& action
+);
+
+
+}
+}
+#endif

SerialPrograms/Source/VideoGames/PokemonSV/PokemonSV_Tests.cpp

@@ -5,6 +5,7 @@
  */
 
 #include "Common/Cpp/TestRunners/UnitTestDatabase.h"
+#include "Pokemon/AutoStory/Pokemon_AutoStoryRetry.h"
 #include "Pokemon/AutoStory/Pokemon_AutoStorySelect.h"
 #include "Inference/PokemonSV_WhiteButtonDetector.h"
 #include "Inference/Boxes/PokemonSV_BoxDetection.h"
@@ -97,6 +98,141 @@ class Test_AutoStorySelectDatabase : public UnitTest{
     }
 };
 
+// Checkpoint steps that record what runs, and fail the attempts they are told to.
+class ScriptedCheckpoint : public Pokemon::CheckpointRetryHooks{
+public:
+    std::set<size_t> failing_actions;
+    std::set<size_t> failing_checks;
+    bool save_fails = false;
+    std::vector<std::string> steps;
+
+    virtual void log(const std::string&) override{}
+    virtual void wait_for_all_requests() override{}
+    virtual void save_checkpoint() override{
+        steps.push_back("save");
+        if (save_fails){
+            throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "save failed");
+        }
+    }
+    virtual void before_attempt(size_t attempt) override{
+        steps.push_back("before" + std::to_string(attempt));
+    }
+    void action(size_t attempt){
+        steps.push_back("action" + std::to_string(attempt));
+        if (failing_actions.contains(attempt)){
+            throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "action failed");
+        }
+        m_attempt = attempt;
+    }
+    virtual void after_success() override{
+        steps.push_back("check");
+        if (failing_checks.contains(m_attempt)){
+            throw OperationFailedException(ErrorReportMode::NO_ERROR_REPORT, "not in the overworld");
+        }
+    }
+    virtual void report_recoverable(const std::string& message) override{
+        steps.push_back("report(" + message + ")");
+    }
+    [[noreturn]] virtual void fail(const OperationFailedException&) override{
+        steps.push_back("fail");
+        throw std::runtime_error("gave up");
+    }
+    virtual void reset_for_retry() override{
+        steps.push_back("reset");
+    }
+
+private:
+    size_t m_attempt = 0;
+};
+
+struct CheckpointRetryScenario{
+    std::string name;
+    std::set<size_t> failing_actions = {};
+    std::set<size_t> failing_checks = {};
+    bool save_fails = false;
+    std::string expected_steps;
+    bool expected_give_up = false;
+};
+
+class Test_CheckpointRetry : public UnitTest{
+public:
+    Test_CheckpointRetry(CheckpointRetryScenario scenario)
+        : UnitTest("Pokemon::CheckpointRetry - " + scenario.name)
+        , m_scenario(std::move(scenario))
+    {}
+
+    virtual UnitTestResult run(Logger&, CancellableScope&) const override{
+        ScriptedCheckpoint hooks;
+        hooks.failing_actions = m_scenario.failing_actions;
+        hooks.failing_checks = m_scenario.failing_checks;
+        hooks.save_fails = m_scenario.save_fails;
+        Pokemon::CheckpointRetryPolicy policy;
+        policy.max_attempts = 2;
+        policy.report_at_attempt = 1;
+
+        bool gave_up = false;
+        try{
+            Pokemon::run_checkpoint_with_retries(hooks, policy, "Checkpoint 1.", [&](size_t attempt){
+                hooks.action(attempt);
+            });
+        }catch (const std::runtime_error&){
+            gave_up = true;
+        }
+        std::string steps;
+        for (const std::string& step : hooks.steps){
+            steps += (steps.empty() ? "" : " ") + step;
+        }
+        if (gave_up != m_scenario.expected_give_up){
+            return std::string("Expected ") + (m_scenario.expected_give_up ? "to give up" : "to finish") + ", steps: " + steps;
+        }
+        if (steps != m_scenario.expected_steps){
+            return "Expected steps: " + m_scenario.expected_steps + "\nActual steps:   " + steps;
+        }
+        return true;
+    }
+
+private:
+    const CheckpointRetryScenario m_scenario;
+};
+
+void add_tests_CheckpointRetry(UnitTestDatabase& database){
+    const CheckpointRetryScenario scenarios[] = {
+        {
+            .name = "first attempt succeeds",
+            .expected_steps = "save before0 action0 check",
+        },
+        {
+            // Only the first attempt saves; the retry starts from that save.
+            .name = "a failed attempt resets and retries without saving",
+            .failing_actions = {0},
+            .expected_steps = "save before0 action0 reset before1 action1 check",
+        },
+        {
+            // A save that fails is a failed attempt, and is not tried again.
+            .name = "a failed save is a failed attempt",
+            .save_fails = true,
+            .expected_steps = "save reset before1 action1 check",
+        },
+        {
+            // SV checks for the overworld after the action; not finding it retries.
+            .name = "a failed check after the action retries",
+            .failing_checks = {0},
+            .expected_steps = "save before0 action0 check reset before1 action1 check",
+        },
+        {
+            // With max 2 and the report at 1: attempts 0..2 retry, attempt 3 gives up.
+            .name = "reports once and gives up past the maximum",
+            .failing_actions = {0, 1, 2, 3},
+            .expected_steps = "save before0 action0 reset before1 action1 report(1 failed attempts. Checkpoint 1.) reset "
+                "before2 action2 reset before3 action3 fail",
+            .expected_give_up = true,
+        },
+    };
+    for (const CheckpointRetryScenario& scenario : scenarios){
+        database.add<Test_CheckpointRetry>(scenario);
+    }
+}
+
 void add_tests_AutoStory(UnitTestDatabase& database){
     const AutoStoryIndexScenario scenarios[] = {
         {
@@ -127,6 +263,7 @@ void add_tests_AutoStory(UnitTestDatabase& database){
         database.add<Test_AutoStoryIndex>(scenario);
     }
     database.add<Test_AutoStorySelectDatabase>();
+    add_tests_CheckpointRetry(database);
 }
 
 }

SerialPrograms/Source/VideoGames/PokemonSV/Programs/AutoStory/PokemonSV_AutoStoryCheckpointHooks.cpp

@@ -0,0 +1,94 @@
+/*  AutoStory Checkpoint Hooks
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include "CommonFramework/Exceptions/OperationFailedExceptionWithScreenshot.h"
+#include "CommonFramework/Notifications/ProgramNotifications.h"
+#include "CommonFramework/VideoPipeline/VideoFeed.h"
+#include "VideoGames/PokemonSV/Programs/PokemonSV_GameEntry.h"
+#include "VideoGames/PokemonSV/Programs/PokemonSV_MenuNavigation.h"
+#include "VideoGames/PokemonSV/Programs/PokemonSV_SaveGame.h"
+#include "PokemonSV_AutoStoryTools.h"
+#include "PokemonSV_AutoStoryCheckpointHooks.h"
+
+namespace PokemonAutomation{
+namespace NintendoSwitch{
+namespace PokemonSV{
+
+
+SVCheckpointHooks::SVCheckpointHooks(
+    SingleSwitchProgramEnvironment& env,
+    ProControllerContext& context,
+    EventNotificationOption& notif_status_update,
+    EventNotificationOption& notif_error_recoverable,
+    Pokemon::AutoStoryStats& stats,
+    std::string fail_message,
+    SVCheckpointMode mode
+)
+    : m_env(env), m_context(context)
+    , m_notif_status_update(notif_status_update)
+    , m_notif_error_recoverable(notif_error_recoverable)
+    , m_stats(stats)
+    , m_fail_message(std::move(fail_message))
+    , m_mode(mode)
+{}
+
+void SVCheckpointHooks::log(const std::string& message){
+    m_env.console.log(message);
+}
+void SVCheckpointHooks::wait_for_all_requests(){
+    m_context.wait_for_all_requests();
+}
+void SVCheckpointHooks::save_checkpoint(){
+    if (m_mode != SVCheckpointMode::TUTORIAL){
+        checkpoint_save(m_env, m_context, m_notif_status_update, m_stats);
+        return;
+    }
+    save_game_tutorial(m_env.program_info(), m_env.console, m_context);
+    m_stats.m_checkpoint++;
+    m_env.update_stats();
+    send_program_status_notification(m_env, m_notif_status_update, "Saved at checkpoint.");
+}
+void SVCheckpointHooks::before_attempt(size_t attempt){
+    if (m_mode == SVCheckpointMode::MAIN_STORY_DAY_SKIP && (attempt > 0 || ENABLE_TEST)){
+        day_skip_from_overworld(m_env.console, m_context);
+        save_game_from_overworld(m_env.program_info(), m_env.console, m_context);
+    }
+}
+void SVCheckpointHooks::after_success(){
+    if (m_mode != SVCheckpointMode::TUTORIAL){
+        enter_menu_from_overworld(m_env.program_info(), m_env.console, m_context, -1);
+    }
+}
+void SVCheckpointHooks::report_recoverable(const std::string& message){
+    //  Send an error report for debugging.
+    auto snapshot = m_env.console.video().snapshot().frame;
+    send_program_recoverable_error_notification_and_telemetry_report(
+        m_env, &m_env.logger(), m_env.program_info(),
+        m_notif_error_recoverable,
+        ErrorReportMode::SEND_ERROR_REPORT,
+        message,
+        "OperationFailedExceptionWithScreenshot",
+        *snapshot,
+        &m_env.console.history()
+    );
+}
+void SVCheckpointHooks::fail(const OperationFailedException& e){
+    OperationFailedExceptionWithScreenshot::fire(
+        ErrorReportMode::SEND_ERROR_REPORT,
+        m_fail_message + e.message(),
+        m_env.console
+    );
+}
+void SVCheckpointHooks::reset_for_retry(){
+    reset_game(m_env.program_info(), m_env.console, m_context);
+    m_stats.m_reset++;
+    m_env.update_stats();
+}
+
+
+}
+}
+}