FRLG AutoStory: replay the 00.27 Route 10 door failures from their frames

SerialPrograms/Source/ComputerPrograms/ComputerPrograms_UnitTestRunner.cpp

@@ -56,7 +56,6 @@ namespace ComputerPrograms{
 
 UnitTestDatabase make_UNIT_TESTS_ALL(){
     UnitTestDatabase ret;
-
     add_tests_QtEventThreadPool(ret, GlobalThreadPools::computation_normal());
     add_tests_Sockets(ret, GlobalThreadPools::unlimited_normal());
     add_tests_BlackBorderDetector(ret);
@@ -83,6 +82,7 @@ UnitTestDatabase make_UNIT_TESTS_ALL(){
     Pokemon::add_tests_ScreenEvents(ret);
     Pokemon::add_tests_Interrupts(ret);
     Pokemon::add_tests_Transitions(ret);
+    Pokemon::add_tests_FrameReplay(ret);
     Pokemon::add_tests_StoryActions(ret);
     Pokemon::add_tests_StoryInterrupts(ret);
     Pokemon::add_tests_StoryEvents(ret);

SerialPrograms/Source/Pokemon/AutoStory/Pokemon_AutoStory_Tests.h

@@ -21,6 +21,8 @@ void add_tests_ScreenEvents(UnitTestDatabase& database);
 void add_tests_Interrupts(UnitTestDatabase& database);
 //  The transition tracker and the games' fade profiles.
 void add_tests_Transitions(UnitTestDatabase& database);
+//  Recorded frames replayed through the detectors on a virtual clock.
+void add_tests_FrameReplay(UnitTestDatabase& database);
 //  Story action tables, the variant parser and the table executor.
 void add_tests_StoryActions(UnitTestDatabase& database);
 //  The table executor's looks for interrupts between steps.

SerialPrograms/Source/Pokemon/AutoStory/Pokemon_FrameReplay.cpp

@@ -0,0 +1,121 @@
+/*  Frame Replay
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ */
+
+#include <algorithm>
+#include <cctype>
+#include "Common/Cpp/Exceptions.h"
+#include "Common/Cpp/Filesystem/Filesystem.h"
+#include "Pokemon_FrameReplay.h"
+
+namespace PokemonAutomation{
+namespace Pokemon{
+
+
+namespace{
+
+//  The time a frame file's name gives ("1250.png" -> 1250), or -1 when the
+//  name is not all digits or the file not a .png.
+long long frame_time_of(const Filesystem::Path& file){
+    if (file.extension().string() != ".png"){
+        return -1;
+    }
+    const std::string stem = file.stem().string();
+    if (stem.empty() || stem.size() > 9){
+        return -1;
+    }
+    for (char c : stem){
+        if (!std::isdigit((unsigned char)c)){
+            return -1;
+        }
+    }
+    return std::stoll(stem);
+}
+
+} // namespace
+
+
+ReplayFrameSource::ReplayFrameSource(std::vector<ReplayFrame> frames)
+    : m_frames(std::move(frames))
+{
+    std::stable_sort(m_frames.begin(), m_frames.end(), [](const ReplayFrame& a, const ReplayFrame& b){
+        return a.at < b.at;
+    });
+}
+
+ReplayFrameSource ReplayFrameSource::from_folder(const std::string& folder){
+    std::vector<ReplayFrame> frames;
+    if (Filesystem::is_directory(folder)){
+        for (const Filesystem::Path& file : Filesystem::list_directory(folder)){
+            const long long ms = frame_time_of(file);
+            if (ms >= 0){
+                frames.push_back(ReplayFrame{.at = Milliseconds(ms), .file = file.string()});
+            }
+        }
+    }
+    if (frames.empty()){
+        throw FileException(nullptr, PA_CURRENT_FUNCTION, "No <ms>.png frames to replay.", folder);
+    }
+    return ReplayFrameSource(std::move(frames));
+}
+
+Milliseconds ReplayFrameSource::last_frame() const{
+    return m_frames.empty() ? Milliseconds(0) : m_frames.back().at;
+}
+
+VideoSnapshot ReplayFrameSource::frame_at(Milliseconds at, WallClock origin) const{
+    auto after = std::upper_bound(m_frames.begin(), m_frames.end(), at, [](Milliseconds time, const ReplayFrame& frame){
+        return time < frame.at;
+    });
+    if (after == m_frames.begin()){
+        return VideoSnapshot();
+    }
+    ReplayFrame& frame = *(after - 1);
+    if (!frame.image){
+        frame.image = std::make_shared<const ImageRGB32>(frame.file);
+    }
+    VideoSnapshot out;
+    out.frame = frame.image;
+    out.timestamp = origin + at;
+    return out;
+}
+
+
+ReplayHit replay_until(
+    const ReplayFrameSource& source, ReplayWindow window,
+    const std::vector<VisualInferenceCallback*>& callbacks,
+    WallClock origin, Milliseconds period
+){
+    for (Milliseconds now = window.start; now < window.end; now += period){
+        const VideoSnapshot frame = source.frame_at(now, origin);
+        if (!frame){
+            continue;
+        }
+        for (size_t c = 0; c < callbacks.size(); c++){
+            if (callbacks[c]->process_frame(frame)){
+                return ReplayHit{.index = (int)c, .at = now};
+            }
+        }
+    }
+    return ReplayHit{};
+}
+
+ScreenEventLog replay_events(
+    const ReplayFrameSource& source, ReplayWindow window,
+    ScreenEventObserver& observer,
+    WallClock origin, Milliseconds period
+){
+    ScreenEventLog log;
+    for (Milliseconds now = window.start; now < window.end; now += period){
+        const VideoSnapshot frame = source.frame_at(now, origin);
+        log.record(observer.observe(frame, origin + now), origin + now);
+    }
+    return log;
+}
+
+
+
+} // namespace Pokemon
+} // namespace PokemonAutomation

SerialPrograms/Source/Pokemon/AutoStory/Pokemon_FrameReplay.h

@@ -0,0 +1,115 @@
+/*  Frame Replay
+ *
+ *  From: https://github.com/PokemonAutomation/
+ *
+ *  Recorded console frames, replayed through the real detectors on a
+ *  virtual clock, so a console failure becomes a deterministic test. The
+ *  story sims script what a screen shows (a label per frame); a replay
+ *  shows what the console showed and lets the game's own watchers and
+ *  ScreenEventObserver judge it.
+ *
+ *  A recording is a folder of frames named by their time in ms from its
+ *  start ("0.png", "1250.png"), as the captured transitions
+ *  (Pokemon_Transitions_Tests.cpp). A frame shows until the next one.
+ *  Recordings are too big for this repo; tests read them from
+ *  CommandLineTests/<game>/FrameReplay/<case>/ and skip when it is missing.
+ *
+ *  From the stream history (an error report's Video.mp4, kept when the
+ *  "Stream History" setting is on), cut the seconds around the failure at
+ *  20 frames a second, then rename frame N to its time, (N - 1) * 50 ms:
+ *      ffmpeg -ss <start s> -t <seconds> -i Video.mp4 -vf fps=20 %d.png
+ *
+ *  One thread, no real time: the caller says which window of the recording
+ *  each action watched, and the replay feeds those frames at the inference
+ *  period, as run_until / wait_until would.
+ *
+ */
+
+#ifndef PokemonAutomation_Pokemon_FrameReplay_H
+#define PokemonAutomation_Pokemon_FrameReplay_H
+
+#include <memory>
+#include <string>
+#include <vector>
+#include "Common/Cpp/Time.h"
+#include "CommonFramework/ImageTypes/ImageRGB32.h"
+#include "CommonFramework/VideoPipeline/VideoFeed.h"
+#include "CommonTools/InferenceCallbacks/VisualInferenceCallback.h"
+#include "Pokemon_ScreenEvents.h"
+
+namespace PokemonAutomation{
+namespace Pokemon{
+
+
+//  One recorded frame: when it showed, from the recording's start, and the
+//  image, or the file it loads from on first use.
+struct ReplayFrame{
+    Milliseconds at = Milliseconds(0);
+    std::string file = {};
+    std::shared_ptr<const ImageRGB32> image = {};
+};
+
+
+//  Recorded frames, served by virtual time. Not thread-safe: images load
+//  on first use.
+class ReplayFrameSource{
+public:
+    explicit ReplayFrameSource(std::vector<ReplayFrame> frames);
+
+    //  Every "<ms>.png" in `folder`. Throws FileException if it has none.
+    static ReplayFrameSource from_folder(const std::string& folder);
+
+    size_t size() const{
+        return m_frames.size();
+    }
+    //  When the last frame showed.
+    Milliseconds last_frame() const;
+
+    //  The frame showing at `at`: the last one at or before it, stamped
+    //  `origin + at`. None before the first frame.
+    VideoSnapshot frame_at(Milliseconds at, WallClock origin) const;
+
+private:
+    mutable std::vector<ReplayFrame> m_frames;
+};
+
+
+//  How often a replay looks: the inference loop's default period.
+constexpr Milliseconds REPLAY_PERIOD = Milliseconds(50);
+
+//  Part of a recording an action watched, in ms from its start, end
+//  excluded: the hold of a leg run_until watched, or a wait_until after it.
+struct ReplayWindow{
+    Milliseconds start = Milliseconds(0);
+    Milliseconds end = Milliseconds(0);
+};
+
+//  The first callback that stopped a replay, and when.
+struct ReplayHit{
+    //  Its index in the callbacks; -1 if none did.
+    int index = -1;
+    Milliseconds at = Milliseconds(0);
+};
+
+//  Feeds `callbacks` the frames of `window`, one look per `period`, each
+//  callback in order on each look, until one returns true: wait_until on a
+//  recording. Frames are stamped `origin + time`.
+ReplayHit replay_until(
+    const ReplayFrameSource& source, ReplayWindow window,
+    const std::vector<VisualInferenceCallback*>& callbacks,
+    WallClock origin, Milliseconds period = REPLAY_PERIOD
+);
+
+//  What `observer` sees over `window`, one look per `period`, as a
+//  ScreenEventWatcher's log; the judges (Pokemon_Interrupts.h) read it.
+ScreenEventLog replay_events(
+    const ReplayFrameSource& source, ReplayWindow window,
+    ScreenEventObserver& observer,
+    WallClock origin, Milliseconds period = REPLAY_PERIOD
+);
+
+
+
+} // namespace Pokemon
+} // namespace PokemonAutomation
+#endif