Fix PartyKit esbuild advisory, clear PoseEstimator complexity, add iOS test target (#26)

.github/workflows/ci.yml

@@ -66,3 +66,18 @@ jobs:
         env:
           MOTION_INDEX_STORE: ${{ runner.temp }}/motion-derived/Index.noindex/DataStore
         run: pnpm quality:swift
+      - name: Run iOS unit tests
+        run: |
+          udid="$(xcrun simctl list devices available --json | python3 -c '
+          import json, sys
+          devices = json.load(sys.stdin)["devices"]
+          print(next(d["udid"] for runtime, items in sorted(devices.items(), reverse=True)
+                     if ".iOS-" in runtime for d in items if ".iPhone-" in d["deviceTypeIdentifier"]))
+          ')"
+          xcodebuild \
+            -workspace ios/Motion.xcworkspace \
+            -scheme Motion \
+            -destination "id=$udid" \
+            -derivedDataPath "$RUNNER_TEMP/motion-derived" \
+            CODE_SIGNING_ALLOWED=NO \
+            test

ios/Podfile

@@ -26,6 +26,11 @@ target 'Motion' do
   # `MPImage`, VIDEO running mode, etc. Pinned to a minor range so `pod install` is
   # reproducible while still picking up patch fixes.
   pod 'MediaPipeTasksVision', '~> 0.10.14'
+
+  # Unit tests run inside the app host, so they reuse its pods (search paths only).
+  target 'MotionTests' do
+    inherit! :search_paths
+  end
 end
 
 post_install do |installer|
@@ -39,3 +44,16 @@ post_install do |installer|
     end
   end
 end
+
+# The test bundle runs inside the app, which already force-loads MediaPipe's graph library.
+# CocoaPods copies that -force_load into the test target too; loading the graph twice aborts
+# the host at launch (duplicate class/graph registration). Keep it in the app only.
+post_integrate do |installer|
+  installer.aggregate_targets.select { |t| t.name == 'Pods-MotionTests' }.each do |target|
+    target.user_build_configurations.each_key do |config|
+      path = target.xcconfig_path(config)
+      lines = File.readlines(path).reject { |line| line.include?('-force_load') }
+      File.write(path, lines.join)
+    end
+  end
+end

ios/Podfile.lock

@@ -15,6 +15,6 @@ SPEC CHECKSUMS:
   MediaPipeTasksCommon: ae27d88c4d185db114a1a768c504f8270dda79b5
   MediaPipeTasksVision: 1d6c3efa89830a131dbd0dc4fb80dea283fac521
 
-PODFILE CHECKSUM: f8e3d2a0ce3856587e72d7a5725865203bd44dec
+PODFILE CHECKSUM: ecc473230b10c4c49d821baf000f8c481c49fdf9
 
 COCOAPODS: 1.17.0

ios/Sources/Motion/Vision/PoseMath.swift

@@ -0,0 +1,139 @@
+//
+//  PoseMath.swift
+//  Motion
+//
+//  The pure parts of `PoseEstimator`: joint mapping (y flip + mirrored left/right
+//  swap), exponential smoothing, quality, ROI geometry and ROI→full-image lifting.
+//  Nothing here runs a Vision request or touches a pixel buffer, so it is covered by
+//  `MotionTests/PoseMathTests.swift` without a camera. See PoseEstimator's header for
+//  why each convention holds; those comments are the source of truth.
+//
+
+import CoreGraphics
+import Vision
+
+/// A converted point (top-left origin, mirror space) and its Vision confidence.
+typealias PoseReading = (point: Point2, confidence: Double)
+
+/// The body wrists handed to the hand pass. A confidence of 0 marks a position carried
+/// over from a dropout, which is too stale to crop around or to assign hands by.
+struct WristReadings {
+    let left: Point2?
+    let leftConfidence: Double?
+    let right: Point2?
+    let rightConfidence: Double?
+
+    /// The left wrist only when it was detected this frame (confidence > 0).
+    var freshLeft: Point2? { (leftConfidence ?? 0) > 0 ? left : nil }
+    /// The right wrist only when it was detected this frame (confidence > 0).
+    var freshRight: Point2? { (rightConfidence ?? 0) > 0 ? right : nil }
+}
+
+enum PoseMath {
+    typealias BodyLookup = (VNHumanBodyPoseObservation.JointName) -> PoseReading?
+
+    /// Vision's bottom-left normalized point → the protocol's top-left. x is already in
+    /// mirror space (the connection mirrors the buffer), so only y flips.
+    static func topLeft(x: Double, y: Double) -> Point2 {
+        [x, 1.0 - y]
+    }
+
+    /// Midpoint of two optional readings (needs both). Confidence = min of the two.
+    static func midpoint(_ a: PoseReading?, _ b: PoseReading?) -> PoseReading? {
+        guard let a, let b else { return nil }
+        return ([(a.point[0] + b.point[0]) / 2, (a.point[1] + b.point[1]) / 2], min(a.confidence, b.confidence))
+    }
+
+    /// The 8 protocol joints from thresholded Vision points. Sides are SWAPPED because the
+    /// mirrored buffer puts Vision's LEFT on the player's RIGHT. Missing joints are absent.
+    static func bodyJoints(_ pt: BodyLookup) -> [JointName: PoseReading] {
+        var raw: [JointName: PoseReading] = [:]
+        // HEAD: prefer the nose; fall back to neck if the face is turned/occluded.
+        raw[.head] = pt(.nose) ?? pt(.neck)
+        raw[.rightHand] = pt(.leftWrist)
+        raw[.leftHand] = pt(.rightWrist)
+        // TORSO: `.root` is the pelvis; then the shoulder midpoint, then the hip midpoint.
+        raw[.torso] =
+            pt(.root)
+            ?? midpoint(pt(.leftShoulder), pt(.rightShoulder))
+            ?? midpoint(pt(.leftHip), pt(.rightHip))
+        raw[.rightKnee] = pt(.leftKnee)
+        raw[.leftKnee] = pt(.rightKnee)
+        // FEET: the 2D body model has no toe joint, so ankle == foot.
+        raw[.rightFoot] = pt(.leftAnkle)
+        raw[.leftFoot] = pt(.rightAnkle)
+        return raw
+    }
+
+    /// The optional arm chain (shoulders + elbows), swapped exactly like `bodyJoints`.
+    static func armJoints(_ pt: BodyLookup) -> [ArmJointName: PoseReading] {
+        var raw: [ArmJointName: PoseReading] = [:]
+        raw[.rightShoulder] = pt(.leftShoulder)
+        raw[.leftShoulder] = pt(.rightShoulder)
+        raw[.rightElbow] = pt(.leftElbow)
+        raw[.leftElbow] = pt(.rightElbow)
+        return raw
+    }
+
+    /// Exponential smoothing per joint: new = alpha*current + (1-alpha)*previous. A joint
+    /// seen for the first time adopts its value; a joint missing this frame reuses its last
+    /// smoothed value with confidence 0; a joint never seen stays absent.
+    static func smooth<Key: Hashable & CaseIterable>(
+        _ raw: [Key: PoseReading],
+        previous: inout [Key: Point2],
+        alpha a: Double
+    ) -> (points: [Key: Point2], confidences: [Key: Double]) {
+        var points: [Key: Point2] = [:]
+        var confidences: [Key: Double] = [:]
+        for name in Key.allCases {
+            if let (p, conf) = raw[name] {
+                if let prev = previous[name] {
+                    let sx = a * p[0] + (1 - a) * prev[0]
+                    let sy = a * p[1] + (1 - a) * prev[1]
+                    points[name] = [sx, sy]
+                    previous[name] = [sx, sy]
+                } else {
+                    points[name] = p
+                    previous[name] = p
+                }
+                confidences[name] = conf
+            } else if let prev = previous[name] {
+                points[name] = prev
+                confidences[name] = 0
+            }
+        }
+        return (points, confidences)
+    }
+
+    /// Aggregate confidence over ALL protocol joints (absent joints count as 0).
+    static func quality(_ confidences: [JointName: Double]) -> Double {
+        confidences.values.isEmpty
+            ? 0
+            : confidences.values.reduce(0, +) / Double(JointName.allCases.count)
+    }
+
+    /// A `size`×`size` Vision ROI (bottom-left origin) centered on a top-left wrist, with the
+    /// center clamped so the whole box stays inside [0,1] (Vision rejects an overflowing ROI).
+    static func roi(aroundTopLeftWrist wrist: Point2, size: CGFloat) -> CGRect {
+        let cx = CGFloat(wrist[0])  // mirror-space x, same as Vision
+        let cy = 1.0 - CGFloat(wrist[1])  // top-left y → Vision bottom-left y
+        let half = size / 2
+        let clampedCx = min(max(cx, half), 1 - half)
+        let clampedCy = min(max(cy, half), 1 - half)
+        return CGRect(x: clampedCx - half, y: clampedCy - half, width: size, height: size)
+    }
+
+    /// Lift an ROI-relative hand landmark (confirmed on device: Vision returns hand points
+    /// relative to the crop) into the full image, then flip to top-left. A full-frame ROI
+    /// is an identity apart from the flip.
+    static func liftFromROI(_ location: CGPoint, roi: CGRect) -> Point2 {
+        let fullX = Double(roi.minX + location.x * roi.width)  // full-image, mirror-space
+        let fullYBottomLeft = Double(roi.minY + location.y * roi.height)
+        return [fullX, 1.0 - fullYBottomLeft]  // bottom-left → top-left
+    }
+
+    /// Fingertips for the packet: omitted entirely only when BOTH sides are nil.
+    static func fingertips(left: Point2?, right: Point2?) -> Fingertips? {
+        (left == nil && right == nil) ? nil : Fingertips(left: left, right: right)
+    }
+}

ios/Tests/MotionTests/PoseMathTests.swift

@@ -0,0 +1,165 @@
+//
+//  PoseMathTests.swift
+//  MotionTests
+//
+//  Pins the coordinate and smoothing contract `PoseEstimator` relies on: y flip,
+//  mirrored left/right swap, fallbacks, dropout reuse, quality, ROI clamping and
+//  ROI→full-image lifting. Camera, Vision requests and MediaPipe stay device-only.
+//
+
+import CoreGraphics
+import Testing
+import Vision
+
+@testable import Motion
+
+private typealias VisionJoint = VNHumanBodyPoseObservation.JointName
+
+private func lookup(_ readings: [VisionJoint: PoseReading]) -> PoseMath.BodyLookup {
+    { readings[$0] }
+}
+
+private func approx(_ a: Point2?, _ b: Point2, tolerance: Double = 1e-12) -> Bool {
+    guard let a, a.count == b.count else { return false }
+    return zip(a, b).allSatisfy { abs($0 - $1) <= tolerance }
+}
+
+@Suite struct PoseMathMappingTests {
+    @Test func topLeftFlipsOnlyY() {
+        #expect(PoseMath.topLeft(x: 0.2, y: 0.9) == [0.2, 1.0 - 0.9])
+    }
+
+    @Test func bodyJointsSwapSidesForTheMirroredBuffer() {
+        let raw = PoseMath.bodyJoints(
+            lookup([
+                .leftWrist: ([0.1, 0.1], 0.9), .rightWrist: ([0.9, 0.1], 0.8),
+                .leftKnee: ([0.2, 0.7], 0.7), .rightKnee: ([0.8, 0.7], 0.6),
+                .leftAnkle: ([0.2, 0.9], 0.5), .rightAnkle: ([0.8, 0.9], 0.4),
+            ]))
+        #expect(raw[.rightHand]?.point == [0.1, 0.1])
+        #expect(raw[.leftHand]?.point == [0.9, 0.1])
+        #expect(raw[.rightKnee]?.confidence == 0.7)
+        #expect(raw[.leftKnee]?.confidence == 0.6)
+        #expect(raw[.rightFoot]?.confidence == 0.5)
+        #expect(raw[.leftFoot]?.confidence == 0.4)
+        #expect(raw[.head] == nil)
+        #expect(raw[.torso] == nil)
+    }
+
+    @Test func headPrefersNoseThenNeck() {
+        let both = PoseMath.bodyJoints(lookup([.nose: ([0.5, 0.1], 0.9), .neck: ([0.5, 0.2], 0.8)]))
+        #expect(both[.head]?.point == [0.5, 0.1])
+        let neckOnly = PoseMath.bodyJoints(lookup([.neck: ([0.5, 0.2], 0.8)]))
+        #expect(neckOnly[.head]?.point == [0.5, 0.2])
+    }
+
+    @Test func torsoFallsBackFromRootToShouldersToHips() {
+        let root = PoseMath.bodyJoints(
+            lookup([.root: ([0.5, 0.5], 0.9), .leftShoulder: ([0.4, 0.3], 0.9), .rightShoulder: ([0.6, 0.3], 0.9)]))
+        #expect(root[.torso]?.point == [0.5, 0.5])
+
+        let shoulders = PoseMath.bodyJoints(
+            lookup([.leftShoulder: ([0.4, 0.3], 0.9), .rightShoulder: ([0.6, 0.5], 0.7), .leftHip: ([0, 0], 1)]))
+        #expect(approx(shoulders[.torso]?.point, [0.5, 0.4]))
+        #expect(shoulders[.torso]?.confidence == 0.7)
+
+        let hips = PoseMath.bodyJoints(
+            lookup([.leftShoulder: ([0.4, 0.3], 0.9), .leftHip: ([0.4, 0.6], 0.5), .rightHip: ([0.6, 0.6], 0.6)]))
+        #expect(approx(hips[.torso]?.point, [0.5, 0.6]))
+        #expect(hips[.torso]?.confidence == 0.5)
+    }
+
+    @Test func armJointsSwapSides() {
+        let arms = PoseMath.armJoints(
+            lookup([
+                .leftShoulder: ([0.4, 0.3], 0.2), .rightShoulder: ([0.6, 0.3], 0.3),
+                .leftElbow: ([0.3, 0.4], 0.15),
+            ]))
+        #expect(arms[.rightShoulder]?.point == [0.4, 0.3])
+        #expect(arms[.leftShoulder]?.point == [0.6, 0.3])
+        #expect(arms[.rightElbow]?.confidence == 0.15)
+        #expect(arms[.leftElbow] == nil)
+    }
+}
+
+@Suite struct PoseMathSmoothingTests {
+    @Test func firstSightingAdoptsThenBlendsThenReusesOnDropout() {
+        var previous: [JointName: Point2] = [:]
+
+        let first = PoseMath.smooth([.head: ([0.2, 0.4], 0.9)], previous: &previous, alpha: 0.5)
+        #expect(first.points[.head] == [0.2, 0.4])
+        #expect(first.confidences[.head] == 0.9)
+        #expect(first.points[.torso] == nil)
+
+        let second = PoseMath.smooth([.head: ([0.6, 0.8], 0.7)], previous: &previous, alpha: 0.5)
+        #expect(approx(second.points[.head], [0.4, 0.6]))
+        #expect(approx(previous[.head], [0.4, 0.6]))
+        #expect(second.confidences[.head] == 0.7)
+
+        let dropout = PoseMath.smooth([:] as [JointName: PoseReading], previous: &previous, alpha: 0.5)
+        #expect(approx(dropout.points[.head], [0.4, 0.6]))
+        #expect(dropout.confidences[.head] == 0)
+        #expect(approx(previous[.head], [0.4, 0.6]))
+    }
+
+    @Test func alphaWeightsTheCurrentFrame() {
+        var previous: [ArmJointName: Point2] = [.leftElbow: [0, 0]]
+        let out = PoseMath.smooth([.leftElbow: ([1, 1], 0.2)], previous: &previous, alpha: 0.25)
+        #expect(approx(out.points[.leftElbow], [0.25, 0.25]))
+    }
+
+    @Test func qualityAveragesOverAllEightJoints() {
+        #expect(PoseMath.quality([:]) == 0)
+        #expect(PoseMath.quality([.head: 0.8, .torso: 0.8]) == 1.6 / 8)
+        #expect(PoseMath.quality([.head: 0]) == 0)
+    }
+}
+
+@Suite struct PoseMathGeometryTests {
+    @Test func roiIsCenteredWithFlippedY() {
+        let roi = PoseMath.roi(aroundTopLeftWrist: [0.5, 0.25], size: 0.3)
+        #expect(abs(roi.midX - 0.5) < 1e-9)
+        #expect(abs(roi.midY - 0.75) < 1e-9)
+        #expect(abs(roi.width - 0.3) < 1e-9 && abs(roi.height - 0.3) < 1e-9)
+    }
+
+    @Test func roiClampsInsideTheFrameWithoutShrinking() {
+        for wrist: Point2 in [[0, 0], [1, 1], [-0.2, 1.4], [0.99, 0.01]] {
+            let roi = PoseMath.roi(aroundTopLeftWrist: wrist, size: 0.3)
+            #expect(roi.minX >= -1e-9 && roi.minY >= -1e-9)
+            #expect(roi.maxX <= 1 + 1e-9 && roi.maxY <= 1 + 1e-9)
+            #expect(abs(roi.width - 0.3) < 1e-9)
+        }
+    }
+
+    @Test func liftFromROIMapsCropRelativePointsToTheFullImage() {
+        let roi = CGRect(x: 0.2, y: 0.4, width: 0.3, height: 0.3)
+        #expect(approx(PoseMath.liftFromROI(CGPoint(x: 0, y: 0), roi: roi), [0.2, 0.6]))
+        #expect(approx(PoseMath.liftFromROI(CGPoint(x: 1, y: 1), roi: roi), [0.5, 0.3]))
+        let full = CGRect(x: 0, y: 0, width: 1, height: 1)
+        #expect(approx(PoseMath.liftFromROI(CGPoint(x: 0.3, y: 0.8), roi: full), [0.3, 0.2]))
+    }
+
+    @Test func roiCenterRoundTripsThroughLift() {
+        let wrist: Point2 = [0.42, 0.37]
+        let roi = PoseMath.roi(aroundTopLeftWrist: wrist, size: 0.3)
+        #expect(approx(PoseMath.liftFromROI(CGPoint(x: 0.5, y: 0.5), roi: roi), wrist, tolerance: 1e-9))
+    }
+}
+
+@Suite struct HandInputTests {
+    @Test func onlyWristsDetectedThisFrameAreFresh() {
+        let wrists = WristReadings(left: [0.3, 0.5], leftConfidence: 0, right: [0.7, 0.5], rightConfidence: 0.6)
+        #expect(wrists.freshLeft == nil)
+        #expect(wrists.freshRight == [0.7, 0.5])
+        let missing = WristReadings(left: nil, leftConfidence: nil, right: nil, rightConfidence: 0.9)
+        #expect(missing.freshLeft == nil && missing.freshRight == nil)
+    }
+
+    @Test func fingertipsAreOmittedOnlyWhenBothSidesAreMissing() {
+        #expect(PoseMath.fingertips(left: nil, right: nil) == nil)
+        let one = PoseMath.fingertips(left: [0.1, 0.2], right: nil)
+        #expect(one?.left == [0.1, 0.2])
+        #expect(one?.right == nil)
+    }
+}

ios/project.yml

@@ -27,6 +27,12 @@ settings:
 targets:
   Motion:
     type: application
+    # Unit tests for the pure pose math (no camera or device). Run with:
+    #   xcodebuild test -workspace Motion.xcworkspace -scheme Motion \
+    #     -destination 'platform=iOS Simulator,name=<an installed iPhone>' CODE_SIGNING_ALLOWED=NO
+    scheme:
+      testTargets:
+        - MotionTests
     platform: iOS
     deploymentTarget: "17.0"
     sources:
@@ -102,3 +108,18 @@ targets:
       - sdk: ReplayKit.framework
       # Photos: saving the recorded (game + camera) screen video to the library.
       - sdk: Photos.framework
+
+  MotionTests:
+    type: bundle.unit-test
+    platform: iOS
+    deploymentTarget: "17.0"
+    sources:
+      - path: Tests/MotionTests
+    dependencies:
+      - target: Motion
+    settings:
+      base:
+        PRODUCT_BUNDLE_IDENTIFIER: com.motion.controller.tests
+        GENERATE_INFOPLIST_FILE: YES
+        TEST_HOST: "$(BUILT_PRODUCTS_DIR)/Motion.app/$(BUNDLE_EXECUTABLE_FOLDER_PATH)/Motion"
+        BUNDLE_LOADER: "$(TEST_HOST)"