diff --git a/.swift-format b/.swift-format new file mode 100644 index 0000000..1e41de5 --- /dev/null +++ b/.swift-format @@ -0,0 +1,8 @@ +{ + "version": 1, + "lineLength": 120, + "indentation": { + "spaces": 4 + }, + "respectsExistingLineBreaks": true +} diff --git a/ios/Sources/Motion/App/AppModel.swift b/ios/Sources/Motion/App/AppModel.swift index c73be77..ade7515 100644 --- a/ios/Sources/Motion/App/AppModel.swift +++ b/ios/Sources/Motion/App/AppModel.swift @@ -23,14 +23,15 @@ import Foundation import Observation + #if canImport(UIKit) -import UIKit + import UIKit #endif /// High-level screen the app should show. Drives `ContentView` routing. enum Phase: Sendable, Equatable { - case setup // camera up, guiding the player into frame - case game // full-screen web game; pose streams in-process via PoseBridge + case setup // camera up, guiding the player into frame + case game // full-screen web game; pose streams in-process via PoseBridge } /// Detects a "clap" purely from the two tracked HAND joints (`joints.leftHand` / @@ -70,7 +71,7 @@ struct ClapDetector { // Fire on the closing edge: armed + hands now together + past the debounce window. if armed, separation < closedThreshold, now - lastClapAt >= minInterval { - armed = false // require a re-separation before the next clap + armed = false // require a re-separation before the next clap lastClapAt = now return true } @@ -82,10 +83,12 @@ struct ClapDetector { /// treat a hand sitting exactly at the origin as "not present" to avoid false claps /// when one hand isn't tracked. private func handSeparation(_ j: Joints) -> Double? { - let l = j.leftHand, r = j.rightHand + let l = j.leftHand + let r = j.rightHand guard l.count == 2, r.count == 2 else { return nil } if isFallback(l) || isFallback(r) { return nil } - let dx = l[0] - r[0], dy = l[1] - r[1] + let dx = l[0] - r[0] + let dy = l[1] - r[1] return (dx * dx + dy * dy).squareRoot() } @@ -110,7 +113,7 @@ final class AppModel { var gameURL: URL? { switch GameConfig.source { case .devServer: return GameConfig.devServerURL(host: devServerIP) - case .bundled: return GameConfig.bundledIndexURL() + case .bundled: return GameConfig.bundledIndexURL() } } @@ -252,8 +255,10 @@ final class AppModel { /// - tracking: setup verdict from `SetupEvaluator` /// - guidance: matching human guidance string /// - ready: whether setup has been good long enough - func ingest(joints: Joints?, quality: Double, tracking: TrackingState, - guidance: String, ready: Bool, hands: HandState?, fingertips: Fingertips? = nil) { + func ingest( + joints: Joints?, quality: Double, tracking: TrackingState, + guidance: String, ready: Bool, hands: HandState?, fingertips: Fingertips? = nil + ) { self.joints = joints self.quality = quality self.tracking = tracking @@ -270,8 +275,9 @@ final class AppModel { // Skip in `.game`: the web game handles its own input, so claps are ignored there. // We always feed the detector so its arm/debounce state stays coherent, but only // publish the event outside the game phase. - let clapped = clapDetector.update(joints: joints, - now: ProcessInfo.processInfo.systemUptime) + let clapped = clapDetector.update( + joints: joints, + now: ProcessInfo.processInfo.systemUptime) if clapped, phase != .game { clapCount &+= 1 } @@ -279,8 +285,9 @@ final class AppModel { // While playing, forward pose + tracking (with hands + fingertips) into the web game // in-process. PoseBridge handles change-detection for tracking and the ~30 Hz throttle. if phase == .game { - bridge.pushLivePose(joints: joints, quality: quality, tracking: tracking, - hands: self.hands, fingertips: self.fingertips) + bridge.pushLivePose( + joints: joints, quality: quality, tracking: tracking, + hands: self.hands, fingertips: self.fingertips) } // Stream to the browser relay IN ADDITION to (and independent of) the local game. @@ -330,7 +337,7 @@ final class AppModel { /// streaming poses (see `ingest`). The relay is on the SAME Mac as the dev server, so we /// reuse `devServerIP` as the host. Idempotent-ish: replaces any existing socket. private func startStreaming() { - stopStreaming() // clear any stale socket first + stopStreaming() // clear any stale socket first let code = roomCode.trimmingCharacters(in: .whitespaces).uppercased() guard !code.isEmpty else { streamToWebsite = false @@ -348,7 +355,7 @@ final class AppModel { // background the app and drop the relay socket. This does NOT stop iOS suspending a // fully-backgrounded app, but it prevents the common idle auto-lock disconnect. #if canImport(UIKit) - UIApplication.shared.isIdleTimerDisabled = true + UIApplication.shared.isIdleTimerDisabled = true #endif } @@ -362,7 +369,7 @@ final class AppModel { // Re-enable auto-lock now that we're no longer streaming, so the phone can sleep // normally when idle. Paired with the enable in `startStreaming()`. #if canImport(UIKit) - UIApplication.shared.isIdleTimerDisabled = false + UIApplication.shared.isIdleTimerDisabled = false #endif } } diff --git a/ios/Sources/Motion/Game/GameWebView.swift b/ios/Sources/Motion/Game/GameWebView.swift index 2e7da0f..8203c40 100644 --- a/ios/Sources/Motion/Game/GameWebView.swift +++ b/ios/Sources/Motion/Game/GameWebView.swift @@ -157,7 +157,7 @@ struct GameWebView: UIViewRepresentable { /// as a JSON STRING (the web bridge JSON.parses strings), which sidesteps any JS /// object-literal escaping concerns — we only have to escape the JSON string once. func pushPose(jsonString: String) { - guard ready else { return } // no bridge yet; poses before start are dropped + guard ready else { return } // no bridge yet; poses before start are dropped let js = "window.__motion && window.__motion.pushPose(\(Self.jsStringLiteral(jsonString)));" eval(js) } @@ -172,12 +172,18 @@ struct GameWebView: UIViewRepresentable { /// Begin / restart the game. If the web app isn't `ready` yet, this is queued and /// fired the moment the `ready` event arrives. func start() { - guard ready else { pendingStart = true; return } + guard ready else { + pendingStart = true + return + } eval("window.__motion && window.__motion.start();") } func stop() { - guard ready else { pendingStart = false; return } + guard ready else { + pendingStart = false + return + } eval("window.__motion && window.__motion.stop();") } @@ -195,13 +201,15 @@ struct GameWebView: UIViewRepresentable { /// all handled correctly — the result is a valid JS/JSON string expression. static func jsStringLiteral(_ s: String) -> String { if let data = try? JSONSerialization.data(withJSONObject: [s], options: []), - let arr = String(data: data, encoding: .utf8), - arr.hasPrefix("["), arr.hasSuffix("]") { + let arr = String(data: data, encoding: .utf8), + arr.hasPrefix("["), arr.hasSuffix("]") + { // arr == `["...escaped..."]`; strip the array brackets to get the literal. return String(arr.dropFirst().dropLast()) } // Fallback: minimal manual escaping (should never be hit for our inputs). - let escaped = s + let escaped = + s .replacingOccurrences(of: "\\", with: "\\\\") .replacingOccurrences(of: "\"", with: "\\\"") .replacingOccurrences(of: "\n", with: "\\n") @@ -210,8 +218,10 @@ struct GameWebView: UIViewRepresentable { // MARK: Web → native (WKScriptMessageHandler) - func userContentController(_ userContentController: WKUserContentController, - didReceive message: WKScriptMessage) { + func userContentController( + _ userContentController: WKUserContentController, + didReceive message: WKScriptMessage + ) { guard message.name == "motion" else { return } guard let event = Self.decodeEvent(message.body) else { return } if case .ready = event { ready = true } @@ -221,11 +231,12 @@ struct GameWebView: UIViewRepresentable { /// Decode a `{ event: ..., ... }` message body (a JS object → NSDictionary). private static func decodeEvent(_ body: Any) -> GameEvent? { guard let dict = body as? [String: Any], - let event = dict["event"] as? String else { return nil } + let event = dict["event"] as? String + else { return nil } switch event { - case "ready": return .ready + case "ready": return .ready case "gameStart": return .gameStart - case "restart": return .restart + case "restart": return .restart case "score": let value = (dict["value"] as? NSNumber)?.doubleValue ?? 0 return .score(value) @@ -233,9 +244,10 @@ struct GameWebView: UIViewRepresentable { // `result` is arbitrary JSON; re-serialize it to a string for the UI. var resultJSON: String? if let result = dict["result"], - JSONSerialization.isValidJSONObject(result), - let data = try? JSONSerialization.data(withJSONObject: result), - let str = String(data: data, encoding: .utf8) { + JSONSerialization.isValidJSONObject(result), + let data = try? JSONSerialization.data(withJSONObject: result), + let str = String(data: data, encoding: .utf8) + { resultJSON = str } return .gameOver(resultJSON: resultJSON) @@ -246,8 +258,10 @@ struct GameWebView: UIViewRepresentable { // MARK: Navigation - func webView(_ webView: WKWebView, didFailProvisionalNavigation navigation: WKNavigation!, - withError error: Error) { + func webView( + _ webView: WKWebView, didFailProvisionalNavigation navigation: WKNavigation!, + withError error: Error + ) { // Surfaced as a black screen; the ContentView shows a dev-server hint on failure. onEvent(.gameOver(resultJSON: nil)) } diff --git a/ios/Sources/Motion/Game/PoseBridge.swift b/ios/Sources/Motion/Game/PoseBridge.swift index 66be82e..946c5cc 100644 --- a/ios/Sources/Motion/Game/PoseBridge.swift +++ b/ios/Sources/Motion/Game/PoseBridge.swift @@ -89,8 +89,10 @@ final class PoseBridge { /// - tracking: setup verdict from `SetupEvaluator` /// - hands: latest per-hand openness 0..1 (nil if never detected), carried in the packet /// - fingertips: latest precise index-fingertips (nil if never detected), carried too - func pushLivePose(joints: Joints?, quality: Double, tracking: TrackingState, - hands: HandState? = nil, fingertips: Fingertips? = nil) { + func pushLivePose( + joints: Joints?, quality: Double, tracking: TrackingState, + hands: HandState? = nil, fingertips: Fingertips? = nil + ) { guard let coordinator else { return } // Tracking transitions are cheap and important (they pause/resume the game), so diff --git a/ios/Sources/Motion/Net/Protocol.swift b/ios/Sources/Motion/Net/Protocol.swift index 8f529c4..580fec9 100644 --- a/ios/Sources/Motion/Net/Protocol.swift +++ b/ios/Sources/Motion/Net/Protocol.swift @@ -47,12 +47,12 @@ enum Role: String, Codable, Sendable { /// Reason the controller has paused sending usable input. Mirrors `TrackingState`. enum TrackingState: String, Codable, Sendable, CaseIterable { case ok - case lost // body not detected / confidence collapsed - case partial // some required joints missing - case tooClose = "too_close" - case tooFar = "too_far" - case raisePhone = "raise_phone" - case lowLight = "low_light" + case lost // body not detected / confidence collapsed + case partial // some required joints missing + case tooClose = "too_close" + case tooFar = "too_far" + case raisePhone = "raise_phone" + case lowLight = "low_light" } // MARK: - Joints @@ -173,8 +173,10 @@ struct PosePacket: Codable, Sendable { /// the TS optional field `fingertips?: Fingertips` so old/new relays both accept the packet. let fingertips: Fingertips? - init(seq: Int, sentAt: Double, quality: Double, joints: Joints, - hands: HandState? = nil, fingertips: Fingertips? = nil) { + init( + seq: Int, sentAt: Double, quality: Double, joints: Joints, + hands: HandState? = nil, fingertips: Fingertips? = nil + ) { self.v = PROTOCOL_VERSION self.type = "pose" self.seq = seq @@ -209,7 +211,7 @@ struct CalibMessage: Codable, Sendable { let v: Int let type: String let stage: Stage - let progress: Double // 0..1 within the whole calibration + let progress: Double // 0..1 within the whole calibration init(stage: Stage, progress: Double) { self.v = PROTOCOL_VERSION @@ -294,10 +296,10 @@ struct RecMetaMessage: Codable, Sendable { let v: Int let type: String let sessionId: String - let mime: String // e.g. "video/webm;codecs=vp9" or "video/mp4" + let mime: String // e.g. "video/webm;codecs=vp9" or "video/mp4" let totalBytes: Int - let chunks: Int // number of RecChunkMessages to expect - let durationMs: Double // gameplay clip duration + let chunks: Int // number of RecChunkMessages to expect + let durationMs: Double // gameplay clip duration /// ms from the session anchor to the first recorded gameplay frame (for sync). let startOffsetMs: Double } @@ -308,8 +310,8 @@ struct RecChunkMessage: Codable, Sendable { let v: Int let type: String let sessionId: String - let i: Int // 0-based chunk index - let data: String // base64 of this slice + let i: Int // 0-based chunk index + let data: String // base64 of this slice } // MARK: - Inbound messages (server → controller) @@ -331,9 +333,9 @@ struct StartMessage: Codable, Sendable { /// Room-level errors (e.g. a second controller tried to join). struct ErrorMessage: Codable, Sendable { enum Code: String, Codable, Sendable { - case roomFull = "room_full" - case badRole = "bad_role" - case badMessage = "bad_message" + case roomFull = "room_full" + case badRole = "bad_role" + case badMessage = "bad_message" case versionMismatch = "version_mismatch" } let v: Int diff --git a/ios/Sources/Motion/Net/RoomSocket.swift b/ios/Sources/Motion/Net/RoomSocket.swift index 4165244..0ceef3c 100644 --- a/ios/Sources/Motion/Net/RoomSocket.swift +++ b/ios/Sources/Motion/Net/RoomSocket.swift @@ -55,7 +55,7 @@ final class RoomSocket { private let session: URLSession private var task: URLSessionWebSocketTask? - private var isRunning = false // user wants a connection to exist + private var isRunning = false // user wants a connection to exist private var reconnectAttempt = 0 private var pingTimer: Task? private var receiveActive = false @@ -113,8 +113,10 @@ final class RoomSocket { delegate?.roomSocket(self, didChangeState: .failed(reason: "Invalid room URL.")) return } - delegate?.roomSocket(self, didChangeState: - reconnectAttempt == 0 ? .connecting : .reconnecting(attempt: reconnectAttempt)) + delegate?.roomSocket( + self, + didChangeState: + reconnectAttempt == 0 ? .connecting : .reconnecting(attempt: reconnectAttempt)) let ws = session.webSocketTask(with: url) task = ws @@ -134,9 +136,9 @@ final class RoomSocket { send(JoinMessage(role: .controller, name: playerName)) } - func send(_ pose: PosePacket) { send(encodable: pose) } + func send(_ pose: PosePacket) { send(encodable: pose) } func send(_ status: StatusMessage) { send(encodable: status) } - func send(_ calib: CalibMessage) { send(encodable: calib) } + func send(_ calib: CalibMessage) { send(encodable: calib) } /// Send a recording control frame (arm/start/stop/cancel) to the display. func send(_ rec: RecControlMessage) { send(encodable: rec) } private func send(_ join: JoinMessage) { send(encodable: join) } @@ -210,7 +212,7 @@ final class RoomSocket { pingTimer?.cancel() pingTimer = Task { [weak self] in while !Task.isCancelled { - try? await Task.sleep(nanoseconds: 2_000_000_000) // 2s + try? await Task.sleep(nanoseconds: 2_000_000_000) // 2s guard let self, self.isRunning else { return } await MainActor.run { self.send(encodable: PingMessage(t: self.nowMS())) diff --git a/ios/Sources/Motion/Recording/CameraRecorder.swift b/ios/Sources/Motion/Recording/CameraRecorder.swift index b3a873f..d78f8a8 100644 --- a/ios/Sources/Motion/Recording/CameraRecorder.swift +++ b/ios/Sources/Motion/Recording/CameraRecorder.swift @@ -68,13 +68,16 @@ final class CameraRecorder: NSObject, CameraSampleBufferTap, @unchecked Sendable private let stateLock = NSLock() private var _isRecording = false private var isRecording: Bool { - stateLock.lock(); defer { stateLock.unlock() } + stateLock.lock() + defer { stateLock.unlock() } return _isRecording } /// Synchronous setter for the recording flag. Kept as a non-async method so it can /// be called from `async` contexts (Swift 6 forbids `NSLock.lock()` directly there). private func setRecording(_ value: Bool) { - stateLock.lock(); _isRecording = value; stateLock.unlock() + stateLock.lock() + _isRecording = value + stateLock.unlock() } /// The video dimensions we configure the writer for. Front camera at 720p portrait @@ -137,12 +140,13 @@ final class CameraRecorder: NSObject, CameraSampleBufferTap, @unchecked Sendable let first = self.firstPTS.seconds let last = self.lastPTS.seconds let duration = max(0, last - first) - cont.resume(returning: CameraRecording( - url: url, - firstFrameWallClockMs: self.firstFrameWallClockMs, - firstFramePTSSeconds: first, - durationSeconds: duration - )) + cont.resume( + returning: CameraRecording( + url: url, + firstFrameWallClockMs: self.firstFrameWallClockMs, + firstFramePTSSeconds: first, + durationSeconds: duration + )) } } } diff --git a/ios/Sources/Motion/Recording/ClipReceiver.swift b/ios/Sources/Motion/Recording/ClipReceiver.swift index dfd596d..40248db 100644 --- a/ios/Sources/Motion/Recording/ClipReceiver.swift +++ b/ios/Sources/Motion/Recording/ClipReceiver.swift @@ -103,7 +103,7 @@ final class ClipReceiver { } else if meta.chunks > 1 { chunkSize = (meta.totalBytes - slice.count) / (meta.chunks - 1) } else { - chunkSize = max(slice.count, meta.totalBytes) // single-chunk clip + chunkSize = max(slice.count, meta.totalBytes) // single-chunk clip } } guard let stride = chunkSize, stride > 0 else { return nil } diff --git a/ios/Sources/Motion/Recording/CompositeExporter.swift b/ios/Sources/Motion/Recording/CompositeExporter.swift index 6324272..a8a6aec 100644 --- a/ios/Sources/Motion/Recording/CompositeExporter.swift +++ b/ios/Sources/Motion/Recording/CompositeExporter.swift @@ -88,13 +88,11 @@ enum CompositeExporter { // Do NOT touch the WebM file with AVAsset — decode WILL fail. Export the camera // clip alone so the user still gets their performance, and tell them why. guard isDecodableByAVFoundation(mime: clip.mime) else { - let message = "Gameplay clip was WebM; on-device compositing needs an MP4 " + - "recording — use a Safari/Chrome display that records MP4. Saved your " + - "camera clip on its own." - if let url = await exportCameraOnly(camera: camera) { - return .cameraOnlyFallback(url: url, message: message) - } - return .failed(message: message) + let message = + "Gameplay clip was WebM; on-device compositing needs an MP4 " + + "recording — use a Safari/Chrome display that records MP4. Saved your " + + "camera clip on its own." + return await cameraOnlyFallback(camera: camera, message: message, failure: message) } // ── Normal PiP composite ───────────────────────────────────────────────── @@ -107,13 +105,10 @@ enum CompositeExporter { let gameplayTrack = await firstVideoTrack(of: gameplayAsset) else { // If the "MP4" turned out to be undecodable after all, degrade to camera-only. - if let url = await exportCameraOnly(camera: camera) { - return .cameraOnlyFallback( - url: url, - message: "Couldn't read the gameplay clip; saved your camera clip alone." - ) - } - return .failed(message: "Couldn't read either video track.") + return await cameraOnlyFallback( + camera: camera, + message: "Couldn't read the gameplay clip; saved your camera clip alone.", + failure: "Couldn't read either video track.") } // Natural sizes/transforms. @@ -130,11 +125,8 @@ enum CompositeExporter { let gameplayStartMs = clip.startOffsetMs // The later start defines t=0; trim the earlier track's head by the difference. - let cameraHeadTrimMs = max(0, gameplayStartMs - cameraStartMs) - let gameplayHeadTrimMs = max(0, cameraStartMs - gameplayStartMs) - - let cameraHeadTrim = CMTime(seconds: cameraHeadTrimMs / 1000.0, preferredTimescale: 600) - let gameplayHeadTrim = CMTime(seconds: gameplayHeadTrimMs / 1000.0, preferredTimescale: 600) + let (cameraHeadTrim, gameplayHeadTrim) = headTrims( + cameraStartMs: cameraStartMs, gameplayStartMs: gameplayStartMs) // Remaining playable duration of each after the head trim. let cameraRemaining = CMTimeSubtract(cameraDuration, cameraHeadTrim) @@ -143,15 +135,71 @@ enum CompositeExporter { let overlap = CMTimeMinimum(cameraRemaining, gameplayRemaining) guard overlap.isValid, overlap.seconds > 0.05 else { // No meaningful overlap (clocks too far apart / one clip empty) → camera only. - if let url = await exportCameraOnly(camera: camera) { - return .cameraOnlyFallback( - url: url, - message: "Clips didn't overlap in time; saved your camera clip alone." - ) - } - return .failed(message: "Recordings didn't overlap in time.") + return await cameraOnlyFallback( + camera: camera, + message: "Clips didn't overlap in time; saved your camera clip alone.", + failure: "Recordings didn't overlap in time.") + } + + let timeline: ComposedTimeline + switch await composeTimeline( + cameraTrack: cameraTrack, gameplayTrack: gameplayTrack, gameplayAsset: gameplayAsset, + cameraHeadTrim: cameraHeadTrim, gameplayHeadTrim: gameplayHeadTrim, overlap: overlap) + { + case .success(let built): timeline = built + case .failure(let failure): return .failed(message: failure.message) } + // ── Render sizes (apply preferredTransform so rotated sources render upright) ── + let gameplayRenderSize = applyingTransformSize(gameplayNatural, gameplayTransform) + let cameraRenderSize = applyingTransformSize(cameraNatural, cameraTransform) + + // Output frame = gameplay's upright size (full-frame background). + let outputSize = CGSize( + width: abs(gameplayRenderSize.width).rounded(), + height: abs(gameplayRenderSize.height).rounded()) + guard outputSize.width > 0, outputSize.height > 0 else { + return .failed(message: "Gameplay clip had zero dimensions.") + } + + let cameraFinal = cameraInsetTransform( + cameraTransform: cameraTransform, cameraRenderSize: cameraRenderSize, + outputSize: outputSize, layout: layout) + let videoComposition = makeVideoComposition( + compCamera: timeline.camera, compGameplay: timeline.gameplay, + cameraFinal: cameraFinal, gameplayTransform: gameplayTransform, + overlap: overlap, outputSize: outputSize) + + return await exportComposite(timeline.composition, videoComposition: videoComposition) + } + + // MARK: - Composite helpers + + /// Degrade to a camera-only export; `failure` is reported if even that fails. + private static func cameraOnlyFallback( + camera: CameraRecording, message: String, failure: String + ) async -> CompositeResult { + if let url = await exportCameraOnly(camera: camera) { + return .cameraOnlyFallback(url: url, message: message) + } + return .failed(message: failure) + } + + private struct TimelineFailure: Error { + let message: String + } + + private struct ComposedTimeline { + let composition: AVMutableComposition + let camera: AVMutableCompositionTrack + let gameplay: AVMutableCompositionTrack + } + + /// Build the composition: both trimmed video ranges at t=0, plus optional gameplay audio. + private static func composeTimeline( + cameraTrack: AVAssetTrack, gameplayTrack: AVAssetTrack, gameplayAsset: AVAsset, + cameraHeadTrim: CMTime, gameplayHeadTrim: CMTime, overlap: CMTime + ) async -> Result { let composition = AVMutableComposition() guard let compCamera = composition.addMutableTrack( @@ -159,7 +207,7 @@ enum CompositeExporter { let compGameplay = composition.addMutableTrack( withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid) else { - return .failed(message: "Couldn't build the composition.") + return .failure(TimelineFailure(message: "Couldn't build the composition.")) } // Insert both trimmed ranges at composition t=0. @@ -171,37 +219,39 @@ enum CompositeExporter { CMTimeRange(start: cameraHeadTrim, duration: overlap), of: cameraTrack, at: .zero) } catch { - return .failed(message: "Couldn't assemble the composite timeline.") + return .failure(TimelineFailure(message: "Couldn't assemble the composite timeline.")) } // Optionally carry the gameplay audio if present (game SFX/music). if let gameplayAudio = await firstAudioTrack(of: gameplayAsset), - let compAudio = composition.addMutableTrack( - withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) { + let compAudio = composition.addMutableTrack( + withMediaType: .audio, preferredTrackID: kCMPersistentTrackID_Invalid) + { try? compAudio.insertTimeRange( CMTimeRange(start: gameplayHeadTrim, duration: overlap), of: gameplayAudio, at: .zero) } + return .success( + ComposedTimeline(composition: composition, camera: compCamera, gameplay: compGameplay)) + } - // ── Render sizes (apply preferredTransform so rotated sources render upright) ── - let gameplayRenderSize = applyingTransformSize(gameplayNatural, gameplayTransform) - let cameraRenderSize = applyingTransformSize(cameraNatural, cameraTransform) - - // Output frame = gameplay's upright size (full-frame background). - let outputSize = CGSize( - width: abs(gameplayRenderSize.width).rounded(), - height: abs(gameplayRenderSize.height).rounded()) - guard outputSize.width > 0, outputSize.height > 0 else { - return .failed(message: "Gameplay clip had zero dimensions.") - } + /// Head trims that put both clips in phase: the later start defines t=0, and the + /// earlier clip is trimmed at its head by the difference. + private static func headTrims(cameraStartMs: Double, gameplayStartMs: Double) -> (CMTime, CMTime) { + let cameraHeadTrimMs = max(0, gameplayStartMs - cameraStartMs) + let gameplayHeadTrimMs = max(0, cameraStartMs - gameplayStartMs) - // Gameplay layer instruction: apply its own transform so it renders upright, - // full-frame at the origin. - let gameplayLayer = AVMutableVideoCompositionLayerInstruction(assetTrack: compGameplay) - gameplayLayer.setTransform(gameplayTransform, at: .zero) + let cameraHeadTrim = CMTime(seconds: cameraHeadTrimMs / 1000.0, preferredTimescale: 600) + let gameplayHeadTrim = CMTime(seconds: gameplayHeadTrimMs / 1000.0, preferredTimescale: 600) + return (cameraHeadTrim, gameplayHeadTrim) + } - // Camera inset transform: start from the camera's upright transform, then scale - // down to the inset width and translate into the chosen corner. + /// Camera inset transform: start from the camera's upright transform, then scale + /// down to the inset width and translate into the chosen corner. + private static func cameraInsetTransform( + cameraTransform: CGAffineTransform, cameraRenderSize: CGSize, + outputSize: CGSize, layout: PiPLayout + ) -> CGAffineTransform { let insetWidth = outputSize.width * layout.widthFraction let cameraUprightW = abs(cameraRenderSize.width) let cameraUprightH = abs(cameraRenderSize.height) @@ -218,6 +268,19 @@ enum CompositeExporter { var cameraFinal = cameraTransform cameraFinal = cameraFinal.concatenating(CGAffineTransform(scaleX: scale, y: scale)) cameraFinal = cameraFinal.concatenating(CGAffineTransform(translationX: tx, y: ty)) + return cameraFinal + } + + /// Layer instructions + video composition: gameplay full-frame, camera inset on top. + private static func makeVideoComposition( + compCamera: AVMutableCompositionTrack, compGameplay: AVMutableCompositionTrack, + cameraFinal: CGAffineTransform, gameplayTransform: CGAffineTransform, + overlap: CMTime, outputSize: CGSize + ) -> AVMutableVideoComposition { + // Gameplay layer instruction: apply its own transform so it renders upright, + // full-frame at the origin. + let gameplayLayer = AVMutableVideoCompositionLayerInstruction(assetTrack: compGameplay) + gameplayLayer.setTransform(gameplayTransform, at: .zero) let cameraLayer = AVMutableVideoCompositionLayerInstruction(assetTrack: compCamera) cameraLayer.setTransform(cameraFinal, at: .zero) @@ -230,16 +293,23 @@ enum CompositeExporter { let videoComposition = AVMutableVideoComposition() videoComposition.instructions = [instruction] - videoComposition.frameDuration = CMTime(value: 1, timescale: 30) // 30 fps output + videoComposition.frameDuration = CMTime(value: 1, timescale: 30) // 30 fps output videoComposition.renderSize = outputSize + return videoComposition + } + /// Export the composed timeline to a temp mp4. + private static func exportComposite( + _ composition: AVMutableComposition, videoComposition: AVMutableVideoComposition + ) async -> CompositeResult { // ── Export ─────────────────────────────────────────────────────────────── let outURL = FileManager.default.temporaryDirectory .appendingPathComponent("motion-composite-\(UUID().uuidString).mp4") try? FileManager.default.removeItem(at: outURL) - guard let export = AVAssetExportSession( - asset: composition, presetName: AVAssetExportPresetHighestQuality) + guard + let export = AVAssetExportSession( + asset: composition, presetName: AVAssetExportPresetHighestQuality) else { return .failed(message: "Couldn't create the exporter.") } @@ -276,8 +346,9 @@ enum CompositeExporter { .appendingPathComponent("motion-cam-only-\(UUID().uuidString).mp4") try? FileManager.default.removeItem(at: outURL) - guard let export = AVAssetExportSession( - asset: asset, presetName: AVAssetExportPresetHighestQuality) + guard + let export = AVAssetExportSession( + asset: asset, presetName: AVAssetExportPresetHighestQuality) else { return nil } export.outputURL = outURL export.outputFileType = .mp4 @@ -323,8 +394,10 @@ enum CompositeExporter { case .bottomLeading: return (margin, outputSize.height - insetHeight - margin) case .bottomTrailing: - return (outputSize.width - insetWidth - margin, - outputSize.height - insetHeight - margin) + return ( + outputSize.width - insetWidth - margin, + outputSize.height - insetHeight - margin + ) } } } diff --git a/ios/Sources/Motion/Recording/PhotosVideoSaver.swift b/ios/Sources/Motion/Recording/PhotosVideoSaver.swift new file mode 100644 index 0000000..b0314b0 --- /dev/null +++ b/ios/Sources/Motion/Recording/PhotosVideoSaver.swift @@ -0,0 +1,44 @@ +// +// PhotosVideoSaver.swift +// Motion +// +// Shared "save a finished video to Photos" step for the v1 ReplayKit recorder and the +// parked v2 composite recorder. Requests ADD-ONLY authorization (matches +// `NSPhotoLibraryAddUsageDescription`); it never reads the library. +// + +import Foundation +import Photos + +enum PhotosVideoSaver { + /// Outcome of a save attempt; `message` is user-facing. + enum Outcome: Sendable { + case saved + case failed(String) + } + + /// Save the video at `url` to Photos, requesting add-only permission first. + static func save(url: URL) async -> Outcome { + let status = await requestAddPermission() + guard status == .authorized || status == .limited else { + return .failed("Photos access denied — enable it in Settings to save your clip.") + } + do { + try await PHPhotoLibrary.shared().performChanges { + let req = PHAssetCreationRequest.forAsset() + req.addResource(with: .video, fileURL: url, options: nil) + } + return .saved + } catch { + return .failed("Couldn't save to Photos: \(error.localizedDescription)") + } + } + + private static func requestAddPermission() async -> PHAuthorizationStatus { + let current = PHPhotoLibrary.authorizationStatus(for: .addOnly) + if current == .notDetermined { + return await PHPhotoLibrary.requestAuthorization(for: .addOnly) + } + return current + } +} diff --git a/ios/Sources/Motion/Recording/RecordingController.swift b/ios/Sources/Motion/Recording/RecordingController.swift index 38af83b..0212f29 100644 --- a/ios/Sources/Motion/Recording/RecordingController.swift +++ b/ios/Sources/Motion/Recording/RecordingController.swift @@ -22,7 +22,6 @@ import Foundation import Observation -import Photos @MainActor @Observable @@ -30,13 +29,13 @@ final class RecordingController { /// Coarse state machine surfaced to the UI. enum State: Sendable, Equatable { - case idle // not recording - case armed // user opted in; waiting for the display to start - case recording // camera recording locally, gameplay in progress - case receiving(Double) // gameplay clip transferring (0..1 progress) - case compositing // building the PiP video / exporting - case saved // saved to Photos - case failed(String) // something went wrong (message for the UI) + case idle // not recording + case armed // user opted in; waiting for the display to start + case recording // camera recording locally, gameplay in progress + case receiving(Double) // gameplay clip transferring (0..1 progress) + case compositing // building the PiP video / exporting + case saved // saved to Photos + case failed(String) // something went wrong (message for the UI) } private(set) var state: State = .idle @@ -164,7 +163,10 @@ final class RecordingController { private func startRecording(sessionId sid: String, anchorMs: Double) { // Only start if this matches our armed session (or adopt if the display drives it). - if sessionId == nil { sessionId = sid; sessionAnchorMs = anchorMs } + if sessionId == nil { + sessionId = sid + sessionAnchorMs = anchorMs + } guard sessionId == sid else { return } camera.start() state = .recording @@ -191,14 +193,14 @@ final class RecordingController { /// Idempotent + order-independent: called after the camera finishes AND after each /// clip chunk, so whichever completes last actually triggers the export. private func maybeComposite() { - if case .compositing = state { return } // already running - guard let cameraRecording else { return } // camera not finished yet - guard clipReceiver.isComplete else { return } // clip still arriving - guard let clip = completedClip else { return } // finalized clip in hand + if case .compositing = state { return } // already running + guard let cameraRecording else { return } // camera not finished yet + guard clipReceiver.isComplete else { return } // clip still arriving + guard let clip = completedClip else { return } // finalized clip in hand guard let anchor = sessionAnchorMs else { return } state = .compositing - cameraTapHost?.setSampleBufferTap(nil) // no longer need the tap + cameraTapHost?.setSampleBufferTap(nil) // no longer need the tap Task { [weak self] in let result = await CompositeExporter.export( @@ -225,34 +227,16 @@ final class RecordingController { /// Save a finished video to the Photos library, requesting add-only permission first. private func save(url: URL, note: String?) async { self.note = note - let status = await requestPhotosAddPermission() - guard status == .authorized || status == .limited else { - state = .failed("Photos access denied — enable it in Settings to save your clip.") - cleanupSession() - return - } - do { - try await PHPhotoLibrary.shared().performChanges { - let req = PHAssetCreationRequest.forAsset() - req.addResource(with: .video, fileURL: url, options: nil) - } + switch await PhotosVideoSaver.save(url: url) { + case .saved: lastSavedURL = url state = .saved - } catch { - state = .failed("Couldn't save to Photos: \(error.localizedDescription)") + case .failed(let message): + state = .failed(message) } cleanupSession() } - /// Request ADD-ONLY Photos authorization (matches `NSPhotoLibraryAddUsageDescription`). - private func requestPhotosAddPermission() async -> PHAuthorizationStatus { - let current = PHPhotoLibrary.authorizationStatus(for: .addOnly) - if current == .notDetermined { - return await PHPhotoLibrary.requestAuthorization(for: .addOnly) - } - return current - } - /// Clear per-session scratch state but keep `lastSavedURL`/`note` for the UI. private func cleanupSession() { camera.cancel() diff --git a/ios/Sources/Motion/Recording/ScreenRecorder.swift b/ios/Sources/Motion/Recording/ScreenRecorder.swift index fb1c0b2..c55aac0 100644 --- a/ios/Sources/Motion/Recording/ScreenRecorder.swift +++ b/ios/Sources/Motion/Recording/ScreenRecorder.swift @@ -44,7 +44,6 @@ import AVFoundation import Foundation import Observation -import Photos import ReplayKit @MainActor @@ -53,12 +52,12 @@ final class ScreenRecorder { /// Coarse state surfaced to the UI. enum State: Sendable, Equatable { - case idle // not recording; not armed - case armed // user opted in; waiting for the game to start - case recording // ReplayKit capture running, writing to the mp4 - case saving // capture stopped; finalizing + saving to Photos - case saved // saved to Photos - case failed(String) // something went wrong (message for the UI) + case idle // not recording; not armed + case armed // user opted in; waiting for the game to start + case recording // ReplayKit capture running, writing to the mp4 + case saving // capture stopped; finalizing + saving to Photos + case saved // saved to Photos + case failed(String) // something went wrong (message for the UI) } private(set) var state: State = .idle @@ -198,29 +197,13 @@ final class ScreenRecorder { /// Save a finished mp4 to Photos (add-only authorization). private func save(url: URL) async { - let status = await requestPhotosAddPermission() - guard status == .authorized || status == .limited else { - state = .failed("Photos access denied — enable it in Settings to save your clip.") - return - } - do { - try await PHPhotoLibrary.shared().performChanges { - let req = PHAssetCreationRequest.forAsset() - req.addResource(with: .video, fileURL: url, options: nil) - } + switch await PhotosVideoSaver.save(url: url) { + case .saved: lastSavedURL = url state = .saved - } catch { - state = .failed("Couldn't save to Photos: \(error.localizedDescription)") - } - } - - private func requestPhotosAddPermission() async -> PHAuthorizationStatus { - let current = PHPhotoLibrary.authorizationStatus(for: .addOnly) - if current == .notDetermined { - return await PHPhotoLibrary.requestAuthorization(for: .addOnly) + case .failed(let message): + state = .failed(message) } - return current } // GitHub issue #20 tracks physical-device verification that ReplayKit includes both @@ -265,13 +248,14 @@ private final class ScreenWriter: @unchecked Sendable { guard self.buildWriter(width: width, height: height) else { return } } guard let writer = self.writer, let input = self.videoInput, - writer.status != .failed else { return } + writer.status != .failed + else { return } if !self.sessionStarted { writer.startSession(atSourceTime: pts) self.sessionStarted = true } - guard input.isReadyForMoreMediaData else { return } // drop rather than block + guard input.isReadyForMoreMediaData else { return } // drop rather than block input.append(retained) } } @@ -282,7 +266,8 @@ private final class ScreenWriter: @unchecked Sendable { await withCheckedContinuation { (cont: CheckedContinuation) in queue.async { [weak self] in guard let self, let writer = self.writer, let input = self.videoInput, - self.sessionStarted, writer.status == .writing else { + self.sessionStarted, writer.status == .writing + else { cont.resume(returning: nil) return } diff --git a/ios/Sources/Motion/Views/ContentView.swift b/ios/Sources/Motion/Views/ContentView.swift index bf5df38..2a732f0 100644 --- a/ios/Sources/Motion/Views/ContentView.swift +++ b/ios/Sources/Motion/Views/ContentView.swift @@ -14,8 +14,9 @@ // import SwiftUI + #if canImport(UIKit) -import UIKit + import UIKit #endif struct ContentView: View { @@ -42,8 +43,10 @@ struct ContentView: View { // scene's `interfaceOrientation` (authoritative + already de-noised, unlike raw // device motion which reports face-up/down) and refresh it on every orientation // change notification. This also drives the initial mode on first appearance. - .onReceive(NotificationCenter.default.publisher( - for: UIDevice.orientationDidChangeNotification)) { _ in + .onReceive( + NotificationCenter.default.publisher( + for: UIDevice.orientationDidChangeNotification) + ) { _ in refreshOrientation() } .task { @@ -68,7 +71,8 @@ struct ContentView: View { /// Resolve the current interface orientation from the active window scene and push /// landscape-ness into the model. Falls back to portrait when no scene is available. private func refreshOrientation() { - let scene = UIApplication.shared.connectedScenes + let scene = + UIApplication.shared.connectedScenes .compactMap { $0 as? UIWindowScene } .first { $0.activationState == .foregroundActive } ?? UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }.first diff --git a/ios/Sources/Motion/Views/GameView.swift b/ios/Sources/Motion/Views/GameView.swift index 907a102..cc84bdf 100644 --- a/ios/Sources/Motion/Views/GameView.swift +++ b/ios/Sources/Motion/Views/GameView.swift @@ -52,7 +52,7 @@ struct GameView: View { } Spacer() } - .padding(.top, 88) // clear the top control bar + .padding(.top, 88) // clear the top control bar .padding(.trailing, 16) // 3. Top control bar: exit + record toggle + status. @@ -131,7 +131,9 @@ struct GameView: View { // Saved-clip quick open (after a game). if case .saved = model.recorder.state, let url = model.recorder.lastSavedURL { - Button { shareURL = url } label: { + Button { + shareURL = url + } label: { Label("Video", systemImage: "play.rectangle.fill") .font(.caption.bold()) .foregroundStyle(.white) diff --git a/ios/Sources/Motion/Views/PairingView.swift b/ios/Sources/Motion/Views/PairingView.swift index 9911461..950ee16 100644 --- a/ios/Sources/Motion/Views/PairingView.swift +++ b/ios/Sources/Motion/Views/PairingView.swift @@ -13,125 +13,129 @@ #if MOTION_V2 -import SwiftUI - -struct PairingView: View { - @Environment(AppModel.self) private var model - @FocusState private var codeFocused: Bool - - var body: some View { - @Bindable var model = model - - VStack(spacing: 28) { - Spacer(minLength: 12) - - VStack(spacing: 6) { - Text("Motion") - .font(.largeTitle.bold()) - Text("Turn your device into a motion controller") - .font(.subheadline) - .foregroundStyle(.secondary) - } - - // Six-box code display driven by a hidden text field. - codeBoxes - .onTapGesture { codeFocused = true } - - // Hidden field that actually captures input. - TextField("", text: Binding( - get: { model.roomCode }, - set: { model.roomCode = sanitize($0) } - )) - .keyboardType(.asciiCapable) - .textInputAutocapitalization(.characters) - .autocorrectionDisabled() - .focused($codeFocused) - .frame(width: 1, height: 1) - .opacity(0.01) - - VStack(alignment: .leading, spacing: 6) { - Text("Server host") - .font(.caption) - .foregroundStyle(.secondary) - TextField("192.168.x.x", text: $model.partyHost) - .textFieldStyle(.roundedBorder) - .keyboardType(.URL) - .autocorrectionDisabled() - .textInputAutocapitalization(.never) - } - .padding(.horizontal, 24) - - Button(action: { model.connect() }) { - Text("Connect") - .font(.headline) - .frame(maxWidth: .infinity) - .padding(.vertical, 14) + import SwiftUI + + struct PairingView: View { + @Environment(AppModel.self) private var model + @FocusState private var codeFocused: Bool + + var body: some View { + @Bindable var model = model + + VStack(spacing: 28) { + Spacer(minLength: 12) + + VStack(spacing: 6) { + Text("Motion") + .font(.largeTitle.bold()) + Text("Turn your device into a motion controller") + .font(.subheadline) + .foregroundStyle(.secondary) + } + + // Six-box code display driven by a hidden text field. + codeBoxes + .onTapGesture { codeFocused = true } + + // Hidden field that actually captures input. + TextField( + "", + text: Binding( + get: { model.roomCode }, + set: { model.roomCode = sanitize($0) } + ) + ) + .keyboardType(.asciiCapable) + .textInputAutocapitalization(.characters) + .autocorrectionDisabled() + .focused($codeFocused) + .frame(width: 1, height: 1) + .opacity(0.01) + + VStack(alignment: .leading, spacing: 6) { + Text("Server host") + .font(.caption) + .foregroundStyle(.secondary) + TextField("192.168.x.x", text: $model.partyHost) + .textFieldStyle(.roundedBorder) + .keyboardType(.URL) + .autocorrectionDisabled() + .textInputAutocapitalization(.never) + } + .padding(.horizontal, 24) + + Button(action: { model.connect() }) { + Text("Connect") + .font(.headline) + .frame(maxWidth: .infinity) + .padding(.vertical, 14) + } + .buttonStyle(.borderedProminent) + .disabled(!isValidRoomCode(model.roomCode)) + .padding(.horizontal, 24) + + statusLine + + Spacer() } - .buttonStyle(.borderedProminent) - .disabled(!isValidRoomCode(model.roomCode)) - .padding(.horizontal, 24) - - statusLine - - Spacer() + .onAppear { codeFocused = true } } - .onAppear { codeFocused = true } - } - // MARK: - Pieces - - private var codeBoxes: some View { - HStack(spacing: 10) { - ForEach(0.. String { - let up = s.uppercased() - let filtered = up.filter { CODE_ALPHABET.contains($0) } - return String(filtered.prefix(CODE_LENGTH)) + /// Keep only alphabet chars, uppercase, clamp to code length. + private func sanitize(_ s: String) -> String { + let up = s.uppercased() + let filtered = up.filter { CODE_ALPHABET.contains($0) } + return String(filtered.prefix(CODE_LENGTH)) + } } -} -#endif // MOTION_V2 +#endif // MOTION_V2 diff --git a/ios/Sources/Motion/Views/RecordingView.swift b/ios/Sources/Motion/Views/RecordingView.swift index adf2b5d..63d0cab 100644 --- a/ios/Sources/Motion/Views/RecordingView.swift +++ b/ios/Sources/Motion/Views/RecordingView.swift @@ -35,10 +35,13 @@ struct RecordingView: View { .font(.subheadline.bold()) .foregroundStyle(.white) Spacer() - Toggle("", isOn: Binding( - get: { rec.isArmed }, - set: { _ in rec.toggle() } - )) + Toggle( + "", + isOn: Binding( + get: { rec.isArmed }, + set: { _ in rec.toggle() } + ) + ) .labelsHidden() .tint(.red) } diff --git a/ios/Sources/Motion/Views/SetupView.swift b/ios/Sources/Motion/Views/SetupView.swift index 6398288..d659fc4 100644 --- a/ios/Sources/Motion/Views/SetupView.swift +++ b/ios/Sources/Motion/Views/SetupView.swift @@ -17,7 +17,6 @@ import SwiftUI struct SetupView: View { @Environment(AppModel.self) private var model - @Environment(\.horizontalSizeClass) private var hSizeClass @Environment(\.verticalSizeClass) private var vSizeClass let session: PoseSession @@ -114,15 +113,18 @@ struct SetupView: View { model.cameraFacing = next session.switchCamera(to: next) } label: { - Image(systemName: model.cameraFacing == .front - ? "arrow.triangle.2.circlepath.camera.fill" // on front → tap to go wide-rear - : "camera.fill") // on wide-rear → tap to go front - .font(.title3) - .foregroundStyle(.white.opacity(0.8)) + Image( + systemName: model.cameraFacing == .front + ? "arrow.triangle.2.circlepath.camera.fill" // on front → tap to go wide-rear + : "camera.fill" + ) // on wide-rear → tap to go front + .font(.title3) + .foregroundStyle(.white.opacity(0.8)) } - .accessibilityLabel(model.cameraFacing == .front - ? "Switch to wide rear camera" - : "Switch to front camera") + .accessibilityLabel( + model.cameraFacing == .front + ? "Switch to wide rear camera" + : "Switch to front camera") } /// Expanded settings: Mac LAN IP (used for BOTH the game and the relay), the "Stream @@ -215,7 +217,7 @@ struct SetupView: View { // never shows green while the laptop shows nothing. Keeps the two coherent. if model.peerConnected { return .green } switch model.streamConnection { - case .connected: return .yellow // on the relay, but the laptop isn't here yet + case .connected: return .yellow // on the relay, but the laptop isn't here yet case .connecting, .reconnecting: return .yellow case .failed: return .red case .idle: return .gray @@ -287,10 +289,12 @@ struct SetupView: View { /// orientation-suggestive icon. Deliberately quiet — it's context, not a control. private var modeChip: some View { HStack(spacing: 4) { - Image(systemName: model.framingMode == .fullBody - ? "figure.stand" - : "hand.raised.fill") - .font(.caption2) + Image( + systemName: model.framingMode == .fullBody + ? "figure.stand" + : "hand.raised.fill" + ) + .font(.caption2) Text(model.framingMode.label) .font(.caption2.bold()) } diff --git a/ios/Sources/Motion/Vision/CameraController.swift b/ios/Sources/Motion/Vision/CameraController.swift index c3832c6..b68302f 100644 --- a/ios/Sources/Motion/Vision/CameraController.swift +++ b/ios/Sources/Motion/Vision/CameraController.swift @@ -250,7 +250,8 @@ final class CameraController: NSObject, @unchecked Sendable { // would be un-mirrored and pose would appear left/right-swapped on the rear camera — // that's the #1 rear-camera on-device verify item. if let connection = videoOutput.connection(with: .video), - connection.isVideoMirroringSupported { + connection.isVideoMirroringSupported + { connection.automaticallyAdjustsVideoMirroring = false connection.isVideoMirrored = true } @@ -348,7 +349,8 @@ final class CameraController: NSObject, @unchecked Sendable { /// which is the invariant PoseEstimator's coordinate mapping depends on. private func applyCaptureAngle(_ angle: CGFloat) { guard let connection = videoOutput.connection(with: .video), - connection.isVideoRotationAngleSupported(angle) else { return } + connection.isVideoRotationAngleSupported(angle) + else { return } connection.videoRotationAngle = angle } @@ -357,7 +359,8 @@ final class CameraController: NSObject, @unchecked Sendable { private func applyPreviewAngle(_ angle: CGFloat) { DispatchQueue.main.async { [weak self] in guard let connection = self?.previewLayer?.connection, - connection.isVideoRotationAngleSupported(angle) else { return } + connection.isVideoRotationAngleSupported(angle) + else { return } connection.videoRotationAngle = angle } } @@ -366,9 +369,11 @@ final class CameraController: NSObject, @unchecked Sendable { // MARK: - Sample buffer delegate extension CameraController: AVCaptureVideoDataOutputSampleBufferDelegate { - func captureOutput(_ output: AVCaptureOutput, - didOutput sampleBuffer: CMSampleBuffer, - from connection: AVCaptureConnection) { + func captureOutput( + _ output: AVCaptureOutput, + didOutput sampleBuffer: CMSampleBuffer, + from connection: AVCaptureConnection + ) { guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } // The connection is kept horizon-level by the RotationCoordinator (see // `installRotationCoordinator`) and ALWAYS mirrored (both front and wide-rear cameras diff --git a/ios/Sources/Motion/Vision/HandLandmarkerEstimator.swift b/ios/Sources/Motion/Vision/HandLandmarkerEstimator.swift index 5936571..9c332ca 100644 --- a/ios/Sources/Motion/Vision/HandLandmarkerEstimator.swift +++ b/ios/Sources/Motion/Vision/HandLandmarkerEstimator.swift @@ -106,9 +106,9 @@ final class HandLandmarkerEstimator: @unchecked Sendable { // MediaPipe's 21-landmark indices (the ordering is a stable, documented MediaPipe // contract — same as the `HandLandmark` enum: 0 = wrist … 20 = pinky tip). We use raw // indices to avoid depending on the exact Swift-imported enum name. - private let wristIdx = 0 // WRIST - private let middleMCPIdx = 9 // MIDDLE_FINGER_MCP (palm-length reference) - private let indexTipIdx = 8 // INDEX_FINGER_TIP + private let wristIdx = 0 // WRIST + private let middleMCPIdx = 9 // MIDDLE_FINGER_MCP (palm-length reference) + private let indexTipIdx = 8 // INDEX_FINGER_TIP private let nonThumbTipIdx: [Int] = [8, 12, 16, 20] // INDEX/MIDDLE/RING/PINKY tips // MARK: - Init @@ -118,7 +118,9 @@ final class HandLandmarkerEstimator: @unchecked Sendable { /// can't initialize — so the caller degrades to Vision instead of crashing. init() { guard let modelPath = Bundle.main.path(forResource: "hand_landmarker", ofType: "task") else { - NSLog("[Motion] MediaPipe HandLandmarker: model 'hand_landmarker.task' not found in bundle — falling back to Vision hand path.") + NSLog( + "[Motion] MediaPipe HandLandmarker: model 'hand_landmarker.task' not found in bundle — falling back to Vision hand path." + ) landmarker = nil return } @@ -137,7 +139,9 @@ final class HandLandmarkerEstimator: @unchecked Sendable { landmarker = try HandLandmarker(options: options) NSLog("[Motion] MediaPipe HandLandmarker initialized (VIDEO mode, numHands=2).") } catch { - NSLog("[Motion] MediaPipe HandLandmarker init failed: \(error.localizedDescription) — falling back to Vision hand path.") + NSLog( + "[Motion] MediaPipe HandLandmarker init failed: \(error.localizedDescription) — falling back to Vision hand path." + ) landmarker = nil } } @@ -264,7 +268,7 @@ final class HandLandmarkerEstimator: @unchecked Sendable { let dl = leftWrist.map { sqDist(h.wrist, $0) } ?? Double.greatestFiniteMagnitude let dr = rightWrist.map { sqDist(h.wrist, $0) } ?? Double.greatestFiniteMagnitude if dl <= dr { - if left == nil || dl < left!.d { // closer left claim wins; loser tries right + if left == nil || dl < left!.d { // closer left claim wins; loser tries right if let prev = left { assignFallback(prev.hand, into: &right) } left = (h, dl) } else { @@ -349,12 +353,14 @@ final class HandLandmarkerEstimator: @unchecked Sendable { private func clamp01(_ v: Double) -> Double { min(1.0, max(0.0, v)) } private func dist(_ a: NormalizedLandmark, _ b: NormalizedLandmark) -> Double { - let dx = Double(a.x - b.x), dy = Double(a.y - b.y) + let dx = Double(a.x - b.x) + let dy = Double(a.y - b.y) return (dx * dx + dy * dy).squareRoot() } private func sqDist(_ a: Point2, _ b: Point2) -> Double { - let dx = a[0] - b[0], dy = a[1] - b[1] + let dx = a[0] - b[0] + let dy = a[1] - b[1] return dx * dx + dy * dy } } diff --git a/ios/Sources/Motion/Vision/HandPoseEstimator.swift b/ios/Sources/Motion/Vision/HandPoseEstimator.swift index c2c3256..8d72f90 100644 --- a/ios/Sources/Motion/Vision/HandPoseEstimator.swift +++ b/ios/Sources/Motion/Vision/HandPoseEstimator.swift @@ -134,7 +134,8 @@ final class HandPoseEstimator: @unchecked Sendable { mapPoint: (VNRecognizedPoint) -> Point2 ) -> (openness: Double, indexTip: Point2?) { if observation != nil { hasEverSeenHand = true } - let reading = observation.map { measure(observation: $0, mapPoint: mapPoint) } + let reading = + observation.map { measure(observation: $0, mapPoint: mapPoint) } ?? HandSideReading(openness: nil, indexTip: nil) switch side { @@ -224,7 +225,7 @@ final class HandPoseEstimator: @unchecked Sendable { return next } // Not detected this frame. - guard let prev = previous else { return 1.0 } // never seen → open, never grabs + guard let prev = previous else { return 1.0 } // never seen → open, never grabs // Decay the held value toward neutral so a lost hand relaxes instead of sticking. let next = prev + (neutralOpenness - prev) * 0.1 previous = next diff --git a/ios/Sources/Motion/Vision/PoseEstimator.swift b/ios/Sources/Motion/Vision/PoseEstimator.swift index 12dcdee..dd006bf 100644 --- a/ios/Sources/Motion/Vision/PoseEstimator.swift +++ b/ios/Sources/Motion/Vision/PoseEstimator.swift @@ -157,9 +157,10 @@ final class PoseEstimator: @unchecked Sendable { let now = ProcessInfo.processInfo.systemUptime guard now - self.lastEmit >= self.minEmitInterval else { return } - let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, - orientation: orientation, - options: [:]) + let handler = VNImageRequestHandler( + cvPixelBuffer: pixelBuffer, + orientation: orientation, + options: [:]) // ── PASS 1: BODY (full frame) ──────────────────────────────────────────────── // We need the wrist positions before we can build the per-hand ROIs, so the body @@ -182,8 +183,9 @@ final class PoseEstimator: @unchecked Sendable { self.lastEmit = now // Pass the pixel buffer + frame time through so the MediaPipe hand path can run on // the SAME frame (VIDEO mode needs a monotonic ms timestamp; `now` is our clock). - self.buildAndEmit(from: points, handler: handler, - pixelBuffer: pixelBuffer, timestampSeconds: now) + self.buildAndEmit( + from: points, handler: handler, + pixelBuffer: pixelBuffer, timestampSeconds: now) } } @@ -191,16 +193,18 @@ final class PoseEstimator: @unchecked Sendable { /// Map Vision's recognized points to the 8 protocol joints, applying coordinate /// flip, left/right swap, smoothing, and confidence thresholding. - private func buildAndEmit(from points: [VNHumanBodyPoseObservation.JointName: VNRecognizedPoint], - handler: VNImageRequestHandler, - pixelBuffer: CVPixelBuffer, - timestampSeconds: TimeInterval) { + private func buildAndEmit( + from points: [VNHumanBodyPoseObservation.JointName: VNRecognizedPoint], + handler: VNImageRequestHandler, + pixelBuffer: CVPixelBuffer, + timestampSeconds: TimeInterval + ) { // Pull a Vision point if it clears the confidence threshold, converting to // top-left origin. x is kept as-is (buffer already mirrored); y is flipped. func pt(_ name: VNHumanBodyPoseObservation.JointName) -> (Point2, Double)? { guard let p = points[name], p.confidence >= confidenceThreshold else { return nil } - let x = Double(p.location.x) // already mirror-space - let y = 1.0 - Double(p.location.y) // bottom-left → top-left + let x = Double(p.location.x) // already mirror-space + let y = 1.0 - Double(p.location.y) // bottom-left → top-left return ([x, y], Double(p.confidence)) } @@ -228,22 +232,23 @@ final class PoseEstimator: @unchecked Sendable { // HANDS: wrists. SWAP because the mirrored buffer flips visual sides — // Vision's LEFT wrist is the player's RIGHT hand, and vice-versa. raw[.rightHand] = pt(.leftWrist) - raw[.leftHand] = pt(.rightWrist) + raw[.leftHand] = pt(.rightWrist) // TORSO: Vision's `.root` is the pelvis/hip center. Fall back to the midpoint // of the shoulders if root is missing, then to the midpoint of the hips. - raw[.torso] = pt(.root) + raw[.torso] = + pt(.root) ?? mid(pt(.leftShoulder), pt(.rightShoulder)) ?? mid(pt(.leftHip), pt(.rightHip)) // KNEES (swapped, same reason as hands). raw[.rightKnee] = pt(.leftKnee) - raw[.leftKnee] = pt(.rightKnee) + raw[.leftKnee] = pt(.rightKnee) // FEET: ankles are the lowest reliable body-pose joint (no toe joint in the // 2D body model), so ankle == foot. (Swapped.) raw[.rightFoot] = pt(.leftAnkle) - raw[.leftFoot] = pt(.rightAnkle) + raw[.leftFoot] = pt(.rightAnkle) // ARM CHAIN (shoulders + elbows) — OPTIONAL. SWAPPED exactly like hands/knees/feet // above: the mirrored buffer flips visual sides, so Vision's LEFT is the player's @@ -251,12 +256,15 @@ final class PoseEstimator: @unchecked Sendable { // stay absent here, so the packet omits them rather than sending garbage. var rawArms: [ArmJointName: (Point2, Double)] = [:] rawArms[.rightShoulder] = ptArm(.leftShoulder) - rawArms[.leftShoulder] = ptArm(.rightShoulder) - rawArms[.rightElbow] = ptArm(.leftElbow) - rawArms[.leftElbow] = ptArm(.rightElbow) + rawArms[.leftShoulder] = ptArm(.rightShoulder) + rawArms[.rightElbow] = ptArm(.leftElbow) + rawArms[.leftElbow] = ptArm(.rightElbow) // Nothing usable this frame. - guard !raw.isEmpty else { emitLost(); return } + guard !raw.isEmpty else { + emitLost() + return + } // Exponential smoothing per joint. A joint that was absent adopts the new // value directly; a joint absent this frame keeps its previous smoothed value @@ -283,7 +291,9 @@ final class PoseEstimator: @unchecked Sendable { } } - let quality = confidences.values.isEmpty ? 0 + let quality = + confidences.values.isEmpty + ? 0 : confidences.values.reduce(0, +) / Double(JointName.allCases.count) // Arm-chain smoothing — same exponential filter (`a`) as the required joints above. @@ -327,8 +337,9 @@ final class PoseEstimator: @unchecked Sendable { timestampSeconds: timestampSeconds ) - let frame = PoseFrame(joints: outJoints, armJoints: outArms, quality: quality, - confidences: confidences, hands: hands, fingertips: fingertips) + let frame = PoseFrame( + joints: outJoints, armJoints: outArms, quality: quality, + confidences: confidences, hands: hands, fingertips: fingertips) Task { @MainActor in self.delegate?.poseEstimator(self, didProduce: frame) } } @@ -402,10 +413,12 @@ final class PoseEstimator: @unchecked Sendable { // Resolve each side's single observation, preferring the ROI result and falling back // to the nearest full-frame observation by wrist when the ROI side was absent. - let leftObs: VNHumanHandPoseObservation? = haveLeft + let leftObs: VNHumanHandPoseObservation? = + haveLeft ? leftHandRequest.results?.first : nearestFallbackObservation(toTopLeftWrist: leftWrist) - let rightObs: VNHumanHandPoseObservation? = haveRight + let rightObs: VNHumanHandPoseObservation? = + haveRight ? rightHandRequest.results?.first : nearestFallbackObservation(toTopLeftWrist: rightWrist) @@ -419,15 +432,18 @@ final class PoseEstimator: @unchecked Sendable { // full-frame request, its ROI is the full frame (identity remap). let leftROI = haveLeft ? leftHandRequest.regionOfInterest : fullFrameROI let rightROI = haveRight ? rightHandRequest.regionOfInterest : fullFrameROI - let leftReading = handEstimator.analyzeSide(observation: leftObs, side: .left, - mapPoint: { self.mapRecognizedPoint($0, roi: leftROI) }) - let rightReading = handEstimator.analyzeSide(observation: rightObs, side: .right, - mapPoint: { self.mapRecognizedPoint($0, roi: rightROI) }) + let leftReading = handEstimator.analyzeSide( + observation: leftObs, side: .left, + mapPoint: { self.mapRecognizedPoint($0, roi: leftROI) }) + let rightReading = handEstimator.analyzeSide( + observation: rightObs, side: .right, + mapPoint: { self.mapRecognizedPoint($0, roi: rightROI) }) let hands = HandState(left: leftReading.openness, right: rightReading.openness) // Fingertips: omit the whole field only when BOTH sides are nil. - let fingertips: Fingertips? = (leftReading.indexTip == nil && rightReading.indexTip == nil) + let fingertips: Fingertips? = + (leftReading.indexTip == nil && rightReading.indexTip == nil) ? nil : Fingertips(left: leftReading.indexTip, right: rightReading.indexTip) @@ -437,7 +453,8 @@ final class PoseEstimator: @unchecked Sendable { /// Pick the full-frame (fallback) hand observation whose wrist is nearest the given /// body wrist (top-left space). Used only for a side that had no ROI observation. private func nearestFallbackObservation(toTopLeftWrist wrist: Point2?) - -> VNHumanHandPoseObservation? { + -> VNHumanHandPoseObservation? + { let candidates = fallbackHandRequest.results ?? [] guard !candidates.isEmpty else { return nil } guard let wrist, wrist.count == 2 else { return candidates.first } @@ -452,9 +469,13 @@ final class PoseEstimator: @unchecked Sendable { // Map the fallback wrist into the same top-left/mirror frame for a fair compare. // Fallback observations are full-image, so use the full-frame (identity) ROI. let mapped = mapRecognizedPoint(w, roi: fullFrameROI) - let dx = mapped[0] - Double(target.x), dy = mapped[1] - Double(target.y) + let dx = mapped[0] - Double(target.x) + let dy = mapped[1] - Double(target.y) let d = dx * dx + dy * dy - if d < bestDist { bestDist = d; best = obs } + if d < bestDist { + bestDist = d + best = obs + } } return best ?? candidates.first } @@ -474,8 +495,8 @@ final class PoseEstimator: @unchecked Sendable { /// [0,1] (Vision REJECTS an ROI that exceeds the image — that would throw in /// `perform` and we'd lose the hand). private func roi(aroundTopLeftWrist wrist: Point2) -> CGRect { - let cx = CGFloat(wrist[0]) // mirror-space x, same as Vision - let cy = 1.0 - CGFloat(wrist[1]) // top-left y → Vision bottom-left y + let cx = CGFloat(wrist[0]) // mirror-space x, same as Vision + let cy = 1.0 - CGFloat(wrist[1]) // top-left y → Vision bottom-left y let half = roiSize / 2 // Clamp the CENTER so a full-size box fits, then place the box. This keeps the box // exactly `roiSize` wide (rather than shrinking it) while staying inside [0,1]. @@ -506,9 +527,9 @@ final class PoseEstimator: @unchecked Sendable { // full-image space using the ROI rect (Vision bottom-left), then flip to top-left. // For the full-frame fallback (roi = 0,0,1,1) this is an identity — so both the // ROI-zoomed and fallback paths use the exact same helper. - let fullX = Double(roi.minX + point.location.x * roi.width) // full-image, mirror-space + let fullX = Double(roi.minX + point.location.x * roi.width) // full-image, mirror-space let fullYBottomLeft = Double(roi.minY + point.location.y * roi.height) - return [fullX, 1.0 - fullYBottomLeft] // bottom-left → top-left + return [fullX, 1.0 - fullYBottomLeft] // bottom-left → top-left } /// The full-frame ROI — used for fallback (full-image) observations, where the diff --git a/ios/Sources/Motion/Vision/SetupEvaluator.swift b/ios/Sources/Motion/Vision/SetupEvaluator.swift index b196a06..c8d7d26 100644 --- a/ios/Sources/Motion/Vision/SetupEvaluator.swift +++ b/ios/Sources/Motion/Vision/SetupEvaluator.swift @@ -100,10 +100,11 @@ final class SetupEvaluator { // Distance from head→feet span in frame. Too tall a span = too close; // too short = too far. if let head = frame.joints[.head], - let lf = frame.joints[.leftFoot], - let rf = frame.joints[.rightFoot] { + let lf = frame.joints[.leftFoot], + let rf = frame.joints[.rightFoot] + { let footY = max(lf[1], rf[1]) - let bodySpan = footY - head[1] // both top-left origin, feet below head + let bodySpan = footY - head[1] // both top-left origin, feet below head if bodySpan > 0.95 { return verdict(.tooClose, "Step back a little.") } @@ -153,9 +154,10 @@ final class SetupEvaluator { /// Evaluate a lost frame (no body detected at all). func evaluateLost() -> SetupVerdict { goodSince = nil - return SetupVerdict(tracking: .lost, - guidance: "No one in view. Point the camera at your body.", - ready: false) + return SetupVerdict( + tracking: .lost, + guidance: "No one in view. Point the camera at your body.", + ready: false) } // MARK: - Debounce diff --git a/scripts/check-code-health.mjs b/scripts/check-code-health.mjs index 134db20..bc4e1cc 100644 --- a/scripts/check-code-health.mjs +++ b/scripts/check-code-health.mjs @@ -131,10 +131,10 @@ function checkCoverage() { ); // Ratcheted legacy debt: https://github.com/Significant-Hobbies/motion/issues/26 checkMinimums("TypeScript coverage", observed, { - lines: 6.4, - branches: 10.06, - functions: 7.01, - statements: 6.49, + lines: 39.4, + branches: 37.7, + functions: 43.5, + statements: 39.9, }); } @@ -170,9 +170,9 @@ function checkComplexity() { ); // Ratcheted legacy debt: https://github.com/Significant-Hobbies/motion/issues/26 failRegressions("Complexity", observed, { - violations: 3, + violations: 2, maxCcn: 22, - maxLength: 180, + maxLength: 149, maxParams: 7, }); } @@ -210,9 +210,9 @@ function checkDuplication() { ); // Ratcheted legacy debt: https://github.com/Significant-Hobbies/motion/issues/26 failRegressions("Duplication", observed, { - clones: 5, - duplicatedLines: 65, - percentage: 0.5543237250554324, + clones: 0, + duplicatedLines: 0, + percentage: 0, }); } @@ -276,13 +276,21 @@ function checkHygiene() { function checkSwiftFormat() { const result = run( "xcrun", - ["swift-format", "lint", "--strict", "--recursive", "ios/Sources"], + [ + "swift-format", + "lint", + "--strict", + "--configuration", + ".swift-format", + "--recursive", + "ios/Sources", + ], { allowFailure: true } ); const diagnostics = countSwiftFormatDiagnostics(result); log(`Swift format debt: ${diagnostics} diagnostics.`); // Ratcheted legacy debt: https://github.com/Significant-Hobbies/motion/issues/26 - failRegressions("Swift format", { diagnostics }, { diagnostics: 4553 }); + failRegressions("Swift format", { diagnostics }, { diagnostics: 8 }); } function checkSwiftUnused() { @@ -323,7 +331,7 @@ function checkSwiftUnused() { `Swift unused-code debt: ${observed.findings} Periphery ${version} findings.` ); // Ratcheted legacy debt: https://github.com/Significant-Hobbies/motion/issues/26 - failRegressions("Swift unused code", observed, { findings: 68 }); + failRegressions("Swift unused code", observed, { findings: 66 }); } const checks = { diff --git a/server/src/server.test.ts b/server/src/server.test.ts new file mode 100644 index 0000000..f5da2bc --- /dev/null +++ b/server/src/server.test.ts @@ -0,0 +1,273 @@ +import type * as Party from "partykit/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { PROTOCOL_VERSION } from "../../protocol/protocol"; +import MotionServer from "./server"; + +type ConnState = { role: "display" | "controller" }; + +/** Minimal in-memory stand-in for a PartyKit connection. */ +class FakeConnection { + readonly sent: string[] = []; + state: ConnState | null = null; + closed: { code?: number; reason?: string } | null = null; + throwOnSend = false; + + constructor( + readonly id: string, + private readonly room: FakeRoom + ) {} + + send(payload: string): void { + if (this.throwOnSend) throw new Error("Can't call send() after close()"); + this.sent.push(payload); + } + + setState(state: ConnState): ConnState { + this.state = state; + return state; + } + + close(code?: number, reason?: string): void { + this.closed = { code, reason }; + this.room.connections.delete(this.id); + } + + messages(): Record[] { + return this.sent.map((s) => JSON.parse(s) as Record); + } +} + +class FakeRoom { + readonly id = "BCDFGH"; + readonly connections = new Map(); + + connect(id: string): FakeConnection { + const conn = new FakeConnection(id, this); + this.connections.set(id, conn); + return conn; + } + + *getConnections(): IterableIterator { + yield* this.connections.values(); + } +} + +function asConn(c: FakeConnection): Party.Connection { + return c as unknown as Party.Connection; +} + +const joints = { + head: [0.5, 0.2], + leftHand: [0.3, 0.5], + rightHand: [0.7, 0.5], + torso: [0.5, 0.5], + leftKnee: [0.4, 0.7], + rightKnee: [0.6, 0.7], + leftFoot: [0.4, 0.9], + rightFoot: [0.6, 0.9], +}; + +function pose(seq: number): string { + return JSON.stringify({ + v: 1, + type: "pose", + seq, + sentAt: seq, + quality: 0.9, + joints, + }); +} + +function join(role: string, v: number = PROTOCOL_VERSION): string { + return JSON.stringify({ v, type: "join", role }); +} + +describe("MotionServer relay", () => { + let room: FakeRoom; + let server: MotionServer; + + function connectAs(id: string, role: "display" | "controller") { + const conn = room.connect(id); + server.onConnect(asConn(conn)); + server.onMessage(join(role), asConn(conn)); + return conn; + } + + beforeEach(() => { + vi.spyOn(console, "log").mockImplementation(() => undefined); + room = new FakeRoom(); + server = new MotionServer(room as unknown as Party.Room); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it("requires a join before accepting any other message", () => { + const conn = room.connect("a"); + server.onMessage( + JSON.stringify({ v: 1, type: "ping", t: 1 }), + asConn(conn) + ); + server.onMessage("not json", asConn(conn)); + server.onMessage(new ArrayBuffer(4), asConn(conn)); + + expect(conn.messages()).toEqual([ + expect.objectContaining({ type: "error", code: "bad_message" }), + expect.objectContaining({ type: "error", code: "bad_message" }), + ]); + expect(conn.state).toBeNull(); + }); + + it("rejects protocol mismatches and unknown roles", () => { + const old = room.connect("old"); + server.onMessage(join("display", 99), asConn(old)); + expect(old.messages()[0]).toMatchObject({ code: "version_mismatch" }); + expect(old.closed).not.toBeNull(); + + const odd = room.connect("odd"); + server.onMessage(join("spectator"), asConn(odd)); + expect(odd.messages()[0]).toMatchObject({ code: "bad_role" }); + expect(odd.closed).not.toBeNull(); + }); + + it("broadcasts authoritative presence to both roles", () => { + const display = connectAs("d", "display"); + expect(display.messages()).toEqual([ + { v: 1, type: "peer", role: "controller", connected: false }, + ]); + + const controller = connectAs("c", "controller"); + expect(display.messages().at(-1)).toEqual({ + v: 1, + type: "peer", + role: "controller", + connected: true, + }); + expect(controller.messages().at(-1)).toEqual({ + v: 1, + type: "peer", + role: "display", + connected: true, + }); + + server.onClose(asConn(controller)); + room.connections.delete("c"); + expect(display.messages().at(-1)).toMatchObject({ + role: "controller", + connected: false, + }); + }); + + it("rejects a second display but lets a reconnecting controller win", () => { + connectAs("d1", "display"); + const second = connectAs("d2", "display"); + expect(second.messages()[0]).toMatchObject({ code: "room_full" }); + expect(second.closed).not.toBeNull(); + + const stale = connectAs("c1", "controller"); + const fresh = connectAs("c2", "controller"); + expect(stale.closed).toEqual({ + code: 1000, + reason: "replaced by a newer connection", + }); + expect(fresh.state).toEqual({ role: "controller" }); + }); + + it("relays each message type only in its allowed direction", () => { + const display = connectAs("d", "display"); + const controller = connectAs("c", "controller"); + display.sent.length = 0; + controller.sent.length = 0; + + const status = JSON.stringify({ v: 1, type: "status", state: "ok" }); + const start = JSON.stringify({ v: 1, type: "start" }); + const rec = JSON.stringify({ v: 1, type: "rec", action: "arm" }); + const chunk = JSON.stringify({ v: 1, type: "recchunk", seq: 0 }); + + server.onMessage(status, asConn(controller)); + server.onMessage(status, asConn(display)); // wrong direction: dropped + server.onMessage(start, asConn(display)); + server.onMessage(start, asConn(controller)); // wrong direction: dropped + server.onMessage(rec, asConn(controller)); + server.onMessage(rec, asConn(display)); + server.onMessage(chunk, asConn(display)); + server.onMessage(chunk, asConn(controller)); // wrong direction: dropped + server.onMessage(join("display"), asConn(display)); // duplicate join ignored + server.onMessage(JSON.stringify({ v: 1, type: "future" }), asConn(display)); + + expect(display.sent).toEqual([status, rec]); + expect(controller.sent).toEqual([start, rec, chunk]); + }); + + it("answers ping with pong without relaying it", () => { + const display = connectAs("d", "display"); + const controller = connectAs("c", "controller"); + display.sent.length = 0; + controller.sent.length = 0; + + server.onMessage( + JSON.stringify({ v: 1, type: "ping", t: 42 }), + asConn(controller) + ); + server.onMessage( + JSON.stringify({ v: 1, type: "pong", t: 42 }), + asConn(controller) + ); + + expect(controller.messages()).toEqual([{ v: 1, type: "pong", t: 42 }]); + expect(display.sent).toEqual([]); + }); + + it("validates and rate-limits pose packets per controller", () => { + vi.useFakeTimers(); + vi.setSystemTime(10_000); + const display = connectAs("d", "display"); + const controller = connectAs("c", "controller"); + display.sent.length = 0; + + server.onMessage(pose(0), asConn(display)); // displays never send pose + server.onMessage( + JSON.stringify({ v: 1, type: "pose", seq: 1, joints: {} }), + asConn(controller) + ); + expect(display.sent).toEqual([]); + + for (let seq = 1; seq <= 45; seq++) { + server.onMessage(pose(seq), asConn(controller)); + } + expect(display.sent).toHaveLength(40); + + vi.setSystemTime(11_000); + server.onMessage(pose(46), asConn(controller)); + expect(display.sent).toHaveLength(41); + expect(display.sent.at(-1)).toBe(pose(46)); + }); + + it("never lets a send to a closing socket escape the handler", () => { + const display = connectAs("d", "display"); + const controller = connectAs("c", "controller"); + display.throwOnSend = true; + + expect(() => server.onMessage(pose(1), asConn(controller))).not.toThrow(); + + server.onError(asConn(display), new Error("reset")); + expect(controller.messages().at(-1)).toMatchObject({ + role: "display", + connected: false, + }); + }); + + it("ignores close/error from connections that never joined", () => { + const display = connectAs("d", "display"); + const anon = room.connect("anon"); + display.sent.length = 0; + + server.onClose(asConn(anon)); + server.onError(asConn(anon), new Error("reset")); + + expect(display.sent).toEqual([]); + }); +}); diff --git a/web/src/games/canvas-utils.test.ts b/web/src/games/canvas-utils.test.ts new file mode 100644 index 0000000..d451672 --- /dev/null +++ b/web/src/games/canvas-utils.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from "vitest"; + +import type { BodyController, Renderer } from "../sdk"; +import { + beginScoreHud, + distPointToSegment, + handJoint, + roundRect, +} from "./canvas-utils"; + +type Call = [string, ...unknown[]]; + +/** Records every 2D-context call and property write, in order. */ +function recordingContext(calls: Call[]): CanvasRenderingContext2D { + return new Proxy({} as CanvasRenderingContext2D, { + get: + (_t, prop) => + (...args: unknown[]) => { + calls.push([String(prop), ...args]); + }, + set: (_t, prop, value) => { + calls.push([`=${String(prop)}`, value]); + return true; + }, + }); +} + +describe("distPointToSegment", () => { + it("measures to the nearest point on the segment, clamped to its ends", () => { + expect(distPointToSegment(0.5, 1, 0, 0, 1, 0)).toBeCloseTo(1); + expect(distPointToSegment(-3, 4, 0, 0, 1, 0)).toBeCloseTo(5); + expect(distPointToSegment(4, 4, 0, 0, 1, 0)).toBeCloseTo(5); + }); + + it("treats a zero-length segment as a point", () => { + expect(distPointToSegment(3, 4, 0, 0, 0, 0)).toBeCloseTo(5); + }); +}); + +describe("roundRect", () => { + it("clamps the radius to half the shortest side", () => { + const calls: Call[] = []; + roundRect(recordingContext(calls), 0, 0, 10, 4, 99); + expect(calls[0]).toEqual(["beginPath"]); + expect(calls[1]).toEqual(["moveTo", 2, 0]); + expect(calls.filter(([name]) => name === "arcTo")).toHaveLength(4); + expect( + calls.every(([name, ...args]) => name !== "arcTo" || args[4] === 2) + ).toBe(true); + expect(calls.at(-1)).toEqual(["closePath"]); + }); +}); + +describe("handJoint", () => { + it("returns the tracked hand joint for each side", () => { + const body = { + joints: { leftHand: [0.1, 0.2], rightHand: [0.8, 0.9] }, + } as unknown as BodyController; + expect(handJoint(body, "left")).toEqual([0.1, 0.2]); + expect(handJoint(body, "right")).toEqual([0.8, 0.9]); + }); +}); + +describe("beginScoreHud", () => { + it("opens a saved HUD layer and draws the zero-padded score top-left", () => { + const calls: Call[] = []; + const renderer = { + ctx: recordingContext(calls), + area: { x: 10, y: 20, w: 1000, h: 500 }, + sx: (n: number) => n * 1000, + sy: (n: number) => n * 500, + } as unknown as Renderer; + + beginScoreHud(renderer, 42, 5); + + expect(calls).toEqual([ + ["save"], + ["=textBaseline", "top"], + ["=fillStyle", "#f4f7ff"], + ["=font", "bold 50px system-ui"], + ["=textAlign", "left"], + ["fillText", "00042", 40, 35], + ]); + }); +}); diff --git a/web/src/games/canvas-utils.ts b/web/src/games/canvas-utils.ts index aaf118b..9f7fb0a 100644 --- a/web/src/games/canvas-utils.ts +++ b/web/src/games/canvas-utils.ts @@ -1,5 +1,7 @@ // Shared canvas drawing helpers used by multiple games. -// Extracted to avoid duplication of geometry and path utilities. +// Extracted to avoid duplication of geometry, path, HUD and hand-input utilities. + +import type { BodyController, Renderer } from "../sdk"; /** Shortest distance from point (px,py) to the segment (ax,ay)–(bx,by). */ export function distPointToSegment( @@ -37,3 +39,34 @@ export function roundRect( ctx.arcTo(x, y, x + w, y, rr); ctx.closePath(); } + +/** Normalized position of the tracked HAND joint for `which` (the grab/blade point). */ +export function handJoint( + body: BodyController, + which: "left" | "right" +): readonly [number, number] { + return which === "left" ? body.joints.leftHand : body.joints.rightHand; +} + +/** + * Open the HUD layer (`ctx.save()`, top baseline) and draw the zero-padded score in + * the play area's top-left corner. The caller draws the rest and calls `ctx.restore()`. + */ +export function beginScoreHud( + r: Renderer, + score: number, + digits: number +): void { + const { ctx } = r; + const p = r.area; + ctx.save(); + ctx.textBaseline = "top"; + ctx.fillStyle = "#f4f7ff"; + ctx.font = `bold ${Math.round(r.sx(0.05))}px system-ui`; + ctx.textAlign = "left"; + ctx.fillText( + String(score).padStart(digits, "0"), + p.x + r.sx(0.03), + p.y + r.sy(0.03) + ); +} diff --git a/web/src/games/motion-maker/index.test.ts b/web/src/games/motion-maker/index.test.ts new file mode 100644 index 0000000..dd83ab8 --- /dev/null +++ b/web/src/games/motion-maker/index.test.ts @@ -0,0 +1,105 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { BodyController } from "../../sdk"; +import { MotionMaker } from "."; + +const FRAME_MS = 1000 / 60; + +/** Left hand at (x, y) with openness `open`; the right hand rests far away, open. */ +function body(x: number, y: number, open: number): BodyController { + return { + joints: { + head: [0.5, 0.18], + leftHand: [x, y], + rightHand: [0.95, 0.05], + torso: [0.5, 0.5], + leftKnee: [0.42, 0.72], + rightKnee: [0.58, 0.72], + leftFoot: [0.42, 0.95], + rightFoot: [0.58, 0.95], + }, + squatAmount: 0, + leanAmount: 0, + leftHandOpen: open, + rightHandOpen: 1, + trackingQuality: 1, + health: "ok", + hasRequiredJoints: true, + ageMs: 0, + tick: () => undefined, + dispose: () => undefined, + }; +} + +function hold(game: MotionMaker, frames: number, input: BodyController): void { + for (let f = 0; f < frames; f++) game.update(FRAME_MS, input); +} + +/** Open the hand at (x, y) to arm it, then close it to grab. */ +function grabAt(game: MotionMaker, x: number, y: number): void { + hold(game, 3, body(x, y, 1)); + hold(game, 3, body(x, y, 0)); +} + +function stat(game: MotionMaker, label: string): string | undefined { + return game.result().stats?.find((s) => s.label === label)?.value; +} + +describe("MotionMaker", () => { + let game: MotionMaker; + + beforeEach(() => { + // random 0.5 → every ball spawns at (0.5, 0.29) with no drift. + vi.spyOn(Math, "random").mockReturnValue(0.5); + game = new MotionMaker(); + game.init({ width: 1280, height: 720 }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("never ends: it is a live mirror", () => { + hold(game, 600, body(0.5, 0.5, 1)); + expect(game.isOver()).toBe(false); + }); + + it("scores a ball carried into the bin and released", () => { + grabAt(game, 0.5, 0.29); + // Carry it down to the bin, then let the hand settle so the release is gentle. + for (let y = 0.29; y <= 0.82; y += 0.01) + game.update(FRAME_MS, body(0.5, y, 0)); + hold(game, 30, body(0.5, 0.82, 0)); + hold(game, 5, body(0.5, 0.82, 1)); + + expect(stat(game, "dropped in bin")).toBe("1"); + expect(game.result().score).toBe(100); + }); + + it("does not grab without a deliberate open→close gesture", () => { + // Starts closed (never armed) over the balls, then carries "them" to the bin. + hold(game, 3, body(0.5, 0.29, 0)); + for (let y = 0.29; y <= 0.82; y += 0.01) + game.update(FRAME_MS, body(0.5, y, 0)); + hold(game, 30, body(0.5, 0.82, 1)); + expect(stat(game, "dropped in bin")).toBe("0"); + }); + + it("slices balls with a swung sword", () => { + grabAt(game, 0.16, 0.6); // the sword rests at (0.16, 0.6) + for (let x = 0.16; x <= 0.9; x += 0.05) + game.update(FRAME_MS, body(x, 0.29, 0)); + + expect(Number(stat(game, "sliced"))).toBeGreaterThan(0); + expect(game.result().score).toBeGreaterThanOrEqual(150); + }); + + it("resets the score and respawns objects", () => { + grabAt(game, 0.16, 0.6); + for (let x = 0.16; x <= 0.9; x += 0.05) + game.update(FRAME_MS, body(x, 0.29, 0)); + game.reset(); + expect(game.result().score).toBe(0); + expect(stat(game, "sliced")).toBe("0"); + }); +}); diff --git a/web/src/games/motion-maker/index.ts b/web/src/games/motion-maker/index.ts index ed85249..26da3ef 100644 --- a/web/src/games/motion-maker/index.ts +++ b/web/src/games/motion-maker/index.ts @@ -16,7 +16,12 @@ import type { GameResult, Renderer, } from "../../sdk"; -import { distPointToSegment, roundRect } from "../canvas-utils"; +import { + beginScoreHud, + distPointToSegment, + handJoint, + roundRect, +} from "../canvas-utils"; // ── Interaction tuning ──────────────────────────────────────────────────────── @@ -240,13 +245,9 @@ export class MotionMaker implements Game { * but it's too jittery at body distance to drive grab or a cursor.) */ private trackHands(dt: number, body: BodyController): void { - const map: Record = { - left: body.joints.leftHand, - right: body.joints.rightHand, - }; for (const which of ["left", "right"] as const) { const h = this.hands[which]; - const [nx, ny] = map[which]; + const [nx, ny] = handJoint(body, which); if (dt > 0) { const ivx = (nx - h.x) / dt; const ivy = (ny - h.y) / dt; @@ -744,16 +745,7 @@ export class MotionMaker implements Game { private renderHud(r: Renderer): void { const { ctx } = r; const p = r.area; - ctx.save(); - ctx.textBaseline = "top"; - ctx.fillStyle = "#f4f7ff"; - ctx.font = `bold ${Math.round(r.sx(0.05))}px system-ui`; - ctx.textAlign = "left"; - ctx.fillText( - String(this.score).padStart(4, "0"), - p.x + r.sx(0.03), - p.y + r.sy(0.03) - ); + beginScoreHud(r, this.score, 4); ctx.fillStyle = "#8a95b5"; ctx.font = `${Math.round(r.sx(0.024))}px system-ui`; diff --git a/web/src/games/reach-dodge/index.test.ts b/web/src/games/reach-dodge/index.test.ts new file mode 100644 index 0000000..f33b50c --- /dev/null +++ b/web/src/games/reach-dodge/index.test.ts @@ -0,0 +1,109 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { Joints } from "../../../../protocol/protocol"; +import type { BodyController } from "../../sdk"; +import { ReachDodge } from "."; + +const FRAME_MS = 1000 / 60; + +function body( + overrides: Partial = {}, + squatAmount = 0 +): BodyController { + return { + joints: { + head: [0.5, 0.18], + leftHand: [0.4, 0.95], + rightHand: [0.6, 0.95], + torso: [0.5, 0.5], + leftKnee: [0.42, 0.72], + rightKnee: [0.58, 0.72], + leftFoot: [0.42, 0.95], + rightFoot: [0.58, 0.95], + ...overrides, + }, + squatAmount, + leanAmount: 0, + leftHandOpen: 1, + rightHandOpen: 1, + trackingQuality: 1, + health: "ok", + hasRequiredJoints: true, + ageMs: 0, + tick: () => undefined, + dispose: () => undefined, + }; +} + +function play(game: ReachDodge, seconds: number, input: BodyController): void { + for (let t = 0; t < seconds * 60; t++) game.update(FRAME_MS, input); +} + +function stat(game: ReachDodge, label: string): string | undefined { + return game.result().stats?.find((s) => s.label === label)?.value; +} + +describe("ReachDodge", () => { + let game: ReachDodge; + + beforeEach(() => { + game = new ReachDodge(); + game.init({ width: 1280, height: 720 }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("scores a target when the matching hand reaches it", () => { + // random 0 → targets spawn on the LEFT edge at (0.16, 0.28); obstacles in the high lane. + vi.spyOn(Math, "random").mockReturnValue(0); + play(game, 1, body({ leftHand: [0.16, 0.28] })); + + expect(Number(stat(game, "hits"))).toBeGreaterThan(0); + expect(game.result().score).toBeGreaterThanOrEqual(110); + }); + + it("does not score a target reached by the wrong hand", () => { + vi.spyOn(Math, "random").mockReturnValue(0); + play(game, 1, body({ rightHand: [0.16, 0.28] })); + expect(stat(game, "hits")).toBe("0"); + }); + + it("counts a dodge when the player leans out of a bar's path", () => { + vi.spyOn(Math, "random").mockReturnValue(0); + play(game, 6, body({ torso: [0.8, 0.5] })); + expect(Number(stat(game, "dodges"))).toBeGreaterThan(0); + }); + + it("clips a centered player and breaks the combo", () => { + vi.spyOn(Math, "random").mockReturnValue(0); + play(game, 6, body()); + expect(stat(game, "dodges")).toBe("0"); + expect(stat(game, "best combo")).toBe("x0"); + }); + + it("dodges a duck-lane bar only by squatting", () => { + // random 0.99 → duck lane; targets spawn right (out of reach of the resting hands). + vi.spyOn(Math, "random").mockReturnValue(0.99); + play(game, 6, body({}, 0.6)); + expect(Number(stat(game, "dodges"))).toBeGreaterThan(0); + + game.reset(); + play(game, 6, body({ torso: [0.8, 0.5] }, 0)); + expect(stat(game, "dodges")).toBe("0"); + }); + + it("ends after the session length and resets cleanly", () => { + vi.spyOn(Math, "random").mockReturnValue(0); + play(game, 76, body({ leftHand: [0.16, 0.28] })); + expect(game.isOver()).toBe(true); + const finalScore = game.result().score; + game.update(FRAME_MS, body({ leftHand: [0.16, 0.28] })); + expect(game.result().score).toBe(finalScore); + + game.reset(); + expect(game.isOver()).toBe(false); + expect(game.result().score).toBe(0); + }); +}); diff --git a/web/src/games/reach-dodge/index.ts b/web/src/games/reach-dodge/index.ts index 6dba98b..a5e7f78 100644 --- a/web/src/games/reach-dodge/index.ts +++ b/web/src/games/reach-dodge/index.ts @@ -20,7 +20,7 @@ import type { GameResult, Renderer, } from "../../sdk"; -import { roundRect } from "../canvas-utils"; +import { beginScoreHud, roundRect } from "../canvas-utils"; const SESSION_MS = 75_000; // ~75s const TARGET_RADIUS = 0.075; // normalized (x-relative) @@ -326,18 +326,7 @@ export class ReachDodge implements Game { private renderHud(r: Renderer): void { const { ctx } = r; const p = r.area; - ctx.save(); - ctx.textBaseline = "top"; - - // Score - ctx.fillStyle = "#f4f7ff"; - ctx.font = `bold ${Math.round(r.sx(0.05))}px system-ui`; - ctx.textAlign = "left"; - ctx.fillText( - String(this.score).padStart(5, "0"), - p.x + r.sx(0.03), - p.y + r.sy(0.03) - ); + beginScoreHud(r, this.score, 5); // Combo if (this.combo > 1) { diff --git a/web/src/games/slice/index.test.ts b/web/src/games/slice/index.test.ts new file mode 100644 index 0000000..8d3c490 --- /dev/null +++ b/web/src/games/slice/index.test.ts @@ -0,0 +1,106 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { BodyController } from "../../sdk"; +import { Slice } from "."; + +const FRAME_MS = 1000 / 60; + +type Hand = { x: number; y: number; active?: boolean }; + +function body(left: Hand, right: Hand): BodyController { + return { + joints: { + head: [0.5, 0.18], + leftHand: [left.x, left.y], + rightHand: [right.x, right.y], + torso: [0.5, 0.5], + leftKnee: [0.42, 0.72], + rightKnee: [0.58, 0.72], + leftFoot: [0.42, 0.95], + rightFoot: [0.58, 0.95], + }, + squatAmount: 0, + leanAmount: 0, + leftHandOpen: 1, + rightHandOpen: 1, + leftHandActive: left.active, + rightHandActive: right.active, + trackingQuality: 1, + health: "ok", + hasRequiredJoints: true, + ageMs: 0, + tick: () => undefined, + dispose: () => undefined, + }; +} + +/** Swipe the left hand up and down through column `x` every frame (a fast blade). */ +function swipe(game: Slice, seconds: number, x: number, active = true): void { + const parked = { x: 0.9, y: 0.1 }; + for (let f = 0; f < seconds * 60; f++) { + const y = f % 2 === 0 ? 0.1 : 1.1; + game.update(FRAME_MS, body({ x, y, active }, parked)); + } +} + +function stat(game: Slice, label: string): string | undefined { + return game.result().stats?.find((s) => s.label === label)?.value; +} + +describe("Slice", () => { + let game: Slice; + + beforeEach(() => { + game = new Slice(); + game.init({ width: 1280, height: 720 }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("slices fruit a fast swipe passes through", () => { + // random 0.5 → every fruit launches straight up the x=0.5 column; no bombs. + vi.spyOn(Math, "random").mockReturnValue(0.5); + swipe(game, 4, 0.5); + + expect(Number(stat(game, "sliced"))).toBeGreaterThan(0); + expect(game.result().score).toBeGreaterThan(0); + expect(game.isOver()).toBe(false); + }); + + it("loses a life for every fruit that falls unsliced", () => { + vi.spyOn(Math, "random").mockReturnValue(0.5); + swipe(game, 12, 0.05); // blade nowhere near the fruit column + expect(stat(game, "sliced")).toBe("0"); + expect(game.isOver()).toBe(true); + }); + + it("costs a life when the blade hits a bomb", () => { + // random 0.05 → each wave is one fruit plus one bomb near x≈0.18. + vi.spyOn(Math, "random").mockReturnValue(0.05); + swipe(game, 8, 0.176); + expect(Number(stat(game, "sliced"))).toBeGreaterThan(0); + expect(game.isOver()).toBe(true); + }); + + it("cannot slice with an untracked hand", () => { + vi.spyOn(Math, "random").mockReturnValue(0.5); + swipe(game, 2, 0.5, false); + expect(stat(game, "sliced")).toBe("0"); + }); + + it("resets score and lives", () => { + vi.spyOn(Math, "random").mockReturnValue(0.5); + swipe(game, 12, 0.05); + game.reset(); + expect(game.isOver()).toBe(false); + expect(game.result()).toEqual({ + score: 0, + stats: [ + { label: "sliced", value: "0" }, + { label: "best combo", value: "0×" }, + ], + }); + }); +}); diff --git a/web/src/games/slice/index.ts b/web/src/games/slice/index.ts index 9bd3e30..ed6a4f7 100644 --- a/web/src/games/slice/index.ts +++ b/web/src/games/slice/index.ts @@ -15,7 +15,7 @@ import type { GameResult, Renderer, } from "../../sdk"; -import { distPointToSegment } from "../canvas-utils"; +import { beginScoreHud, distPointToSegment, handJoint } from "../canvas-utils"; // ── Tuning ────────────────────────────────────────────────────────────────────── @@ -203,13 +203,9 @@ export class Slice implements Game { } private trackHands(dt: number, body: BodyController): void { - const map: Record = { - left: body.joints.leftHand, - right: body.joints.rightHand, - }; for (const which of ["left", "right"] as const) { const h = this.hands[which]; - const [nx, ny] = map[which]; + const [nx, ny] = handJoint(body, which); if (!handActive(body, which)) { // Hand not tracked: freeze it, drop its trail, and it can't slice (speed 0). @@ -644,16 +640,7 @@ export class Slice implements Game { private renderHud(r: Renderer): void { const { ctx } = r; const p = r.area; - ctx.save(); - ctx.textBaseline = "top"; - ctx.fillStyle = "#f4f7ff"; - ctx.font = `bold ${Math.round(r.sx(0.05))}px system-ui`; - ctx.textAlign = "left"; - ctx.fillText( - String(this.score).padStart(4, "0"), - p.x + r.sx(0.03), - p.y + r.sy(0.03) - ); + beginScoreHud(r, this.score, 4); // Lives as hearts, top-right. ctx.textAlign = "right"; diff --git a/web/src/sdk/bridge.test.ts b/web/src/sdk/bridge.test.ts new file mode 100644 index 0000000..1f495a1 --- /dev/null +++ b/web/src/sdk/bridge.test.ts @@ -0,0 +1,121 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { Joints, PosePacket } from "../../../protocol/protocol"; +import { + BridgeController, + type BridgeHostCallbacks, + hasNativeBridge, + type MotionNativeApi, + NativeBridge, +} from "./bridge"; + +const joints: Joints = { + head: [0.5, 0.2], + leftHand: [0.3, 0.5], + rightHand: [0.7, 0.5], + torso: [0.5, 0.5], + leftKnee: [0.4, 0.7], + rightKnee: [0.6, 0.7], + leftFoot: [0.4, 0.9], + rightFoot: [0.6, 0.9], +}; + +function packet(seq: number): PosePacket { + return { v: 1, type: "pose", seq, sentAt: 0, quality: 0.9, joints }; +} + +type FakeWindow = { + webkit?: { messageHandlers?: { motion?: { postMessage(m: unknown): void } } }; + __motion?: MotionNativeApi; +}; + +function callbacks(): BridgeHostCallbacks { + return { + onStart: vi.fn(), + onStop: vi.fn(), + onCalibrated: vi.fn(), + onTracking: vi.fn(), + }; +} + +describe("BridgeController", () => { + it("reuses pose smoothing and lets native force a tracking loss", () => { + const controller = new BridgeController(); + controller.pushPose(packet(1)); + controller.tick(10); + expect(controller.health).toBe("ok"); + + controller.setTracking("lost"); + controller.tick(20); + expect(controller.health).toBe("stale"); + + // A fresh frame clears the native loss flag. + controller.pushPose(packet(2)); + controller.tick(30); + expect(controller.health).toBe("ok"); + }); +}); + +describe("NativeBridge", () => { + let win: FakeWindow; + + beforeEach(() => { + win = {}; + vi.stubGlobal("window", win); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("detects the WKWebView message handler", () => { + expect(hasNativeBridge()).toBe(false); + win.webkit = { messageHandlers: { motion: { postMessage: vi.fn() } } }; + expect(hasNativeBridge()).toBe(true); + }); + + it("installs window.__motion and routes native calls to the host", () => { + const controller = new BridgeController(); + const host = callbacks(); + const bridge = new NativeBridge(controller, host); + const api = win.__motion; + if (!api) throw new Error("bridge did not install window.__motion"); + + api.start(); + api.stop(); + api.calibrated(); + api.setTracking("too_far"); + expect(host.onStart).toHaveBeenCalledOnce(); + expect(host.onStop).toHaveBeenCalledOnce(); + expect(host.onCalibrated).toHaveBeenCalledOnce(); + expect(host.onTracking).toHaveBeenCalledWith("too_far"); + + bridge.dispose(); + expect(win.__motion).toBeUndefined(); + }); + + it("accepts pose frames as objects or JSON and drops junk", () => { + const controller = new BridgeController(); + const ingest = vi.spyOn(controller, "pushPose"); + const bridge = new NativeBridge(controller, callbacks()); + const api = win.__motion as MotionNativeApi; + + api.pushPose(packet(1)); + api.pushPose(JSON.stringify(packet(2))); + api.pushPose("{not json"); + api.pushPose(JSON.stringify({ type: "status" })); + + expect(ingest.mock.calls.map(([p]) => p.seq)).toEqual([1, 2]); + bridge.dispose(); + }); + + it("emits lifecycle events only when a native host is present", () => { + const bridge = new NativeBridge(new BridgeController(), callbacks()); + expect(() => bridge.emit({ event: "ready" })).not.toThrow(); + + const postMessage = vi.fn(); + win.webkit = { messageHandlers: { motion: { postMessage } } }; + bridge.emit({ event: "score", value: 7 }); + expect(postMessage).toHaveBeenCalledWith({ event: "score", value: 7 }); + }); +}); diff --git a/web/src/sdk/keyboard-debug.test.ts b/web/src/sdk/keyboard-debug.test.ts new file mode 100644 index 0000000..8990afe --- /dev/null +++ b/web/src/sdk/keyboard-debug.test.ts @@ -0,0 +1,108 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { KeyboardDebugController } from "./keyboard-debug"; + +/** A 200×100 target at (0, 0); Node's EventTarget delivers the synthetic events. */ +class FakeTarget extends EventTarget { + getBoundingClientRect() { + return { left: 0, top: 0, width: 200, height: 100 }; + } +} + +function event(type: string, fields: T): Event & T { + return Object.assign(new Event(type), fields); +} + +function settle(controller: KeyboardDebugController, frames = 120): void { + for (let f = 1; f <= frames; f++) controller.tick(f * 16); +} + +describe("KeyboardDebugController", () => { + let target: FakeTarget; + let win: EventTarget; + let controller: KeyboardDebugController; + + beforeEach(() => { + target = new FakeTarget(); + win = new EventTarget(); + vi.stubGlobal("window", win); + controller = new KeyboardDebugController(target as unknown as HTMLElement); + }); + + afterEach(() => { + controller.dispose(); + vi.unstubAllGlobals(); + }); + + it("reports no signal until the pointer moves", () => { + settle(controller, 2); + expect(controller.health).toBe("no_signal"); + expect(controller.hasRequiredJoints).toBe(false); + expect(controller.leftFingertip).toBeUndefined(); + + target.dispatchEvent(event("mousemove", { clientX: 100, clientY: 50 })); + settle(controller, 2); + expect(controller.health).toBe("ok"); + expect(controller.hasRequiredJoints).toBe(true); + expect(controller.trackingQuality).toBe(1); + }); + + it("mirrors both hands around the cursor", () => { + target.dispatchEvent(event("mousemove", { clientX: 100, clientY: 25 })); + settle(controller); + expect(controller.joints.leftHand[0]).toBeCloseTo(0.38); + expect(controller.joints.rightHand[0]).toBeCloseTo(0.62); + expect(controller.joints.leftHand[1]).toBeCloseTo(0.25); + expect(controller.leftFingertip?.[0]).toBeCloseTo(0.38); + + target.dispatchEvent( + event("touchmove", { touches: [{ clientX: 0, clientY: 100 }] }) + ); + settle(controller); + expect(controller.joints.leftHand[0]).toBeCloseTo(0); + expect(controller.joints.rightHand[1]).toBeCloseTo(1); + }); + + it("ramps lean and squat from held keys and releases them", () => { + win.dispatchEvent(event("keydown", { key: "ArrowLeft" })); + win.dispatchEvent(event("keydown", { key: "s" })); + settle(controller); + expect(controller.leanAmount).toBeCloseTo(-1); + expect(controller.squatAmount).toBeCloseTo(1); + expect(controller.joints.torso[0]).toBeLessThan(0.3); + + // Opposing lean keys cancel out. + win.dispatchEvent(event("keydown", { key: "d" })); + win.dispatchEvent(event("keyup", { key: "s" })); + settle(controller); + expect(controller.leanAmount).toBeCloseTo(0); + expect(controller.squatAmount).toBeCloseTo(0); + }); + + it("closes both hands while the left button or space is held", () => { + target.dispatchEvent(event("mousedown", { button: 2 })); // right button: ignored + settle(controller); + expect(controller.leftHandOpen).toBeCloseTo(1); + + target.dispatchEvent(event("mousedown", { button: 0 })); + settle(controller); + expect(controller.leftHandOpen).toBeCloseTo(0); + expect(controller.rightHandOpen).toBeCloseTo(0); + + win.dispatchEvent(event("mouseup", { button: 0 })); + win.dispatchEvent(event("keydown", { key: " " })); + settle(controller); + expect(controller.leftHandOpen).toBeCloseTo(0); + + win.dispatchEvent(event("keyup", { key: " " })); + settle(controller); + expect(controller.leftHandOpen).toBeCloseTo(1); + }); + + it("stops listening after dispose", () => { + controller.dispose(); + win.dispatchEvent(event("keydown", { key: "ArrowRight" })); + settle(controller); + expect(controller.leanAmount).toBeCloseTo(0); + }); +});