Skip to content
This repository was archived by the owner on Oct 16, 2026. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .swift-format
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"version": 1,
"lineLength": 120,
"indentation": {
"spaces": 4
},
"respectsExistingLineBreaks": true
}
39 changes: 23 additions & 16 deletions ios/Sources/Motion/App/AppModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,15 @@

import Foundation
import Observation

#if canImport(UIKit)
import UIKit
import UIKit
#endif

/// High-level screen the app should show. Drives `ContentView` routing.
enum Phase: Sendable, Equatable {
case setup // camera up, guiding the player into frame
case game // full-screen web game; pose streams in-process via PoseBridge
case setup // camera up, guiding the player into frame
case game // full-screen web game; pose streams in-process via PoseBridge
}

/// Detects a "clap" purely from the two tracked HAND joints (`joints.leftHand` /
Expand Down Expand Up @@ -70,7 +71,7 @@ struct ClapDetector {

// Fire on the closing edge: armed + hands now together + past the debounce window.
if armed, separation < closedThreshold, now - lastClapAt >= minInterval {
armed = false // require a re-separation before the next clap
armed = false // require a re-separation before the next clap
lastClapAt = now
return true
}
Expand All @@ -82,10 +83,12 @@ struct ClapDetector {
/// treat a hand sitting exactly at the origin as "not present" to avoid false claps
/// when one hand isn't tracked.
private func handSeparation(_ j: Joints) -> Double? {
let l = j.leftHand, r = j.rightHand
let l = j.leftHand
let r = j.rightHand
guard l.count == 2, r.count == 2 else { return nil }
if isFallback(l) || isFallback(r) { return nil }
let dx = l[0] - r[0], dy = l[1] - r[1]
let dx = l[0] - r[0]
let dy = l[1] - r[1]
return (dx * dx + dy * dy).squareRoot()
}

Expand All @@ -110,7 +113,7 @@ final class AppModel {
var gameURL: URL? {
switch GameConfig.source {
case .devServer: return GameConfig.devServerURL(host: devServerIP)
case .bundled: return GameConfig.bundledIndexURL()
case .bundled: return GameConfig.bundledIndexURL()
}
}

Expand Down Expand Up @@ -252,8 +255,10 @@ final class AppModel {
/// - tracking: setup verdict from `SetupEvaluator`
/// - guidance: matching human guidance string
/// - ready: whether setup has been good long enough
func ingest(joints: Joints?, quality: Double, tracking: TrackingState,
guidance: String, ready: Bool, hands: HandState?, fingertips: Fingertips? = nil) {
func ingest(
joints: Joints?, quality: Double, tracking: TrackingState,
guidance: String, ready: Bool, hands: HandState?, fingertips: Fingertips? = nil
) {
self.joints = joints
self.quality = quality
self.tracking = tracking
Expand All @@ -270,17 +275,19 @@ final class AppModel {
// Skip in `.game`: the web game handles its own input, so claps are ignored there.
// We always feed the detector so its arm/debounce state stays coherent, but only
// publish the event outside the game phase.
let clapped = clapDetector.update(joints: joints,
now: ProcessInfo.processInfo.systemUptime)
let clapped = clapDetector.update(
joints: joints,
now: ProcessInfo.processInfo.systemUptime)
if clapped, phase != .game {
clapCount &+= 1
}

// While playing, forward pose + tracking (with hands + fingertips) into the web game
// in-process. PoseBridge handles change-detection for tracking and the ~30 Hz throttle.
if phase == .game {
bridge.pushLivePose(joints: joints, quality: quality, tracking: tracking,
hands: self.hands, fingertips: self.fingertips)
bridge.pushLivePose(
joints: joints, quality: quality, tracking: tracking,
hands: self.hands, fingertips: self.fingertips)
}

// Stream to the browser relay IN ADDITION to (and independent of) the local game.
Expand Down Expand Up @@ -330,7 +337,7 @@ final class AppModel {
/// streaming poses (see `ingest`). The relay is on the SAME Mac as the dev server, so we
/// reuse `devServerIP` as the host. Idempotent-ish: replaces any existing socket.
private func startStreaming() {
stopStreaming() // clear any stale socket first
stopStreaming() // clear any stale socket first
let code = roomCode.trimmingCharacters(in: .whitespaces).uppercased()
guard !code.isEmpty else {
streamToWebsite = false
Expand All @@ -348,7 +355,7 @@ final class AppModel {
// background the app and drop the relay socket. This does NOT stop iOS suspending a
// fully-backgrounded app, but it prevents the common idle auto-lock disconnect.
#if canImport(UIKit)
UIApplication.shared.isIdleTimerDisabled = true
UIApplication.shared.isIdleTimerDisabled = true
#endif
}

Expand All @@ -362,7 +369,7 @@ final class AppModel {
// Re-enable auto-lock now that we're no longer streaming, so the phone can sleep
// normally when idle. Paired with the enable in `startStreaming()`.
#if canImport(UIKit)
UIApplication.shared.isIdleTimerDisabled = false
UIApplication.shared.isIdleTimerDisabled = false
#endif
}
}
Expand Down
46 changes: 30 additions & 16 deletions ios/Sources/Motion/Game/GameWebView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ struct GameWebView: UIViewRepresentable {
/// as a JSON STRING (the web bridge JSON.parses strings), which sidesteps any JS
/// object-literal escaping concerns — we only have to escape the JSON string once.
func pushPose(jsonString: String) {
guard ready else { return } // no bridge yet; poses before start are dropped
guard ready else { return } // no bridge yet; poses before start are dropped
let js = "window.__motion && window.__motion.pushPose(\(Self.jsStringLiteral(jsonString)));"
eval(js)
}
Expand All @@ -172,12 +172,18 @@ struct GameWebView: UIViewRepresentable {
/// Begin / restart the game. If the web app isn't `ready` yet, this is queued and
/// fired the moment the `ready` event arrives.
func start() {
guard ready else { pendingStart = true; return }
guard ready else {
pendingStart = true
return
}
eval("window.__motion && window.__motion.start();")
}

func stop() {
guard ready else { pendingStart = false; return }
guard ready else {
pendingStart = false
return
}
eval("window.__motion && window.__motion.stop();")
}

Expand All @@ -195,13 +201,15 @@ struct GameWebView: UIViewRepresentable {
/// all handled correctly — the result is a valid JS/JSON string expression.
static func jsStringLiteral(_ s: String) -> String {
if let data = try? JSONSerialization.data(withJSONObject: [s], options: []),
let arr = String(data: data, encoding: .utf8),
arr.hasPrefix("["), arr.hasSuffix("]") {
let arr = String(data: data, encoding: .utf8),
arr.hasPrefix("["), arr.hasSuffix("]")
{
// arr == `["...escaped..."]`; strip the array brackets to get the literal.
return String(arr.dropFirst().dropLast())
}
// Fallback: minimal manual escaping (should never be hit for our inputs).
let escaped = s
let escaped =
s
.replacingOccurrences(of: "\\", with: "\\\\")
.replacingOccurrences(of: "\"", with: "\\\"")
.replacingOccurrences(of: "\n", with: "\\n")
Expand All @@ -210,8 +218,10 @@ struct GameWebView: UIViewRepresentable {

// MARK: Web → native (WKScriptMessageHandler)

func userContentController(_ userContentController: WKUserContentController,
didReceive message: WKScriptMessage) {
func userContentController(
_ userContentController: WKUserContentController,
didReceive message: WKScriptMessage
) {
guard message.name == "motion" else { return }
guard let event = Self.decodeEvent(message.body) else { return }
if case .ready = event { ready = true }
Expand All @@ -221,21 +231,23 @@ struct GameWebView: UIViewRepresentable {
/// Decode a `{ event: ..., ... }` message body (a JS object → NSDictionary).
private static func decodeEvent(_ body: Any) -> GameEvent? {
guard let dict = body as? [String: Any],
let event = dict["event"] as? String else { return nil }
let event = dict["event"] as? String
else { return nil }
switch event {
case "ready": return .ready
case "ready": return .ready
case "gameStart": return .gameStart
case "restart": return .restart
case "restart": return .restart
case "score":
let value = (dict["value"] as? NSNumber)?.doubleValue ?? 0
return .score(value)
case "gameOver":
// `result` is arbitrary JSON; re-serialize it to a string for the UI.
var resultJSON: String?
if let result = dict["result"],
JSONSerialization.isValidJSONObject(result),
let data = try? JSONSerialization.data(withJSONObject: result),
let str = String(data: data, encoding: .utf8) {
JSONSerialization.isValidJSONObject(result),
let data = try? JSONSerialization.data(withJSONObject: result),
let str = String(data: data, encoding: .utf8)
{
resultJSON = str
}
return .gameOver(resultJSON: resultJSON)
Expand All @@ -246,8 +258,10 @@ struct GameWebView: UIViewRepresentable {

// MARK: Navigation

func webView(_ webView: WKWebView, didFailProvisionalNavigation navigation: WKNavigation!,
withError error: Error) {
func webView(
_ webView: WKWebView, didFailProvisionalNavigation navigation: WKNavigation!,
withError error: Error
) {
// Surfaced as a black screen; the ContentView shows a dev-server hint on failure.
onEvent(.gameOver(resultJSON: nil))
}
Expand Down
6 changes: 4 additions & 2 deletions ios/Sources/Motion/Game/PoseBridge.swift
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,10 @@ final class PoseBridge {
/// - tracking: setup verdict from `SetupEvaluator`
/// - hands: latest per-hand openness 0..1 (nil if never detected), carried in the packet
/// - fingertips: latest precise index-fingertips (nil if never detected), carried too
func pushLivePose(joints: Joints?, quality: Double, tracking: TrackingState,
hands: HandState? = nil, fingertips: Fingertips? = nil) {
func pushLivePose(
joints: Joints?, quality: Double, tracking: TrackingState,
hands: HandState? = nil, fingertips: Fingertips? = nil
) {
guard let coordinator else { return }

// Tracking transitions are cheap and important (they pause/resume the game), so
Expand Down
36 changes: 19 additions & 17 deletions ios/Sources/Motion/Net/Protocol.swift
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,12 @@ enum Role: String, Codable, Sendable {
/// Reason the controller has paused sending usable input. Mirrors `TrackingState`.
enum TrackingState: String, Codable, Sendable, CaseIterable {
case ok
case lost // body not detected / confidence collapsed
case partial // some required joints missing
case tooClose = "too_close"
case tooFar = "too_far"
case raisePhone = "raise_phone"
case lowLight = "low_light"
case lost // body not detected / confidence collapsed
case partial // some required joints missing
case tooClose = "too_close"
case tooFar = "too_far"
case raisePhone = "raise_phone"
case lowLight = "low_light"
}

// MARK: - Joints
Expand Down Expand Up @@ -173,8 +173,10 @@ struct PosePacket: Codable, Sendable {
/// the TS optional field `fingertips?: Fingertips` so old/new relays both accept the packet.
let fingertips: Fingertips?

init(seq: Int, sentAt: Double, quality: Double, joints: Joints,
hands: HandState? = nil, fingertips: Fingertips? = nil) {
init(
seq: Int, sentAt: Double, quality: Double, joints: Joints,
hands: HandState? = nil, fingertips: Fingertips? = nil
) {
self.v = PROTOCOL_VERSION
self.type = "pose"
self.seq = seq
Expand Down Expand Up @@ -209,7 +211,7 @@ struct CalibMessage: Codable, Sendable {
let v: Int
let type: String
let stage: Stage
let progress: Double // 0..1 within the whole calibration
let progress: Double // 0..1 within the whole calibration

init(stage: Stage, progress: Double) {
self.v = PROTOCOL_VERSION
Expand Down Expand Up @@ -294,10 +296,10 @@ struct RecMetaMessage: Codable, Sendable {
let v: Int
let type: String
let sessionId: String
let mime: String // e.g. "video/webm;codecs=vp9" or "video/mp4"
let mime: String // e.g. "video/webm;codecs=vp9" or "video/mp4"
let totalBytes: Int
let chunks: Int // number of RecChunkMessages to expect
let durationMs: Double // gameplay clip duration
let chunks: Int // number of RecChunkMessages to expect
let durationMs: Double // gameplay clip duration
/// ms from the session anchor to the first recorded gameplay frame (for sync).
let startOffsetMs: Double
}
Expand All @@ -308,8 +310,8 @@ struct RecChunkMessage: Codable, Sendable {
let v: Int
let type: String
let sessionId: String
let i: Int // 0-based chunk index
let data: String // base64 of this slice
let i: Int // 0-based chunk index
let data: String // base64 of this slice
}

// MARK: - Inbound messages (server → controller)
Expand All @@ -331,9 +333,9 @@ struct StartMessage: Codable, Sendable {
/// Room-level errors (e.g. a second controller tried to join).
struct ErrorMessage: Codable, Sendable {
enum Code: String, Codable, Sendable {
case roomFull = "room_full"
case badRole = "bad_role"
case badMessage = "bad_message"
case roomFull = "room_full"
case badRole = "bad_role"
case badMessage = "bad_message"
case versionMismatch = "version_mismatch"
}
let v: Int
Expand Down
14 changes: 8 additions & 6 deletions ios/Sources/Motion/Net/RoomSocket.swift
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ final class RoomSocket {
private let session: URLSession

private var task: URLSessionWebSocketTask?
private var isRunning = false // user wants a connection to exist
private var isRunning = false // user wants a connection to exist
private var reconnectAttempt = 0
private var pingTimer: Task<Void, Never>?
private var receiveActive = false
Expand Down Expand Up @@ -113,8 +113,10 @@ final class RoomSocket {
delegate?.roomSocket(self, didChangeState: .failed(reason: "Invalid room URL."))
return
}
delegate?.roomSocket(self, didChangeState:
reconnectAttempt == 0 ? .connecting : .reconnecting(attempt: reconnectAttempt))
delegate?.roomSocket(
self,
didChangeState:
reconnectAttempt == 0 ? .connecting : .reconnecting(attempt: reconnectAttempt))

let ws = session.webSocketTask(with: url)
task = ws
Expand All @@ -134,9 +136,9 @@ final class RoomSocket {
send(JoinMessage(role: .controller, name: playerName))
}

func send(_ pose: PosePacket) { send(encodable: pose) }
func send(_ pose: PosePacket) { send(encodable: pose) }
func send(_ status: StatusMessage) { send(encodable: status) }
func send(_ calib: CalibMessage) { send(encodable: calib) }
func send(_ calib: CalibMessage) { send(encodable: calib) }
/// Send a recording control frame (arm/start/stop/cancel) to the display.
func send(_ rec: RecControlMessage) { send(encodable: rec) }
private func send(_ join: JoinMessage) { send(encodable: join) }
Expand Down Expand Up @@ -210,7 +212,7 @@ final class RoomSocket {
pingTimer?.cancel()
pingTimer = Task { [weak self] in
while !Task.isCancelled {
try? await Task.sleep(nanoseconds: 2_000_000_000) // 2s
try? await Task.sleep(nanoseconds: 2_000_000_000) // 2s
guard let self, self.isRunning else { return }
await MainActor.run {
self.send(encodable: PingMessage(t: self.nowMS()))
Expand Down
Loading
Loading