diff --git a/crates/xcrs/src/mcp.rs b/crates/xcrs/src/mcp.rs index 3bae78a..39bc9e7 100644 --- a/crates/xcrs/src/mcp.rs +++ b/crates/xcrs/src/mcp.rs @@ -568,6 +568,24 @@ pub struct ButtonArgs { pub button: ButtonName, } +#[derive(Debug, Deserialize, JsonSchema)] +pub struct AppleTargetArgs { + /// Exact Apple simulator name. Omit to use the target selected with + /// `device_select`. + #[serde(default)] + pub simulator_name: Option, + /// Apple simulator UDID. Omit to use the target selected with `device_select`. + #[serde(default)] + pub simulator_udid: Option, + /// ControlKit host of a physical device or remote runner. Omit for a local + /// simulator. + #[serde(default)] + pub host: Option, + /// Local ControlKit JSON-RPC port. Defaults to 12004. + #[serde(default)] + pub controlkit_port: Option, +} + #[derive(Debug, Deserialize, JsonSchema)] pub struct UiTargetArgs { /// Exact Apple simulator name. Omit to use the target selected with @@ -584,6 +602,9 @@ pub struct UiTargetArgs { /// Local ControlKit JSON-RPC port. Defaults to 12004. #[serde(default)] pub controlkit_port: Option, + /// Bundle identifier of the app whose accessibility hierarchy should be read. + #[serde(default)] + pub bundle_id: Option, } #[derive(Debug, Deserialize, JsonSchema)] @@ -676,6 +697,25 @@ macro_rules! xcrs_mcp_tools { .and_then(|slot| slot.clone()) } + fn require_bundle_id( + tool_name: &str, + bundle_id: Option, + ) -> ::std::result::Result { + let bundle_id = bundle_id + .as_deref() + .map(str::trim) + .filter(|bundle_id| !bundle_id.is_empty()) + .ok_or_else(|| { + ::rmcp::model::ErrorData::invalid_request( + format!( + "{tool_name} requires bundle_id. Pass the bundle identifier of the foreground Apple app." + ), + None, + ) + })?; + Ok(bundle_id.to_string()) + } + /// Validate and tag a target from per-call fields, falling back to the /// target selected with `device_select` when no field is provided. fn dispatch_target( @@ -957,9 +997,9 @@ macro_rules! xcrs_mcp_tools { #[::rmcp::tool( name = $app_install_launch_name, - title = "Install and launch iOS app", - annotations(title = "Install and launch iOS app", read_only_hint = false, destructive_hint = true, idempotent_hint = false), - description = "Purpose: one-shot end-to-end setup for a freshly built iOS app: boot the simulator, install the .app bundle, optionally terminate a previous instance, launch it, and return its app container path. When to use vs siblings: use this once to get a build under test onto a simulator; use app_launch/app_terminate afterwards for an already-installed app, and screen_capture/ui_describe to inspect it. Behavior: boots the target simulator if it is not already booted, installs app_path, optionally force-terminates bundle_id first, launches bundle_id, then reads back the app's container directory. Prerequisites: Xcode command line tools installed; app_path must point to an existing .app bundle built for the simulator (not a device) architecture. Failure modes: errors if neither simulator_name nor simulator_udid is given, if the simulator cannot be found, or if any underlying `simctl` step fails. Limitations: iOS-simulator-only; there is no Android or physical-device equivalent in this tool." + title = "Install and launch Apple simulator app", + annotations(title = "Install and launch Apple simulator app", read_only_hint = false, destructive_hint = true, idempotent_hint = false), + description = "Purpose: one-shot end-to-end setup for a freshly built Apple-platform app: boot an iOS, tvOS, watchOS, or visionOS simulator, install the .app bundle, optionally terminate a previous instance, launch it, and return its app container path. When to use vs siblings: use this once to get a build under test onto a simulator; use app_launch/app_terminate afterwards for an already-installed app, and screen_capture/ui_describe to inspect it. Behavior: boots the target simulator if it is not already booted, installs app_path, optionally force-terminates bundle_id first, launches bundle_id, then reads back the app's container directory. Prerequisites: Xcode command line tools installed; app_path must point to an existing .app bundle built for the selected simulator platform and architecture. Failure modes: errors if neither simulator_name nor simulator_udid is given, if the simulator cannot be found, if the app bundle targets a different platform, or if any underlying `simctl` step fails. Limitations: simulator-only; there is no Android or physical-device equivalent in this tool." )] async fn app_install_launch( &self, @@ -1317,7 +1357,7 @@ macro_rules! xcrs_mcp_tools { name = $ui_describe_name, title = "Describe UI", annotations(title = "Describe UI", read_only_hint = true, idempotent_hint = true), - description = "Purpose: return the full accessibility hierarchy of the foreground app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: calls the resolved target's ControlKit `device.dump.ui` method and returns the raw hierarchy alongside the resolved simulator (if any). Prerequisites: a reachable ControlKit endpoint. Failure modes: errors if no target can be resolved, or if the resolved target is Android (Apple-only tool; ControlKit UI introspection has no Android equivalent), or if the ControlKit call fails." + description = "Purpose: return the full accessibility hierarchy of an Apple app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method and returns the raw hierarchy alongside the resolved simulator (if any). Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool; ControlKit UI introspection has no Android equivalent), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable." )] async fn ui_describe( &self, @@ -1330,6 +1370,7 @@ macro_rules! xcrs_mcp_tools { ::rmcp::model::CallToolResult, ::rmcp::model::ErrorData, > { + let bundle_id = Self::require_bundle_id($ui_describe_name, args.bundle_id)?; let target = Self::dispatch_apple_target( $ui_describe_name, args.simulator_name, @@ -1339,7 +1380,13 @@ macro_rules! xcrs_mcp_tools { )?; let (simulator, controlkit) = Self::controlkit_for_target(&target)?; let result = controlkit - .call("device.dump.ui", ::serde_json::json!({ "format": "json" })) + .call( + "device.dump.ui", + ::serde_json::json!({ + "format": "json", + "bundleId": bundle_id, + }), + ) .await .map_err(|error| { ::rmcp::model::ErrorData::internal_error(error.to_string(), None) @@ -1356,7 +1403,7 @@ macro_rules! xcrs_mcp_tools { name = $ui_element_list_name, title = "List UI elements", annotations(title = "List UI elements", read_only_hint = true, idempotent_hint = true), - description = "Purpose: list just the actionable accessibility elements of the foreground app with their labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: calls the resolved target's ControlKit `device.dump.ui` method, then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: a reachable ControlKit endpoint. Failure modes: errors if no target can be resolved, or if the resolved target is Android (Apple-only tool), or if the ControlKit call fails." + description = "Purpose: list just the actionable accessibility elements of an Apple app with their labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method, then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable." )] async fn ui_element_list( &self, @@ -1369,6 +1416,7 @@ macro_rules! xcrs_mcp_tools { ::rmcp::model::CallToolResult, ::rmcp::model::ErrorData, > { + let bundle_id = Self::require_bundle_id($ui_element_list_name, args.bundle_id)?; let target = Self::dispatch_apple_target( $ui_element_list_name, args.simulator_name, @@ -1378,7 +1426,13 @@ macro_rules! xcrs_mcp_tools { )?; let (simulator, controlkit) = Self::controlkit_for_target(&target)?; let ui = controlkit - .call("device.dump.ui", ::serde_json::json!({ "format": "json" })) + .call( + "device.dump.ui", + ::serde_json::json!({ + "format": "json", + "bundleId": bundle_id, + }), + ) .await .map_err(|error| { ::rmcp::model::ErrorData::internal_error(error.to_string(), None) @@ -1760,7 +1814,7 @@ macro_rules! xcrs_mcp_tools { ::rmcp::handler::server::wrapper::Parameters( args, ): ::rmcp::handler::server::wrapper::Parameters< - $crate::mcp::UiTargetArgs, + $crate::mcp::AppleTargetArgs, >, ) -> ::std::result::Result< ::rmcp::model::CallToolResult, @@ -2137,6 +2191,70 @@ mod tests { } } + #[test] + fn ui_tools_keep_bundle_id_optional_in_the_input_schema() { + let tools = XcrsMcpServer::xcrs_tool_router().list_all(); + for tool_name in ["ui_describe", "ui_element_list"] { + let tool = tools + .iter() + .find(|tool| tool.name.as_ref() == tool_name) + .unwrap_or_else(|| panic!("{tool_name} should be registered")); + let required = tool + .input_schema + .get("required") + .and_then(serde_json::Value::as_array) + .cloned() + .unwrap_or_default(); + + assert!(tool.input_schema["properties"].get("bundle_id").is_some()); + assert!(!required.contains(&serde_json::json!("bundle_id"))); + } + } + + #[test] + fn orientation_get_does_not_advertise_bundle_id() { + let tools = XcrsMcpServer::xcrs_tool_router().list_all(); + let tool = tools + .iter() + .find(|tool| tool.name.as_ref() == "orientation_get") + .expect("orientation_get should be registered"); + let properties = tool.input_schema["properties"] + .as_object() + .expect("orientation_get properties should be an object"); + + for property in [ + "simulator_name", + "simulator_udid", + "host", + "controlkit_port", + ] { + assert!( + properties.contains_key(property), + "orientation_get should advertise {property}" + ); + } + assert!(!properties.contains_key("bundle_id")); + } + + #[test] + fn ui_target_args_accept_omitted_bundle_id_and_validate_it_explicitly() { + let args: UiTargetArgs = + serde_json::from_value(serde_json::json!({})).expect("arguments should deserialize"); + + assert!(args.bundle_id.is_none()); + let error = XcrsMcpServer::require_bundle_id("ui_describe", args.bundle_id) + .expect_err("missing bundle_id should fail validation"); + assert!(error.to_string().contains("ui_describe requires bundle_id")); + assert_eq!( + XcrsMcpServer::require_bundle_id( + "ui_describe", + Some(" com.example.app ".to_string()) + ) + .expect("non-empty bundle_id should pass validation"), + "com.example.app" + ); + } + #[test] fn output_schema_is_present_only_where_structured_content_is_guaranteed() { let tools = XcrsMcpServer::xcrs_tool_router().list_all(); diff --git a/crates/xcrs/src/xcrs.rs b/crates/xcrs/src/xcrs.rs index 0ac8cea..2f4f5de 100644 --- a/crates/xcrs/src/xcrs.rs +++ b/crates/xcrs/src/xcrs.rs @@ -10,6 +10,8 @@ use std::time::Duration; pub mod mcp; const IOS_SIMULATOR_DESTINATION_PREFIX: &str = "platform=iOS Simulator,id="; +const CONTROLKIT_METHOD_NOT_FOUND: i64 = -32601; +const CONTROLKIT_RUNNER_INFO_TIMEOUT: Duration = Duration::from_secs(2); pub fn encode_base64(data: impl AsRef<[u8]>) -> String { use base64::{engine::general_purpose::STANDARD, Engine}; @@ -46,7 +48,7 @@ fn collect_controlkit_elements(element: &serde_json::Value, elements: &mut Vec Result { + let body = self.send(method, params).await?; + + if let Some(error) = body.get("error") { + // `send` cannot recurse; this guard only avoids repeating an unsupported probe. + let should_probe_runner = + is_controlkit_method_not_found(error) && method != "device.info"; + let (runner_info, runner_info_error) = if should_probe_runner { + match tokio::time::timeout( + CONTROLKIT_RUNNER_INFO_TIMEOUT, + self.send("device.info", serde_json::json!({})), + ) + .await + { + Ok(Ok(body)) => match body.get("result") { + Some(result) => (Some(result.clone()), None), + None => { + let reason = body + .get("error") + .map(|error| format!("device.info returned {error}")) + .unwrap_or_else(|| { + "device.info response did not contain a result".to_string() + }); + (None, Some(reason)) + } + }, + Ok(Err(error)) => (None, Some(error.to_string())), + Err(_) => ( + None, + Some(format!( + "device.info timed out after {} seconds", + CONTROLKIT_RUNNER_INFO_TIMEOUT.as_secs() + )), + ), + } + } else { + (None, None) + }; + return Err(anyhow!(controlkit_rpc_error( + method, + error, + runner_info.as_ref(), + runner_info_error.as_deref() + ))); + } + + body.get("result") + .cloned() + .ok_or_else(|| anyhow!("ControlKit response for '{method}' did not contain a result")) + } + + async fn send(&self, method: &str, params: serde_json::Value) -> Result { let response = self .client .post(format!("{}/rpc", self.base_url)) @@ -150,16 +207,59 @@ impl ControlKit { return Err(anyhow!("ControlKit returned HTTP {}: {}", status, body)); } - if let Some(error) = body.get("error") { - return Err(anyhow!("ControlKit method '{method}' failed: {error}")); - } - - body.get("result") - .cloned() - .ok_or_else(|| anyhow!("ControlKit response for '{method}' did not contain a result")) + Ok(body) } } +fn is_controlkit_method_not_found(error: &serde_json::Value) -> bool { + let Some(code) = error.get("code") else { + return false; + }; + + // Some runners serialise the JSON-RPC code as an integral float (e.g. `-32601.0`), + // which `as_i64` rejects, so fall back to an exact `as_f64` comparison for that case. + code.as_i64() == Some(CONTROLKIT_METHOD_NOT_FOUND) + || code.as_f64() == Some(CONTROLKIT_METHOD_NOT_FOUND as f64) +} + +fn controlkit_rpc_error( + method: &str, + error: &serde_json::Value, + runner_info: Option<&serde_json::Value>, + runner_info_error: Option<&str>, +) -> String { + let code = error + .get("code") + .and_then(serde_json::Value::as_i64) + .unwrap_or_default(); + let message = error + .get("message") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown JSON-RPC error"); + + if !is_controlkit_method_not_found(error) { + return format!("ControlKit method '{method}' failed ({code}): {message}"); + } + + let runner = runner_info + .and_then(|info| info.get("runner")) + .and_then(serde_json::Value::as_str) + .unwrap_or("The connected ControlKit runner"); + let protocol = runner_info + .and_then(|info| info.get("protocolVersion")) + .and_then(serde_json::Value::as_u64) + .map(|version| format!(" protocol {version}")) + .unwrap_or_default(); + + let runner_info_note = runner_info_error + .map(|error| format!(" Runner metadata could not be read: {error}.")) + .unwrap_or_default(); + + format!( + "{runner}{protocol} does not implement ControlKit method '{method}'.{runner_info_note} Rebuild or upgrade xcrs-controlkit, restart its runner, and retry." + ) +} + #[derive(Debug, Clone)] pub struct XcodeCommandLineTools { xcrun_path: PathBuf, @@ -297,14 +397,14 @@ impl Simctl<'_> { self.list_simulators()? .into_iter() .find(|simulator| simulator.name == name) - .ok_or_else(|| anyhow!("iOS simulator named '{name}' was not found")) + .ok_or_else(|| anyhow!("Apple simulator named '{name}' was not found")) } pub fn find_simulator_by_udid(&self, udid: &str) -> Result { self.list_simulators()? .into_iter() .find(|simulator| simulator.udid == udid) - .ok_or_else(|| anyhow!("iOS simulator with UDID '{udid}' was not found")) + .ok_or_else(|| anyhow!("Apple simulator with UDID '{udid}' was not found")) } pub fn boot(&self, udid: &str) -> Result<()> { @@ -1273,6 +1373,88 @@ mod tests { assert_eq!(controlkit.base_url, "http://[fdb4:e020:7377::1]:12006"); } + #[test] + fn reports_actionable_controlkit_method_mismatch() { + let error = serde_json::json!({ + "code": -32601, + "message": "Method not found" + }); + let runner_info = serde_json::json!({ + "runner": "XCRSControlKit", + "protocolVersion": 1 + }); + + let message = controlkit_rpc_error("device.dump.ui", &error, Some(&runner_info), None); + + assert!(message.contains("XCRSControlKit protocol 1")); + assert!(message.contains("device.dump.ui")); + assert!(message.contains("Rebuild or upgrade")); + } + + #[test] + fn recognizes_integral_float_controlkit_method_not_found_code() { + let error = serde_json::json!({ + "code": -32601.0, + "message": "Method not found" + }); + + assert!(is_controlkit_method_not_found(&error)); + let message = controlkit_rpc_error("device.dump.ui", &error, None, None); + assert!(message.contains("does not implement ControlKit method 'device.dump.ui'")); + } + + #[test] + fn reports_when_controlkit_runner_metadata_is_unavailable() { + let error = serde_json::json!({ + "code": -32601, + "message": "Method not found" + }); + + let message = controlkit_rpc_error( + "device.dump.ui", + &error, + None, + Some("device.info returned a JSON-RPC error"), + ); + + assert!(message.contains("The connected ControlKit runner")); + assert!(message.contains("Runner metadata could not be read")); + assert!(message.contains("device.info returned a JSON-RPC error")); + assert!(!message.contains("unknown ControlKit runner")); + } + + #[test] + fn extracts_identified_visible_controlkit_elements() { + let hierarchy = serde_json::json!({ + "type": "Application", + "rect": { "x": 0, "y": 0, "width": 1920, "height": 1080 }, + "children": [ + { + "type": "Button", + "label": "Play", + "rawIdentifier": "play-button", + "rect": { "x": 100, "y": 200, "width": 80, "height": 40 }, + "enabled": true, + "selected": false, + "hittable": true, + "children": [] + }, + { + "type": "Image", + "label": "", + "rect": { "x": 0, "y": 0, "width": 40, "height": 40 }, + "children": [] + } + ] + }); + + let elements = extract_controlkit_elements(&hierarchy); + + assert_eq!(elements.len(), 1); + assert_eq!(elements[0]["label"], serde_json::json!("Play")); + assert_eq!(elements[0]["hittable"], serde_json::json!(true)); + } + #[test] fn parses_device_tunnel_address() { let output = "• Device Name: iPhone\n• Tunnel IP Address: fd55:33ce:ad87::1\n"; diff --git a/docs/controlkit.md b/docs/controlkit.md index cabfeb2..b1a82b2 100644 --- a/docs/controlkit.md +++ b/docs/controlkit.md @@ -117,8 +117,13 @@ expose the same unprefixed tool names. | `input_button` | iOS / tvOS, simulator or physical | Press a Home or tvOS remote button. | | `app_launch`/`app_terminate` | simulator or physical | Launch/terminate an app by bundle ID. On a physical device this calls the runner's `device.apps.launch`/`device.apps.terminate` RPC methods instead of `simctl`. | | `screen_capture` | Simulator or physical | Capture a PNG screenshot. Uses `simctl` for a simulator, or `devicectl device capture screenshot` when a `device` identifier is given. | -| `ui_describe` | All UI-test runners | Read the accessibility hierarchy. | -| `ui_element_list` | All UI-test runners | Read only actionable elements with tap coordinates. | +| `ui_describe` | All UI-test runners | Read the accessibility hierarchy for the supplied `bundle_id`. | +| `ui_element_list` | All UI-test runners | Read actionable elements with tap coordinates for the supplied `bundle_id`. | + +UI introspection requires a recent ControlKit runner that implements +`device.dump.ui`. Both UI tools require the target app's `bundle_id`; this +lets XCTest attach to an already-running app instead of accidentally reading +the ControlKit host app. For a local macOS runner, omit simulator fields and pass `host` and `controlkit_port` when needed: