diff --git a/Cargo.lock b/Cargo.lock index d122275159..fbdc481501 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1633,6 +1633,13 @@ dependencies = [ "log", ] +[[package]] +name = "litebox_common" +version = "0.1.0" +dependencies = [ + "spin 0.9.8", +] + [[package]] name = "litebox_common_linux" version = "0.1.0" @@ -1643,6 +1650,7 @@ dependencies = [ "int-enum", "litebox", "litebox_broker_protocol", + "litebox_common", "litebox_syscall_rewriter", "litebox_util_log", "rangemap", @@ -1766,6 +1774,7 @@ dependencies = [ "bindgen", "bitflags", "litebox", + "litebox_common", "litebox_common_linux", "litebox_platform", "litebox_util_log", @@ -1786,6 +1795,7 @@ dependencies = [ "cfg-if", "libc", "litebox", + "litebox_common", "litebox_common_linux", "litebox_common_optee", "litebox_platform", @@ -1873,6 +1883,7 @@ name = "litebox_platform_windows_userland" version = "0.1.0" dependencies = [ "litebox", + "litebox_common", "litebox_common_linux", "litebox_common_windows", "litebox_platform", diff --git a/Cargo.toml b/Cargo.toml index 538203ecbf..8cbe47e970 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "litebox", + "litebox_common", "litebox_broker_local", "litebox_broker_local_userland", "litebox_broker_core", @@ -60,6 +61,7 @@ members = [ ] default-members = [ "litebox", + "litebox_common", "litebox_broker_local", "litebox_broker_local_userland", "litebox_broker_core", diff --git a/dev_tests/src/ratchet.rs b/dev_tests/src/ratchet.rs index 2921cce158..87f6867072 100644 --- a/dev_tests/src/ratchet.rs +++ b/dev_tests/src/ratchet.rs @@ -72,15 +72,16 @@ fn ratchet_globals() -> Result<()> { ("litebox_broker_core/", 1), ("litebox_broker_transport_linux_userland/", 1), ("litebox_broker_userland/", 1), + ("litebox_common/", 1), ("litebox_hal/", 1), ("litebox_platform/", 2), ("litebox_platform_linux_kernel/", 5), - ("litebox_platform_linux_userland/", 6), + ("litebox_platform_linux_userland/", 7), ("litebox_platform_macos_userland/", 7), ("litebox_platform_lvbs/", 23), ("litebox_platform_vm_kernel/", 4), ("litebox_platform_vm_userland/", 1), - ("litebox_platform_windows_userland/", 11), + ("litebox_platform_windows_userland/", 10), ("litebox_runner_lvbs/", 8), ("litebox_runner_optee_on_qemu/", 1), ("litebox_runner_optee_on_vm_userland/", 1), diff --git a/litebox/src/platform/vector.rs b/litebox/src/platform/vector.rs index 3c93f3aeb7..33777a409a 100644 --- a/litebox/src/platform/vector.rs +++ b/litebox/src/platform/vector.rs @@ -18,7 +18,7 @@ pub trait GuestVectorStateProvider { // TODO: replace this blanket with architecture-specific providers before // implementing guest vector-state preservation outside AArch64. -#[cfg(not(target_arch = "aarch64"))] +#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] impl GuestVectorStateProvider for T { type GuestVectorState = (); diff --git a/litebox_common/Cargo.toml b/litebox_common/Cargo.toml new file mode 100644 index 0000000000..c8bef47493 --- /dev/null +++ b/litebox_common/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "litebox_common" +version = "0.1.0" +edition = "2024" +license = "MIT" +repository = "https://github.com/microsoft/litebox" +description = "Common definitions shared across LiteBox shims and platforms" + +[dependencies] +spin = { version = "0.9.8", default-features = false, features = ["once"] } + +[lints] +workspace = true \ No newline at end of file diff --git a/litebox_common/src/arch/mod.rs b/litebox_common/src/arch/mod.rs new file mode 100644 index 0000000000..f431af155c --- /dev/null +++ b/litebox_common/src/arch/mod.rs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Architecture-specific helpers independent of host operating systems and guest ABIs. + +#[cfg(target_arch = "x86_64")] +pub mod x86_64; diff --git a/litebox_common/src/arch/x86_64/mod.rs b/litebox_common/src/arch/x86_64/mod.rs new file mode 100644 index 0000000000..608ca7f68d --- /dev/null +++ b/litebox_common/src/arch/x86_64/mod.rs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Shared x86-64 CPU-state helpers. + +pub mod xstate; diff --git a/litebox_common/src/arch/x86_64/xstate.rs b/litebox_common/src/arch/x86_64/xstate.rs new file mode 100644 index 0000000000..6798be742d --- /dev/null +++ b/litebox_common/src/arch/x86_64/xstate.rs @@ -0,0 +1,332 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Host-independent x86-64 XSAVE layout discovery and aligned state storage. + +use alloc::{boxed::Box, vec, vec::Vec}; + +/// Size of the architectural legacy x87/SSE save area. +const XSAVE_LEGACY_SIZE: usize = core::mem::size_of::(); +/// Offset of the XSAVE header in a standard-layout save area. +const XSAVE_HEADER_OFFSET: usize = XSAVE_LEGACY_SIZE; +/// Size of the architectural XSAVE header. +const XSAVE_HEADER_SIZE: usize = core::mem::size_of::(); + +/// Architectural x86-64 FXSAVE area at the start of a standard XSAVE area. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct XsaveLegacyArea { + pub control_word: u16, + pub status_word: u16, + pub tag_word: u8, + pub reserved1: u8, + pub error_opcode: u16, + pub error_offset: u32, + pub error_selector: u16, + pub reserved2: u16, + pub data_offset: u32, + pub data_selector: u16, + pub reserved3: u16, + pub mxcsr: u32, + pub mxcsr_mask: u32, + pub float_registers: [[u8; 16]; 8], + pub xmm_registers: [[u8; 16]; 16], + pub reserved4: [u8; 96], +} + +impl Default for XsaveLegacyArea { + fn default() -> Self { + Self { + control_word: 0, + status_word: 0, + tag_word: 0, + reserved1: 0, + error_opcode: 0, + error_offset: 0, + error_selector: 0, + reserved2: 0, + data_offset: 0, + data_selector: 0, + reserved3: 0, + mxcsr: 0, + mxcsr_mask: 0, + float_registers: [[0; 16]; 8], + xmm_registers: [[0; 16]; 16], + reserved4: [0; 96], + } + } +} + +/// Architectural header following the legacy area in an XSAVE area. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct XsaveHeader { + /// Bitmap of state components that are not in their initial state. + pub xstate_bv: u64, + /// Compacted-format indicator and bitmap. Zero for standard format. + pub xcomp_bv: u64, + pub reserved: [u8; 48], +} + +impl Default for XsaveHeader { + fn default() -> Self { + Self { + xstate_bv: 0, + xcomp_bv: 0, + reserved: [0; 48], + } + } +} + +/// An initialized, 64-byte-aligned storage chunk for CPU or host context buffers. +#[repr(C, align(64))] +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct XsaveChunk( + /// The chunk's raw storage bytes. + pub [u8; 64], +); + +/// Standard-layout XSAVE requirements for the host's enabled CPU state. +pub struct XsaveLayout { + /// Required buffer size in bytes for the state components in `mask`. + pub size: usize, + /// Saved user-state feature mask, suitable for XSAVE and XRSTOR. + pub mask: u64, + /// Locations of saved state components beyond the legacy x87/SSE area. + pub components: Vec, + /// Whether the CPU supports XSAVEOPT. + pub xsaveopt: bool, +} + +/// An extended state component in the standard XSAVE layout. +pub struct XsaveComponent { + /// Architectural feature bit in XCR0 and XSTATE_BV. + pub id: u32, + /// Byte offset of this component in the save area. + pub offset: usize, + /// Size of this component in bytes. + pub size: usize, +} + +impl XsaveLayout { + /// AMX tile state (XTILECFG and XTILEDATA), excluded to avoid about 8 KiB + /// of additional storage per save area. + /// + /// Excluding these bits does not disable AMX execution. Platforms must + /// prevent guest enablement or manage AMX state separately. On Linux, + /// tile-data use requires explicit permission and is lazily enabled via XFD. + pub const EXCLUDED_FEATURES: u64 = (1 << 17) | (1 << 18); + + /// Returns the host layout, detecting it on first use. + /// + /// # Panics + /// Panics under the same conditions as [`Self::detect`]. + #[must_use] + pub fn get() -> &'static Self { + static LAYOUT: spin::Once = spin::Once::new(); + LAYOUT.call_once(Self::detect) + } + + /// Detects the current host's enabled XSAVE features and standard layout, + /// excluding [`Self::EXCLUDED_FEATURES`]. + /// + /// # Panics + /// Panics if XSAVE is unavailable, disabled by the host OS, or reports an + /// invalid layout. Both x87 and SSE state must be enabled. + #[must_use] + pub fn detect() -> Self { + const CPUID_XSAVE: u32 = 1 << 26; + const CPUID_OSXSAVE: u32 = 1 << 27; + + assert!(core::arch::x86_64::__cpuid(0).eax >= 0x0d); + let feature_info = core::arch::x86_64::__cpuid(1); + assert_eq!( + feature_info.ecx & (CPUID_XSAVE | CPUID_OSXSAVE), + CPUID_XSAVE | CPUID_OSXSAVE, + "XSAVE must be supported and enabled by the host OS", + ); + let features = core::arch::x86_64::__cpuid_count(0x0d, 0); + // SAFETY: CPUID confirms that the OS has enabled XSAVE and XGETBV. + let mask = unsafe { core::arch::x86_64::_xgetbv(0) } & !Self::EXCLUDED_FEATURES; + assert_eq!(mask & 3, 3, "x87 and SSE state must be enabled"); + let max_size = features.ebx as usize; + assert!(max_size >= XSAVE_HEADER_OFFSET + XSAVE_HEADER_SIZE); + let components: Vec = (2..64) + .filter(|id| mask & (1 << id) != 0) + .map(|id| { + let feature = core::arch::x86_64::__cpuid_count(0x0d, id); + let component = XsaveComponent { + id, + offset: feature.ebx as usize, + size: feature.eax as usize, + }; + assert!(component.offset + component.size <= max_size); + component + }) + .collect(); + let size = components + .iter() + .map(|component| component.offset + component.size) + .fold(XSAVE_HEADER_OFFSET + XSAVE_HEADER_SIZE, usize::max); + let xsaveopt = core::arch::x86_64::__cpuid_count(0x0d, 1).eax & 1 != 0; + Self { + size, + mask, + components, + xsaveopt, + } + } +} + +/// Represents the standard-layout XSAVE area for a guest context. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct XsaveArea { + storage: Box<[XsaveChunk]>, + size: usize, +} + +impl XsaveArea { + /// Architectural initial x87 control word, used when x87 is in init state. + pub const GUEST_INITIAL_X87_CONTROL_WORD: u16 = 0x037f; + /// Architectural initial MXCSR, with all SIMD exceptions masked. + pub const GUEST_INITIAL_MXCSR: u32 = 0x1f80; + + /// Allocates initial CPU state for the given layout. + /// + /// The zero XSTATE_BV requests architectural init state on XRSTOR. MXCSR + /// is initialized explicitly because standard XRSTOR loads it even when + /// SSE is marked as initial. Extended components and header padding are zero. + /// + /// # Panics + /// Panics if the layout cannot hold the legacy area and XSAVE header. + #[must_use] + pub fn initial(layout: &XsaveLayout) -> Self { + assert!( + layout.size >= XSAVE_HEADER_OFFSET + XSAVE_HEADER_SIZE, + "XSAVE layout must include the legacy area and header", + ); + let chunks = layout.size.div_ceil(size_of::()); + let mut area = Self { + storage: vec![XsaveChunk([0; 64]); chunks].into_boxed_slice(), + size: layout.size, + }; + area.legacy_area_mut().mxcsr = Self::GUEST_INITIAL_MXCSR; + area + } + + /// Discards saved state and restores architectural initial state without + /// replacing the aligned allocation. + pub fn reset_to_initial(&mut self) { + for chunk in &mut self.storage { + chunk.0.fill(0); + } + self.legacy_area_mut().mxcsr = Self::GUEST_INITIAL_MXCSR; + } + + /// Returns the architectural x87/SSE state at the start of this area. + #[must_use] + pub fn legacy_area(&self) -> &XsaveLegacyArea { + // SAFETY: storage is 64-byte aligned and always contains the 512-byte legacy area. + unsafe { &*self.as_ptr().cast() } + } + + /// Returns mutable access to the architectural x87/SSE state. + pub fn legacy_area_mut(&mut self) -> &mut XsaveLegacyArea { + // SAFETY: storage is 64-byte aligned and always contains the 512-byte legacy area. + unsafe { &mut *self.as_mut_ptr().cast() } + } + + /// Returns the architectural XSAVE header. + #[must_use] + pub fn header(&self) -> &XsaveHeader { + // SAFETY: storage contains the header at its architectural offset. + unsafe { &*self.as_ptr().add(XSAVE_HEADER_OFFSET).cast() } + } + + /// Returns mutable access to the architectural XSAVE header. + pub fn header_mut(&mut self) -> &mut XsaveHeader { + // SAFETY: storage contains the header at its architectural offset. + unsafe { &mut *self.as_mut_ptr().add(XSAVE_HEADER_OFFSET).cast() } + } + + /// Returns a complete legacy area with architectural initial state + /// substituted for components that are not present in `XSTATE_BV`. + #[must_use] + pub fn materialized_legacy_area(&self) -> XsaveLegacyArea { + let saved = self.legacy_area(); + let mut state = XsaveLegacyArea { + control_word: Self::GUEST_INITIAL_X87_CONTROL_WORD, + mxcsr: saved.mxcsr, + mxcsr_mask: saved.mxcsr_mask, + ..Default::default() + }; + let xstate_bv = self.xstate_bv(); + if xstate_bv & 1 != 0 { + state.control_word = saved.control_word; + state.status_word = saved.status_word; + state.tag_word = saved.tag_word; + state.error_opcode = saved.error_opcode; + state.error_offset = saved.error_offset; + state.error_selector = saved.error_selector; + state.data_offset = saved.data_offset; + state.data_selector = saved.data_selector; + state.float_registers = saved.float_registers; + } + if xstate_bv & 2 != 0 { + state.xmm_registers = saved.xmm_registers; + } + state + } + + /// Returns the aligned buffer address, valid for this area's lifetime. + #[must_use] + pub fn as_ptr(&self) -> *const u8 { + self.storage.as_ptr().cast() + } + + /// Returns the aligned buffer address for exclusive CPU-state capture. + /// + /// Writing through the pointer requires ensuring that captures and reads + /// do not overlap and that the contents remain valid standard-layout state. + pub fn as_mut_ptr(&mut self) -> *mut u8 { + self.storage.as_mut_ptr().cast() + } + + /// Returns the logical XSAVE bytes, excluding trailing allocation padding. + #[must_use] + pub fn as_bytes(&self) -> &[u8] { + // SAFETY: `size` is the requested layout size used to allocate enough + // whole chunks, and the allocation remains live for the returned borrow. + unsafe { core::slice::from_raw_parts(self.as_ptr(), self.size) } + } + + /// Reconstructs a standard-layout XSAVE area from untrusted bytes. + /// + /// Returns `None` if the size, component mask, format, or reserved fields + /// could make restoring the state invalid for `layout`. + #[must_use] + pub fn from_bytes(bytes: &[u8], layout: &XsaveLayout) -> Option { + if bytes.len() != layout.size { + return None; + } + let mut area = Self::initial(layout); + // SAFETY: source and destination are distinct valid buffers of exactly + // `layout.size` bytes. + unsafe { core::ptr::copy_nonoverlapping(bytes.as_ptr(), area.as_mut_ptr(), bytes.len()) }; + let header = area.header(); + if header.xcomp_bv != 0 + || header.xstate_bv & !layout.mask != 0 + || header.reserved != [0; 48] + || area.legacy_area().mxcsr & !0xffff != 0 + { + return None; + } + Some(area) + } + + /// Returns the header's bitmap of state components that are not initial. + #[must_use] + pub fn xstate_bv(&self) -> u64 { + self.header().xstate_bv + } +} diff --git a/litebox_common/src/lib.rs b/litebox_common/src/lib.rs new file mode 100644 index 0000000000..31fb0556fe --- /dev/null +++ b/litebox_common/src/lib.rs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! Host- and guest-ABI-neutral implementations shared by LiteBox platforms and shims. + +#![no_std] + +extern crate alloc; + +pub mod arch; diff --git a/litebox_common_linux/Cargo.toml b/litebox_common_linux/Cargo.toml index af7b6c577c..20d43db374 100644 --- a/litebox_common_linux/Cargo.toml +++ b/litebox_common_linux/Cargo.toml @@ -8,6 +8,7 @@ bitflags = "2.9.0" cfg-if = "1.0.0" elf = { version = "0.8.0", default-features = false } litebox = { path = "../litebox/", version = "0.1.0" } +litebox_common = { path = "../litebox_common", version = "0.1.0", default-features = false } litebox_broker_protocol = { path = "../litebox_broker_protocol", version = "0.1.0" } litebox_util_log = { version = "0.1.0", path = "../litebox_util_log" } thiserror = { version = "2.0.6", default-features = false } diff --git a/litebox_common_linux/src/lib.rs b/litebox_common_linux/src/lib.rs index 7cff5d2ae4..675df88b20 100644 --- a/litebox_common_linux/src/lib.rs +++ b/litebox_common_linux/src/lib.rs @@ -24,10 +24,18 @@ pub mod mm; pub mod physical_pointers; pub mod program_startup; pub mod signal; -#[cfg(target_arch = "aarch64")] -pub use signal::aarch64::GuestVectorState; -#[cfg(not(target_arch = "aarch64"))] -pub type GuestVectorState = (); +mod guest_vector_state { + cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + pub type Selected = litebox_common::arch::x86_64::xstate::XsaveArea; + } else if #[cfg(target_arch = "aarch64")] { + pub type Selected = crate::signal::aarch64::GuestVectorState; + } else { + pub type Selected = (); + } + } +} +pub use guest_vector_state::Selected as GuestVectorState; pub mod user_pointers; pub mod vmap; pub mod vmem; diff --git a/litebox_common_linux/src/program_startup.rs b/litebox_common_linux/src/program_startup.rs index f5f578931c..ba0c9ff3d0 100644 --- a/litebox_common_linux/src/program_startup.rs +++ b/litebox_common_linux/src/program_startup.rs @@ -148,7 +148,7 @@ pub struct LinuxForkStartup { pub alternate_signal_stack: SigAltStack, /// Registers at the `fork` system call, which the child returns from. pub registers: PtRegs, - /// Floating-point and vector state at the `fork` system call (encoded only on AArch64). + /// Floating-point and vector state at the `fork` system call. pub vector_state: GuestVectorState, /// Thread pointer, such as the FS base on x86-64 or `TPIDR_EL0` on AArch64. pub thread_pointer: usize, @@ -383,7 +383,7 @@ impl LinuxForkStartup { output.extend_from_slice(self.signal_actions.as_bytes()); output.extend_from_slice(self.alternate_signal_stack.as_bytes()); output.extend_from_slice(self.registers.as_bytes()); - #[cfg(target_arch = "aarch64")] + #[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] push_vector_state(&mut output, &self.vector_state); for value in [ self.thread_pointer, @@ -443,7 +443,7 @@ impl LinuxForkStartup { let signal_actions = read_value(&mut input)?; let alternate_signal_stack = read_value(&mut input)?; let registers = read_value(&mut input)?; - #[cfg(target_arch = "aarch64")] + #[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] let vector_state = read_vector_state(&mut input)?; let thread_pointer = read_usize(&mut input)?; let syscall_entry_point = read_usize(&mut input)?; @@ -506,9 +506,9 @@ impl LinuxForkStartup { signal_actions, alternate_signal_stack, registers, - #[cfg(target_arch = "aarch64")] + #[cfg(any(target_arch = "aarch64", target_arch = "x86_64"))] vector_state, - #[cfg(not(target_arch = "aarch64"))] + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] vector_state: (), thread_pointer, syscall_entry_point, @@ -675,6 +675,25 @@ fn read_vector_state(input: &mut &[u8]) -> Result, state: &GuestVectorState) { + push_u32( + output, + u32::try_from(state.as_bytes().len()).expect("XSAVE area length must fit in u32"), + ); + output.extend_from_slice(state.as_bytes()); +} + +#[cfg(target_arch = "x86_64")] +fn read_vector_state(input: &mut &[u8]) -> Result { + let length = + usize::try_from(read_u32(input)?).map_err(|_| LinuxProgramStartupError::Malformed)?; + let bytes = take_bytes(input, length)?; + let layout = litebox_common::arch::x86_64::xstate::XsaveLayout::get(); + GuestVectorState::from_bytes(bytes, layout).ok_or(LinuxProgramStartupError::Malformed) +} + fn read_u32(input: &mut &[u8]) -> Result { let bytes = take_bytes(input, size_of::())?; Ok(u32::from_le_bytes( @@ -807,7 +826,12 @@ mod tests { state } - #[cfg(not(target_arch = "aarch64"))] + #[cfg(target_arch = "x86_64")] + fn fork_vector_state() -> GuestVectorState { + GuestVectorState::initial(litebox_common::arch::x86_64::xstate::XsaveLayout::get()) + } + + #[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))] fn fork_vector_state() -> GuestVectorState {} fn fork_startup() -> LinuxForkStartup { diff --git a/litebox_platform_linux_kernel/Cargo.toml b/litebox_platform_linux_kernel/Cargo.toml index fc512ec085..cc45783744 100644 --- a/litebox_platform_linux_kernel/Cargo.toml +++ b/litebox_platform_linux_kernel/Cargo.toml @@ -11,6 +11,7 @@ bindgen = "0.71.0" [dependencies] bitflags = "2.13.1" litebox = { path = "../litebox/", version = "0.1.0" } +litebox_common = { path = "../litebox_common", version = "0.1.0", default-features = false } litebox_common_linux = { path = "../litebox_common_linux/", version = "0.1.0" } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } litebox_util_log = { version = "0.1.0", path = "../litebox_util_log" } diff --git a/litebox_platform_linux_kernel/src/host/snp/snp_impl.rs b/litebox_platform_linux_kernel/src/host/snp/snp_impl.rs index c34e1e2dff..799fe7f399 100644 --- a/litebox_platform_linux_kernel/src/host/snp/snp_impl.rs +++ b/litebox_platform_linux_kernel/src/host/snp/snp_impl.rs @@ -268,6 +268,19 @@ impl litebox::platform::ThreadProvider for SnpLinuxKernel { } } +// TODO: Preserve guest XSAVE state once SNP floating-point support is implemented. +impl litebox::platform::GuestVectorStateProvider for SnpLinuxKernel { + type GuestVectorState = litebox_common_linux::GuestVectorState; + + fn get_guest_vector_state(&self) -> Self::GuestVectorState { + litebox_common_linux::GuestVectorState::initial( + litebox_common::arch::x86_64::xstate::XsaveLayout::get(), + ) + } + + fn set_guest_vector_state(&self, _state: &Self::GuestVectorState) {} +} + impl litebox::platform::TimerProvider for SnpLinuxKernel { type TimerHandle = litebox::platform::trivial_providers::UnsupportedTimerHandle; type Signal = litebox_common_linux::signal::Signal; diff --git a/litebox_platform_linux_userland/Cargo.toml b/litebox_platform_linux_userland/Cargo.toml index 6904c72204..c66bd369be 100644 --- a/litebox_platform_linux_userland/Cargo.toml +++ b/litebox_platform_linux_userland/Cargo.toml @@ -8,6 +8,7 @@ arrayvec = { version = "0.7.6", default-features = false } cfg-if = "1.0.0" libc = { version = "0.2.169", default-features = false } litebox = { path = "../litebox/", version = "0.1.0" } +litebox_common = { path = "../litebox_common", version = "0.1.0", default-features = false } litebox_common_linux = { path = "../litebox_common_linux", version = "0.1.0", features = ["aarch64_gate_recovery"] } litebox_common_optee = { path = "../litebox_common_optee", version = "0.1.0", default-features = false, optional = true } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } diff --git a/litebox_platform_linux_userland/src/aarch64.rs b/litebox_platform_linux_userland/src/aarch64.rs index 4eed80862c..a29a30b89c 100644 --- a/litebox_platform_linux_userland/src/aarch64.rs +++ b/litebox_platform_linux_userland/src/aarch64.rs @@ -32,7 +32,6 @@ macro_rules! load_tls_block_base { ) }; } -pub(super) use load_tls_block_base; macro_rules! tprel_offset { ($var:ident) => { @@ -222,7 +221,8 @@ pub(super) fn set_guest_thread_pointer(value: usize) { } } -pub(super) fn set_is_guest_thread(value: bool) { +/// Updates the marker from host execution context. +pub(super) fn set_guest_thread_marker(value: bool) { // SAFETY: an own-thread [`tls_offset`] slot access. unsafe { core::arch::asm! { @@ -236,7 +236,27 @@ pub(super) fn set_is_guest_thread(value: bool) { } } -pub(super) fn is_guest_thread() -> bool { +/// Replaces the marker from host execution context. +pub(super) fn replace_guest_thread_marker(value: bool) -> bool { + let previous: u32; + // SAFETY: an own-thread [`tls_offset`] slot access. + unsafe { + core::arch::asm! { + load_tls_block_base!("{tmp}"), + "ldrb {previous:w}, [{tmp}, #{off}]", + "strb {value:w}, [{tmp}, #{off}]", + tmp = out(reg) _, + previous = out(reg) previous, + value = in(reg) u32::from(value), + off = const tls_offset::IS_GUEST_THREAD, + options(nostack, preserves_flags) + } + } + previous != 0 +} + +/// Reads the marker from a signal context or host execution context. +pub(super) fn is_guest_thread_from_signal_context() -> bool { let value: u32; // SAFETY: an own-thread [`tls_offset`] slot access. Also safe from a signal // handler: `TPIDR_EL0` is the host anchor at all times, guest code included. @@ -253,6 +273,102 @@ pub(super) fn is_guest_thread() -> bool { value != 0 } +pub(super) fn take_pending_host_signals_bits() -> u32 { + let signals: u32; + // Atomic against this thread's signal handlers; use the ARMv8.0 baseline + // rather than ARMv8.1 LSE. + // SAFETY: reads and clears a naturally aligned `u32` in this thread's own + // TLS control block, whose offset from `TPIDR_EL0` is checked by + // `assert_tls_block_placement`. + unsafe { + core::arch::asm!( + load_tls_block_base!("{addr}"), + "add {addr}, {addr}, #{off}", + "2:", + "ldaxr {val:w}, [{addr}]", + "stlxr {status:w}, wzr, [{addr}]", + "cbnz {status:w}, 2b", + addr = out(reg) _, + val = out(reg) signals, + status = out(reg) _, + off = const tls_offset::PENDING_HOST_SIGNALS, + options(nostack) + ); + } + signals +} + +pub(super) fn swap_wait_waker_addr(new_addr: usize) -> usize { + let old_addr: usize; + // Atomic against `record_pending_signal` in this thread's handler. + // SAFETY: exchanges a naturally aligned `u64` in this thread's own TLS + // control block, whose offset from `TPIDR_EL0` is checked by + // `assert_tls_block_placement`. + unsafe { + core::arch::asm!( + load_tls_block_base!("{addr}"), + "add {addr}, {addr}, #{off}", + "2:", + "ldaxr {old}, [{addr}]", + "stlxr {status:w}, {new}, [{addr}]", + "cbnz {status:w}, 2b", + addr = out(reg) _, + old = out(reg) old_addr, + new = in(reg) new_addr, + status = out(reg) _, + off = const tls_offset::WAIT_WAKER_ADDR, + options(nostack), + ); + } + old_addr +} + +pub(super) fn clear_interrupt() { + // SAFETY: writes a single byte in this thread's own TLS control block. + unsafe { + core::arch::asm!( + load_tls_block_base!("{tmp}"), + "strb wzr, [{tmp}, #{off}]", + tmp = out(reg) _, + off = const tls_offset::INTERRUPT, + options(nostack, preserves_flags) + ); + } +} + +pub(super) unsafe fn record_pending_signal(mask: u32) -> usize { + let waker_addr: usize; + + // Atomic against an interrupted exchange; a plain load/or/store can lose a bit. + // + // SAFETY: both slots are in this thread's own TLS control block at offsets + // checked by `assert_tls_block_placement`, and both accesses are naturally aligned. + // The exclusive monitor reservation is opened and closed within the loop. + unsafe { + core::arch::asm!( + load_tls_block_base!("{block}"), + "add {addr}, {block}, #{pending_off}", + "2:", + "ldaxr {old:w}, [{addr}]", + "orr {new:w}, {old:w}, {mask:w}", + "stlxr {status:w}, {new:w}, [{addr}]", + "cbnz {status:w}, 2b", + "ldr {waker}, [{block}, #{waker_off}]", + block = out(reg) _, + addr = out(reg) _, + old = out(reg) _, + new = out(reg) _, + status = out(reg) _, + waker = out(reg) waker_addr, + mask = in(reg) mask, + pending_off = const tls_offset::PENDING_HOST_SIGNALS, + waker_off = const tls_offset::WAIT_WAKER_ADDR, + options(nostack) + ); + } + waker_addr +} + pub(super) fn get_guest_thread_pointer() -> usize { let value: usize; // SAFETY: an own-thread [`tls_offset`] slot access. @@ -1338,8 +1454,15 @@ pub(super) fn get_guest_vector_state() -> GuestVectorState { } } +/// Prepares guest vector state for entry, optionally using inherited state. +pub(super) fn prepare_guest_vector_state(inherited: Option<&GuestVectorState>) { + if let Some(state) = inherited { + set_guest_vector_state(state); + } +} + /// Replaces the current thread's platform-owned guest vector state. -pub(super) fn set_guest_vector_state(state: &GuestVectorState) { +fn set_guest_vector_state(state: &GuestVectorState) { let block: usize; // SAFETY: callers run on an initialized guest thread with guest execution // stopped, giving exclusive access to this aligned TLS slot. Volatile @@ -1356,12 +1479,54 @@ pub(super) fn set_guest_vector_state(state: &GuestVectorState) { } unsafe extern "C" { + pub(super) fn syscall_callback() -> isize; + pub(super) fn syscall_callback_in_guest_cleared(); + pub(super) fn exception_callback(); + pub(super) fn interrupt_callback(); + pub(super) fn switch_to_guest_via_outbound_stub_start(); + pub(super) fn switch_to_guest_via_outbound_stub_end(); + pub(super) fn switch_to_guest_via_sigreturn_start(); + pub(super) fn switch_to_guest_via_sigreturn_end(); + #[cfg(test)] + fn switch_to_guest_stage_x16(); + #[cfg(test)] + fn switch_to_guest_stage_x16_end(); + #[cfg(test)] + fn switch_to_guest_stage_x16_fixup(); + /// The compiler runtime's cache-synchronization helper: cleans the data /// cache to the point of unification and invalidates the instruction cache /// over `[start, end)`. fn __clear_cache(start: *mut core::ffi::c_char, end: *mut core::ffi::c_char); } +pub(super) fn syscall_entry_point() -> usize { + syscall_callback as *const () as usize +} + +pub(super) fn in_switch_to_guest(pc: usize) -> bool { + fn body(start: unsafe extern "C" fn(), end: unsafe extern "C" fn()) -> core::ops::Range { + start as *const () as usize..end as *const () as usize + } + + body( + switch_to_guest_via_outbound_stub_start, + switch_to_guest_via_outbound_stub_end, + ) + .contains(&pc) + || body( + switch_to_guest_via_sigreturn_start, + switch_to_guest_via_sigreturn_end, + ) + .contains(&pc) +} + +pub(super) fn in_syscall_callback_prologue(ip: usize) -> bool { + // The label keeps the multi-instruction bound tied to the asm. + let start = syscall_callback as *const () as usize; + (start..syscall_callback_in_guest_cleared as *const () as usize).contains(&ip) +} + /// Make code just written to `range` visible to the instruction fetch path, /// before that range becomes executable. /// @@ -1695,6 +1860,73 @@ mod tests { use litebox_common_linux::PtRegs; use litebox_common_linux::signal::{SigSet, Signal}; + #[test] + fn spawned_thread_installs_inherited_vector_state() { + use litebox::platform::{GuestVectorStateProvider as _, ThreadProvider as _}; + use litebox::shim::InitThread; + + struct ObserveInit(std::sync::mpsc::Sender); + struct ObserveShim(std::sync::mpsc::Sender); + + impl InitThread for ObserveInit { + type ExecutionContext = litebox_common_linux::PtRegs; + + fn init( + self: Box, + ) -> Box> { + Box::new(ObserveShim(self.0)) + } + } + + impl EnterShim for ObserveShim { + type ExecutionContext = litebox_common_linux::PtRegs; + + fn init(&self, _ctx: &mut Self::ExecutionContext) -> ContinueOperation { + self.0.send(get_guest_vector_state()).unwrap(); + ContinueOperation::Terminate + } + + fn syscall(&self, _ctx: &mut Self::ExecutionContext) -> ContinueOperation { + unreachable!() + } + + fn exception( + &self, + _ctx: &mut Self::ExecutionContext, + _info: &litebox::shim::ExceptionInfo, + ) -> ContinueOperation { + unreachable!() + } + + fn interrupt(&self, _ctx: &mut Self::ExecutionContext) -> ContinueOperation { + unreachable!() + } + } + + let platform = LinuxUserland::new(); + let original = platform.get_guest_vector_state(); + let expected = litebox_common_linux::GuestVectorState { + registers: core::array::from_fn(|index| index as u128 * 0x101), + fpsr: 0x1234, + fpcr: 0x5678, + }; + platform.set_guest_vector_state(&expected); + let (send, receive) = std::sync::mpsc::channel(); + unsafe { + platform + .spawn_thread( + &litebox_common_linux::PtRegs::default(), + Box::new(ObserveInit(send)), + ) + .unwrap(); + } + let observed = receive + .recv_timeout(core::time::Duration::from_secs(5)) + .unwrap(); + platform.set_guest_vector_state(&original); + assert_eq!(observed, expected); + } + fn live_fp_state() -> FpState { let status: u64; let control: u64; @@ -2071,7 +2303,7 @@ mod tests { _ => unreachable!(), }; let mut reader = fixture_reader(&code, &trampoline, &[], &guest_tp); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2123,7 +2355,7 @@ mod tests { }; context.uc_mcontext.regs[5] = saved.regs[5] as u64; let mut reader = fixture_reader(&code, &trampoline, &frame, &guest_tp); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2165,7 +2397,7 @@ mod tests { } else { 0x1004 }; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2196,7 +2428,7 @@ mod tests { for offset in [36usize, 40, 44] { context.uc_mcontext.pc = 0x400010 + offset as u64; for (stub, pc) in [(0, 0x1004), (0x400010 + 36, 0), (0x400010 + 36, 0x2004)] { - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2870,7 +3102,7 @@ mod tests { // `NotGate`. No failure may mutate the saved registers. for denied_address in [0x400010usize, 0x1000, 0x8000] { let mut reader = fixture_reader(&code, &trampoline, &frame, &[]); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2891,7 +3123,7 @@ mod tests { let mut wrong_branch = code.clone(); wrong_branch.copy_from_slice(&0x1400_0000u32.to_le_bytes()); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2904,7 +3136,7 @@ mod tests { let mut wrong_template = trampoline.clone(); wrong_template[16 + 8..16 + 12].copy_from_slice(&0xd503_201fu32.to_le_bytes()); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2916,7 +3148,7 @@ mod tests { assert_eq!(ptregs_bytes(&saved), before); frame[16..24].copy_from_slice(&0xdead_beef_u64.to_le_bytes()); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2928,7 +3160,7 @@ mod tests { assert_eq!(ptregs_bytes(&saved), before); context.uc_mcontext.pc = 0x400010 + 24; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2974,7 +3206,7 @@ mod tests { (0x400010 + 60, false, "SVC metadata"), ] { context.uc_mcontext.pc = pc; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -2999,7 +3231,7 @@ mod tests { ] { let mut mutated = trampoline.clone(); mutated[16 + word_offset..16 + word_offset + 4].copy_from_slice(&0u32.to_le_bytes()); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3012,7 +3244,7 @@ mod tests { let mut bad_metadata = trampoline.clone(); bad_metadata[16 + 60] ^= 0x80; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3028,7 +3260,7 @@ mod tests { (0x1400_0001, "inbound B wrong target"), ] { let wrong_code = branch.to_le_bytes(); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3058,7 +3290,7 @@ mod tests { assert_eq!(trapped, []); second_slot.copy_from_slice(&relocated[16..80]); context.uc_mcontext.pc = second_pc as u64; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3089,7 +3321,7 @@ mod tests { // Gate stack arithmetic wraps exactly like the AArch64 ADD/SUB pair. context.uc_mcontext.pc = 0x400010 + 4; context.uc_mcontext.sp = u64::MAX - 15; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3108,7 +3340,7 @@ mod tests { let (code, trampoline, mut context, saved) = gate_fixture(0xd53b_d049); context.uc_mcontext.pc = 0x400010 + 4; let before = (saved.regs, saved.sp, saved.pc, saved.pstate, saved.orig_x0); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3144,7 +3376,7 @@ mod tests { } else { 0xfeed_f000_0000_0010 }; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3177,7 +3409,7 @@ mod tests { 0x8000 }; context.uc_mcontext.regs[5] = saved.regs[5] as u64; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3202,7 +3434,7 @@ mod tests { context.uc_mcontext.regs[5] = saved.regs[5] as u64; let captured = saved.regs[5].to_ne_bytes(); let mut reader = fixture_reader(&msr_code, &msr_slot, &[], &[]); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3230,7 +3462,7 @@ mod tests { for offset in [12usize, 20, 24] { context.uc_mcontext.pc = 0x400010 + offset as u64; context.uc_mcontext.sp = 0x8000; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3248,7 +3480,7 @@ mod tests { for offset in (0usize..=32).step_by(4) { context.uc_mcontext.pc = 0x400010 + offset as u64; context.uc_mcontext.sp = if offset == 0 { 0x8020 } else { 0x8000 }; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3319,7 +3551,7 @@ mod tests { trampoline[28..32].copy_from_slice(&0x1f11_b807u32.to_le_bytes()); context.uc_mcontext.pc = 0x400010; - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3344,7 +3576,7 @@ mod tests { frame[8..16].copy_from_slice(&0x1717u64.to_ne_bytes()); frame[16..24].copy_from_slice(&(expected as u64).to_ne_bytes()); - let result = super::aarch64::canonicalize_aarch64_gate_signal_context( + let result = super::canonicalize_aarch64_gate_signal_context( &context, &saved, 0x500000, @@ -3390,13 +3622,14 @@ mod tests { // waker slot is null on this thread (asserted below), so no waker is // dereferenced. assert_eq!(read_tls_slot!(usize, tls_offset::WAIT_WAKER_ADDR), 0); + let alarm_bit = 1u32 << (Signal::SIGALRM.as_i32() - 1); + let interrupt_bit = 1u32 << (Signal::SIGINT.as_i32() - 1); unsafe { - super::record_pending_signal(Signal::SIGALRM); - super::record_pending_signal(Signal::SIGINT); + super::record_pending_signal(alarm_bit); + super::record_pending_signal(interrupt_bit); } - let expected_bits = - (1u32 << (Signal::SIGALRM.as_i32() - 1)) | (1u32 << (Signal::SIGINT.as_i32() - 1)); + let expected_bits = alarm_bit | interrupt_bit; assert_eq!( read_tls_slot!(u32, tls_offset::PENDING_HOST_SIGNALS), expected_bits, @@ -3488,38 +3721,44 @@ mod tests { assert!(result.is_err(), "the panic must propagate to the caller"); assert!( - !super::is_guest_thread(), + !super::is_guest_thread_from_signal_context(), "unwinding must run `GuestThreadMarker`'s drop" ); } #[test] fn test_guest_thread_marker() { - assert!(!is_guest_thread(), "a plain test thread is not a guest"); + assert!( + !is_guest_thread_from_signal_context(), + "a plain test thread is not a guest" + ); { let _outer = GuestThreadMarker::enter(); - assert!(is_guest_thread()); + assert!(is_guest_thread_from_signal_context()); { let _inner = GuestThreadMarker::enter(); - assert!(is_guest_thread()); + assert!(is_guest_thread_from_signal_context()); } assert!( - is_guest_thread(), + is_guest_thread_from_signal_context(), "dropping a nested marker must not un-mark the thread while \ an outer frame is still running the guest" ); } - assert!(!is_guest_thread(), "the outermost marker must restore"); + assert!( + !is_guest_thread_from_signal_context(), + "the outermost marker must restore" + ); let result = std::panic::catch_unwind(|| { let _marker = GuestThreadMarker::enter(); - assert!(is_guest_thread()); + assert!(is_guest_thread_from_signal_context()); panic!("unwind out of guest execution"); }); assert!(result.is_err()); assert!( - !is_guest_thread(), + !is_guest_thread_from_signal_context(), "the marker must be restored when the guest frame unwinds" ); } @@ -3624,7 +3863,7 @@ mod tests { let output = std::process::Command::new(std::env::current_exe().unwrap()) .args([ "--exact", - "aarch64::tests::nested_exception_during_sigsys_preserves_guest_context", + "arch::tests::nested_exception_during_sigsys_preserves_guest_context", "--nocapture", ]) .env(CHILD_ENV, "1") diff --git a/litebox_platform_linux_userland/src/lib.rs b/litebox_platform_linux_userland/src/lib.rs index 6800f3b08b..9d6550fcdc 100644 --- a/litebox_platform_linux_userland/src/lib.rs +++ b/litebox_platform_linux_userland/src/lib.rs @@ -31,25 +31,30 @@ use std::time::Duration; use zerocopy::{FromBytes, IntoBytes}; +extern crate alloc; + +#[cfg_attr(target_arch = "aarch64", path = "aarch64.rs")] +#[cfg_attr(target_arch = "x86_64", path = "x86_64.rs")] +mod arch; +mod page_mgmt; + #[cfg(target_arch = "aarch64")] -mod aarch64; -#[cfg(all(test, target_arch = "aarch64"))] -use aarch64::get_guest_vector_state; -#[cfg(target_arch = "aarch64")] -use aarch64::{ +use arch::{ Aarch64GateSignalResult, GateInterruption, assert_tls_block_placement, canonicalize_runtime_aarch64_gate_signal_context, fatal_aarch64_runtime_state, - guest_thread_pointer_tp_offset, is_guest_thread, load_tls_block_base, run_thread_arch, - set_guest_vector_state, set_is_guest_thread, set_signal_return, - signal_handler_capture_guest_vector_state, signal_handler_set_interrupt, - signal_handler_take_guest, switch_to_guest, sync_instruction_stream, tls_offset, + guest_thread_pointer_tp_offset, signal_handler_capture_guest_vector_state, + signal_handler_set_interrupt, signal_handler_take_guest, sync_instruction_stream, +}; +#[cfg(target_arch = "x86_64")] +use arch::{copy_signal_context, interrupt_callback_no_xsave, signal_handler_exit_guest}; +use arch::{ + exception_callback, interrupt_callback, is_guest_thread_from_signal_context, + prepare_guest_vector_state, replace_guest_thread_marker, run_thread_arch, + set_guest_thread_marker, set_signal_return, switch_to_guest, }; - -extern crate alloc; #[cfg(target_arch = "aarch64")] const AT_FDCWD: usize = (litebox_common_linux::AT_FDCWD as isize).cast_unsigned(); -mod page_mgmt; /// The admitted open syscall and its flags index must remain paired. #[cfg(target_arch = "x86_64")] @@ -60,67 +65,6 @@ const OPEN_FLAGS_ARG: u8 = 1; const OPEN_SYSNO: i64 = libc::SYS_openat; #[cfg(target_arch = "aarch64")] const OPEN_FLAGS_ARG: u8 = 2; -// --------------------------------------------------------------------------- -// TLS (`.tbss`) access helpers -// -// On x86_64, the ELF TLS model uses `@tpoff`; on x86 it uses `@ntpoff`. -// At guest-host transitions we swap `fs` and `gs`, so after the swap the host TLS base -// is in the normal segment register. Before the swap (e.g. in a signal -// handler that fires while the guest is running), the host TLS base is -// in the *saved* segment register (`gs` on x86_64, `fs` on x86). -// -// The macros below produce string literals so they can be used inside -// `concat!()` within `core::arch::asm!()`. -// --------------------------------------------------------------------------- - -/// TLS relocation suffix: `"@tpoff"` on x86_64, `"@ntpoff"` on x86. -#[cfg(target_arch = "x86_64")] -macro_rules! tls_suffix { - () => { - "@tpoff" - }; -} - -/// Segment register used for TLS after the fs/gs swap (normal host context). -#[cfg(target_arch = "x86_64")] -macro_rules! tls_seg { - () => { - "fs" - }; -} - -/// Segment register where the host TLS base is saved before the swap -/// (signal handler context while the guest is running). -#[cfg(target_arch = "x86_64")] -macro_rules! saved_tls_seg { - () => { - "gs" - }; -} - -/// Full TLS memory operand for a `.tbss` variable in normal host context -/// (after the fs/gs swap). -/// -/// Example: `tls!("pending_host_signals")` expands to -/// `"fs:pending_host_signals@tpoff"` on x86_64. -#[cfg(target_arch = "x86_64")] -macro_rules! tls { - ($var:literal) => { - concat!(tls_seg!(), ":", $var, tls_suffix!()) - }; -} - -/// Full TLS memory operand for a `.tbss` variable accessed via the *saved* -/// segment register (before the fs/gs swap, e.g. from a signal handler). -/// -/// Example: `saved_tls!("in_guest")` expands to -/// `"gs:in_guest@tpoff"` on x86_64. -#[cfg(target_arch = "x86_64")] -macro_rules! saved_tls { - ($var:literal) => { - concat!(saved_tls_seg!(), ":", $var, tls_suffix!()) - }; -} /// The userland Linux platform. /// @@ -507,37 +451,7 @@ impl litebox::platform::SignalProvider for LinuxUserland { fn take_pending_host_signals() -> litebox_common_linux::signal::SigSet { // Atomically swap the per-thread pending signals with zero. // Only the low 32 bits are used (covers traditional signals 1-31). - let lo: u32; - #[cfg(target_arch = "x86_64")] - unsafe { - core::arch::asm!( - "xor {tmp:e}, {tmp:e}", - concat!("xchg DWORD PTR ", tls!("pending_host_signals"), ", {tmp:e}"), - tmp = out(reg) lo, - options(nostack) - ); - } - // Atomic against this thread's signal handlers; use the ARMv8.0 baseline - // rather than ARMv8.1 LSE. - #[cfg(target_arch = "aarch64")] - // SAFETY: reads and clears a naturally aligned `u32` in this thread's own - // TLS control block, whose offset from `TPIDR_EL0` is checked by - // `assert_tls_block_placement`. - unsafe { - core::arch::asm!( - load_tls_block_base!("{addr}"), - "add {addr}, {addr}, #{off}", - "2:", - "ldaxr {val:w}, [{addr}]", - "stlxr {status:w}, wzr, [{addr}]", - "cbnz {status:w}, 2b", - addr = out(reg) _, - val = out(reg) lo, - status = out(reg) _, - off = const tls_offset::PENDING_HOST_SIGNALS, - options(nostack) - ); - } + let lo = arch::take_pending_host_signals_bits(); litebox_common_linux::signal::SigSet::from_u64(u64::from(lo)) } @@ -551,7 +465,7 @@ pub unsafe fn run_thread(shim: T, ctx: &mut litebox_common_linux::PtRegs) where T: litebox::shim::EnterShim, { - run_thread_inner(&shim, ctx, false); + run_thread_inner(&shim, ctx, GuestVectorStateMode::Fresh); } /// Run a guest thread using a reference to the shim. @@ -565,7 +479,7 @@ pub unsafe fn run_thread_ref(shim: &T, ctx: &mut litebox_common_linux::PtRegs where T: litebox::shim::EnterShim, { - run_thread_inner(shim, ctx, false); + run_thread_inner(shim, ctx, GuestVectorStateMode::Fresh); } /// Re-enter a guest thread using a reference to the shim. @@ -579,7 +493,13 @@ pub unsafe fn reenter_thread(shim: &T, ctx: &mut litebox_common_linux::PtRegs where T: litebox::shim::EnterShim, { - run_thread_inner(shim, ctx, true); + run_thread_inner(shim, ctx, GuestVectorStateMode::Resume); +} + +enum GuestVectorStateMode { + Fresh, + Resume, + Inherited(Box), } struct ThreadContext<'a> { @@ -590,397 +510,59 @@ struct ThreadContext<'a> { fn run_thread_inner( shim: &dyn litebox::shim::EnterShim, ctx: &mut litebox_common_linux::PtRegs, - reenter: bool, + vector_state_mode: GuestVectorStateMode, ) { + prepare_guest_vector_state(match &vector_state_mode { + GuestVectorStateMode::Inherited(state) => Some(state.as_ref()), + GuestVectorStateMode::Fresh | GuestVectorStateMode::Resume => None, + }); let ctx_ptr = core::ptr::from_mut(ctx); let mut thread_ctx = ThreadContext { shim, ctx }; let _guest_thread = GuestThreadMarker::enter(); ThreadHandle::run_with_handle(|| { with_signal_alt_stack(|| unsafe { - run_thread_arch(&mut thread_ctx, ctx_ptr, u8::from(reenter)); + run_thread_arch( + &mut thread_ctx, + ctx_ptr, + u8::from(matches!(vector_state_mode, GuestVectorStateMode::Resume)), + ); }); }); } -#[cfg(target_arch = "x86_64")] -core::arch::global_asm!( - " - .section .tbss - .align 8 -scratch: - .quad 0 -host_sp: - .quad 0 -host_bp: - .quad 0 -guest_context_top: - .quad 0 -.globl guest_fsbase -guest_fsbase: - .quad 0 -.globl guest_thread -guest_thread: - .byte 0 -in_guest: - .byte 0 -.globl interrupt -interrupt: - .byte 0 - .align 4 -.globl pending_host_signals -pending_host_signals: - .long 0 - .align 8 -.globl wait_waker_addr -wait_waker_addr: - .quad 0 - " -); - -#[cfg(target_arch = "x86_64")] -fn set_guest_fsbase(value: usize) { - unsafe { - core::arch::asm! { - "mov fs:guest_fsbase@tpoff, {}", - in(reg) value, - options(nostack, preserves_flags) - } - } -} - -#[cfg(target_arch = "x86_64")] -fn get_guest_fsbase() -> usize { - let value: usize; - unsafe { - core::arch::asm! { - "mov {}, fs:guest_fsbase@tpoff", - out(reg) value, - options(nostack, preserves_flags) - } - } - value -} - -#[cfg(target_arch = "x86_64")] -fn is_guest_thread() -> bool { - macro_rules! read_guest_thread { - ($access:ident) => {{ - let marker: u8; - core::arch::asm!( - concat!("mov {marker}, BYTE PTR ", $access!("guest_thread")), - marker = out(reg_byte) marker, - options(nostack, preserves_flags), - ); - marker - }}; - } - - let marker = match GUEST_TLS_MODE.load(Ordering::Relaxed) { - mode if mode == GuestTlsMode::Windows as u8 => { - // SAFETY: Windows guests leave host TLS active in FS. - unsafe { read_guest_thread!(tls) } - } - _ => { - // SAFETY: this platform requires FSGSBASE support. - if unsafe { litebox_common_linux::rdgsbase() } == 0 { - return false; - } - // SAFETY: nonzero GS holds this thread's host TLS in Linux guest mode, - // including after guest exit, when the lifetime marker has been cleared. - unsafe { read_guest_thread!(saved_tls) } - } - }; - marker != 0 -} - /// Tracks the whole guest-thread lifetime and preserves nested-entry state. struct GuestThreadMarker { - #[cfg(target_arch = "x86_64")] - previous: u8, - #[cfg(target_arch = "aarch64")] previous: bool, } impl GuestThreadMarker { fn enter() -> Self { - #[cfg(target_arch = "x86_64")] - { - let previous: u8; - // SAFETY: entry runs with host TLS active in FS and accesses this thread's marker. - unsafe { - core::arch::asm!( - concat!("mov {previous}, BYTE PTR ", tls!("guest_thread")), - concat!("mov BYTE PTR ", tls!("guest_thread"), ", 1"), - previous = out(reg_byte) previous, - options(nostack, preserves_flags), - ); - } - Self { previous } - } - #[cfg(target_arch = "aarch64")] - { - let previous = is_guest_thread(); - set_is_guest_thread(true); - Self { previous } + Self { + previous: replace_guest_thread_marker(true), } } } impl Drop for GuestThreadMarker { fn drop(&mut self) { - #[cfg(target_arch = "x86_64")] - // SAFETY: drop runs after host FS is restored and accesses this thread's marker. - unsafe { - core::arch::asm!( - concat!("mov BYTE PTR ", tls!("guest_thread"), ", {previous}"), - previous = in(reg_byte) self.previous, - options(nostack, preserves_flags), - ); - } - #[cfg(target_arch = "aarch64")] - set_is_guest_thread(self.previous); + set_guest_thread_marker(self.previous); } } -/// Runs the guest thread until it terminates. -/// -/// This saves all non-volatile register state then switches to the guest -/// context. When the guest makes a syscall, it jumps back into the middle of -/// this routine, at `syscall_callback`. This code then updates the guest -/// context structure, switches back to the host stack, and calls the syscall -/// handler. -/// -/// When the guest thread terminates, this function returns after restoring -/// non-volatile register state. -#[cfg(target_arch = "x86_64")] -#[unsafe(naked)] -unsafe extern "C-unwind" fn run_thread_arch( - thread_ctx: &mut ThreadContext, - ctx: *mut litebox_common_linux::PtRegs, - reenter: u8, -) { - core::arch::naked_asm!( - " - .macro restore_guest_context scratch - pop r15 - pop r14 - pop r13 - pop r12 - pop rbp - pop rbx - pop r11 - pop r10 - pop r9 - pop r8 - pop rax - pop rcx - pop rdx - pop rsi - pop rdi - add rsp, 8 // skip orig_rax - pop \\scratch // read rip into scratch - add rsp, 8 // skip cs - popfq - pop rsp - jmp \\scratch // jump to the guest - .endm - - .cfi_startproc - // Push all non-volatiles. - push rbp - mov rbp, rsp - .cfi_def_cfa rbp, 16 - push rbx - push r12 - push r13 - push r14 - push r15 - push rdi // save thread context - - // Save host rsp and rbp and guest context top in TLS. - mov fs:host_sp@tpoff, rsp - mov fs:host_bp@tpoff, rbp - lea r8, [rsi + {GUEST_CONTEXT_SIZE}] - mov fs:guest_context_top@tpoff, r8 - - // Linux guests use FS, so preserve host FS in GS. Windows guests retain - // their GS base and leave host FS active. - cmp BYTE PTR [rip + GUEST_TLS_MODE], {WINDOWS_TLS_MODE} - je 2f - rdfsbase r8 - wrgsbase r8 -2: - - // Call init_handler or reenter_handler based on reenter flag (in dl). - test dl, dl - jnz 1f - call {init_handler} - jmp .Ldone -1: - call {reenter_handler} - jmp .Ldone - - // This entry point is called from the guest when it issues a syscall - // instruction. - // - // At entry, the register context is the guest context with the - // return address in rcx. r11 is an available scratch register (it would - // contain rflags if the syscall instruction had actually been issued). - .globl syscall_callback -syscall_callback: - // Clear in_guest flag. This must be the first instruction to match the - // expectations of `interrupt_signal_handler`. - mov BYTE PTR gs:in_guest@tpoff, 0 - - // Restore host fs base. - rdfsbase r11 - mov gs:guest_fsbase@tpoff, r11 - rdgsbase r11 - wrfsbase r11 - jmp .Lhost_tls_restored - - .globl syscall_callback_guest_gs -syscall_callback_guest_gs: - // Windows guests use GS, different from Linux hosts which use FS. - // Thus no need to swap FS and GS like we do for Linux guests. - mov BYTE PTR fs:in_guest@tpoff, 0 - -.Lhost_tls_restored: - - // Switch to the top of the guest context. - mov r11, rsp - mov rsp, fs:guest_context_top@tpoff - - // TODO: save float and vector registers (xsave or fxsave) - // Save caller-saved registers - push 0x2b // pt_regs->ss = __USER_DS - push r11 // pt_regs->sp - pushfq // pt_regs->eflags - push 0x33 // pt_regs->cs = __USER_CS - push rcx // pt_regs->ip - push rax // pt_regs->orig_ax - - push rdi // pt_regs->di - push rsi // pt_regs->si - push rdx // pt_regs->dx - push rcx // pt_regs->cx - push -38 // pt_regs->ax = ENOSYS - push r8 // pt_regs->r8 - push r9 // pt_regs->r9 - push r10 // pt_regs->r10 - push [rsp + 88] // pt_regs->r11 = rflags - push rbx // pt_regs->bx - push rbp // pt_regs->bp - push r12 // pt_regs->r12 - push r13 // pt_regs->r13 - push r14 // pt_regs->r14 - push r15 // pt_regs->r15 - - // Restore the stack and frame pointer. - mov rsp, fs:host_sp@tpoff - mov rbp, fs:host_bp@tpoff - - // Handle the syscall. This will jump back to the guest but - // will return if the thread is exiting. - mov rdi, [rsp] // pass thread_ctx - call {syscall_handler} - // This thread is done. Return. - jmp .Ldone - -exception_callback: - // Restore the stack and frame pointer. - mov rsp, fs:host_sp@tpoff - mov rbp, fs:host_bp@tpoff - - mov rdi, [rsp] // pass thread_ctx - call {exception_handler} - jmp .Ldone - -interrupt_callback: - // Restore the stack and frame pointer. - mov rsp, fs:host_sp@tpoff - mov rbp, fs:host_bp@tpoff - - mov rdi, [rsp] // pass thread_ctx - call {interrupt_handler} - -.Ldone: - - lea rsp, [rbp - 5*8] - pop r15 - pop r14 - pop r13 - pop r12 - pop rbx - pop rbp - .cfi_def_cfa rsp, 8 - ret - .cfi_endproc -", - GUEST_CONTEXT_SIZE = const core::mem::size_of::(), - init_handler = sym init_handler, - reenter_handler = sym reenter_handler, - syscall_handler = sym syscall_handler, - exception_handler = sym exception_handler, - interrupt_handler = sym interrupt_handler, - WINDOWS_TLS_MODE = const GuestTlsMode::Windows as u8, - ); -} - -/// Switches to the provided guest context. -/// -/// # Safety -/// The context must be valid guest context. This can only be called if -/// `run_thread_arch` is on the stack; after the guest exits, it will return to -/// the interior of `run_thread_arch`. -/// -/// Do not call this at a point where the stack needs to be unwound to run -/// destructors. -#[cfg(target_arch = "x86_64")] -#[unsafe(naked)] -unsafe extern "C" fn switch_to_guest(ctx: &litebox_common_linux::PtRegs) -> ! { - core::arch::naked_asm!( - "switch_to_guest_start:", - // Set `in_guest` now, then check if there is a pending interrupt. If an - // interrupt arrives while `in_guest` is set, the signal handler will - // see that the IP is between `switch_to_guest_start` and - // `switch_to_guest_end` and will set `interrupt` and jump to - // `interrupt_callback`. - // - // If an interrupt is already pending, clear `in_guest` and jump to - // `interrupt_callback` without entering the guest. `interrupt_callback` - // runs host code, and a signal arriving there with `in_guest` still set - // would be taken for a guest interrupt and overwrite the saved guest - // context with host registers. - "mov BYTE PTR fs:in_guest@tpoff, 1", - "cmp BYTE PTR fs:interrupt@tpoff, 0", - "je 2f", - "mov BYTE PTR fs:in_guest@tpoff, 0", - "jmp interrupt_callback", - "2:", - // Restore guest context from ctx. - "mov rsp, rdi", - "cmp BYTE PTR [rip + GUEST_TLS_MODE], {WINDOWS_TLS_MODE}", - "je 2f", - // Linux guest: switch FS and use saved host FS through GS. - "mov rdx, fs:guest_fsbase@tpoff", - "wrfsbase rdx", - "restore_guest_context gs:scratch@tpoff", - "2:", - // Windows guest: host FS and guest GS are already active. - "restore_guest_context fs:scratch@tpoff", - "switch_to_guest_end:", - ".purgem restore_guest_context", - WINDOWS_TLS_MODE = const GuestTlsMode::Windows as u8, - ); -} - +/// Non-guest threads (e.g., network workers, background tasks) should call this +/// function at the start of their execution so the kernel only delivers +/// `SIGALRM` / `SIGINT` to guest threads, which have the proper signal-handler +/// context to re-enter the shim. fn guest_signal_set() -> libc::sigset_t { unsafe { let mut set: libc::sigset_t = std::mem::zeroed(); libc::sigemptyset(&raw mut set); libc::sigaddset(&raw mut set, libc::SIGALRM); libc::sigaddset(&raw mut set, libc::SIGINT); + let interrupt_signal = INTERRUPT_SIGNAL_NUMBER.load(Ordering::Relaxed); + if interrupt_signal != 0 { + libc::sigaddset(&raw mut set, interrupt_signal); + } set } } @@ -1065,14 +647,14 @@ fn thread_start( dyn litebox::shim::InitThread, >, mut ctx: litebox_common_linux::PtRegs, - #[cfg(target_arch = "aarch64")] vector_state: litebox_common_linux::GuestVectorState, + vector_state: Box, ) { - #[cfg(target_arch = "aarch64")] - set_guest_vector_state(&vector_state); - // Allow caller to run some code before we return to the new thread. let shim = init_thread.init(); - - run_thread_inner(shim.as_ref(), &mut ctx, false); + run_thread_inner( + shim.as_ref(), + &mut ctx, + GuestVectorStateMode::Inherited(vector_state), + ); // TODO: have syscall_callback return if we need to terminate the process. // We should return this value to the caller so load_program can return it // to the user. @@ -1139,18 +721,11 @@ impl litebox::platform::ThreadProvider for LinuxUserland { >, ) -> Result<(), Self::ThreadSpawnError> { let ctx = ctx.clone(); - #[cfg(target_arch = "aarch64")] let vector_state = - litebox::platform::GuestVectorStateProvider::get_guest_vector_state(self); + Box::new(litebox::platform::GuestVectorStateProvider::get_guest_vector_state(self)); // TODO: do we need to wait for the handle in the main thread? - let _handle = std::thread::Builder::new().spawn(move || { - thread_start( - init_thread, - ctx, - #[cfg(target_arch = "aarch64")] - vector_state, - ); - })?; + let _handle = std::thread::Builder::new() + .spawn(move || thread_start(init_thread, ctx, vector_state))?; Ok(()) } @@ -1165,7 +740,7 @@ impl litebox::platform::ThreadProvider for LinuxUserland { #[cfg(debug_assertions)] fn run_test_thread(f: impl FnOnce() -> R) -> R { - // Sets `gsbase = fsbase` (x86_64) or `fs = gs` (x86) on the current thread + // Sets `gsbase = fsbase` (x86_64) on the current thread // to mirror the TLS base used in guest context, so that test threads can use the // same TLS access code as guest threads. #[cfg(target_arch = "x86_64")] @@ -1264,37 +839,8 @@ impl RawMutexProvider for LinuxUserland { impl WaitWakerProvider for LinuxUserland { fn update_waker(&self, waker: Option) { - let mut waker_ptr = waker.map_or(std::ptr::null_mut(), |w| Box::into_raw(Box::new(w))); - #[cfg(target_arch = "x86_64")] - unsafe { - core::arch::asm!( - concat!("xchg ", tls!("wait_waker_addr"), ", {}"), - inout(reg) waker_ptr, - options(nostack), - ); - } - // Atomic against `record_pending_signal` in this thread's handler. - #[cfg(target_arch = "aarch64")] - // SAFETY: exchanges a naturally aligned `u64` in this thread's own TLS - // control block, whose offset from `TPIDR_EL0` is checked by - // `assert_tls_block_placement`. - unsafe { - let new_ptr = waker_ptr; - core::arch::asm!( - load_tls_block_base!("{addr}"), - "add {addr}, {addr}, #{off}", - "2:", - "ldaxr {old}, [{addr}]", - "stlxr {status:w}, {new}, [{addr}]", - "cbnz {status:w}, 2b", - addr = out(reg) _, - old = out(reg) waker_ptr, - new = in(reg) new_ptr, - status = out(reg) _, - off = const tls_offset::WAIT_WAKER_ADDR, - options(nostack), - ); - } + let waker_ptr = waker.map_or(std::ptr::null_mut(), |w| Box::into_raw(Box::new(w))); + let waker_ptr = arch::swap_wait_waker_addr(waker_ptr as usize) as *mut core::task::Waker; if !waker_ptr.is_null() { // SAFETY: old waker_ptr was created by Box::into_raw in a previous call to update_waker. unsafe { drop(Box::from_raw(waker_ptr)) }; @@ -1439,81 +985,6 @@ impl SystemTimeTrait for SystemTime { } } -#[cfg(target_arch = "x86_64")] -impl litebox::platform::ArchSpecificProvider for LinuxUserland { - fn set_arch_specific_register( - &self, - reg: &litebox::platform::ArchSpecificRegister, - val: usize, - ) -> Result<(), litebox::platform::ArchSpecificError> { - match reg { - litebox::platform::ArchSpecificRegister::FsBase - if Self::guest_tls_mode() == GuestTlsMode::Linux => - { - if litebox_common_linux::arch::is_valid_user_fs_base(val) { - // We swap gs and fs before and after a syscall, so while handling a guest - // syscall the guest's fs base is stored in the gs base register; the - // per-thread `guest_fsbase` slot holds the value that will be programmed - // into fs base on guest re-entry. - set_guest_fsbase(val); - Ok(()) - } else { - Err(litebox::platform::ArchSpecificError::RegisterUnpermittedValue) - } - } - litebox::platform::ArchSpecificRegister::FsBase => { - // Windows guests leave host TLS active in FS. - Err(litebox::platform::ArchSpecificError::RegisterReserved) - } - litebox::platform::ArchSpecificRegister::GsBase => { - match Self::guest_tls_mode() { - GuestTlsMode::Windows => { - if litebox_common_linux::arch::is_valid_user_fs_base(val) { - // SAFETY: this platform requires FSGSBASE support. - unsafe { litebox_common_linux::wrgsbase(val) }; - Ok(()) - } else { - Err(litebox::platform::ArchSpecificError::RegisterUnpermittedValue) - } - } - GuestTlsMode::Linux => { - // GS holds host TLS across guest/host fs-gs swaps for Linux guests. - Err(litebox::platform::ArchSpecificError::RegisterReserved) - } - } - } - _ => Err(litebox::platform::ArchSpecificError::RegisterUnsupported), - } - } - fn get_arch_specific_register( - &self, - reg: &litebox::platform::ArchSpecificRegister, - ) -> Result { - match reg { - litebox::platform::ArchSpecificRegister::FsBase - if Self::guest_tls_mode() == GuestTlsMode::Linux => - { - Ok(get_guest_fsbase()) - } - litebox::platform::ArchSpecificRegister::FsBase => { - Err(litebox::platform::ArchSpecificError::RegisterReserved) - } - litebox::platform::ArchSpecificRegister::GsBase => { - match Self::guest_tls_mode() { - GuestTlsMode::Windows => { - // SAFETY: this platform requires FSGSBASE support. - Ok(unsafe { litebox_common_linux::rdgsbase() }) - } - GuestTlsMode::Linux => { - Err(litebox::platform::ArchSpecificError::RegisterReserved) - } - } - } - _ => Err(litebox::platform::ArchSpecificError::RegisterUnsupported), - } - } -} - type UserMutPtr = litebox::platform::common_providers::userspace_pointers::UserMutPtr< litebox::platform::common_providers::userspace_pointers::NoValidation, T, @@ -1605,34 +1076,6 @@ fn futex_val2( } } -unsafe extern "C" { - fn syscall_callback() -> isize; - #[cfg(target_arch = "x86_64")] - fn syscall_callback_guest_gs() -> isize; - #[cfg(target_arch = "aarch64")] - fn syscall_callback_in_guest_cleared(); - fn exception_callback(); - fn interrupt_callback(); - #[cfg(target_arch = "x86_64")] - fn switch_to_guest_start(); - #[cfg(target_arch = "x86_64")] - fn switch_to_guest_end(); - #[cfg(target_arch = "aarch64")] - fn switch_to_guest_via_outbound_stub_start(); - #[cfg(target_arch = "aarch64")] - fn switch_to_guest_via_outbound_stub_end(); - #[cfg(target_arch = "aarch64")] - fn switch_to_guest_via_sigreturn_start(); - #[cfg(target_arch = "aarch64")] - fn switch_to_guest_via_sigreturn_end(); - #[cfg(all(test, target_arch = "aarch64"))] - fn switch_to_guest_stage_x16(); - #[cfg(all(test, target_arch = "aarch64"))] - fn switch_to_guest_stage_x16_end(); - #[cfg(all(test, target_arch = "aarch64"))] - fn switch_to_guest_stage_x16_fixup(); -} - /// Whether `pc` is inside a sequence switching this thread to guest state. /// /// From the store that sets `in_guest` to the final branch into the guest, @@ -1643,27 +1086,7 @@ unsafe extern "C" { /// AArch64 has two such sequences to x86-64's one, in separate functions, so /// this is a union of two ranges: one span would depend on link order. fn in_switch_to_guest(pc: usize) -> bool { - fn body(start: unsafe extern "C" fn(), end: unsafe extern "C" fn()) -> core::ops::Range { - start as *const () as usize..end as *const () as usize - } - - #[cfg(target_arch = "x86_64")] - { - body(switch_to_guest_start, switch_to_guest_end).contains(&pc) - } - #[cfg(target_arch = "aarch64")] - { - body( - switch_to_guest_via_outbound_stub_start, - switch_to_guest_via_outbound_stub_end, - ) - .contains(&pc) - || body( - switch_to_guest_via_sigreturn_start, - switch_to_guest_via_sigreturn_end, - ) - .contains(&pc) - } + arch::in_switch_to_guest(pc) } /// Whether `ip` is inside `syscall_callback`'s prologue, i.e. before the guest @@ -1676,22 +1099,7 @@ fn in_switch_to_guest(pc: usize) -> bool { /// frame's shifted sp, and `syscallno = NO_SYSCALL` — silently dropping the /// in-flight guest syscall. fn in_syscall_callback_prologue(ip: usize) -> bool { - #[cfg(target_arch = "x86_64")] - { - let start = match GUEST_TLS_MODE.load(Ordering::Relaxed) { - mode if mode == GuestTlsMode::Windows as u8 => { - syscall_callback_guest_gs as *const () as usize - } - _ => syscall_callback as *const () as usize, - }; - ip == start - } - // The label keeps the multi-instruction AArch64 bound tied to the asm. - #[cfg(target_arch = "aarch64")] - { - let start = syscall_callback as *const () as usize; - (start..syscall_callback_in_guest_cleared as *const () as usize).contains(&ip) - } + arch::in_syscall_callback_prologue(ip) } unsafe extern "C-unwind" fn init_handler(thread_ctx: &mut ThreadContext) { @@ -1780,24 +1188,7 @@ impl ThreadContext<'_> { // Clear the interrupt flag before calling the shim, since we've handled it // now (by calling into the shim), and it might be set again by the shim // before returning. - #[cfg(target_arch = "x86_64")] - unsafe { - core::arch::asm!( - concat!("mov BYTE PTR ", tls!("interrupt"), ", 0"), - options(nostack, preserves_flags) - ); - } - #[cfg(target_arch = "aarch64")] - // SAFETY: writes a single byte in this thread's own TLS control block. - unsafe { - core::arch::asm!( - load_tls_block_base!("{tmp}"), - "strb wzr, [{tmp}, #{off}]", - tmp = out(reg) _, - off = const tls_offset::INTERRUPT, - options(nostack, preserves_flags) - ); - } + arch::clear_interrupt(); let op = f(self.shim, self.ctx); match op { ContinueOperation::Resume => unsafe { switch_to_guest(self.ctx) }, @@ -1808,17 +1199,7 @@ impl ThreadContext<'_> { impl litebox::platform::SystemInfoProvider for LinuxUserland { fn get_syscall_entry_point(&self) -> usize { - #[cfg(target_arch = "x86_64")] - { - match Self::guest_tls_mode() { - GuestTlsMode::Windows => syscall_callback_guest_gs as *const () as usize, - GuestTlsMode::Linux => syscall_callback as *const () as usize, - } - } - #[cfg(target_arch = "aarch64")] - { - syscall_callback as *const () as usize - } + arch::syscall_entry_point() } fn get_vdso_address(&self) -> Option { @@ -2026,134 +1407,6 @@ fn with_signal_alt_stack(f: impl FnOnce() -> R) -> R { f() } -/// Called from signal handlers to fix up thread state after potentially running -/// in the guest. -/// -/// Restores the proper host `fsbase` so that TLS can be used. Clears `in_guest` -/// and optionally sets `interrupt`. If `in_guest` was previously set, returns -/// the guest context pointer (which does not necessarily have up-to-date guest -/// register state yet). -#[cfg(target_arch = "x86_64")] -fn signal_handler_exit_guest( - _context: &libc::ucontext_t, - set_interrupt: bool, - _capture_vector_state: bool, -) -> Option<*mut litebox_common_linux::PtRegs> { - unsafe { - macro_rules! take_guest { - ($access:ident) => {{ - let in_guest: u8; - core::arch::asm!( - concat!("mov {in_guest}, BYTE PTR ", $access!("in_guest")), - concat!("mov BYTE PTR ", $access!("in_guest"), ", 0"), - in_guest = out(reg_byte) in_guest, - options(nostack, preserves_flags) - ); - if set_interrupt { - core::arch::asm!( - concat!("mov BYTE PTR ", $access!("interrupt"), ", 1"), - options(nostack, preserves_flags) - ); - } - in_guest != 0 - }}; - } - match GUEST_TLS_MODE.load(Ordering::Relaxed) { - mode if mode == GuestTlsMode::Windows as u8 => { - if !take_guest!(tls) { - return None; - } - } - _ => { - let gsbase = litebox_common_linux::rdgsbase(); - if gsbase == 0 || !take_guest!(saved_tls) { - return None; - } - litebox_common_linux::wrfsbase(gsbase); - } - } - let guest_context_top: *mut litebox_common_linux::PtRegs; - core::arch::asm!( - concat!("mov {guest_context_top}, ", tls!("guest_context_top")), - guest_context_top = out(reg) guest_context_top, - options(nostack, preserves_flags) - ); - Some(guest_context_top.sub(1)) - } -} - -/// Copies register state from a Linux signal context to a LiteBox PtRegs -/// structure. -#[cfg(target_arch = "x86_64")] -fn copy_signal_context(regs: &mut litebox_common_linux::PtRegs, context: &libc::ucontext_t) { - let litebox_common_linux::PtRegs { - r15, - r14, - r13, - r12, - rbp, - rbx, - r11, - r10, - r9, - r8, - rax, - rcx, - rdx, - rsi, - rdi, - orig_rax, - rip, - cs: _, - eflags, - rsp, - ss: _, - } = regs; - for (reg, sig_reg) in [ - (r15, libc::REG_R15), - (r14, libc::REG_R14), - (r13, libc::REG_R13), - (r12, libc::REG_R12), - (rbp, libc::REG_RBP), - (rbx, libc::REG_RBX), - (r11, libc::REG_R11), - (r10, libc::REG_R10), - (r9, libc::REG_R9), - (r8, libc::REG_R8), - (rax, libc::REG_RAX), - (rcx, libc::REG_RCX), - (rdx, libc::REG_RDX), - (rsi, libc::REG_RSI), - (rdi, libc::REG_RDI), - (rip, libc::REG_RIP), - (rsp, libc::REG_RSP), - (eflags, libc::REG_EFL), - ] { - *reg = context.uc_mcontext.gregs[sig_reg.reinterpret_as_unsigned() as usize] - .reinterpret_as_unsigned() - .trunc(); - } - *orig_rax = *rax; -} - -/// Updates a Linux signal context to return to `f` with the given arguments. -#[cfg(target_arch = "x86_64")] -fn set_signal_return( - context: &mut libc::ucontext_t, - f: unsafe extern "C" fn(), - p0: isize, - p1: isize, - p2: isize, - p3: isize, -) { - let sigctx = &mut context.uc_mcontext; - sigctx.gregs[libc::REG_RIP as usize] = (f as usize).reinterpret_as_signed() as i64; - sigctx.gregs[libc::REG_RDI as usize] = p0 as i64; - sigctx.gregs[libc::REG_RSI as usize] = p1 as i64; - sigctx.gregs[libc::REG_RDX as usize] = p2 as i64; - sigctx.gregs[libc::REG_RCX as usize] = p3 as i64; -} - #[cfg(target_arch = "aarch64")] fn is_synchronous_memory_fault(signum: libc::c_int, code: libc::c_int) -> bool { match signum { @@ -2294,7 +1547,7 @@ unsafe extern "C" fn exception_signal_handler( } #[cfg(target_arch = "x86_64")] - let Some(regs) = signal_handler_exit_guest(context, false, true) else { + let Some(regs) = signal_handler_exit_guest(context, false) else { return unsafe { next_signal_handler(signum, info, context) }; }; #[cfg(target_arch = "aarch64")] @@ -2430,61 +1683,9 @@ unsafe fn next_signal_handler( /// through FS in Windows guest mode. unsafe fn record_pending_signal(signal: litebox_common_linux::signal::Signal) { let mask: u32 = 1u32 << (signal.as_i32() - 1); - let waker_addr: usize; - - // SAFETY: the bitmask and waker slot are reached through this thread's own - // saved host TLS segment, which the caller guarantees is valid, and both - // accesses are naturally aligned. - #[cfg(target_arch = "x86_64")] - unsafe { - macro_rules! record { - ($access:ident) => {{ - let waker: usize; - core::arch::asm!( - concat!("lock or DWORD PTR ", $access!("pending_host_signals"), ", {mask:e}"), - concat!("mov {waker}, ", $access!("wait_waker_addr")), - mask = in(reg) mask, - waker = out(reg) waker, - options(nostack) - ); - waker - }}; - } - waker_addr = match GUEST_TLS_MODE.load(Ordering::Relaxed) { - mode if mode == GuestTlsMode::Windows as u8 => record!(tls), - _ => record!(saved_tls), - }; - } - - // Atomic against an interrupted exchange; a plain load/or/store can lose a bit. - // - // SAFETY: both slots are in this thread's own TLS control block at offsets - // checked by `assert_tls_block_placement`, and both accesses are naturally aligned. - // The exclusive monitor reservation is opened and closed within the loop. - #[cfg(target_arch = "aarch64")] - unsafe { - core::arch::asm!( - load_tls_block_base!("{block}"), - "add {addr}, {block}, #{pending_off}", - "2:", - "ldaxr {old:w}, [{addr}]", - "orr {new:w}, {old:w}, {mask:w}", - "stlxr {status:w}, {new:w}, [{addr}]", - "cbnz {status:w}, 2b", - "ldr {waker}, [{block}, #{waker_off}]", - block = out(reg) _, - addr = out(reg) _, - old = out(reg) _, - new = out(reg) _, - status = out(reg) _, - waker = out(reg) waker_addr, - mask = in(reg) mask, - pending_off = const tls_offset::PENDING_HOST_SIGNALS, - waker_off = const tls_offset::WAIT_WAKER_ADDR, - options(nostack) - ); - } - + // SAFETY: the caller satisfies the architecture-specific TLS access + // requirements documented above. + let waker_addr = unsafe { arch::record_pending_signal(mask) }; if waker_addr == 0 { return; } @@ -2539,7 +1740,7 @@ unsafe fn interrupt_signal_handler( if let Ok(signal) = litebox_common_linux::signal::Signal::try_from(guest_signum) { // Check whether this is a guest thread. If not, re-raise the signal // process-wide. - if is_guest_thread() { + if is_guest_thread_from_signal_context() { // SAFETY: we verified above that this is a guest thread. unsafe { record_pending_signal(signal) }; } else { @@ -2578,7 +1779,7 @@ unsafe fn interrupt_signal_handler( let in_switch_to_guest = in_switch_to_guest(ip); #[cfg(target_arch = "x86_64")] - let Some(regs) = signal_handler_exit_guest(context, true, !in_switch_to_guest) else { + let Some(regs) = signal_handler_exit_guest(context, true) else { return; }; #[cfg(target_arch = "aarch64")] @@ -2604,7 +1805,16 @@ unsafe fn interrupt_signal_handler( GateInterruption::Asynchronous, ); } - set_signal_return(context, interrupt_callback, 0, 0, 0, 0); + // Cases 3 and 4: jump to interrupt handler. + #[cfg(target_arch = "x86_64")] + let callback = if in_switch_to_guest { + interrupt_callback_no_xsave + } else { + interrupt_callback + }; + #[cfg(target_arch = "aarch64")] + let callback = interrupt_callback; + set_signal_return(context, callback, 0, 0, 0, 0); } impl litebox::platform::DerivedKeyProvider for LinuxUserland { @@ -2692,74 +1902,6 @@ mod tests { extern crate std; - #[cfg(target_arch = "aarch64")] - #[test] - fn spawned_thread_installs_inherited_vector_state() { - use litebox::platform::{GuestVectorStateProvider as _, ThreadProvider as _}; - use litebox::shim::{ContinueOperation, EnterShim, InitThread}; - - struct ObserveInit(std::sync::mpsc::Sender); - struct TerminateShim; - - impl InitThread for ObserveInit { - type ExecutionContext = litebox_common_linux::PtRegs; - - fn init( - self: Box, - ) -> Box> { - self.0.send(super::get_guest_vector_state()).unwrap(); - Box::new(TerminateShim) - } - } - - impl EnterShim for TerminateShim { - type ExecutionContext = litebox_common_linux::PtRegs; - - fn init(&self, _ctx: &mut Self::ExecutionContext) -> ContinueOperation { - ContinueOperation::Terminate - } - - fn syscall(&self, _ctx: &mut Self::ExecutionContext) -> ContinueOperation { - unreachable!() - } - - fn exception( - &self, - _ctx: &mut Self::ExecutionContext, - _info: &litebox::shim::ExceptionInfo, - ) -> ContinueOperation { - unreachable!() - } - - fn interrupt(&self, _ctx: &mut Self::ExecutionContext) -> ContinueOperation { - unreachable!() - } - } - - let platform = LinuxUserland::new(); - let original = platform.get_guest_vector_state(); - let expected = litebox_common_linux::GuestVectorState { - registers: core::array::from_fn(|index| index as u128 * 0x101), - fpsr: 0x1234, - fpcr: 0x5678, - }; - platform.set_guest_vector_state(&expected); - let (send, receive) = std::sync::mpsc::channel(); - unsafe { - platform - .spawn_thread( - &litebox_common_linux::PtRegs::default(), - Box::new(ObserveInit(send)), - ) - .unwrap(); - } - let observed = receive - .recv_timeout(core::time::Duration::from_secs(5)) - .unwrap(); - platform.set_guest_vector_state(&original); - assert_eq!(observed, expected); - } - /// An interrupt pending at guest entry diverts to the interrupt handler /// without entering the guest, so a further interrupt taken while that /// handler runs host code must not be treated as a guest interrupt, which diff --git a/litebox_platform_linux_userland/src/x86_64.rs b/litebox_platform_linux_userland/src/x86_64.rs new file mode 100644 index 0000000000..ae2a437659 --- /dev/null +++ b/litebox_platform_linux_userland/src/x86_64.rs @@ -0,0 +1,1397 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +//! x86-64 guest/host transitions and signal-context helpers. + +use std::cell::RefCell; + +use litebox::utils::{ReinterpretSignedExt, ReinterpretUnsignedExt as _, TruncateExt}; +use litebox_common::arch::x86_64::xstate::{XsaveArea, XsaveLayout}; + +use super::{ + LinuxUserland, ThreadContext, exception_handler, init_handler, interrupt_handler, + reenter_handler, syscall_handler, +}; + +const RFLAGS_TF: u32 = 1 << 8; +const RFLAGS_DF: u32 = 1 << 10; +const RFLAGS_AC: u32 = 1 << 18; +const HOST_UNSAFE_RFLAGS: u32 = RFLAGS_TF | RFLAGS_DF | RFLAGS_AC; +const HOST_SAFE_RFLAGS_MASK: u32 = !HOST_UNSAFE_RFLAGS; + +// Clear TF, DF, and AC in the live flags before entering host code. Callers +// must preserve the original guest flags before invoking this macro. +macro_rules! sanitize_host_rflags { + () => { + " + pushfq + and DWORD PTR [rsp], {HOST_SAFE_RFLAGS_MASK} + popfq +" + }; +} + +/// Full TLS memory operand for a `.tbss` variable in normal host context +/// (after the fs/gs swap). +/// +/// Example: `tls!("pending_host_signals")` expands to +/// `"fs:pending_host_signals@tpoff"`. +macro_rules! tls { + ($var:literal) => { + concat!("fs:", $var, "@tpoff") + }; +} + +/// Full TLS memory operand for a `.tbss` variable accessed via the *saved* +/// segment register (before the fs/gs swap, e.g. from a signal handler). +/// +/// Example: `saved_tls!("in_guest")` expands to `"gs:in_guest@tpoff"`. +macro_rules! saved_tls { + ($var:literal) => { + concat!("gs:", $var, "@tpoff") + }; +} + +thread_local! { + pub(super) static GUEST_XSTATE: RefCell> = const { RefCell::new(None) }; +} + +fn install_xstate(guest: &mut XsaveArea, layout: &XsaveLayout) { + // SAFETY: The buffer remains owned by this thread's TLS until thread exit. + // No guest is running while these assembly TLS slots are initialized. + unsafe { + core::arch::asm!( + "mov fs:guest_xsave@tpoff, {guest}", + "mov fs:xsave_mask@tpoff, {mask}", + "mov BYTE PTR fs:use_xsaveopt@tpoff, {xsaveopt}", + guest = in(reg) guest.as_mut_ptr(), + mask = in(reg) layout.mask, + xsaveopt = in(reg_byte) u8::from(layout.xsaveopt), + options(nostack, preserves_flags), + ); + } +} + +impl litebox::platform::GuestVectorStateProvider for LinuxUserland { + type GuestVectorState = XsaveArea; + + fn get_guest_vector_state(&self) -> XsaveArea { + GUEST_XSTATE.with_borrow(|guest| { + guest + .clone() + .unwrap_or_else(|| XsaveArea::initial(XsaveLayout::get())) + }) + } + + fn set_guest_vector_state(&self, state: &XsaveArea) { + set_guest_vector_state(state); + } +} + +fn set_guest_vector_state(state: &XsaveArea) { + let layout = XsaveLayout::get(); + GUEST_XSTATE.with_borrow_mut(|guest| { + let guest = guest.insert(state.clone()); + install_xstate(guest, layout); + }); +} + +/// Prepares guest vector state for entry, optionally using inherited state. +pub(super) fn prepare_guest_vector_state(inherited: Option<&XsaveArea>) { + if let Some(state) = inherited { + set_guest_vector_state(state); + } else { + let layout = XsaveLayout::get(); + GUEST_XSTATE.with_borrow_mut(|guest| { + if guest.is_some() { + return; + } + let guest = guest.insert(XsaveArea::initial(layout)); + install_xstate(guest, layout); + }); + } +} + +core::arch::global_asm!( + " + .section .tbss + .align 8 +scratch: + .quad 0 +host_sp: + .quad 0 +host_bp: + .quad 0 +.globl guest_xsave +guest_xsave: + .quad 0 +.globl xsave_mask +xsave_mask: + .quad 0 +.globl use_xsaveopt +use_xsaveopt: + .byte 0 + .align 2 +host_x87_control_word: + .word 0 + .align 4 +host_mxcsr: + .long 0 +.globl guest_context_top +guest_context_top: + .quad 0 +.globl guest_fsbase +guest_fsbase: + .quad 0 +.globl guest_thread +guest_thread: + .byte 0 +.globl in_guest +in_guest: + .byte 0 +.globl interrupt +interrupt: + .byte 0 + .align 4 +.globl pending_host_signals +pending_host_signals: + .long 0 + .align 8 +.globl wait_waker_addr +wait_waker_addr: + .quad 0 + " +); + +pub(super) fn take_pending_host_signals_bits() -> u32 { + let signals: u32; + // SAFETY: atomically exchanges this thread's naturally aligned TLS slot. + unsafe { + core::arch::asm!( + "xor {tmp:e}, {tmp:e}", + concat!("xchg DWORD PTR ", tls!("pending_host_signals"), ", {tmp:e}"), + tmp = out(reg) signals, + options(nostack) + ); + } + signals +} + +pub(super) fn swap_wait_waker_addr(mut new_addr: usize) -> usize { + // SAFETY: atomically exchanges this thread's naturally aligned TLS slot. + unsafe { + core::arch::asm!( + concat!("xchg ", tls!("wait_waker_addr"), ", {addr}"), + addr = inout(reg) new_addr, + options(nostack), + ); + } + new_addr +} + +pub(super) fn clear_interrupt() { + // SAFETY: accesses this thread's marker with host TLS active in FS. + unsafe { + core::arch::asm!( + concat!("mov BYTE PTR ", tls!("interrupt"), ", 0"), + options(nostack, preserves_flags) + ); + } +} + +pub(super) unsafe fn record_pending_signal(mask: u32) -> usize { + macro_rules! record { + ($access:ident) => {{ + let waker: usize; + core::arch::asm!( + concat!("lock or DWORD PTR ", $access!("pending_host_signals"), ", {mask:e}"), + concat!("mov {waker}, ", $access!("wait_waker_addr")), + mask = in(reg) mask, + waker = out(reg) waker, + options(nostack) + ); + waker + }}; + } + + // SAFETY: the bitmask and waker slot are reached through this thread's own + // saved host TLS segment, which the caller guarantees is valid, and both + // accesses are naturally aligned. + unsafe { + match super::GUEST_TLS_MODE.load(core::sync::atomic::Ordering::Relaxed) { + mode if mode == super::GuestTlsMode::Windows as u8 => record!(tls), + _ => record!(saved_tls), + } + } +} + +pub(super) fn set_guest_fsbase(value: usize) { + unsafe { + core::arch::asm! { + "mov fs:guest_fsbase@tpoff, {}", + in(reg) value, + options(nostack, preserves_flags) + } + } +} + +pub(super) fn get_guest_fsbase() -> usize { + let value: usize; + unsafe { + core::arch::asm! { + "mov {}, fs:guest_fsbase@tpoff", + out(reg) value, + options(nostack, preserves_flags) + } + } + value +} + +/// Updates the marker while host TLS is active in FS. +pub(super) fn set_guest_thread_marker(value: bool) { + // SAFETY: accesses this thread's marker with host TLS active in FS. + unsafe { + core::arch::asm!( + concat!("mov BYTE PTR ", tls!("guest_thread"), ", {value}"), + value = in(reg_byte) u8::from(value), + options(nostack, preserves_flags), + ); + } +} + +/// Replaces the marker while host TLS is active in FS. +pub(super) fn replace_guest_thread_marker(value: bool) -> bool { + let previous: u8; + // SAFETY: accesses this thread's marker with host TLS active in FS. + unsafe { + core::arch::asm!( + concat!("mov {previous}, BYTE PTR ", tls!("guest_thread")), + concat!("mov BYTE PTR ", tls!("guest_thread"), ", {value}"), + previous = out(reg_byte) previous, + value = in(reg_byte) u8::from(value), + options(nostack, preserves_flags), + ); + } + previous != 0 +} + +/// Reads the marker from a signal context, where Linux guest TLS may be active in FS. +pub(super) fn is_guest_thread_from_signal_context() -> bool { + macro_rules! read_guest_thread { + ($access:ident) => {{ + let marker: u8; + core::arch::asm!( + concat!("mov {marker}, BYTE PTR ", $access!("guest_thread")), + marker = out(reg_byte) marker, + options(nostack, preserves_flags), + ); + marker + }}; + } + + let marker = match super::GUEST_TLS_MODE.load(core::sync::atomic::Ordering::Relaxed) { + mode if mode == super::GuestTlsMode::Windows as u8 => { + // SAFETY: Windows guests leave host TLS active in FS. + unsafe { read_guest_thread!(tls) } + } + _ => { + // SAFETY: this platform requires FSGSBASE support. + if unsafe { litebox_common_linux::rdgsbase() } == 0 { + return false; + } + // SAFETY: nonzero GS holds this thread's host TLS in Linux guest mode. + unsafe { read_guest_thread!(saved_tls) } + } + }; + marker != 0 +} + +/// Saves the guest's extended state to its TLS save area. Clobbers rax, rdx, r10, and flags. +macro_rules! save_guest_xstate { + () => { + " + mov r10, fs:guest_xsave@tpoff + mov eax, DWORD PTR fs:xsave_mask@tpoff + mov edx, DWORD PTR fs:xsave_mask@tpoff+4 + cmp BYTE PTR fs:use_xsaveopt@tpoff, 0 + je 8f + xsaveopt64 [r10] + jmp 9f +8: + xsave64 [r10] +9: +" + }; +} + +/// Clears the x87 exception flags and restores the host x87 FPU and SSE control words. +macro_rules! restore_host_fp_controls { + () => { + " + fnclex + fldcw WORD PTR fs:host_x87_control_word@tpoff + ldmxcsr DWORD PTR fs:host_mxcsr@tpoff +" + }; +} + +/// Runs the guest thread until it terminates. +/// +/// This saves all non-volatile register state then switches to the guest +/// context. When the guest makes a syscall, it jumps back into the middle of +/// this routine, at `syscall_callback`. This code then updates the guest +/// context structure, switches back to the host stack, and calls the syscall +/// handler. +/// +/// When the guest thread terminates, this function returns after restoring +/// non-volatile register state. +#[unsafe(naked)] +pub(super) unsafe extern "C-unwind" fn run_thread_arch( + thread_ctx: &mut ThreadContext, + ctx: *mut litebox_common_linux::PtRegs, + reenter: u8, +) { + core::arch::naked_asm!( + " + .cfi_startproc + // Push all non-volatiles. + push rbp + mov rbp, rsp + .cfi_def_cfa rbp, 16 + push rbx + push r12 + push r13 + push r14 + push r15 + push rdi // save thread context + + // Save host rsp and rbp and guest context top in TLS. + mov fs:host_sp@tpoff, rsp + mov fs:host_bp@tpoff, rbp + lea r8, [rsi + {GUEST_CONTEXT_SIZE}] + mov fs:guest_context_top@tpoff, r8 + + // Linux guests use FS, so preserve host FS in GS. Windows guests retain + // their GS base and leave host FS active. + cmp BYTE PTR [rip + GUEST_TLS_MODE], {WINDOWS_TLS_MODE} + je 2f + rdfsbase r8 + wrgsbase r8 +2: + + // Preserve the host FP controls required by the SysV AMD64 ABI. FP data + // registers are caller-saved, so they do not require a host XSAVE area. + fnstcw WORD PTR fs:host_x87_control_word@tpoff + stmxcsr DWORD PTR fs:host_mxcsr@tpoff + + // Call init_handler or reenter_handler based on reenter flag (in dl). + test dl, dl + jnz 1f + call {init_handler} + jmp .Ldone +1: + call {reenter_handler} + jmp .Ldone + + // This entry point is called from the guest when it issues a syscall + // instruction. + // + // At entry, the register context is the guest context with the + // return address in rcx. r11 is an available scratch register (it would + // contain rflags if the syscall instruction had actually been issued). + .globl syscall_callback +syscall_callback: + // Clear in_guest flag. This must be the first instruction to match the + // expectations of `interrupt_signal_handler`. + mov BYTE PTR gs:in_guest@tpoff, 0 + + // Restore host fs base. + rdfsbase r11 + mov gs:guest_fsbase@tpoff, r11 + rdgsbase r11 + wrfsbase r11 + jmp .Lhost_tls_restored + + .globl syscall_callback_guest_gs +syscall_callback_guest_gs: + // Windows guests use GS, while host TLS remains active in FS. + mov BYTE PTR fs:in_guest@tpoff, 0 + +.Lhost_tls_restored: + + // Switch to the top of the guest context. + mov r11, rsp + mov rsp, fs:guest_context_top@tpoff + + // Save caller-saved registers + push 0x2b // pt_regs->ss = __USER_DS + push r11 // pt_regs->sp + pushfq // pt_regs->eflags +", + sanitize_host_rflags!(), +" + push 0x33 // pt_regs->cs = __USER_CS + push rcx // pt_regs->ip + push rax // pt_regs->orig_ax + + push rdi // pt_regs->di + push rsi // pt_regs->si + push rdx // pt_regs->dx + push rcx // pt_regs->cx + push -38 // pt_regs->ax = ENOSYS + push r8 // pt_regs->r8 + push r9 // pt_regs->r9 + push r10 // pt_regs->r10 + push [rsp + 88] // pt_regs->r11 = rflags + push rbx // pt_regs->bx + push rbp // pt_regs->bp + push r12 // pt_regs->r12 + push r13 // pt_regs->r13 + push r14 // pt_regs->r14 + push r15 // pt_regs->r15 +", + save_guest_xstate!(), + restore_host_fp_controls!(), +" + // Restore the stack and frame pointer. + mov rsp, fs:host_sp@tpoff + mov rbp, fs:host_bp@tpoff + + // Handle the syscall. This will jump back to the guest but + // will return if the thread is exiting. + mov rdi, [rsp] // pass thread_ctx + call {syscall_handler} + // This thread is done. Return. + jmp .Ldone + +.globl exception_callback +exception_callback: + // rt_sigreturn restored the guest FP state. Save it before entering host code. + mov r8, rdx +", + save_guest_xstate!(), + restore_host_fp_controls!(), +" + mov rdx, r8 + + // Restore the stack and frame pointer. + mov rsp, fs:host_sp@tpoff + mov rbp, fs:host_bp@tpoff + + mov rdi, [rsp] // pass thread_ctx + call {exception_handler} + jmp .Ldone + +.globl interrupt_callback +interrupt_callback: + // rt_sigreturn restored the guest FP state. Save it before entering host code. +", + save_guest_xstate!(), +" +.globl interrupt_callback_no_xsave +interrupt_callback_no_xsave: +", + restore_host_fp_controls!(), +" + // Restore the stack and frame pointer. + mov rsp, fs:host_sp@tpoff + mov rbp, fs:host_bp@tpoff + + mov rdi, [rsp] // pass thread_ctx + call {interrupt_handler} + +.Ldone: + // Shim handlers may leave modified FP controls when terminating the guest. +", + restore_host_fp_controls!(), +" + lea rsp, [rbp - 5*8] + pop r15 + pop r14 + pop r13 + pop r12 + pop rbx + pop rbp + .cfi_def_cfa rsp, 8 + ret + .cfi_endproc +", + GUEST_CONTEXT_SIZE = const core::mem::size_of::(), + init_handler = sym init_handler, + reenter_handler = sym reenter_handler, + syscall_handler = sym syscall_handler, + exception_handler = sym exception_handler, + interrupt_handler = sym interrupt_handler, + HOST_SAFE_RFLAGS_MASK = const HOST_SAFE_RFLAGS_MASK, + WINDOWS_TLS_MODE = const super::GuestTlsMode::Windows as u8, + ); +} + +/// Switches to the provided guest context. +/// +/// # Safety +/// The context must be valid guest context. This can only be called if +/// `run_thread_arch` is on the stack; after the guest exits, it will return to +/// the interior of `run_thread_arch`. +/// +/// Do not call this at a point where the stack needs to be unwound to run +/// destructors. +#[unsafe(naked)] +pub(super) unsafe extern "C" fn switch_to_guest(ctx: &litebox_common_linux::PtRegs) -> ! { + core::arch::naked_asm!( + ".macro restore_guest_context scratch", + "pop r15", + "pop r14", + "pop r13", + "pop r12", + "pop rbp", + "pop rbx", + "pop r11", + "pop r10", + "pop r9", + "pop r8", + "pop rax", + "pop rcx", + "pop rdx", + "pop rsi", + "pop rdi", + "add rsp, 8", + "pop \\scratch", + "add rsp, 8", + "popfq", + "pop rsp", + "jmp \\scratch", + ".endm", + ".globl switch_to_guest_start", + "switch_to_guest_start:", + // Set `in_guest` now, then check if there is a pending interrupt. If an + // interrupt arrives while `in_guest` is set, the signal handler will + // see that the IP is between `switch_to_guest_start` and + // `switch_to_guest_end` and will set `interrupt` and jump to + // `interrupt_callback_no_xsave`. + // + // If an interrupt is already pending, clear `in_guest` and jump to + // `interrupt_callback_no_xsave` without entering the guest. The callback + // runs host code, and a signal arriving there with `in_guest` still set + // would be taken for a guest interrupt and overwrite the saved guest + // context with host registers. + "mov BYTE PTR fs:in_guest@tpoff, 1", + "cmp BYTE PTR fs:interrupt@tpoff, 0", + "je 2f", + "mov BYTE PTR fs:in_guest@tpoff, 0", + "jmp interrupt_callback_no_xsave", + "2:", + "mov r10, fs:guest_xsave@tpoff", + "mov eax, DWORD PTR fs:xsave_mask@tpoff", + "mov edx, DWORD PTR fs:xsave_mask@tpoff+4", + "xrstor64 [r10]", + // Restore guest context from ctx. + "mov rsp, rdi", + "cmp BYTE PTR [rip + GUEST_TLS_MODE], {WINDOWS_TLS_MODE}", + "je 2f", + // Linux guest: switch FS and use saved host FS through GS. + "mov rdx, fs:guest_fsbase@tpoff", + "wrfsbase rdx", + "restore_guest_context gs:scratch@tpoff", + "2:", + // Windows guest: host FS and guest GS are already active. + "restore_guest_context fs:scratch@tpoff", + ".globl switch_to_guest_end", + "switch_to_guest_end:", + ".purgem restore_guest_context", + WINDOWS_TLS_MODE = const super::GuestTlsMode::Windows as u8, + ); +} + +// Defined in the assembly blocks above. +unsafe extern "C" { + pub(super) fn syscall_callback() -> isize; + pub(super) fn syscall_callback_guest_gs() -> isize; + pub(super) fn exception_callback(); + pub(super) fn interrupt_callback(); + pub(super) fn interrupt_callback_no_xsave(); + pub(super) fn switch_to_guest_start(); + pub(super) fn switch_to_guest_end(); +} + +pub(super) fn syscall_entry_point() -> usize { + match LinuxUserland::guest_tls_mode() { + super::GuestTlsMode::Windows => syscall_callback_guest_gs as *const () as usize, + super::GuestTlsMode::Linux => syscall_callback as *const () as usize, + } +} + +pub(super) fn in_switch_to_guest(pc: usize) -> bool { + (switch_to_guest_start as *const () as usize..switch_to_guest_end as *const () as usize) + .contains(&pc) +} + +pub(super) fn in_syscall_callback_prologue(ip: usize) -> bool { + let start = match super::GUEST_TLS_MODE.load(core::sync::atomic::Ordering::Relaxed) { + mode if mode == super::GuestTlsMode::Windows as u8 => { + syscall_callback_guest_gs as *const () as usize + } + _ => syscall_callback as *const () as usize, + }; + ip == start +} + +/// Called from signal handlers to fix up thread state after potentially running +/// in the guest. +/// +/// Restores the proper host `fsbase` so that TLS can be used. Clears `in_guest` +/// and optionally sets `interrupt`. If `in_guest` was previously set, returns +/// the guest context pointer (which does not necessarily have up-to-date guest +/// register state yet). +pub(super) fn signal_handler_exit_guest( + _context: &libc::ucontext_t, + set_interrupt: bool, +) -> Option<*mut litebox_common_linux::PtRegs> { + unsafe { + macro_rules! take_guest { + ($access:ident) => {{ + let in_guest: u8; + core::arch::asm!( + concat!("mov {in_guest}, BYTE PTR ", $access!("in_guest")), + concat!("mov BYTE PTR ", $access!("in_guest"), ", 0"), + in_guest = out(reg_byte) in_guest, + options(nostack, preserves_flags) + ); + if set_interrupt { + core::arch::asm!( + concat!("mov BYTE PTR ", $access!("interrupt"), ", 1"), + options(nostack, preserves_flags) + ); + } + in_guest != 0 + }}; + } + match super::GUEST_TLS_MODE.load(core::sync::atomic::Ordering::Relaxed) { + mode if mode == super::GuestTlsMode::Windows as u8 => { + if !take_guest!(tls) { + return None; + } + } + _ => { + let gsbase = litebox_common_linux::rdgsbase(); + if gsbase == 0 || !take_guest!(saved_tls) { + return None; + } + litebox_common_linux::wrfsbase(gsbase); + } + } + + let guest_context_top: *mut litebox_common_linux::PtRegs; + core::arch::asm!( + concat!("mov {guest_context_top}, ", tls!("guest_context_top")), + guest_context_top = out(reg) guest_context_top, + options(nostack, preserves_flags) + ); + Some(guest_context_top.sub(1)) + } +} + +/// Copies register state from a Linux signal context to a LiteBox PtRegs +/// structure. +pub(super) fn copy_signal_context( + regs: &mut litebox_common_linux::PtRegs, + context: &libc::ucontext_t, +) { + let litebox_common_linux::PtRegs { + r15, + r14, + r13, + r12, + rbp, + rbx, + r11, + r10, + r9, + r8, + rax, + rcx, + rdx, + rsi, + rdi, + orig_rax, + rip, + cs: _, + eflags, + rsp, + ss: _, + } = regs; + for (reg, sig_reg) in [ + (r15, libc::REG_R15), + (r14, libc::REG_R14), + (r13, libc::REG_R13), + (r12, libc::REG_R12), + (rbp, libc::REG_RBP), + (rbx, libc::REG_RBX), + (r11, libc::REG_R11), + (r10, libc::REG_R10), + (r9, libc::REG_R9), + (r8, libc::REG_R8), + (rax, libc::REG_RAX), + (rcx, libc::REG_RCX), + (rdx, libc::REG_RDX), + (rsi, libc::REG_RSI), + (rdi, libc::REG_RDI), + (rip, libc::REG_RIP), + (rsp, libc::REG_RSP), + (eflags, libc::REG_EFL), + ] { + *reg = context.uc_mcontext.gregs[sig_reg.reinterpret_as_unsigned() as usize] + .reinterpret_as_unsigned() + .trunc(); + } + *orig_rax = *rax; +} + +/// Updates a Linux signal context to return to `f` with the given arguments. +pub(super) fn set_signal_return( + context: &mut libc::ucontext_t, + f: unsafe extern "C" fn(), + p0: isize, + p1: isize, + p2: isize, + p3: isize, +) { + let sigctx = &mut context.uc_mcontext; + sigctx.gregs[libc::REG_RIP as usize] = (f as usize).reinterpret_as_signed() as i64; + sigctx.gregs[libc::REG_RDI as usize] = p0 as i64; + sigctx.gregs[libc::REG_RSI as usize] = p1 as i64; + sigctx.gregs[libc::REG_RDX as usize] = p2 as i64; + sigctx.gregs[libc::REG_RCX as usize] = p3 as i64; + // `rt_sigreturn` restores RFLAGS before entering the callback, which must + // satisfy the host ABI rather than inherit guest-controlled execution flags. + sigctx.gregs[libc::REG_EFL as usize] &= !i64::from(HOST_UNSAFE_RFLAGS); +} + +impl litebox::platform::ArchSpecificProvider for LinuxUserland { + fn set_arch_specific_register( + &self, + reg: &litebox::platform::ArchSpecificRegister, + val: usize, + ) -> Result<(), litebox::platform::ArchSpecificError> { + match reg { + litebox::platform::ArchSpecificRegister::FsBase + if super::LinuxUserland::guest_tls_mode() == super::GuestTlsMode::Linux => + { + if litebox_common_linux::arch::is_valid_user_fs_base(val) { + set_guest_fsbase(val); + Ok(()) + } else { + Err(litebox::platform::ArchSpecificError::RegisterUnpermittedValue) + } + } + litebox::platform::ArchSpecificRegister::FsBase => { + Err(litebox::platform::ArchSpecificError::RegisterReserved) + } + litebox::platform::ArchSpecificRegister::GsBase => { + match super::LinuxUserland::guest_tls_mode() { + super::GuestTlsMode::Windows => { + if litebox_common_linux::arch::is_valid_user_fs_base(val) { + // SAFETY: this platform requires FSGSBASE support. + unsafe { litebox_common_linux::wrgsbase(val) }; + Ok(()) + } else { + Err(litebox::platform::ArchSpecificError::RegisterUnpermittedValue) + } + } + super::GuestTlsMode::Linux => { + Err(litebox::platform::ArchSpecificError::RegisterReserved) + } + } + } + _ => Err(litebox::platform::ArchSpecificError::RegisterUnsupported), + } + } + + fn get_arch_specific_register( + &self, + reg: &litebox::platform::ArchSpecificRegister, + ) -> Result { + match reg { + litebox::platform::ArchSpecificRegister::FsBase + if super::LinuxUserland::guest_tls_mode() == super::GuestTlsMode::Linux => + { + Ok(get_guest_fsbase()) + } + litebox::platform::ArchSpecificRegister::FsBase => { + Err(litebox::platform::ArchSpecificError::RegisterReserved) + } + litebox::platform::ArchSpecificRegister::GsBase => { + match super::LinuxUserland::guest_tls_mode() { + super::GuestTlsMode::Windows => { + // SAFETY: this platform requires FSGSBASE support. + Ok(unsafe { litebox_common_linux::rdgsbase() }) + } + super::GuestTlsMode::Linux => { + Err(litebox::platform::ArchSpecificError::RegisterReserved) + } + } + } + _ => Err(litebox::platform::ArchSpecificError::RegisterUnsupported), + } + } +} + +#[cfg(test)] +mod tests { + use core::sync::atomic::Ordering; + + use crate::LinuxUserland; + + #[test] + fn spawned_guest_inherits_xstate() { + use litebox::platform::ThreadProvider as _; + use litebox::shim::{ContinueOperation, EnterShim, ExceptionInfo, InitThread}; + use litebox_common::arch::x86_64::xstate::{XsaveArea, XsaveLayout, XsaveLegacyArea}; + use litebox_common_linux::PtRegs; + use std::sync::mpsc::{Sender, channel}; + + const TEST_CW: u16 = 0x0b7f; + const TEST_MXCSR: u32 = 0x5f80; + // 1.0 in 80-bit floating-point format. + const X87_ONE: [u8; 10] = [0, 0, 0, 0, 0, 0, 0, 0x80, 0xff, 0x3f]; + + #[unsafe(naked)] + unsafe extern "C" fn guest_entry() { + core::arch::naked_asm!( + "test rdi, rdi", + "jnz 3f", + "sub rsp, 40", + "mov WORD PTR [rsp + 32], {control_word}", + "mov DWORD PTR [rsp + 36], {mxcsr}", + "mov rax, 0x5a5a5a5a5a5a5a5a", + "mov [rsp], rax", + "mov [rsp + 8], rax", + "mov [rsp + 16], rax", + "mov [rsp + 24], rax", + "fldcw [rsp + 32]", + "ldmxcsr [rsp + 36]", + "fld1", + "movdqu xmm0, [rsp]", + "add rsp, 40", + "lea rcx, [rip + 4f]", + "jmp {syscall_callback}", + "3:", + "fxsave64 [rsi]", + "4:", + "lea rcx, [rip + 5f]", + "jmp {syscall_callback}", + "5:", + "ud2", + control_word = const TEST_CW, + mxcsr = const TEST_MXCSR, + syscall_callback = sym super::syscall_callback, + ); + } + + struct ChildStorage { + _stack: Box<[u128; 256]>, + snapshot: XsaveArea, + } + + struct Shim { + platform: &'static LinuxUserland, + sender: Sender, + child: Option, + } + + impl InitThread for Shim { + type ExecutionContext = PtRegs; + + fn init(self: Box) -> Box> { + self + } + } + + impl EnterShim for Shim { + type ExecutionContext = PtRegs; + + fn init(&self, _ctx: &mut PtRegs) -> ContinueOperation { + ContinueOperation::Resume + } + + fn syscall(&self, ctx: &mut PtRegs) -> ContinueOperation { + if let Some(child) = &self.child { + self.sender.send(*child.snapshot.legacy_area()).unwrap(); + } else { + // SAFETY: These caller-saved registers belong to the shim; + // the parent's guest state has already been captured. + unsafe { + core::arch::asm!( + "fninit", + "pxor xmm0, xmm0", + out("xmm0") _, + options(nostack), + ); + } + let mut stack = Box::new([0_u128; 256]); + let mut snapshot = XsaveArea::initial(XsaveLayout::get()); + let mut child_ctx = ctx.clone(); + child_ctx.rip = guest_entry as *const () as usize; + child_ctx.rsp = stack.as_mut_ptr().wrapping_add(stack.len()).addr(); + child_ctx.rdi = 1; + child_ctx.rsi = snapshot.as_mut_ptr().addr(); + let child = Box::new(Shim { + platform: self.platform, + sender: self.sender.clone(), + child: Some(ChildStorage { + _stack: stack, + snapshot, + }), + }); + // SAFETY: The child owns its stack and aligned snapshot + // buffer, and its shim terminates after capturing state. + unsafe { self.platform.spawn_thread(&child_ctx, child).unwrap() }; + } + ContinueOperation::Terminate + } + + fn exception(&self, _ctx: &mut PtRegs, info: &ExceptionInfo) -> ContinueOperation { + panic!("unexpected guest exception: {info:?}"); + } + + fn interrupt(&self, _ctx: &mut PtRegs) -> ContinueOperation { + ContinueOperation::Resume + } + } + + let platform = LinuxUserland::new(); + let (sender, receiver) = channel(); + let shim = Shim { + platform, + sender, + child: None, + }; + let mut stack = [0_u128; 256]; + let mut ctx = PtRegs { + rip: guest_entry as *const () as usize, + rsp: stack.as_mut_ptr().wrapping_add(stack.len()).addr(), + eflags: 0x202, + ..Default::default() + }; + // SAFETY: The parent uses a valid stack and terminates after spawning + // the child, which retains ownership of all its guest buffers. + unsafe { crate::run_thread_ref(&shim, &mut ctx) }; + let snapshot = receiver + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert_eq!(snapshot.control_word, TEST_CW); + assert_eq!(snapshot.mxcsr, TEST_MXCSR); + assert_eq!(&snapshot.float_registers[0][..X87_ONE.len()], &X87_ONE); + assert_eq!(snapshot.xmm_registers[0], [0x5a; 16]); + } + + #[test] + fn xsave_preserves_state_across_guest_transitions() { + use litebox::shim::{ContinueOperation, EnterShim, ExceptionInfo}; + use litebox_common::arch::x86_64::xstate::{XsaveArea, XsaveLegacyArea}; + use litebox_common_linux::PtRegs; + use std::cell::Cell; + + const TEST_CW: u16 = 0x077e; + const TEST_MXCSR: u32 = 0x3f80; + const HOST_CW: u16 = 0x0b7f; + const HOST_MXCSR: u32 = 0x5f80; + const SNAPSHOT_COUNT: usize = 5; + const X87_STATUS_INVALID_OPERATION: u16 = 1 << 0; + const X87_STATUS_EXCEPTION_SUMMARY: u16 = 1 << 7; + const X87_INVALID_OPERATION_STATUS: u16 = + X87_STATUS_INVALID_OPERATION | X87_STATUS_EXCEPTION_SUMMARY; + + #[repr(C, align(64))] + #[derive(Clone, Copy)] + struct Snapshot { + legacy: XsaveLegacyArea, + ymm_hi: [u8; 64], + } + + impl Default for Snapshot { + fn default() -> Self { + Self { + legacy: XsaveLegacyArea::default(), + ymm_hi: [0; 64], + } + } + } + + #[repr(C, align(64))] + struct CaptureSet { + snapshots: [Snapshot; SNAPSHOT_COUNT], + host_mxcsr: u32, + host_control: u16, + } + + const SNAPSHOT_SIZE: usize = core::mem::size_of::(); + + #[unsafe(naked)] + unsafe extern "C" fn guest_entry() { + core::arch::naked_asm!( + // Preserve the snapshot buffer, AVX flag, interrupt signal, + // process ID, and thread ID in callee-saved registers. + "mov r12, rsi", + "mov r13, rdi", + "mov ebx, edx", + "mov r14, r8", + "mov r15, r9", + // Record the host controls observed before installing guest + // state. The assertions below verify their exact restoration. + "stmxcsr [r12 + {host_mxcsr_offset}]", + "fnstcw [r12 + {host_control_offset}]", + // Install distinctive x87, SSE, and optional AVX guest state. + "sub rsp, 40", + "mov WORD PTR [rsp + 32], {control_word}", + "mov DWORD PTR [rsp + 36], {mxcsr}", + "mov rax, 0x5a5a5a5a5a5a5a5a", + "mov [rsp], rax", + "mov [rsp + 8], rax", + "mov [rsp + 16], rax", + "mov [rsp + 24], rax", + "fldcw [rsp + 32]", + "ldmxcsr [rsp + 36]", + "movdqu xmm0, [rsp]", + "test r13, r13", // Whether support for AVX is enabled + "jz 2f", + "vmovdqu ymm0, [rsp]", + "2:", + "fldz", + "fldz", + "fdivp st(1), st(0)", + "add rsp, 40", + // Syscall transition: the shim clobbers host FP state before + // resuming at label 3. Capture snapshot 0 after restoration. + "lea rcx, [rip + 3f]", + "jmp {syscall_callback}", + "3:", + "fxsave64 [r12]", + "test r13, r13", + "jz 4f", + "vmovdqu [r12 + 512], ymm0", + "4:", + // Install a state distinct from the preceding syscall so this + // snapshot proves the exception callback saved it. + "sub rsp, 32", + "mov rax, 0x6b6b6b6b6b6b6b6b", + "mov [rsp], rax", + "mov [rsp + 8], rax", + "mov [rsp + 16], rax", + "mov [rsp + 24], rax", + "movdqu xmm0, [rsp]", + "test r13, r13", + "jz 13f", + "vmovdqu ymm0, [rsp]", + "13:", + "add rsp, 32", + // Exception transition: UD2 enters the exception callback, + // which advances RIP. Capture snapshot 1 after resumption. + "ud2", + "fxsave64 [r12 + {snapshot_size}]", + "test r13, r13", + "jz 5f", + "vmovdqu [r12 + {snapshot_size} + 512], ymm0", + "5:", + // Install another distinct state so this snapshot proves the + // interrupt callback saved it rather than restoring snapshot 1. + "sub rsp, 32", + "mov rax, 0x7c7c7c7c7c7c7c7c", + "mov [rsp], rax", + "mov [rsp + 8], rax", + "mov [rsp + 16], rax", + "mov [rsp + 24], rax", + "movdqu xmm0, [rsp]", + "test r13, r13", + "jz 14f", + "vmovdqu ymm0, [rsp]", + "14:", + "add rsp, 32", + // Interrupt transition: signal this guest thread directly and + // capture snapshot 2 after the interrupt callback resumes it. + "mov rdi, r14", + "mov rsi, r15", + "mov edx, ebx", + "mov eax, {tgkill}", + "syscall", + "fxsave64 [r12 + 2*{snapshot_size}]", + "test r13, r13", // Whether support for AVX is enabled + "jz 6f", + "vmovdqu [r12 + 2*{snapshot_size} + 512], ymm0", + "6:", + // The shim terminates on this second syscall. reenter_thread + // resumes at label 7; snapshot 3 verifies persisted XSTATE. + "lea rcx, [rip + 7f]", + "jmp {syscall_callback}", + "7:", + "fxsave64 [r12 + 3*{snapshot_size}]", + "test r13, r13", + "jz 8f", + "vmovdqu [r12 + 3*{snapshot_size} + 512], ymm0", + "8:", + // Replace the guest state, cross another syscall, and capture + // snapshot 4 to prove the newly saved state supersedes it. + "fninit", + "pxor xmm0, xmm0", + "test r13, r13", + "jz 9f", + "vzeroall", + "9:", + "lea rcx, [rip + 10f]", + "jmp {syscall_callback}", + "10:", + "fxsave64 [r12 + 4*{snapshot_size}]", + "test r13, r13", + "jz 11f", + "vmovdqu [r12 + 4*{snapshot_size} + 512], ymm0", + "11:", + // The fourth syscall terminates the test. UD2 is unreachable + // and prevents accidental fallthrough if it unexpectedly resumes. + "lea rcx, [rip + 12f]", + "jmp {syscall_callback}", + "12:", + "ud2", + snapshot_size = const SNAPSHOT_SIZE, + host_mxcsr_offset = const core::mem::offset_of!(CaptureSet, host_mxcsr), + host_control_offset = const core::mem::offset_of!(CaptureSet, host_control), + control_word = const TEST_CW, + mxcsr = const TEST_MXCSR, + tgkill = const libc::SYS_tgkill, + syscall_callback = sym super::syscall_callback, + ); + } + + struct StateShim { + calls: Cell, + exceptions: Cell, + interrupts: Cell, + avx: bool, + } + + impl StateShim { + fn clobber_host_state(&self) { + let mut control = 0_u16; + let status: u16; + let mut mxcsr = 0_u32; + // SAFETY: These instructions access only the current thread's FP + // registers and valid local outputs; AVX is checked before use. + unsafe { + core::arch::asm!( + "fnstcw [{control}]", + "fnstsw ax", + "stmxcsr [{mxcsr}]", + control = in(reg) &raw mut control, + mxcsr = in(reg) &raw mut mxcsr, + out("ax") status, + options(nostack, preserves_flags), + ); + } + assert_eq!(control, HOST_CW); + assert_eq!(status & X87_INVALID_OPERATION_STATUS, 0); + assert_eq!(mxcsr & !0x3f, HOST_MXCSR); + // SAFETY: The clobbers are declared and AVX support was detected. + unsafe { + core::arch::asm!( + "fninit", + "pxor xmm0, xmm0", + out("xmm0") _, + options(nostack), + ); + if self.avx { + core::arch::asm!( + "vpxor ymm0, ymm0, ymm0", + out("ymm0") _, + options(nostack), + ); + } + } + } + } + + impl EnterShim for StateShim { + type ExecutionContext = PtRegs; + + fn init(&self, _ctx: &mut PtRegs) -> ContinueOperation { + self.clobber_host_state(); + ContinueOperation::Resume + } + + fn reenter(&self, _ctx: &mut PtRegs) -> ContinueOperation { + self.clobber_host_state(); + ContinueOperation::Resume + } + + fn syscall(&self, _ctx: &mut PtRegs) -> ContinueOperation { + self.clobber_host_state(); + let call = self.calls.get() + 1; + self.calls.set(call); + if call == 2 || call == 4 { + ContinueOperation::Terminate + } else { + ContinueOperation::Resume + } + } + + fn exception(&self, ctx: &mut PtRegs, info: &ExceptionInfo) -> ContinueOperation { + self.clobber_host_state(); + assert_eq!(info.exception, litebox::shim::Exception(6)); + self.exceptions.set(self.exceptions.get() + 1); + ctx.rip += 2; + ContinueOperation::Resume + } + + fn interrupt(&self, _ctx: &mut PtRegs) -> ContinueOperation { + self.clobber_host_state(); + self.interrupts.set(self.interrupts.get() + 1); + ContinueOperation::Resume + } + } + + let mut original_control = 0_u16; + let mut original_mxcsr = 0_u32; + let host_control = HOST_CW; + let host_mxcsr = HOST_MXCSR; + // SAFETY: Save the test thread's controls and install masked, + // non-default rounding modes to verify exact host restoration. + unsafe { + core::arch::asm!( + "fnstcw [{original_control}]", + "stmxcsr [{original_mxcsr}]", + "fldcw [{host_control}]", + "ldmxcsr [{host_mxcsr}]", + original_control = in(reg) &raw mut original_control, + original_mxcsr = in(reg) &raw mut original_mxcsr, + host_control = in(reg) &raw const host_control, + host_mxcsr = in(reg) &raw const host_mxcsr, + options(nostack, preserves_flags), + ); + } + let _restore_controls = litebox::utils::defer(|| unsafe { + core::arch::asm!( + "fnclex", + "fldcw [{original_control}]", + "ldmxcsr [{original_mxcsr}]", + original_control = in(reg) &raw const original_control, + original_mxcsr = in(reg) &raw const original_mxcsr, + options(nostack, preserves_flags), + ); + }); + + let _platform = LinuxUserland::new(); + let shim = StateShim { + calls: Cell::new(0), + exceptions: Cell::new(0), + interrupts: Cell::new(0), + avx: std::is_x86_feature_detected!("avx"), + }; + let mut stack = [0_u128; 256]; + let mut captures = CaptureSet { + snapshots: [Snapshot::default(); SNAPSHOT_COUNT], + host_mxcsr: 0, + host_control: 0, + }; + let mut ctx = PtRegs { + rip: guest_entry as *const () as usize, + rsp: stack.as_mut_ptr().wrapping_add(stack.len()).addr(), + rsi: captures.snapshots.as_mut_ptr().addr(), + rdi: usize::from(shim.avx), + rdx: usize::try_from(crate::INTERRUPT_SIGNAL_NUMBER.load(Ordering::Relaxed)).unwrap(), + // SAFETY: These calls only query the current process and thread IDs. + r8: usize::try_from(unsafe { libc::getpid() }).unwrap(), + r9: usize::try_from(unsafe { libc::syscall(libc::SYS_gettid) }).unwrap(), + eflags: 0x202, + ..Default::default() + }; + // SAFETY: The assembly guest uses a valid stack and output buffer and the + // shim terminates at known points, preserving a valid reentry context. + unsafe { + crate::run_thread_ref(&shim, &mut ctx); + assert_eq!(shim.calls.get(), 2); + crate::reenter_thread(&shim, &mut ctx); + } + assert_eq!(shim.calls.get(), 4); + assert_eq!(shim.exceptions.get(), 1); + assert_eq!(shim.interrupts.get(), 1); + for (index, snapshot) in captures.snapshots.iter().enumerate() { + let (expected_status, expected_vector) = match index { + 0 => (X87_INVALID_OPERATION_STATUS, 0x5a), + 1 => (X87_INVALID_OPERATION_STATUS, 0x6b), + 2 | 3 => (X87_INVALID_OPERATION_STATUS, 0x7c), + 4 => (0, 0), + _ => unreachable!(), + }; + let expected_control = if index == 4 { 0x037f } else { TEST_CW }; + assert_eq!(snapshot.legacy.control_word, expected_control); + assert_eq!( + snapshot.legacy.status_word & X87_INVALID_OPERATION_STATUS, + expected_status + ); + assert_eq!(snapshot.legacy.mxcsr, TEST_MXCSR); + assert_eq!(snapshot.legacy.xmm_registers[0], [expected_vector; 16]); + if shim.avx { + assert_eq!(&snapshot.ymm_hi[..32], &[expected_vector; 32]); + } + } + assert_eq!(captures.host_mxcsr, XsaveArea::GUEST_INITIAL_MXCSR); + assert_eq!( + captures.host_control, + XsaveArea::GUEST_INITIAL_X87_CONTROL_WORD + ); + } + + /// An interrupt pending at guest entry diverts to the interrupt handler + /// without entering the guest, so a further interrupt taken while that + /// handler runs host code must not be treated as a guest interrupt, which + /// would re-enter the handler. + #[test] + fn interrupt_in_diverted_interrupt_handler_is_not_a_guest_interrupt() { + use litebox::shim::{ContinueOperation, EnterShim, ExceptionInfo}; + use litebox_common_linux::PtRegs; + + // The signal is delivered to the calling thread before `pthread_kill` returns. + fn interrupt_current_thread() { + let signal = crate::INTERRUPT_SIGNAL_NUMBER.load(Ordering::Relaxed); + // SAFETY: `pthread_self` is a live thread with the interrupt handler installed. + assert_eq!( + unsafe { libc::pthread_kill(libc::pthread_self(), signal) }, + 0 + ); + } + + #[derive(Default)] + struct Shim { + interrupts: core::cell::Cell, + } + + impl EnterShim for Shim { + type ExecutionContext = PtRegs; + + fn init(&self, _ctx: &mut PtRegs) -> ContinueOperation { + interrupt_current_thread(); + ContinueOperation::Resume + } + + fn syscall(&self, _ctx: &mut PtRegs) -> ContinueOperation { + unreachable!() + } + + fn exception(&self, _ctx: &mut PtRegs, _info: &ExceptionInfo) -> ContinueOperation { + unreachable!() + } + + fn interrupt(&self, _ctx: &mut PtRegs) -> ContinueOperation { + self.interrupts.set(self.interrupts.get() + 1); + if self.interrupts.get() == 1 { + interrupt_current_thread(); + } + ContinueOperation::Terminate + } + } + + let _platform = LinuxUserland::new(); + let shim = Shim::default(); + // SAFETY: the shim terminates the thread before it enters the guest. + unsafe { crate::run_thread_ref(&shim, &mut PtRegs::default()) }; + assert_eq!(shim.interrupts.get(), 1); + } +} diff --git a/litebox_platform_windows_userland/Cargo.toml b/litebox_platform_windows_userland/Cargo.toml index 62283216be..4f6dc8e1e8 100644 --- a/litebox_platform_windows_userland/Cargo.toml +++ b/litebox_platform_windows_userland/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [target.'cfg(windows)'.dependencies] litebox = { path = "../litebox/", version = "0.1.0" } +litebox_common = { path = "../litebox_common", version = "0.1.0", default-features = false } litebox_common_linux = { path = "../litebox_common_linux", version = "0.1.0" } litebox_common_windows = { path = "../litebox_common_windows", version = "0.1.0", default-features = false } litebox_platform = { path = "../litebox_platform", version = "0.1.0" } diff --git a/litebox_platform_windows_userland/src/lib.rs b/litebox_platform_windows_userland/src/lib.rs index 1da7ecc5b3..1b5177e950 100644 --- a/litebox_platform_windows_userland/src/lib.rs +++ b/litebox_platform_windows_userland/src/lib.rs @@ -24,6 +24,7 @@ use litebox::platform::page_mgmt::{ }; use litebox::shim::{ContinueOperation, Exception}; use litebox::utils::TruncateExt as _; +use litebox_common::arch::x86_64::xstate::{XsaveArea, XsaveChunk, XsaveLayout, XsaveLegacyArea}; use litebox_platform::sync::{ ImmediatelyWokenUp, RawMutex as RawMutexTrait, RawMutexProvider, UnblockedOrTimedOut, WaitWakerProvider, @@ -155,6 +156,19 @@ fn guest_tls_mode() -> GuestTlsMode { } } +/// Clears the x87 exception flags and restores the Windows ABI's host x87 control word and MXCSR. +/// +/// Requires `HOST_X87_CONTROL_WORD` and `HOST_MXCSR` symbol operands. +macro_rules! restore_host_fp_controls { + () => { + " + fnclex + fldcw WORD PTR [rip + {HOST_X87_CONTROL_WORD}] + ldmxcsr DWORD PTR [rip + {HOST_MXCSR}] +" + }; +} + /// Runs the Rust handler with host FP controls #[unsafe(naked)] unsafe extern "system" fn vectored_exception_handler( @@ -167,10 +181,7 @@ unsafe extern "system" fn vectored_exception_handler( ".seh_endprologue", "fnstcw WORD PTR [rsp + 32]", "stmxcsr DWORD PTR [rsp + 36]", - // Clear the x87 exception flags and restore the host FP control state - "fnclex", - "fldcw WORD PTR [rip + {HOST_X87_CONTROL_WORD}]", - "ldmxcsr DWORD PTR [rip + {HOST_MXCSR}]", + restore_host_fp_controls!(), "call {handler}", "fldcw WORD PTR [rsp + 32]", "ldmxcsr DWORD PTR [rsp + 36]", @@ -419,7 +430,15 @@ fn run_thread_inner( shim: &dyn litebox::shim::EnterShim, ctx: &mut litebox_common_linux::PtRegs, ) { - let tls_state = TlsState::new(); + run_thread_inner_with_xstate(shim, ctx, XsaveArea::initial(XsaveLayout::get())); +} + +fn run_thread_inner_with_xstate( + shim: &dyn litebox::shim::EnterShim, + ctx: &mut litebox_common_linux::PtRegs, + guest_xsave_area: XsaveArea, +) { + let tls_state = TlsState::with_guest_xsave_area(guest_xsave_area); tls_state .guest_context_top .set(std::ptr::from_mut(ctx).wrapping_add(1)); @@ -462,177 +481,53 @@ fn debug_assert_host_fx_control_state() { } } -const XSAVE_LEGACY_SIZE: usize = - size_of::(); -const XSAVE_HEADER_OFFSET: usize = XSAVE_LEGACY_SIZE; - -#[repr(C, align(64))] -#[derive(Clone)] -struct XsaveChunk([u8; 64]); - -struct XsaveLayout { - size: usize, - mask: u64, - components: Vec, -} +const _: () = assert!( + size_of::() + == size_of::() +); -struct XsaveComponent { - id: u32, - offset: usize, - size: usize, -} - -impl XsaveLayout { - fn get() -> &'static Self { - static LAYOUT: OnceLock = OnceLock::new(); - LAYOUT.get_or_init(|| { - const CPUID_XSAVE: u32 = 1 << 26; - const CPUID_OSXSAVE: u32 = 1 << 27; +fn restore_xsave_to_context( + xsave: &XsaveArea, + context: &mut windows_sys::Win32::System::Diagnostics::Debug::CONTEXT, +) { + use windows_sys::Win32::System::Diagnostics::Debug::{ + CONTEXT_XSTATE_AMD64, LocateXStateFeature, SetXStateFeaturesMask, XSAVE_FORMAT, + }; - assert!(core::arch::x86_64::__cpuid(0).eax >= 0x0d); - let feature_info = core::arch::x86_64::__cpuid(1); - assert_eq!( - feature_info.ecx & (CPUID_XSAVE | CPUID_OSXSAVE), - CPUID_XSAVE | CPUID_OSXSAVE, - "XSAVE must be supported and enabled by Windows", - ); - let features = core::arch::x86_64::__cpuid_count(0x0d, 0); - let mask = unsafe { core::arch::x86_64::_xgetbv(0) }; - assert_eq!(mask & 3, 3, "x87 and SSE state must be enabled"); - assert!(features.ebx as usize >= XSAVE_HEADER_OFFSET + 64); - let components = (2..64) - .filter(|id| mask & (1 << id) != 0) - .map(|id| { - let component = core::arch::x86_64::__cpuid_count(0x0d, id); - let component = XsaveComponent { - id, - offset: component.ebx as usize, - size: component.eax as usize, - }; - assert!(component.offset + component.size <= features.ebx as usize); - component - }) - .collect(); - XsaveLayout { - size: features.ebx as usize, - mask, - components, - } - }) + let legacy_state = xsave.materialized_legacy_area(); + context.MxCsr = legacy_state.mxcsr; + // SAFETY: XSAVE_FORMAT is Windows' representation of the architectural + // legacy area, and its size is checked above. + context.Anonymous.FltSave = unsafe { + core::ptr::from_ref(&legacy_state) + .cast::() + .read_unaligned() + }; + if context.ContextFlags & CONTEXT_XSTATE_AMD64 != CONTEXT_XSTATE_AMD64 { + return; } -} - -/// Represents the standard-layout XSAVE area for a guest context. -struct XsaveArea { - storage: Box<[XsaveChunk]>, -} -impl XsaveArea { - const GUEST_INITIAL_X87_CONTROL_WORD: u16 = 0x037f; - const GUEST_INITIAL_MXCSR: u32 = core::arch::x86_64::_MM_MASK_MASK; - - fn initial_guest() -> Self { - let chunk_count = XsaveLayout::get().size.div_ceil(size_of::()); - let mut area = Self { - storage: vec![XsaveChunk([0; 64]); chunk_count].into_boxed_slice(), - }; - // SAFETY: The buffer owns aligned, initialized storage for the legacy area. - // Standard XRSTOR loads MXCSR even when XSTATE_BV marks SSE as initial. - unsafe { - (*area - .storage - .as_mut_ptr() - .cast::()) - .MxCsr = Self::GUEST_INITIAL_MXCSR; + let layout = XsaveLayout::get(); + let xstate_bv = xsave.xstate_bv(); + for component in &layout.components { + if xstate_bv & (1 << component.id) == 0 { + continue; } - area - } - - fn as_ptr(&self) -> *const u8 { - self.storage.as_ptr().cast() - } - - fn as_mut_ptr(&mut self) -> *mut u8 { - self.storage.as_mut_ptr().cast() - } - - fn xstate_bv(&self) -> u64 { + let mut length = 0; + // SAFETY: The context has initialized XSTATE storage, checked above. + let destination = + unsafe { LocateXStateFeature(context, component.id, &raw mut length).cast::() }; + assert!(!destination.is_null()); + assert_eq!(length as usize, component.size); + // SAFETY: The component fits both disjoint buffers and is marked valid. unsafe { - self.as_ptr() - .add(XSAVE_HEADER_OFFSET) - .cast::() - .read_unaligned() + destination + .copy_from_nonoverlapping(xsave.as_ptr().add(component.offset), component.size); } } - - fn restore_to_context( - &self, - context: &mut windows_sys::Win32::System::Diagnostics::Debug::CONTEXT, - ) { - use windows_sys::Win32::System::Diagnostics::Debug::{ - CONTEXT_XSTATE_AMD64, LocateXStateFeature, SetXStateFeaturesMask, - }; - - let legacy_state = self.legacy_state_for_context(); - context.Anonymous.FltSave = legacy_state; - context.MxCsr = legacy_state.MxCsr; - if context.ContextFlags & CONTEXT_XSTATE_AMD64 != CONTEXT_XSTATE_AMD64 { - return; - } - - let layout = XsaveLayout::get(); - let xstate_bv = self.xstate_bv(); - for component in &layout.components { - if xstate_bv & (1 << component.id) == 0 { - continue; - } - let mut length = 0; - // SAFETY: The context has initialized XSTATE storage, checked above. - let destination = - unsafe { LocateXStateFeature(context, component.id, &raw mut length).cast::() }; - assert!(!destination.is_null()); - assert_eq!(length as usize, component.size); - // SAFETY: The component fits both disjoint buffers and is marked valid. - unsafe { - destination - .copy_from_nonoverlapping(self.as_ptr().add(component.offset), component.size); - } - } - // SAFETY: The context owns initialized XSTATE storage for the enabled features. - let ok = unsafe { SetXStateFeaturesMask(context, xstate_bv & layout.mask) }; - assert_ne!(ok, 0, "SetXStateFeaturesMask failed"); - } - - fn legacy_state_for_context( - &self, - ) -> windows_sys::Win32::System::Diagnostics::Debug::XSAVE_FORMAT { - use windows_sys::Win32::System::Diagnostics::Debug::XSAVE_FORMAT; - - // SAFETY: `storage` is 64-byte aligned and initialized for at least the legacy XSAVE area. - let saved = unsafe { self.storage.as_ptr().cast::().read() }; - let mut state = XSAVE_FORMAT { - ControlWord: Self::GUEST_INITIAL_X87_CONTROL_WORD, - MxCsr: saved.MxCsr, - MxCsr_Mask: saved.MxCsr_Mask, - ..Default::default() - }; - let xstate_bv = self.xstate_bv(); - if xstate_bv & 1 != 0 { - state.ControlWord = saved.ControlWord; - state.StatusWord = saved.StatusWord; - state.TagWord = saved.TagWord; - state.ErrorOpcode = saved.ErrorOpcode; - state.ErrorOffset = saved.ErrorOffset; - state.ErrorSelector = saved.ErrorSelector; - state.DataOffset = saved.DataOffset; - state.DataSelector = saved.DataSelector; - state.FloatRegisters = saved.FloatRegisters; - } - if xstate_bv & 2 != 0 { - state.XmmRegisters = saved.XmmRegisters; - } - state - } + // SAFETY: The context owns initialized XSTATE storage for the enabled features. + let ok = unsafe { SetXStateFeaturesMask(context, xstate_bv & layout.mask) }; + assert_ne!(ok, 0, "SetXStateFeaturesMask failed"); } /// Represents an extended CPU context, including the XSAVE area. @@ -747,7 +642,10 @@ struct TlsState { impl TlsState { /// Creates a new `TlsState` with all fields zeroed / defaulted. fn new() -> Self { - let mut guest_xsave_area = XsaveArea::initial_guest(); + Self::with_guest_xsave_area(XsaveArea::initial(XsaveLayout::get())) + } + + fn with_guest_xsave_area(mut guest_xsave_area: XsaveArea) -> Self { let guest_xsave_ptr = guest_xsave_area.as_mut_ptr(); Self { host_sp: Cell::new(core::ptr::null_mut()), @@ -886,11 +784,9 @@ syscall_callback: mov r10, QWORD PTR [r11 + {GUEST_XSAVE_PTR}] xsave64 [r10] mov BYTE PTR [r11 + {GUEST_XSTATE_FORMAT}], 0 - // Clear the x87 exception flags and restore the Windows ABI's standard host x87 control word and mxcsr. - fnclex - fldcw WORD PTR [rip + {HOST_X87_CONTROL_WORD}] - ldmxcsr DWORD PTR [rip + {HOST_MXCSR}] - +", + restore_host_fp_controls!(), +" /// Reestablish the stack and frame pointers. mov rsp, [r11 + {HOST_SP}] mov rbp, [r11 + {HOST_BP}] @@ -909,10 +805,9 @@ syscall_callback: jmp .Ldone exception_callback: - // Clear the x87 exception flags and restore the Windows ABI's standard host x87 control word and mxcsr. - fnclex - fldcw WORD PTR [rip + {HOST_X87_CONTROL_WORD}] - ldmxcsr DWORD PTR [rip + {HOST_MXCSR}] +", + restore_host_fp_controls!(), +" // Handle the exception. The stack and frame pointers are already restored, // and the guest context is up to date. rcx contains a pointer to the // guest pt_regs, and rdx contains a pointer to the exception record. @@ -936,10 +831,9 @@ interrupt_callback: mov BYTE PTR [r11 + {IS_IN_GUEST}], 0 mov rsp, [r11 + {HOST_SP}] mov rbp, [r11 + {HOST_BP}] - // Clear the x87 exception flags and restore the Windows ABI's standard host x87 control word and mxcsr. - fnclex - fldcw WORD PTR [rip + {HOST_X87_CONTROL_WORD}] - ldmxcsr DWORD PTR [rip + {HOST_MXCSR}] +", + restore_host_fp_controls!(), +" cmp BYTE PTR [rip + {GUEST_TLS_MODE}], {WINDOWS_TLS_MODE} jne 2f mov rcx, r11 @@ -1114,7 +1008,7 @@ unsafe extern "C" fn switch_to_guest(ctx: &litebox_common_linux::PtRegs) -> ! { win_ctx.R15 = ctx.r15 as u64; win_ctx.Rip = ctx.rip as u64; if native_xstate { - (*tls.guest_xsave_area.get()).restore_to_context(win_ctx); + restore_xsave_to_context(&*tls.guest_xsave_area.get(), win_ctx); } } tls.guest_xstate_format.set(GuestXstateFormat::Windows); @@ -1184,11 +1078,36 @@ fn thread_start( dyn litebox::shim::InitThread, >, mut ctx: litebox_common_linux::PtRegs, + guest_xsave_area: XsaveArea, ) { // Allow caller to run some code before we return to the new thread. let shim = init_thread.init(); - run_thread_inner(shim.as_ref(), &mut ctx); + run_thread_inner_with_xstate(shim.as_ref(), &mut ctx, guest_xsave_area); +} + +impl litebox::platform::GuestVectorStateProvider for WindowsUserland { + type GuestVectorState = XsaveArea; + + fn get_guest_vector_state(&self) -> Self::GuestVectorState { + let tls = get_tls_ptr().expect("current thread is not managed by LiteBox"); + // SAFETY: managed threads own their TLS until `run_thread_arch` returns, + // and guest state is accessed only while execution is on the host. + unsafe { (&*(*tls).guest_xsave_area.get()).clone() } + } + + fn set_guest_vector_state(&self, state: &Self::GuestVectorState) { + let tls = get_tls_ptr().expect("current thread is not managed by LiteBox"); + // SAFETY: managed threads own their TLS until `run_thread_arch` returns, + // and guest state is accessed only while execution is on the host. + let tls = unsafe { &*tls }; + // SAFETY: the guest XSAVE area has no overlapping access while on the host. + unsafe { *tls.guest_xsave_area.get() = state.clone() }; + // Replacing `XsaveArea` changes its backing allocation. + let pointer = unsafe { (&mut *tls.guest_xsave_area.get()).as_mut_ptr() }; + tls.guest_xsave_ptr.set(pointer); + tls.guest_xstate_format.set(GuestXstateFormat::Native); + } } impl litebox::platform::ThreadProvider for WindowsUserland { @@ -1204,8 +1123,11 @@ impl litebox::platform::ThreadProvider for WindowsUserland, ) -> Result<(), Self::ThreadSpawnError> { let ctx = ctx.clone(); + let guest_xsave_area = + litebox::platform::GuestVectorStateProvider::get_guest_vector_state(self); // TODO: do we need to wait for the handle in the main thread? - let _handle = std::thread::Builder::new().spawn(move || thread_start(init_thread, ctx))?; + let _handle = std::thread::Builder::new() + .spawn(move || thread_start(init_thread, ctx, guest_xsave_area))?; Ok(()) } @@ -2436,19 +2358,12 @@ mod tests { self.calls.set(call); // SAFETY: This is the active guest's host callback; capture has finished. let area = unsafe { &*(*crate::get_tls_ptr().unwrap()).guest_xsave_area.get() }; - let legacy = area.legacy_state_for_context(); - assert_eq!(legacy.ControlWord, TEST_CW); - assert_eq!(legacy.StatusWord & pending_status, pending_status); - assert_eq!(legacy.MxCsr, TEST_MXCSR); + let legacy = area.materialized_legacy_area(); + assert_eq!(legacy.control_word, TEST_CW); + assert_eq!(legacy.status_word & pending_status, pending_status); + assert_eq!(legacy.mxcsr, TEST_MXCSR); let expected = if call >= 4 { 0 } else { 0x5a }; - assert_eq!( - legacy.XmmRegisters[0].Low, - u64::from_le_bytes([expected; 8]) - ); - assert_eq!( - legacy.XmmRegisters[0].High.cast_unsigned(), - u64::from_le_bytes([expected; 8]) - ); + assert_eq!(legacy.xmm_registers[0], [expected; 16]); if self.avx { let component = XsaveLayout::get() .components diff --git a/litebox_shim_linux/src/syscalls/process.rs b/litebox_shim_linux/src/syscalls/process.rs index f49fb72c41..76e90a46cb 100644 --- a/litebox_shim_linux/src/syscalls/process.rs +++ b/litebox_shim_linux/src/syscalls/process.rs @@ -1212,10 +1212,6 @@ impl Task { /// `MADV_DONTFORK` regions are copied too. Like Linux, the child has a single thread, a copy /// of the calling one; the parent's other threads pause while the parent is copied. /// - /// Floating-point and vector state is copied only on AArch64; x86-64 needs a real - /// [`GuestVectorStateProvider`](litebox::platform::GuestVectorStateProvider) and a payload - /// encoding first. - /// /// Only a process outside a `vfork` window, with default resource-limit and alarm state, no /// shared memory mappings, no ELF file mid-load, and only descriptors a fresh runner can /// inherit is duplicated; otherwise this fails with `EAGAIN`.